机架管理前后端（rack/）：FastAPI + SQLite + Vue3，经统一门户上线
- 后端 prrack：MQTT（phonerail-backend @ 127.0.0.1:1884）↔ 遥测入库 / 事件增量同步 / WebSocket 推送； 命令白名单 + 重传同 seq；审计记门户 X-Auth-User（只绑 127.0.0.1，main.py 拒绝非回环）。 - 自动恢复：只看电流（平线 / 过低，服务器够不到手机 adb），先长按电源再断电重启， 每小时限次后停手；默认逐台关闭。adb 探活接口 recovery.Prober 预留。 - 前端：总览 / 板子页（五路操作、电流电压曲线）/ 事件 / 操作记录 / 自动恢复；手机宽度可用。 - 部署 deploy/deploy.sh：服务器 Python 3.8 上先跑测试再替换；systemd phonerail-rack，每日备份。 - 门户应用 id 16、key rack（phonerail 已被需求文档站占用），授权 admin + staff。 测试 26 例（本机 3.14 / 服务器 3.8 均过）。线上板 b81f3fc9c8bc 在线，WS 经门户 101 + 1 条/s 遥测。 Co-Authored-By: Claude Opus 5.5 <[email protected]>
这个提交包含在：
zhou yang
@@ -0,0 +1,194 @@
|
||||
"""HTTP + WebSocket 接口。前端静态文件也由这里托管（同源，门户 vhost 直接反代到本进程）。
|
||||
|
||||
身份：统一门户在 vhost 里**覆盖式**注入 `X-Auth-User`（gen-vhosts.py），本进程只绑 127.0.0.1，
|
||||
所以这个头只可能来自门户 —— 与 evse 一样按 L2 信任它，用来记审计「是谁按的」。
|
||||
直接在服务器本机 curl（没有这个头）记作 `local`。
|
||||
⚠️ 这个信任**只在绑 127.0.0.1 时成立**。哪天改绑 0.0.0.0，外面加一个 `X-Auth-User: admin`
|
||||
就能冒名写审计 —— 所以 main.py 拒绝非回环地址的绑定。
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import Awaitable, Callable, List, Optional
|
||||
|
||||
from fastapi import FastAPI, HTTPException, Request, WebSocket, WebSocketDisconnect
|
||||
from fastapi.responses import FileResponse, JSONResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
|
||||
from . import protocol
|
||||
from .db import now_ms
|
||||
from .hub import CmdTimeout, Hub
|
||||
from .recovery import Params, RecoveryManager
|
||||
|
||||
|
||||
def _user(req: Request) -> str:
|
||||
return req.headers.get("x-auth-user") or "local"
|
||||
|
||||
|
||||
def _board_or_404(hub: Hub, board: str):
|
||||
if board not in hub.latest:
|
||||
raise HTTPException(404, "没有这块板子：%s" % board)
|
||||
|
||||
|
||||
Hook = Callable[[], Awaitable[None]]
|
||||
|
||||
|
||||
def create_app(hub: Hub, rec: RecoveryManager, static_dir: Optional[str] = None,
|
||||
startup: Optional[List[Hook]] = None, shutdown: Optional[List[Hook]] = None) -> FastAPI:
|
||||
# 用 lifespan 而不是 on_event：后者在新版 Starlette 里已经删了，服务器上的旧版两种都认。
|
||||
@asynccontextmanager
|
||||
async def lifespan(app):
|
||||
for fn in startup or []:
|
||||
await fn()
|
||||
yield
|
||||
for fn in shutdown or []:
|
||||
await fn()
|
||||
|
||||
app = FastAPI(title="PhoneRail 机架管理", docs_url="/api/docs", openapi_url="/api/openapi.json",
|
||||
lifespan=lifespan)
|
||||
|
||||
@app.get("/api/health")
|
||||
def health():
|
||||
return {"ok": True, "mqtt": hub.mqtt_up, "boards": len(hub.latest), "ts": now_ms()}
|
||||
|
||||
@app.get("/api/me")
|
||||
def me(req: Request):
|
||||
return {"user": _user(req), "roles": req.headers.get("x-auth-roles", "")}
|
||||
|
||||
# ---- 板子 ----------------------------------------------------------------
|
||||
@app.get("/api/boards")
|
||||
def boards():
|
||||
return hub.boards()
|
||||
|
||||
@app.get("/api/boards/{board}")
|
||||
def board(board: str):
|
||||
_board_or_404(hub, board)
|
||||
return hub.snapshot(board)
|
||||
|
||||
@app.patch("/api/boards/{board}")
|
||||
async def patch_board(board: str, req: Request):
|
||||
_board_or_404(hub, board)
|
||||
body = await req.json()
|
||||
for k in ("name", "location"):
|
||||
if k in body:
|
||||
hub.db.x("UPDATE boards SET %s=? WHERE id=?" % k, (str(body[k])[:64], board))
|
||||
snap = hub.snapshot(board)
|
||||
hub.broadcast({"t": "board", "board": snap})
|
||||
return snap
|
||||
|
||||
@app.patch("/api/boards/{board}/rails/{rail}")
|
||||
async def patch_phone(board: str, rail: int, req: Request):
|
||||
_board_or_404(hub, board)
|
||||
if not 1 <= rail <= protocol.RAILS:
|
||||
raise HTTPException(400, "rail 必须是 1~5")
|
||||
body = await req.json()
|
||||
for k in ("label", "adb_addr", "notes"):
|
||||
if k in body:
|
||||
hub.db.x("UPDATE phones SET %s=? WHERE board_id=? AND rail=?" % k,
|
||||
(str(body[k])[:128], board, rail))
|
||||
if "auto_recover" in body:
|
||||
hub.db.x("UPDATE phones SET auto_recover=? WHERE board_id=? AND rail=?",
|
||||
(1 if body["auto_recover"] else 0, board, rail))
|
||||
rec.reload()
|
||||
snap = hub.snapshot(board)
|
||||
hub.broadcast({"t": "board", "board": snap})
|
||||
return snap
|
||||
|
||||
@app.post("/api/boards/{board}/cmd")
|
||||
async def command(board: str, req: Request):
|
||||
_board_or_404(hub, board)
|
||||
try:
|
||||
cmd = protocol.build_command(await req.json())
|
||||
except (protocol.CmdError, ValueError) as e:
|
||||
raise HTTPException(400, str(e))
|
||||
try:
|
||||
return await hub.call(board, cmd, _user(req))
|
||||
except CmdTimeout as e:
|
||||
raise HTTPException(504, str(e))
|
||||
|
||||
@app.get("/api/boards/{board}/rails/{rail}/history")
|
||||
def history(board: str, rail: int, start: int, end: Optional[int] = None):
|
||||
_board_or_404(hub, board)
|
||||
return hub.db.history(board, rail, start, end or now_ms())
|
||||
|
||||
# ---- 事件 / 审计 / 恢复 ----------------------------------------------------
|
||||
@app.get("/api/events")
|
||||
def events(board: Optional[str] = None, limit: int = 200, before: Optional[int] = None):
|
||||
sql = "SELECT * FROM events WHERE 1=1"
|
||||
args = []
|
||||
if board:
|
||||
sql += " AND board_id=?"
|
||||
args.append(board)
|
||||
if before:
|
||||
sql += " AND received<?"
|
||||
args.append(before)
|
||||
sql += " ORDER BY received DESC, seq DESC LIMIT ?"
|
||||
args.append(min(limit, 1000))
|
||||
out = []
|
||||
for r in hub.db.q(sql, args):
|
||||
d = dict(r)
|
||||
code, zh = protocol.EVENT_TYPES.get(d["type"], ("?", "未知事件 %d" % d["type"]))
|
||||
d.update(code=code, name=zh, severe=d["type"] in protocol.EVENT_SEVERE)
|
||||
if d["type"] == 13:
|
||||
d["name"] += "（%s）" % protocol.CMD_NAMES.get(d["arg"], d["arg"])
|
||||
out.append(d)
|
||||
return out
|
||||
|
||||
@app.get("/api/audit")
|
||||
def audit(limit: int = 200, board: Optional[str] = None):
|
||||
sql, args = "SELECT * FROM audit", []
|
||||
if board:
|
||||
sql, args = sql + " WHERE board_id=?", [board]
|
||||
rows = hub.db.q(sql + " ORDER BY id DESC LIMIT ?", args + [min(limit, 1000)])
|
||||
return [dict(r, request=json.loads(r["request"]), response=json.loads(r["response"]))
|
||||
for r in rows]
|
||||
|
||||
@app.get("/api/recovery")
|
||||
def recovery():
|
||||
from dataclasses import asdict
|
||||
log = hub.db.q("SELECT * FROM recovery_log ORDER BY id DESC LIMIT 200")
|
||||
return {"params": asdict(rec.params), "states": rec.states(), "log": [dict(r) for r in log]}
|
||||
|
||||
@app.put("/api/recovery/params")
|
||||
async def recovery_params(req: Request):
|
||||
from dataclasses import asdict
|
||||
p = Params.load(json.dumps(await req.json()))
|
||||
rec.set_params(p)
|
||||
return asdict(p)
|
||||
|
||||
@app.post("/api/recovery/{board}/{rail}/reset")
|
||||
def recovery_reset(board: str, rail: int):
|
||||
rec.reset(board, rail)
|
||||
return {"ok": True}
|
||||
|
||||
# ---- 实时推送 --------------------------------------------------------------
|
||||
@app.websocket("/api/ws")
|
||||
async def ws(sock: WebSocket):
|
||||
await sock.accept()
|
||||
q = hub.subscribe_client()
|
||||
try:
|
||||
await sock.send_json({"t": "hello", "boards": hub.boards(), "mqtt": hub.mqtt_up,
|
||||
"user": sock.headers.get("x-auth-user") or "local"})
|
||||
while hub.is_subscribed(q):
|
||||
try:
|
||||
msg = await asyncio.wait_for(q.get(), 20)
|
||||
except asyncio.TimeoutError:
|
||||
msg = {"t": "ping", "ts": now_ms()} # 保活，顺带让前端判断连接还活着
|
||||
await sock.send_json(msg)
|
||||
except WebSocketDisconnect:
|
||||
pass
|
||||
finally:
|
||||
hub.unsubscribe_client(q)
|
||||
|
||||
# ---- 前端 ------------------------------------------------------------------
|
||||
if static_dir:
|
||||
app.mount("/assets", StaticFiles(directory=static_dir + "/assets"), name="assets")
|
||||
|
||||
@app.get("/{path:path}")
|
||||
def spa(path: str):
|
||||
# index.html 不许缓存（home-server-deploy §关键约定 8 的教训）；assets 带哈希，可以长缓存
|
||||
resp = FileResponse(static_dir + "/index.html")
|
||||
resp.headers["Cache-Control"] = "no-cache"
|
||||
return resp
|
||||
|
||||
return app
|
||||
@@ -0,0 +1,137 @@
|
||||
"""MQTT 总线的薄抽象：真实的 paho，和测试 / 本机开发用的进程内假 broker。
|
||||
|
||||
Hub 只认三件事：`publish(topic, payload, qos, retain)`、`subscribe(pattern)`、
|
||||
收到消息时回调 `on_message(topic, payload_bytes)`。回调**可能在任意线程**里被调
|
||||
（paho 的网络线程），Hub 自己负责把它搬回 asyncio 循环。
|
||||
"""
|
||||
import logging
|
||||
import threading
|
||||
from typing import Callable, Dict, List, Optional, Tuple
|
||||
|
||||
log = logging.getLogger("prrack.bus")
|
||||
|
||||
OnMessage = Callable[[str, bytes], None]
|
||||
|
||||
|
||||
def topic_matches(pattern: str, topic: str) -> bool:
|
||||
"""MQTT 通配：`+` 一级，`#` 余下全部。"""
|
||||
p, t = pattern.split("/"), topic.split("/")
|
||||
for i, seg in enumerate(p):
|
||||
if seg == "#":
|
||||
return True
|
||||
if i >= len(t) or (seg != "+" and seg != t[i]):
|
||||
return False
|
||||
return len(p) == len(t)
|
||||
|
||||
|
||||
class Bus:
|
||||
on_message: Optional[OnMessage] = None
|
||||
on_connect: Optional[Callable[[bool], None]] = None
|
||||
|
||||
def start(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def stop(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def publish(self, topic: str, payload: str, qos: int = 1, retain: bool = False):
|
||||
raise NotImplementedError
|
||||
|
||||
def subscribe(self, pattern: str, qos: int = 1):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class PahoBus(Bus):
|
||||
def __init__(self, host: str, port: int, user: str, password: str, client_id: str):
|
||||
import paho.mqtt.client as mqtt # 只在真连 broker 时才需要
|
||||
|
||||
self._subs: List[Tuple[str, int]] = []
|
||||
self._c = mqtt.Client(mqtt.CallbackAPIVersion.VERSION2, client_id=client_id,
|
||||
clean_session=True)
|
||||
self._c.username_pw_set(user, password)
|
||||
self._c.reconnect_delay_set(1, 30)
|
||||
self._c.on_connect = self._on_connect
|
||||
self._c.on_disconnect = self._on_disconnect
|
||||
self._c.on_message = lambda c, u, m: self.on_message and self.on_message(m.topic, m.payload)
|
||||
self._host, self._port = host, port
|
||||
|
||||
def _on_connect(self, c, userdata, flags, reason, props=None):
|
||||
ok = not reason.is_failure
|
||||
log.info("MQTT %s（%s）", "已连上" if ok else "连接被拒", reason)
|
||||
if ok:
|
||||
# clean_session：每次重连都要重新订阅，否则断一次线就再也收不到东西
|
||||
for pat, qos in self._subs:
|
||||
c.subscribe(pat, qos)
|
||||
if self.on_connect:
|
||||
self.on_connect(ok)
|
||||
|
||||
def _on_disconnect(self, c, userdata, flags, reason, props=None):
|
||||
log.warning("MQTT 断开（%s），paho 会自动重连", reason)
|
||||
if self.on_connect:
|
||||
self.on_connect(False)
|
||||
|
||||
def start(self):
|
||||
self._c.connect_async(self._host, self._port, keepalive=30)
|
||||
self._c.loop_start()
|
||||
|
||||
def stop(self):
|
||||
self._c.loop_stop()
|
||||
self._c.disconnect()
|
||||
|
||||
def publish(self, topic: str, payload: str, qos: int = 1, retain: bool = False):
|
||||
self._c.publish(topic, payload, qos=qos, retain=retain)
|
||||
|
||||
def subscribe(self, pattern: str, qos: int = 1):
|
||||
self._subs.append((pattern, qos))
|
||||
self._c.subscribe(pattern, qos)
|
||||
|
||||
|
||||
class MemBroker:
|
||||
"""进程内的假 broker：多个 MemBus 挂在同一个上面互相收发，支持 retained。
|
||||
投递是**同步**的（publish 返回前订阅者已经收到），测试因此是确定的。"""
|
||||
|
||||
def __init__(self):
|
||||
self._lock = threading.RLock()
|
||||
self.clients: List["MemBus"] = []
|
||||
self.retained: Dict[str, str] = {}
|
||||
self.log: List[Tuple[str, str]] = []
|
||||
|
||||
def deliver(self, topic: str, payload: str, retain: bool):
|
||||
with self._lock:
|
||||
self.log.append((topic, payload))
|
||||
if retain:
|
||||
self.retained[topic] = payload
|
||||
targets = [c for c in self.clients if c.wants(topic)]
|
||||
for c in targets:
|
||||
if c.on_message:
|
||||
c.on_message(topic, payload.encode())
|
||||
|
||||
|
||||
class MemBus(Bus):
|
||||
def __init__(self, broker: MemBroker):
|
||||
self.broker = broker
|
||||
self.subs: List[str] = []
|
||||
self.up = False
|
||||
|
||||
def wants(self, topic: str) -> bool:
|
||||
return self.up and any(topic_matches(p, topic) for p in self.subs)
|
||||
|
||||
def start(self):
|
||||
self.up = True
|
||||
self.broker.clients.append(self)
|
||||
if self.on_connect:
|
||||
self.on_connect(True)
|
||||
|
||||
def stop(self):
|
||||
self.up = False
|
||||
if self in self.broker.clients:
|
||||
self.broker.clients.remove(self)
|
||||
|
||||
def publish(self, topic: str, payload: str, qos: int = 1, retain: bool = False):
|
||||
self.broker.deliver(topic, payload, retain)
|
||||
|
||||
def subscribe(self, pattern: str, qos: int = 1):
|
||||
self.subs.append(pattern)
|
||||
for t, p in list(self.broker.retained.items()):
|
||||
if topic_matches(pattern, t) and self.on_message:
|
||||
self.on_message(t, p.encode())
|
||||
@@ -0,0 +1,181 @@
|
||||
"""SQLite 存储（WAL）。一个文件，备份就是拷它（同门户 / evse 的做法）。
|
||||
|
||||
数据量估算：一块板 5 路 × 1 条/s = 43 万行/天。原始 1 s 数据只留 `RAW_KEEP_S`（2 天），
|
||||
每分钟降采样一行（均值 / 最小 / 最大）留 `MIN_KEEP_S`（90 天）——
|
||||
电流曲线的用途是「这台手机前天夜里是不是卡死过」，一分钟的 min/max 足够看出来。
|
||||
事件、审计、恢复记录**永久保留**（量小，而且是排查的唯一线索）。
|
||||
|
||||
⚠️ 时间戳一律是**服务器收到的时刻**（毫秒）。板子没有墙钟（需求 §10.12：
|
||||
上电从 0 计），它自己的 `t_ms` 只存在事件表的 `t_ms` 列里，作参考。
|
||||
"""
|
||||
import sqlite3
|
||||
import threading
|
||||
import time
|
||||
from typing import Iterable, List, Optional, Sequence
|
||||
|
||||
RAW_KEEP_S = 2 * 86400
|
||||
MIN_KEEP_S = 90 * 86400
|
||||
|
||||
SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS boards (
|
||||
id TEXT PRIMARY KEY, -- STA MAC 12 位小写十六进制
|
||||
name TEXT NOT NULL DEFAULT '',
|
||||
location TEXT NOT NULL DEFAULT '',
|
||||
fw TEXT NOT NULL DEFAULT '',
|
||||
ip TEXT NOT NULL DEFAULT '',
|
||||
online INTEGER NOT NULL DEFAULT 0,
|
||||
last_seen INTEGER NOT NULL DEFAULT 0,
|
||||
ev_cursor INTEGER NOT NULL DEFAULT 0, -- 下一条要拉的事件序号
|
||||
created INTEGER NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS phones (
|
||||
board_id TEXT NOT NULL,
|
||||
rail INTEGER NOT NULL,
|
||||
label TEXT NOT NULL DEFAULT '',
|
||||
adb_addr TEXT NOT NULL DEFAULT '', -- 预留：服务器够得着手机时填 host:port
|
||||
auto_recover INTEGER NOT NULL DEFAULT 0, -- 默认关：逐台打开
|
||||
notes TEXT NOT NULL DEFAULT '',
|
||||
PRIMARY KEY (board_id, rail)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS tele (
|
||||
ts INTEGER NOT NULL, board_id TEXT NOT NULL, rail INTEGER NOT NULL,
|
||||
mv INTEGER, ua INTEGER, en INTEGER, vb INTEGER, pr INTEGER, pg INTEGER, flt INTEGER
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS tele_idx ON tele (board_id, rail, ts);
|
||||
CREATE TABLE IF NOT EXISTS tele_min (
|
||||
ts INTEGER NOT NULL, board_id TEXT NOT NULL, rail INTEGER NOT NULL,
|
||||
mv_avg INTEGER, ua_avg INTEGER, ua_min INTEGER, ua_max INTEGER, n INTEGER,
|
||||
PRIMARY KEY (board_id, rail, ts)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS board_tele (
|
||||
ts INTEGER NOT NULL, board_id TEXT NOT NULL,
|
||||
in_mv INTEGER, in_ua INTEGER, temp0 INTEGER, temp1 INTEGER, fan_duty INTEGER
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS board_tele_idx ON board_tele (board_id, ts);
|
||||
CREATE TABLE IF NOT EXISTS events (
|
||||
board_id TEXT NOT NULL, seq INTEGER NOT NULL,
|
||||
t_ms INTEGER, rail INTEGER, type INTEGER, arg INTEGER, mv INTEGER, ua INTEGER,
|
||||
received INTEGER NOT NULL,
|
||||
PRIMARY KEY (board_id, seq)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS events_recv_idx ON events (received);
|
||||
CREATE TABLE IF NOT EXISTS audit (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
ts INTEGER NOT NULL, user TEXT NOT NULL, board_id TEXT NOT NULL, rail INTEGER,
|
||||
request TEXT NOT NULL, response TEXT NOT NULL, ok INTEGER NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS recovery_log (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
ts INTEGER NOT NULL, board_id TEXT NOT NULL, rail INTEGER NOT NULL,
|
||||
state TEXT NOT NULL, reason TEXT NOT NULL, detail TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS settings (
|
||||
key TEXT PRIMARY KEY, value TEXT NOT NULL
|
||||
);
|
||||
"""
|
||||
|
||||
|
||||
def now_ms() -> int:
|
||||
return int(time.time() * 1000)
|
||||
|
||||
|
||||
class Db:
|
||||
"""一个连接 + 一把锁。后端是单进程（uvicorn 单 worker），写入量是每秒几十行，
|
||||
不值得上连接池；锁是为了 paho 线程之外偶尔还有线程池里的查询。"""
|
||||
|
||||
def __init__(self, path: str):
|
||||
self.path = path
|
||||
self._lock = threading.Lock()
|
||||
self.conn = sqlite3.connect(path, check_same_thread=False, isolation_level=None)
|
||||
self.conn.row_factory = sqlite3.Row
|
||||
self.conn.execute("PRAGMA journal_mode=WAL")
|
||||
self.conn.execute("PRAGMA synchronous=NORMAL")
|
||||
self.conn.executescript(SCHEMA)
|
||||
|
||||
def close(self):
|
||||
with self._lock:
|
||||
self.conn.close()
|
||||
|
||||
# ---- 通用 ----------------------------------------------------------
|
||||
def q(self, sql: str, args: Sequence = ()) -> List[sqlite3.Row]:
|
||||
with self._lock:
|
||||
return self.conn.execute(sql, args).fetchall()
|
||||
|
||||
def x(self, sql: str, args: Sequence = ()) -> int:
|
||||
with self._lock:
|
||||
cur = self.conn.execute(sql, args)
|
||||
return cur.lastrowid
|
||||
|
||||
def many(self, sql: str, rows: Iterable[Sequence]):
|
||||
with self._lock:
|
||||
self.conn.execute("BEGIN")
|
||||
try:
|
||||
self.conn.executemany(sql, rows)
|
||||
self.conn.execute("COMMIT")
|
||||
except Exception:
|
||||
self.conn.execute("ROLLBACK")
|
||||
raise
|
||||
|
||||
# ---- 板子 ----------------------------------------------------------
|
||||
def ensure_board(self, board: str):
|
||||
self.x("INSERT OR IGNORE INTO boards (id, created) VALUES (?, ?)", (board, now_ms()))
|
||||
self.many("INSERT OR IGNORE INTO phones (board_id, rail) VALUES (?, ?)",
|
||||
[(board, r) for r in range(1, 6)])
|
||||
|
||||
def set_board_state(self, board: str, online: bool, fw: str = None, ip: str = None):
|
||||
self.ensure_board(board)
|
||||
self.x("UPDATE boards SET online=?, fw=COALESCE(?, fw), ip=COALESCE(?, ip), last_seen=? "
|
||||
"WHERE id=?", (1 if online else 0, fw, ip, now_ms(), board))
|
||||
|
||||
def setting(self, key: str, default: str) -> str:
|
||||
rows = self.q("SELECT value FROM settings WHERE key=?", (key,))
|
||||
return rows[0]["value"] if rows else default
|
||||
|
||||
def set_setting(self, key: str, value: str):
|
||||
self.x("INSERT INTO settings (key, value) VALUES (?, ?) "
|
||||
"ON CONFLICT(key) DO UPDATE SET value=excluded.value", (key, value))
|
||||
|
||||
# ---- 遥测 ----------------------------------------------------------
|
||||
def downsample(self, upto_ms: Optional[int] = None) -> int:
|
||||
"""把还没聚合的整分钟原始数据聚成 tele_min。幂等（主键冲突就跳过）。
|
||||
只聚到 `upto_ms` 之前的**整分钟**：正在进行的这一分钟数据还不全。"""
|
||||
upto = (upto_ms if upto_ms is not None else now_ms()) // 60000 * 60000
|
||||
with self._lock:
|
||||
row = self.conn.execute("SELECT MAX(ts) FROM tele_min").fetchone()
|
||||
start = (row[0] + 60000) if row[0] is not None else 0
|
||||
cur = self.conn.execute(
|
||||
"INSERT OR IGNORE INTO tele_min (ts, board_id, rail, mv_avg, ua_avg, ua_min, ua_max, n) "
|
||||
"SELECT ts/60000*60000 AS m, board_id, rail, CAST(AVG(mv) AS INTEGER), "
|
||||
"CAST(AVG(ua) AS INTEGER), MIN(ua), MAX(ua), COUNT(*) FROM tele "
|
||||
"WHERE ts >= ? AND ts < ? AND ua IS NOT NULL GROUP BY m, board_id, rail",
|
||||
(start, upto))
|
||||
return cur.rowcount
|
||||
|
||||
def prune(self, now: Optional[int] = None):
|
||||
t = now if now is not None else now_ms()
|
||||
with self._lock:
|
||||
self.conn.execute("DELETE FROM tele WHERE ts < ?", (t - RAW_KEEP_S * 1000,))
|
||||
self.conn.execute("DELETE FROM board_tele WHERE ts < ?", (t - RAW_KEEP_S * 1000,))
|
||||
self.conn.execute("DELETE FROM tele_min WHERE ts < ?", (t - MIN_KEEP_S * 1000,))
|
||||
|
||||
def history(self, board: str, rail: int, t0: int, t1: int, max_points: int = 1500) -> dict:
|
||||
"""[t0, t1) 的电压电流曲线。跨度 ≤ 原始数据保留期且点数不多时回原始 1 s 数据，
|
||||
否则回分钟聚合（带 min/max，界面画成带状）。点太多时按时间桶再抽一次。"""
|
||||
span = max(1, t1 - t0)
|
||||
raw_ok = t0 >= now_ms() - RAW_KEEP_S * 1000 and span <= 6 * 3600 * 1000
|
||||
if raw_ok:
|
||||
bucket = max(1000, span // max_points)
|
||||
rows = self.q(
|
||||
"SELECT ts/?*? AS t, CAST(AVG(mv) AS INTEGER) mv, CAST(AVG(ua) AS INTEGER) ua, "
|
||||
"MIN(ua) ua_min, MAX(ua) ua_max FROM tele WHERE board_id=? AND rail=? "
|
||||
"AND ts>=? AND ts<? GROUP BY t ORDER BY t", (bucket, bucket, board, rail, t0, t1))
|
||||
res = "raw"
|
||||
else:
|
||||
bucket = max(60000, span // max_points // 60000 * 60000)
|
||||
rows = self.q(
|
||||
"SELECT ts/?*? AS t, CAST(AVG(mv_avg) AS INTEGER) mv, CAST(AVG(ua_avg) AS INTEGER) ua, "
|
||||
"MIN(ua_min) ua_min, MAX(ua_max) ua_max FROM tele_min WHERE board_id=? AND rail=? "
|
||||
"AND ts>=? AND ts<? GROUP BY t ORDER BY t", (bucket, bucket, board, rail, t0, t1))
|
||||
res = "minute"
|
||||
return {"resolution": res, "bucket_ms": bucket,
|
||||
"points": [[r["t"], r["mv"], r["ua"], r["ua_min"], r["ua_max"]] for r in rows]}
|
||||
@@ -0,0 +1,262 @@
|
||||
"""服务器侧的核心：MQTT ↔ 状态 ↔ 数据库 ↔ 网页推送。
|
||||
|
||||
四条规矩（都来自板侧协议，需求 §10.8）：
|
||||
1. **重传用同一个 seq。** 超时之后原样再发，板子的判重兜住；换 seq 重发 =
|
||||
「按 10 秒电源键」可能执行两遍。`call()` 是唯一的下发入口。
|
||||
2. **失联不动作。** 板子掉线、遥测中断，服务器只把它标成离线，没有任何自动动作会因此触发
|
||||
（自动恢复在板子离线时直接停摆，recovery.py）。
|
||||
3. **时间一律用服务器收到的时刻。** 板子没有墙钟。
|
||||
4. **事件按序号增量拉**（遥测里的 `ev_next` 比游标大就拉），一次 8 条、拉到追平为止；
|
||||
板子序号比游标还小 = 板子 flash 被清过，游标归零重拉。
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from typing import Callable, Dict, List, Optional, Set
|
||||
|
||||
from . import protocol
|
||||
from .bus import Bus
|
||||
from .db import Db, now_ms
|
||||
|
||||
log = logging.getLogger("prrack.hub")
|
||||
|
||||
STALE_MS = 10000 # 这么久没遥测 = 通信中断（遥测默认 1 s 一条）
|
||||
FLUSH_S = 2.0 # 遥测攒批写库
|
||||
MAINT_S = 60.0 # 降采样 + 清理
|
||||
|
||||
|
||||
class CmdTimeout(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class Hub:
|
||||
def __init__(self, db: Db, bus: Bus, attempt_timeout: float = 1.5, attempts: int = 3):
|
||||
self.db = db
|
||||
self.bus = bus
|
||||
self.attempt_timeout = attempt_timeout
|
||||
self.attempts = attempts
|
||||
self.seq = protocol.SeqGen()
|
||||
self.loop: Optional[asyncio.AbstractEventLoop] = None
|
||||
self.latest: Dict[str, dict] = {} # board → {status, last_tele, online, fw, ip}
|
||||
self.mqtt_up = False
|
||||
self._pending: Dict[tuple, asyncio.Future] = {}
|
||||
self._rows: List[tuple] = []
|
||||
self._brows: List[tuple] = []
|
||||
self._syncing: Set[str] = set()
|
||||
self._clients: Set[asyncio.Queue] = set()
|
||||
self._tasks: List[asyncio.Task] = []
|
||||
self.tele_observers: List[Callable[[str, int, dict], None]] = []
|
||||
|
||||
# ---- 生命周期 ------------------------------------------------------------
|
||||
async def start(self):
|
||||
self.loop = asyncio.get_running_loop()
|
||||
for r in self.db.q("SELECT id, fw, ip, online FROM boards"):
|
||||
self.latest[r["id"]] = {"status": None, "last_tele": 0, "online": False,
|
||||
"fw": r["fw"], "ip": r["ip"]}
|
||||
self.bus.on_message = self._from_bus
|
||||
self.bus.on_connect = lambda ok: self.loop.call_soon_threadsafe(self._set_mqtt, ok)
|
||||
self.bus.start()
|
||||
for kind in ("rsp", "tele", "state"):
|
||||
self.bus.subscribe("%s/+/%s" % (protocol.TOPIC_ROOT, kind), 0 if kind == "tele" else 1)
|
||||
self._tasks = [asyncio.ensure_future(self._flusher()), asyncio.ensure_future(self._maint())]
|
||||
|
||||
async def stop(self):
|
||||
for t in self._tasks:
|
||||
t.cancel()
|
||||
self.flush()
|
||||
self.bus.stop()
|
||||
|
||||
def _set_mqtt(self, ok: bool):
|
||||
self.mqtt_up = ok
|
||||
self.broadcast({"t": "mqtt", "up": ok})
|
||||
|
||||
# ---- 收：paho 线程 → asyncio ----------------------------------------------
|
||||
def _from_bus(self, t: str, payload: bytes):
|
||||
if self.loop is None:
|
||||
return
|
||||
self.loop.call_soon_threadsafe(self._handle, t, payload)
|
||||
|
||||
def _handle(self, t: str, payload: bytes):
|
||||
parsed = protocol.parse_topic(t)
|
||||
if parsed is None:
|
||||
return
|
||||
board, kind = parsed
|
||||
try:
|
||||
msg = json.loads(payload)
|
||||
except ValueError:
|
||||
log.warning("%s 的 %s 不是 JSON：%r", board, kind, payload[:80])
|
||||
return
|
||||
if kind == "rsp":
|
||||
fut = self._pending.get((board, msg.get("seq")))
|
||||
if fut is not None and not fut.done():
|
||||
fut.set_result(msg)
|
||||
elif kind == "tele":
|
||||
self._on_tele(board, msg)
|
||||
elif kind == "state":
|
||||
self._on_state(board, msg)
|
||||
|
||||
def _board(self, board: str) -> dict:
|
||||
b = self.latest.get(board)
|
||||
if b is None:
|
||||
self.db.ensure_board(board)
|
||||
b = self.latest[board] = {"status": None, "last_tele": 0, "online": False, "fw": "", "ip": ""}
|
||||
return b
|
||||
|
||||
def _on_state(self, board: str, msg: dict):
|
||||
b = self._board(board)
|
||||
b["online"] = bool(msg.get("online"))
|
||||
if b["online"]:
|
||||
b["fw"], b["ip"] = msg.get("fw", b["fw"]), msg.get("ip", b["ip"])
|
||||
self.db.set_board_state(board, b["online"], msg.get("fw"), msg.get("ip"))
|
||||
self.broadcast({"t": "board", "board": self.snapshot(board)})
|
||||
|
||||
def _on_tele(self, board: str, st: dict):
|
||||
ts = now_ms()
|
||||
b = self._board(board)
|
||||
b["status"], b["last_tele"] = st, ts
|
||||
if not b["online"]: # 遥测来了但 retained 的 state 还没到（或丢了）：以遥测为准
|
||||
b["online"] = True
|
||||
self.db.set_board_state(board, True, st.get("fw"))
|
||||
for r in st.get("rails", []):
|
||||
self._rows.append((ts, board, r.get("r"), r.get("mv"), r.get("ua"), r.get("en"),
|
||||
r.get("vb"), r.get("pr"), r.get("pg"), r.get("flt")))
|
||||
temp = st.get("temp") or [None, None]
|
||||
self._brows.append((ts, board, (st.get("in") or {}).get("mv"), (st.get("in") or {}).get("ua"),
|
||||
temp[0], temp[1] if len(temp) > 1 else None, (st.get("fan") or {}).get("duty")))
|
||||
for fn in self.tele_observers:
|
||||
try:
|
||||
fn(board, ts, st)
|
||||
except Exception: # 观察者的 bug 不能拖垮遥测
|
||||
log.exception("遥测观察者出错")
|
||||
self.broadcast({"t": "tele", "board": board, "ts": ts, "status": st})
|
||||
ev_next = st.get("ev_next")
|
||||
if isinstance(ev_next, int) and board not in self._syncing:
|
||||
cur = self.db.q("SELECT ev_cursor FROM boards WHERE id=?", (board,))[0]["ev_cursor"]
|
||||
if ev_next != cur:
|
||||
self._syncing.add(board)
|
||||
asyncio.ensure_future(self._sync_events(board, ev_next < cur))
|
||||
|
||||
# ---- 发 ----------------------------------------------------------------
|
||||
async def call(self, board: str, req: dict, user: str, audit: bool = True) -> dict:
|
||||
"""发一条命令并等回应。`req` 已经过 protocol.build_command 校验（或是内部命令）。
|
||||
超时按同一个 seq 重发 `attempts` 次；全部超时抛 CmdTimeout（审计里照样记一条）。"""
|
||||
seq = self.seq.next()
|
||||
msg = dict(req, seq=seq)
|
||||
payload = json.dumps(msg, separators=(",", ":"))
|
||||
fut = self.loop.create_future()
|
||||
self._pending[(board, seq)] = fut
|
||||
rsp = None
|
||||
try:
|
||||
for _ in range(self.attempts):
|
||||
self.bus.publish(protocol.topic(board, "cmd"), payload, qos=1)
|
||||
try:
|
||||
rsp = await asyncio.wait_for(asyncio.shield(fut), self.attempt_timeout)
|
||||
break
|
||||
except asyncio.TimeoutError:
|
||||
continue
|
||||
finally:
|
||||
self._pending.pop((board, seq), None)
|
||||
if audit and protocol.is_mutating(req.get("cmd", "")):
|
||||
self._audit(user, board, msg, rsp)
|
||||
if rsp is None:
|
||||
raise CmdTimeout("板子 %s 没有回应（seq %d，重发 %d 次）" % (board, seq, self.attempts))
|
||||
return rsp
|
||||
|
||||
def _audit(self, user: str, board: str, req: dict, rsp: Optional[dict]):
|
||||
ok = bool(rsp and rsp.get("ok"))
|
||||
body = rsp if rsp is not None else {"ok": False, "err": "timeout"}
|
||||
rid = self.db.x("INSERT INTO audit (ts, user, board_id, rail, request, response, ok) "
|
||||
"VALUES (?,?,?,?,?,?,?)", (now_ms(), user, board, req.get("rail"),
|
||||
json.dumps(req), json.dumps(body), 1 if ok else 0))
|
||||
self.broadcast({"t": "audit", "row": {"id": rid, "ts": now_ms(), "user": user, "board_id": board,
|
||||
"rail": req.get("rail"), "request": req,
|
||||
"response": body, "ok": ok}})
|
||||
|
||||
async def _sync_events(self, board: str, reset: bool):
|
||||
try:
|
||||
if reset:
|
||||
log.warning("板子 %s 的事件序号倒退了（flash 被清过？）：游标归零", board)
|
||||
self.db.x("UPDATE boards SET ev_cursor=0 WHERE id=?", (board,))
|
||||
for _ in range(50): # 一次最多追 400 条，剩下的下一条遥测再追
|
||||
cur = self.db.q("SELECT ev_cursor FROM boards WHERE id=?", (board,))[0]["ev_cursor"]
|
||||
rsp = await self.call(board, {"cmd": "events", "from": cur}, "system", audit=False)
|
||||
ev = rsp.get("ev") or []
|
||||
if not rsp.get("ok"):
|
||||
return
|
||||
now = now_ms()
|
||||
self.db.many("INSERT OR IGNORE INTO events (board_id, seq, t_ms, rail, type, arg, mv, ua, "
|
||||
"received) VALUES (?,?,?,?,?,?,?,?,?)", [(board, *e[:7], now) for e in ev])
|
||||
nxt = max([cur] + [e[0] + 1 for e in ev])
|
||||
if not ev and rsp.get("oldest", 0) > cur:
|
||||
nxt = rsp["oldest"] # 中间那段被板子的环形缓冲盖掉了
|
||||
self.db.x("UPDATE boards SET ev_cursor=? WHERE id=?", (nxt, board))
|
||||
if ev:
|
||||
self.broadcast({"t": "events", "board": board, "ev": ev, "received": now})
|
||||
if nxt >= rsp.get("next", nxt) or not ev:
|
||||
return
|
||||
except CmdTimeout:
|
||||
log.warning("拉 %s 的事件超时，等下一条遥测再试", board)
|
||||
finally:
|
||||
self._syncing.discard(board)
|
||||
|
||||
# ---- 查 ----------------------------------------------------------------
|
||||
def snapshot(self, board: str) -> dict:
|
||||
b = self._board(board)
|
||||
meta = self.db.q("SELECT name, location FROM boards WHERE id=?", (board,))
|
||||
phones = self.db.q("SELECT rail, label, adb_addr, auto_recover, notes FROM phones "
|
||||
"WHERE board_id=? ORDER BY rail", (board,))
|
||||
stale = b["last_tele"] == 0 or now_ms() - b["last_tele"] > STALE_MS
|
||||
return {"id": board, "name": meta[0]["name"] if meta else "", "location": meta[0]["location"] if meta else "",
|
||||
"online": b["online"], "stale": stale, "fw": b["fw"], "ip": b["ip"],
|
||||
"last_tele": b["last_tele"], "status": b["status"],
|
||||
"phones": [dict(p) for p in phones]}
|
||||
|
||||
def boards(self) -> List[dict]:
|
||||
return [self.snapshot(b) for b in sorted(self.latest)]
|
||||
|
||||
# ---- 推送 ----------------------------------------------------------------
|
||||
def subscribe_client(self) -> asyncio.Queue:
|
||||
q: asyncio.Queue = asyncio.Queue(maxsize=200)
|
||||
self._clients.add(q)
|
||||
return q
|
||||
|
||||
def unsubscribe_client(self, q: asyncio.Queue):
|
||||
self._clients.discard(q)
|
||||
|
||||
def broadcast(self, msg: dict):
|
||||
for q in list(self._clients):
|
||||
try:
|
||||
q.put_nowait(msg)
|
||||
except asyncio.QueueFull:
|
||||
# 慢客户端：丢掉它，让它重连后重新拉全量 —— 别让一个卡住的浏览器撑爆内存。
|
||||
# WS 协程发现自己不在集合里了就断开（is_subscribed）。
|
||||
self._clients.discard(q)
|
||||
|
||||
def is_subscribed(self, q: asyncio.Queue) -> bool:
|
||||
return q in self._clients
|
||||
|
||||
# ---- 后台 ----------------------------------------------------------------
|
||||
def flush(self):
|
||||
rows, brows = self._rows, self._brows
|
||||
self._rows, self._brows = [], []
|
||||
if rows:
|
||||
self.db.many("INSERT INTO tele VALUES (?,?,?,?,?,?,?,?,?,?)", rows)
|
||||
if brows:
|
||||
self.db.many("INSERT INTO board_tele VALUES (?,?,?,?,?,?,?)", brows)
|
||||
|
||||
async def _flusher(self):
|
||||
while True:
|
||||
await asyncio.sleep(FLUSH_S)
|
||||
try:
|
||||
self.flush()
|
||||
except Exception:
|
||||
log.exception("遥测写库失败")
|
||||
|
||||
async def _maint(self):
|
||||
while True:
|
||||
await asyncio.sleep(MAINT_S)
|
||||
try:
|
||||
self.db.downsample()
|
||||
self.db.prune()
|
||||
except Exception:
|
||||
log.exception("降采样 / 清理失败")
|
||||
@@ -0,0 +1,111 @@
|
||||
"""入口。
|
||||
|
||||
生产（服务器上，systemd 起）：
|
||||
python -m prrack.main --db /root/phonerail-rack/data/rack.db \\
|
||||
--mqtt-env /root/phonerail/mqtt.env --static /root/phonerail-rack/frontend/dist
|
||||
本机开发（不连 broker，起 N 块模拟板子，每秒一条遥测）：
|
||||
python -m prrack.main --sim 2 --db dev.db --static ../frontend/dist
|
||||
|
||||
⛔ 只许绑回环地址：api.py 按 L2 信任门户注入的 X-Auth-User，这个信任只在「外面够不着本进程」时成立。
|
||||
"""
|
||||
import argparse
|
||||
import asyncio
|
||||
import ipaddress
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
||||
import uvicorn
|
||||
|
||||
from .api import create_app
|
||||
from .bus import MemBroker, MemBus, PahoBus
|
||||
from .db import Db, now_ms
|
||||
from .hub import Hub
|
||||
from .recovery import RecoveryManager
|
||||
from .simboard import HUNG, SimBoard
|
||||
|
||||
log = logging.getLogger("prrack")
|
||||
|
||||
|
||||
def read_env(path: str) -> dict:
|
||||
out = {}
|
||||
with open(path, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line and not line.startswith("#") and "=" in line:
|
||||
k, v = line.split("=", 1)
|
||||
out[k.strip()] = v.strip().strip('"')
|
||||
return out
|
||||
|
||||
|
||||
def build(args):
|
||||
db = Db(args.db)
|
||||
sims = []
|
||||
if args.sim:
|
||||
broker = MemBroker()
|
||||
bus = MemBus(broker)
|
||||
for i in range(args.sim):
|
||||
present = (True, True, True, False, True) if i == 0 else (True,) * 5
|
||||
sb = SimBoard(MemBus(broker), "5157ab%06x" % i, present=present, seed=i + 1)
|
||||
if i == 0:
|
||||
sb.set_phone(2, HUNG) # 留一台「卡死」的，给自动恢复看
|
||||
sims.append(sb)
|
||||
else:
|
||||
env = read_env(args.mqtt_env)
|
||||
bus = PahoBus(args.mqtt_host, args.mqtt_port, env.get("BACKEND_USER", "phonerail-backend"),
|
||||
env["BACKEND_PASS"], "phonerail-rack-%d" % os.getpid())
|
||||
hub = Hub(db, bus)
|
||||
rec = RecoveryManager(hub, now_ms)
|
||||
|
||||
async def up():
|
||||
await hub.start()
|
||||
for sb in sims:
|
||||
sb.start()
|
||||
if sims:
|
||||
async def ticker():
|
||||
while True:
|
||||
for sb in sims:
|
||||
sb.tick(now_ms())
|
||||
await asyncio.sleep(1)
|
||||
asyncio.ensure_future(ticker())
|
||||
|
||||
async def down():
|
||||
await hub.stop()
|
||||
db.close()
|
||||
|
||||
static = args.static if args.static and os.path.isdir(args.static) else None
|
||||
app = create_app(hub, rec, static, startup=[up], shutdown=[down])
|
||||
return app
|
||||
|
||||
|
||||
def ws_library_ok() -> bool:
|
||||
for mod in ("websockets", "wsproto"):
|
||||
try:
|
||||
__import__(mod)
|
||||
return True
|
||||
except ImportError:
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser(description="PhoneRail 机架管理后端")
|
||||
ap.add_argument("--db", default="rack.db")
|
||||
ap.add_argument("--static", default=None)
|
||||
ap.add_argument("--bind", default="127.0.0.1")
|
||||
ap.add_argument("--port", type=int, default=7764)
|
||||
ap.add_argument("--mqtt-host", default="127.0.0.1")
|
||||
ap.add_argument("--mqtt-port", type=int, default=1884)
|
||||
ap.add_argument("--mqtt-env", default="/root/phonerail/mqtt.env")
|
||||
ap.add_argument("--sim", type=int, default=0, help="不连 broker，起 N 块模拟板子")
|
||||
args = ap.parse_args(argv)
|
||||
if not ipaddress.ip_address(args.bind).is_loopback:
|
||||
sys.exit("拒绝绑 %s：只许回环地址（见文件头）" % args.bind)
|
||||
if not ws_library_ok():
|
||||
sys.exit("没装 websockets / wsproto：uvicorn 会把 /api/ws 当普通 GET，界面能开但永远不刷新")
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
|
||||
uvicorn.run(build(args), host=args.bind, port=args.port, log_level="info")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,131 @@
|
||||
"""板侧协议在服务器这一侧的镜像（需求文档 §10.8「A12 / A14 定稿」）。
|
||||
|
||||
改这里之前先看固件：
|
||||
· 命令与错误码 —— firmware/main/cmd.h（数字冻结，只许追加）
|
||||
· 事件类型 —— firmware/main/evlog.h 的 log_ev_t（数字就是 flash 格式，冻结）
|
||||
· topic —— firmware/main/mlink.h
|
||||
两边对不上时**不会报错**：事件会被显示成别的类型、命令会被板子回 unknown_cmd。
|
||||
"""
|
||||
import re
|
||||
import time
|
||||
from typing import Dict, Optional, Tuple
|
||||
|
||||
TOPIC_ROOT = "phonerail"
|
||||
TOPIC_RE = re.compile(r"^phonerail/([0-9a-f]{12})/(cmd|rsp|tele|state)$")
|
||||
|
||||
RAILS = 5
|
||||
|
||||
# log_ev_t（firmware/main/evlog.h）。⚠️ 数字冻结。
|
||||
EVENT_TYPES: Dict[int, Tuple[str, str]] = {
|
||||
0: ("NONE", "（空槽位）"),
|
||||
1: ("BOOT", "板子上电"),
|
||||
2: ("LOST", "事件丢失"),
|
||||
3: ("OVER_VOLT", "过压关路"),
|
||||
4: ("UNDER_VOLT", "欠压关路"),
|
||||
5: ("REVERSE", "反灌断 VBUS"),
|
||||
6: ("PUSHED_UP", "顶高断 VBUS"),
|
||||
7: ("EFUSE_FAULT", "eFuse 故障"),
|
||||
8: ("PG_CHANGE", "PG 变化"),
|
||||
9: ("I2C_LOST", "I²C 失联"),
|
||||
10: ("THERMAL_DERATE", "过温降载"),
|
||||
11: ("FAN_STALL", "风扇停转"),
|
||||
12: ("REBOOT", "重启序列"),
|
||||
13: ("CMD_REJECT", "命令被拒"),
|
||||
14: ("MUX_LOST", "mux 失联全断"),
|
||||
15: ("SLOT_EMPTY", "刀片不在位"),
|
||||
16: ("PD_LIMIT", "PD 限功率"),
|
||||
}
|
||||
|
||||
# 严重程度：界面上标红的那些（会断电 / 断 VBUS / 硬件故障）。
|
||||
EVENT_SEVERE = {3, 4, 5, 6, 7, 9, 11, 14}
|
||||
|
||||
# cmd_id_t（firmware/main/cmd.h），CMD_REJECT 事件的 arg 就是它。
|
||||
CMD_NAMES = {1: "status", 2: "set_voltage", 3: "power_cycle", 4: "key", 5: "edl", 6: "vbus",
|
||||
7: "fan", 8: "events", 9: "power", 10: "wifi", 11: "mqtt", 12: "net"}
|
||||
|
||||
KEYS = ("pwr", "vup", "vdn")
|
||||
KEY_MAX_MS = 15000
|
||||
MV_MIN, MV_MAX = 3524, 4010
|
||||
|
||||
|
||||
class CmdError(ValueError):
|
||||
"""请求在发出去之前就被服务器拒了（参数不对）。"""
|
||||
|
||||
|
||||
def _rail(v) -> int:
|
||||
if not isinstance(v, int) or isinstance(v, bool) or not 1 <= v <= RAILS:
|
||||
raise CmdError("rail 必须是 1~%d" % RAILS)
|
||||
return v
|
||||
|
||||
|
||||
def _bool(v, name: str) -> bool:
|
||||
if not isinstance(v, bool):
|
||||
raise CmdError("%s 必须是 true/false" % name)
|
||||
return v
|
||||
|
||||
|
||||
def build_command(req: dict) -> dict:
|
||||
"""网页 / 自动恢复发来的请求 → 发给板子的命令（不含 seq）。
|
||||
|
||||
**白名单**：只放行管理界面该有的命令。`wifi` / `mqtt` 板子本来就只收 USB，
|
||||
这里再拦一道，免得界面上出现一个永远失败的按钮；`events` 由服务器自己同步。
|
||||
服务器这一侧先校验一遍，是为了让操作者立刻看到中文的错误，而不是等板子回 `range`；
|
||||
**板子的校验仍然是唯一的权威**（SR-13 在固件里）。
|
||||
"""
|
||||
cmd = req.get("cmd")
|
||||
if cmd == "status" or cmd == "net":
|
||||
return {"cmd": cmd}
|
||||
if cmd == "set_voltage":
|
||||
mv = req.get("mv")
|
||||
if not isinstance(mv, int) or isinstance(mv, bool) or not MV_MIN <= mv <= MV_MAX:
|
||||
raise CmdError("电压必须是 %d~%d mV" % (MV_MIN, MV_MAX))
|
||||
return {"cmd": cmd, "rail": _rail(req.get("rail")), "mv": mv}
|
||||
if cmd in ("power_cycle", "edl"):
|
||||
return {"cmd": cmd, "rail": _rail(req.get("rail"))}
|
||||
if cmd == "key":
|
||||
keys = req.get("keys")
|
||||
if not isinstance(keys, str) or not keys or any(k not in KEYS for k in keys.split("+")):
|
||||
raise CmdError("keys 只能是 pwr / vup / vdn 用 + 连起来")
|
||||
ms = req.get("ms")
|
||||
if not isinstance(ms, int) or isinstance(ms, bool) or not 1 <= ms <= KEY_MAX_MS:
|
||||
raise CmdError("按键时长必须是 1~%d ms" % KEY_MAX_MS)
|
||||
return {"cmd": cmd, "rail": _rail(req.get("rail")), "keys": keys, "ms": ms}
|
||||
if cmd in ("vbus", "power"):
|
||||
return {"cmd": cmd, "rail": _rail(req.get("rail")), "on": _bool(req.get("on"), "on")}
|
||||
if cmd == "fan":
|
||||
duty = req.get("duty")
|
||||
if not isinstance(duty, int) or isinstance(duty, bool) or not 0 <= duty <= 1000:
|
||||
raise CmdError("风扇下限必须是 0~1000‰（0 = 自动）")
|
||||
return {"cmd": cmd, "duty": duty}
|
||||
raise CmdError("不支持的命令：%r" % (cmd,))
|
||||
|
||||
|
||||
def is_mutating(cmd: str) -> bool:
|
||||
return cmd not in ("status", "events", "net")
|
||||
|
||||
|
||||
class SeqGen:
|
||||
"""seq 生成器：以毫秒时间戳为底、进程内严格递增，落在 1~2^32−1。
|
||||
|
||||
板子的判重窗口按「值相等」判（cmd.h），不要求单调，只要求**不撞**：
|
||||
服务重启之后从当前时间重新起步，不会撞上重启前刚用过的那几十个号。
|
||||
"""
|
||||
|
||||
def __init__(self, clock=time.time):
|
||||
self._clock = clock
|
||||
self._last = 0
|
||||
|
||||
def next(self) -> int:
|
||||
now = int(self._clock() * 1000)
|
||||
v = max(self._last + 1, now)
|
||||
self._last = v
|
||||
return (v - 1) % 0xFFFFFFFF + 1
|
||||
|
||||
|
||||
def parse_topic(topic: str) -> Optional[Tuple[str, str]]:
|
||||
m = TOPIC_RE.match(topic)
|
||||
return (m.group(1), m.group(2)) if m else None
|
||||
|
||||
|
||||
def topic(board: str, kind: str) -> str:
|
||||
return "%s/%s/%s" % (TOPIC_ROOT, board, kind)
|
||||
@@ -0,0 +1,221 @@
|
||||
"""自动崩溃恢复（需求 §4.1 两级重启：先软后硬），2026-09-27 定为「先只看电流」。
|
||||
|
||||
服务器够不到手机的 adb（手机在机架那边的 NAT 后面），所以这一版的判据只来自板子遥测：
|
||||
· FLAT：窗口内电流几乎不变（max − min < flat_ua）且不低 —— 卡死的手机常见的样子；
|
||||
· LOW ：窗口内电流一直低于 low_ua —— 关机 / 掉到待机，却本该在跑。
|
||||
adb 探活留了接口（`Prober`），手机能从服务器够到（比如经 frp）时再接上。
|
||||
|
||||
⚠️ 这两条阈值**还没有实测依据**（刀片还没插、手机还没上架），默认值是按常识填的。
|
||||
所以自动恢复**默认关闭、逐台打开**，而且：
|
||||
· 只在「这台手机开了自动恢复 + 板子在线 + 刀片在位 + 这一路有电 + 执行器空闲」时才判定，
|
||||
任何一条不满足就把积累的历史清掉 —— 失联不动作（§10.2）；
|
||||
· 每小时最多 `max_per_hour` 次，超了就 GAVE_UP 停手、告警，要人来复位 ——
|
||||
一个误判的阈值最坏的后果是「把一台正常的手机每 5 分钟重启一次」，这条闸就是防它的。
|
||||
|
||||
状态机：IDLE ──判定──▶ SOFT（长按电源 10 s）──等 settle──▶ 观察一个窗口
|
||||
▲ │ 仍然判定 │ 没判定
|
||||
└──── 恢复 ◀──────────┼──────────────────────────────┘
|
||||
▼
|
||||
HARD（断电重启）──仍然判定──▶ GAVE_UP（停手，等人）
|
||||
"""
|
||||
import asyncio
|
||||
import collections
|
||||
import json
|
||||
import logging
|
||||
from dataclasses import asdict, dataclass
|
||||
from typing import Callable, Deque, Dict, List, Optional, Tuple
|
||||
|
||||
log = logging.getLogger("prrack.recovery")
|
||||
|
||||
IDLE, SOFT, HARD, GAVE_UP = "idle", "soft", "hard", "gave_up"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Params:
|
||||
window_s: int = 300 # 判定窗口
|
||||
flat_ua: int = 5000 # 窗口内电流摆幅小于它 = 平线（5 mA）
|
||||
low_ua: int = 30000 # 窗口内电流一直低于它 = 像关机（30 mA）
|
||||
settle_s: int = 120 # 动作之后先等它重启完，这段不看
|
||||
max_per_hour: int = 3 # 每台每小时最多动作几次
|
||||
soft_ms: int = 10000 # 一级：长按电源键时长（需求 §4.1 ≈10 s）
|
||||
|
||||
@staticmethod
|
||||
def load(text: str) -> "Params":
|
||||
p = Params()
|
||||
try:
|
||||
for k, v in json.loads(text or "{}").items():
|
||||
if hasattr(p, k) and isinstance(v, int) and v > 0:
|
||||
setattr(p, k, v)
|
||||
except ValueError:
|
||||
pass
|
||||
return p
|
||||
|
||||
|
||||
class Detector:
|
||||
"""一台手机的电流窗口。样本 1 条/s；窗口没攒满（覆盖不到 window_s 的 90 %）一律不下结论。"""
|
||||
|
||||
def __init__(self, p: Params):
|
||||
self.p = p
|
||||
self.s: Deque[Tuple[int, int]] = collections.deque()
|
||||
|
||||
def reset(self):
|
||||
self.s.clear()
|
||||
|
||||
def add(self, ts: int, ua: int):
|
||||
if self.s and ts <= self.s[-1][0]:
|
||||
return # 时间倒退 / 重复：丢掉（一条迟到的遥测不能把窗口搅乱）
|
||||
self.s.append((ts, ua))
|
||||
horizon = ts - self.p.window_s * 1000
|
||||
while self.s and self.s[0][0] < horizon:
|
||||
self.s.popleft()
|
||||
|
||||
def full(self, now: int) -> bool:
|
||||
if not self.s:
|
||||
return False
|
||||
covered = self.s[-1][0] - self.s[0][0]
|
||||
return covered >= self.p.window_s * 900 and len(self.s) >= self.p.window_s * 0.8
|
||||
|
||||
def verdict(self, now: int) -> Optional[str]:
|
||||
if not self.full(now):
|
||||
return None
|
||||
vals = [u for _, u in self.s]
|
||||
if max(vals) < self.p.low_ua:
|
||||
return "low"
|
||||
if max(vals) - min(vals) < self.p.flat_ua:
|
||||
return "flat"
|
||||
return None
|
||||
|
||||
|
||||
Action = Tuple[str, dict] # (原因, 要发的命令)
|
||||
|
||||
|
||||
class PhoneRecovery:
|
||||
"""一台手机的状态机。纯逻辑：喂样本、吐动作，时钟由调用方给。"""
|
||||
|
||||
def __init__(self, p: Params):
|
||||
self.p = p
|
||||
self.det = Detector(p)
|
||||
self.state = IDLE
|
||||
self.since = 0 # 进入当前状态的时刻
|
||||
self.reason = ""
|
||||
self.attempts: List[int] = []
|
||||
|
||||
def eligible_reset(self):
|
||||
self.det.reset()
|
||||
|
||||
def manual_reset(self):
|
||||
self.state, self.reason, self.attempts = IDLE, "", []
|
||||
self.det.reset()
|
||||
|
||||
def feed(self, now: int, ua: int, rail: int) -> Tuple[Optional[str], Optional[Action]]:
|
||||
"""返回 (新状态或 None, 动作或 None)。"""
|
||||
if self.state == GAVE_UP:
|
||||
return None, None
|
||||
if self.state in (SOFT, HARD) and now - self.since < self.p.settle_s * 1000:
|
||||
return None, None # 还在重启过程中，这段电流不代表任何东西
|
||||
self.det.add(now, ua)
|
||||
v = self.det.verdict(now)
|
||||
if v is None:
|
||||
if self.state in (SOFT, HARD) and self.det.full(now):
|
||||
self.state, self.reason = IDLE, "recovered"
|
||||
return IDLE, None
|
||||
return None, None
|
||||
self.attempts = [t for t in self.attempts if now - t < 3600 * 1000]
|
||||
if len(self.attempts) >= self.p.max_per_hour or self.state == HARD:
|
||||
self.state, self.reason = GAVE_UP, ("rate_limit" if self.state != HARD else v)
|
||||
return GAVE_UP, None
|
||||
nxt = SOFT if self.state == IDLE else HARD
|
||||
self.state, self.since, self.reason = nxt, now, v
|
||||
self.attempts.append(now)
|
||||
self.det.reset()
|
||||
if nxt == SOFT:
|
||||
return SOFT, (v, {"cmd": "key", "rail": rail, "keys": "pwr", "ms": self.p.soft_ms})
|
||||
return HARD, (v, {"cmd": "power_cycle", "rail": rail})
|
||||
|
||||
|
||||
class Prober:
|
||||
"""adb 探活的接口（预留）。返回 True 活着 / False 死了 / None 不知道（没配、够不着）。"""
|
||||
|
||||
async def probe(self, addr: str) -> Optional[bool]:
|
||||
return None
|
||||
|
||||
|
||||
class RecoveryManager:
|
||||
"""把状态机接到 Hub 上：订阅遥测，判定，经 hub.call 下发动作，记录并推送。"""
|
||||
|
||||
def __init__(self, hub, clock: Callable[[], int]):
|
||||
self.hub = hub
|
||||
self.db = hub.db
|
||||
self.clock = clock
|
||||
self.params = Params.load(self.db.setting("recovery", "{}"))
|
||||
self.phones: Dict[Tuple[str, int], PhoneRecovery] = {}
|
||||
self.enabled: Dict[Tuple[str, int], bool] = {}
|
||||
self.reload()
|
||||
hub.tele_observers.append(self.on_tele)
|
||||
|
||||
def reload(self):
|
||||
for r in self.db.q("SELECT board_id, rail, auto_recover FROM phones"):
|
||||
key = (r["board_id"], r["rail"])
|
||||
was = self.enabled.get(key)
|
||||
self.enabled[key] = bool(r["auto_recover"])
|
||||
if was and not self.enabled[key] and key in self.phones:
|
||||
self.phones[key].manual_reset() # 关掉 = 清状态（含 GAVE_UP）
|
||||
|
||||
def set_params(self, p: Params):
|
||||
self.params = p
|
||||
self.db.set_setting("recovery", json.dumps(asdict(p)))
|
||||
for ph in self.phones.values():
|
||||
ph.p = p
|
||||
ph.det.p = p
|
||||
|
||||
def phone(self, board: str, rail: int) -> PhoneRecovery:
|
||||
key = (board, rail)
|
||||
if key not in self.phones:
|
||||
self.phones[key] = PhoneRecovery(self.params)
|
||||
return self.phones[key]
|
||||
|
||||
def states(self) -> List[dict]:
|
||||
return [{"board": b, "rail": r, "state": ph.state, "reason": ph.reason, "since": ph.since,
|
||||
"attempts_1h": len([t for t in ph.attempts if self.clock() - t < 3600 * 1000]),
|
||||
"enabled": self.enabled.get((b, r), False)}
|
||||
for (b, r), ph in sorted(self.phones.items())]
|
||||
|
||||
def reset(self, board: str, rail: int):
|
||||
self.phone(board, rail).manual_reset()
|
||||
self._log(board, rail, IDLE, "manual_reset")
|
||||
|
||||
def on_tele(self, board: str, ts: int, st: dict):
|
||||
now = self.clock()
|
||||
for r in st.get("rails", []):
|
||||
rail = r.get("r")
|
||||
key = (board, rail)
|
||||
if not self.enabled.get(key):
|
||||
continue
|
||||
ph = self.phone(board, rail)
|
||||
ok = r.get("pr") and r.get("en") and r.get("st") == "idle" and isinstance(r.get("ua"), int)
|
||||
if not ok:
|
||||
ph.eligible_reset()
|
||||
continue
|
||||
state, action = ph.feed(now, r["ua"], rail)
|
||||
if state is not None:
|
||||
self._log(board, rail, state, ph.reason, action[1] if action else None)
|
||||
if action is not None:
|
||||
asyncio.ensure_future(self._act(board, rail, action[1]))
|
||||
|
||||
async def _act(self, board: str, rail: int, cmd: dict):
|
||||
try:
|
||||
rsp = await self.hub.call(board, cmd, "auto-recovery")
|
||||
if not rsp.get("ok"):
|
||||
log.warning("自动恢复 %s#%d 的 %s 被板子拒了：%s", board, rail, cmd["cmd"], rsp)
|
||||
except Exception as e: # 超时也只记下来：失联不再加码
|
||||
log.warning("自动恢复 %s#%d 下发失败：%s", board, rail, e)
|
||||
|
||||
def _log(self, board: str, rail: int, state: str, reason: str, cmd: Optional[dict] = None):
|
||||
detail = json.dumps(cmd) if cmd else ""
|
||||
now = self.clock()
|
||||
rid = self.db.x("INSERT INTO recovery_log (ts, board_id, rail, state, reason, detail) "
|
||||
"VALUES (?,?,?,?,?,?)", (now, board, rail, state, reason, detail))
|
||||
self.hub.broadcast({"t": "recovery", "row": {"id": rid, "ts": now, "board_id": board,
|
||||
"rail": rail, "state": state, "reason": reason,
|
||||
"detail": detail}})
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
"""模拟板子：说和真板一样的 MQTT 协议（firmware/main/mlink.h + cmd.h），给测试与本机开发用。
|
||||
|
||||
它**不是**固件的第二份实现，只模拟服务器看得见的那部分行为：
|
||||
· 回应格式、seq 判重（按值、窗口 32）、空槽拒绝、越界拒绝；
|
||||
· 遥测（status 的形状）、retained 上线消息；
|
||||
· 事件日志（带序号，`events` 分页 8 条）；
|
||||
· 手机的电流：正常 = 在 300~900 mA 间游走；卡死 = 一条平线；关机 = ≈0。
|
||||
断电重启 / 长按电源键会让「卡死」的手机恢复正常 —— 自动恢复的测试靠这个。
|
||||
时间由调用方推进（`tick(now_ms)`），测试因此是确定的。
|
||||
"""
|
||||
import json
|
||||
import random
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from .bus import Bus
|
||||
from .protocol import topic
|
||||
|
||||
NORMAL, HUNG, OFF = "normal", "hung", "off"
|
||||
|
||||
|
||||
class SimRail:
|
||||
def __init__(self, rail: int, present: bool = True):
|
||||
self.rail = rail
|
||||
self.present = present
|
||||
self.en = present
|
||||
self.vb = False
|
||||
self.set_mv = 3900
|
||||
self.phone = NORMAL if present else OFF
|
||||
self.ua = 500000
|
||||
self.busy_until = 0
|
||||
self.act = "idle"
|
||||
self.pending: Optional[str] = None # 序列结束时要做的事
|
||||
|
||||
|
||||
class SimBoard:
|
||||
def __init__(self, bus: Bus, board_id: str, present=(True,) * 5, seed: int = 1,
|
||||
fw: str = "0.1.0-sim"):
|
||||
self.bus = bus
|
||||
self.id = board_id
|
||||
self.fw = fw
|
||||
self.rails = [SimRail(i + 1, present[i]) for i in range(5)]
|
||||
self.rng = random.Random(seed)
|
||||
self.seen: List[tuple] = [] # (seq, cmd, rsp)
|
||||
self.events: List[list] = [] # [seq, t, rail, type, arg, mv, ua]
|
||||
self.now = 0
|
||||
self.up_ms = 0
|
||||
self.fan_floor = 0
|
||||
self.received: List[dict] = []
|
||||
self.drop_next_replies = 0 # 模拟网络丢回应（测试重传）
|
||||
bus.on_message = self._on_message
|
||||
self._ev(0, 1, 0) # BOOT
|
||||
|
||||
# ---- 对外 --------------------------------------------------------------
|
||||
def start(self):
|
||||
self.bus.start()
|
||||
self.bus.subscribe(topic(self.id, "cmd"))
|
||||
self.bus.publish(topic(self.id, "state"),
|
||||
json.dumps({"online": True, "fw": self.fw, "ip": "192.168.8.200"}),
|
||||
retain=True)
|
||||
|
||||
def go_offline(self):
|
||||
self.bus.publish(topic(self.id, "state"), '{"online":false}', retain=True)
|
||||
self.bus.stop()
|
||||
|
||||
def set_phone(self, rail: int, state: str):
|
||||
self.rails[rail - 1].phone = state
|
||||
|
||||
def tick(self, now_ms: int, publish: bool = True):
|
||||
dt = now_ms - self.now if self.now else 0
|
||||
self.now = now_ms
|
||||
self.up_ms += dt
|
||||
for r in self.rails:
|
||||
if r.act != "idle" and now_ms >= r.busy_until:
|
||||
self._finish(r)
|
||||
r.ua = self._current(r)
|
||||
if publish:
|
||||
self.bus.publish(topic(self.id, "tele"), json.dumps(self.status_body()), qos=0)
|
||||
|
||||
# ---- 行为 --------------------------------------------------------------
|
||||
def _current(self, r: SimRail) -> int:
|
||||
if not r.present or not r.en or r.act == "hr-off":
|
||||
return 0
|
||||
if r.phone == OFF:
|
||||
return 800 # 关机漏电流
|
||||
if r.phone == HUNG:
|
||||
return 412000 # 一条平线
|
||||
return max(250000, min(950000, r.ua + self.rng.randint(-60000, 60000)))
|
||||
|
||||
def _finish(self, r: SimRail):
|
||||
if r.pending == "restart":
|
||||
r.phone = NORMAL
|
||||
r.ua = 500000
|
||||
r.pending = None
|
||||
r.en = True if r.act == "hr-off" else r.en
|
||||
r.act = "idle"
|
||||
|
||||
def _ev(self, rail: int, typ: int, arg: int, mv: int = 0, ua: int = 0):
|
||||
self.events.append([len(self.events), self.up_ms, rail, typ, arg, mv, ua])
|
||||
|
||||
def status_body(self) -> dict:
|
||||
rails = []
|
||||
for r in self.rails:
|
||||
ok = r.present
|
||||
rails.append({"r": r.rail, "pr": r.present, "en": r.en, "vb": r.vb, "set": r.set_mv,
|
||||
"mv": (r.set_mv if r.en else 0) if ok else None,
|
||||
"ua": r.ua if ok else None, "pg": r.en if ok else None,
|
||||
"flt": False if ok else None, "st": r.act, "res": "-"})
|
||||
return {"ok": True, "fw": self.fw, "up": self.up_ms, "in": {"mv": 12030, "ua": 1500000},
|
||||
"rails": rails, "temp": [35000, 36500],
|
||||
"fan": {"duty": max(300, self.fan_floor), "floor": self.fan_floor, "en": True,
|
||||
"rpm": [3300, 3280], "stall": [False, False]}, "derate": False,
|
||||
"safety": {"cycles": self.up_ms // 20, "skipped": 0, "i2c_err": 0, "events": 0},
|
||||
"ev_next": len(self.events), "ev_pending": 0,
|
||||
"cmd": {"lines": len(self.received), "dups": 0, "rej": 0}}
|
||||
|
||||
def _on_message(self, t: str, payload: bytes):
|
||||
try:
|
||||
req = json.loads(payload)
|
||||
except ValueError:
|
||||
return self._reply({"ok": False, "err": "parse"})
|
||||
self.received.append(req)
|
||||
seq = req.get("seq")
|
||||
for s, c, rsp in self.seen:
|
||||
if s == seq:
|
||||
out = dict(rsp) if c == req.get("cmd") else {"seq": seq, "ok": False, "err": "arg"}
|
||||
out["dup"] = True
|
||||
return self._reply(out)
|
||||
rsp = self._exec(req)
|
||||
if seq is not None:
|
||||
rsp = dict({"seq": seq}, **rsp)
|
||||
if req.get("cmd") not in ("status", "events", "net") and seq is not None:
|
||||
self.seen.append((seq, req.get("cmd"), rsp))
|
||||
self.seen = self.seen[-32:]
|
||||
self._reply(rsp)
|
||||
|
||||
def _reply(self, rsp: dict):
|
||||
if self.drop_next_replies > 0:
|
||||
self.drop_next_replies -= 1
|
||||
return
|
||||
self.bus.publish(topic(self.id, "rsp"), json.dumps(rsp))
|
||||
|
||||
def _exec(self, req: dict) -> dict:
|
||||
cmd = req.get("cmd")
|
||||
if cmd == "status":
|
||||
return self.status_body()
|
||||
if cmd == "events":
|
||||
start = max(0, req.get("from", 0))
|
||||
ev = self.events[start:start + 8]
|
||||
return {"ok": True, "oldest": 0, "next": len(self.events), "ev": ev}
|
||||
if cmd == "fan":
|
||||
self.fan_floor = req["duty"]
|
||||
return {"ok": True, "floor": self.fan_floor}
|
||||
if req.get("seq") is None:
|
||||
return {"ok": False, "err": "no_seq"}
|
||||
rail = req.get("rail")
|
||||
if not isinstance(rail, int) or not 1 <= rail <= 5:
|
||||
return {"ok": False, "err": "range"}
|
||||
r = self.rails[rail - 1]
|
||||
if not r.present:
|
||||
self._ev(rail, 13, 0)
|
||||
return {"ok": False, "err": "empty"}
|
||||
if r.act != "idle" and cmd in ("power_cycle", "key", "edl", "vbus", "power"):
|
||||
return {"ok": False, "err": "busy"}
|
||||
if cmd == "power_cycle":
|
||||
r.en, r.act, r.busy_until, r.pending = False, "hr-off", self.now + 4000, "restart"
|
||||
self._ev(rail, 12, 0x21)
|
||||
return {"ok": True}
|
||||
if cmd == "key":
|
||||
r.act, r.busy_until = "key-hold", self.now + req["ms"]
|
||||
if req["keys"] == "pwr" and req["ms"] >= 10000:
|
||||
r.pending = "restart"
|
||||
self._ev(rail, 12, 0x20)
|
||||
return {"ok": True}
|
||||
if cmd == "edl":
|
||||
r.act, r.busy_until = "edl-enum", self.now + 3000
|
||||
return {"ok": True}
|
||||
if cmd == "vbus":
|
||||
if req["on"] and not r.en:
|
||||
return {"ok": False, "err": "state"}
|
||||
r.vb = req["on"]
|
||||
return {"ok": True}
|
||||
if cmd == "power":
|
||||
r.en = req["on"]
|
||||
if not r.en:
|
||||
r.vb = False
|
||||
return {"ok": True}
|
||||
if cmd == "set_voltage":
|
||||
r.set_mv = req["mv"]
|
||||
return {"ok": True, "set": r.set_mv}
|
||||
return {"ok": False, "err": "unknown_cmd"}
|
||||
@@ -0,0 +1,8 @@
|
||||
# 服务器是 Python 3.8（Ubuntu 20.04 自带），版本按 3.8 还能装的锁死。
|
||||
# 本机开发用更新的版本也行，代码只用两边都有的 API（不用 3.9+ 语法）。
|
||||
fastapi==0.115.14
|
||||
starlette==0.44.0 # 3.8 上能装到的最高版（0.45 起要 3.9）
|
||||
pydantic==2.10.6
|
||||
uvicorn==0.33.0
|
||||
paho-mqtt==2.1.0
|
||||
websockets==13.1 # uvicorn 的 WS 实现；缺了它 /api/ws 会落到 SPA 兜底、回 200 HTML（2026-09-27 线上踩过）
|
||||
@@ -0,0 +1,131 @@
|
||||
"""HTTP / WebSocket 接口：用 TestClient 起完整的应用，Hub 经进程内假 broker 连模拟板子。"""
|
||||
import time
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from prrack.api import create_app
|
||||
from prrack.bus import MemBroker, MemBus
|
||||
from prrack.db import Db, now_ms
|
||||
from prrack.hub import Hub
|
||||
from prrack.recovery import RecoveryManager
|
||||
from prrack.simboard import SimBoard
|
||||
|
||||
B = "b81f3fc9c8bc"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def env(tmp_path):
|
||||
broker = MemBroker()
|
||||
db = Db(str(tmp_path / "a.db"))
|
||||
hub = Hub(db, MemBus(broker), attempt_timeout=0.2)
|
||||
rec = RecoveryManager(hub, now_ms)
|
||||
sim = SimBoard(MemBus(broker), B, present=(True, True, True, False, True))
|
||||
dist = tmp_path / "dist"
|
||||
(dist / "assets").mkdir(parents=True)
|
||||
(dist / "index.html").write_text("<html>rack</html>", encoding="utf-8")
|
||||
async def up():
|
||||
await hub.start()
|
||||
sim.start()
|
||||
sim.tick(now_ms())
|
||||
|
||||
app = create_app(hub, rec, str(dist), startup=[up], shutdown=[hub.stop])
|
||||
with TestClient(app) as c:
|
||||
deadline = time.time() + 2
|
||||
while B not in hub.latest and time.time() < deadline:
|
||||
time.sleep(0.01)
|
||||
yield c, hub, sim, db
|
||||
|
||||
|
||||
def test_boards_and_snapshot(env):
|
||||
c, hub, sim, db = env
|
||||
boards = c.get("/api/boards").json()
|
||||
assert [b["id"] for b in boards] == [B]
|
||||
assert boards[0]["online"] is True and len(boards[0]["phones"]) == 5
|
||||
assert c.get("/api/boards/nope").status_code == 404
|
||||
|
||||
|
||||
def test_command_records_the_portal_user(env):
|
||||
c, hub, sim, db = env
|
||||
r = c.post("/api/boards/%s/cmd" % B, json={"cmd": "power", "rail": 1, "on": False},
|
||||
headers={"X-Auth-User": "zhouyang"})
|
||||
assert r.status_code == 200 and r.json()["ok"] is True
|
||||
assert sim.rails[0].en is False
|
||||
a = c.get("/api/audit").json()[0]
|
||||
assert a["user"] == "zhouyang" and a["request"]["cmd"] == "power" and a["ok"] == 1
|
||||
# 没有门户头（服务器本机直接 curl）记作 local
|
||||
c.post("/api/boards/%s/cmd" % B, json={"cmd": "power", "rail": 1, "on": True})
|
||||
assert c.get("/api/audit").json()[0]["user"] == "local"
|
||||
|
||||
|
||||
def test_bad_commands_never_reach_the_board(env):
|
||||
c, hub, sim, db = env
|
||||
n = len(sim.received)
|
||||
for body in ({"cmd": "wifi", "ssid": "x"}, {"cmd": "set_voltage", "rail": 1, "mv": 4300},
|
||||
{"cmd": "key", "rail": 1, "keys": "pwr", "ms": 60000}, {"cmd": "rm -rf"}):
|
||||
r = c.post("/api/boards/%s/cmd" % B, json=body)
|
||||
assert r.status_code == 400, body
|
||||
assert len(sim.received) == n
|
||||
|
||||
|
||||
def test_board_rejection_is_passed_through(env):
|
||||
c, hub, sim, db = env
|
||||
r = c.post("/api/boards/%s/cmd" % B, json={"cmd": "power_cycle", "rail": 4})
|
||||
assert r.status_code == 200 and r.json()["err"] == "empty"
|
||||
|
||||
|
||||
def test_timeout_is_504(env):
|
||||
c, hub, sim, db = env
|
||||
sim.drop_next_replies = 99
|
||||
r = c.post("/api/boards/%s/cmd" % B, json={"cmd": "power", "rail": 2, "on": True})
|
||||
assert r.status_code == 504
|
||||
|
||||
|
||||
def test_phone_settings_toggle_auto_recovery(env):
|
||||
c, hub, sim, db = env
|
||||
r = c.patch("/api/boards/%s/rails/3" % B, json={"label": "A-03", "auto_recover": True})
|
||||
assert r.status_code == 200
|
||||
ph = [p for p in r.json()["phones"] if p["rail"] == 3][0]
|
||||
assert ph["label"] == "A-03" and ph["auto_recover"] == 1
|
||||
rec = c.get("/api/recovery").json()
|
||||
assert rec["params"]["window_s"] == 300
|
||||
r = c.put("/api/recovery/params", json={"window_s": 120, "flat_ua": -5, "bogus": 1})
|
||||
assert r.json()["window_s"] == 120 and r.json()["flat_ua"] == 5000 # 负数不收
|
||||
|
||||
|
||||
def test_history_after_telemetry(env):
|
||||
c, hub, sim, db = env
|
||||
for _ in range(3):
|
||||
sim.tick(now_ms())
|
||||
time.sleep(0.1)
|
||||
hub.flush()
|
||||
h = c.get("/api/boards/%s/rails/1/history" % B, params={"start": now_ms() - 60000}).json()
|
||||
assert h["resolution"] == "raw" and len(h["points"]) >= 1
|
||||
assert h["points"][0][2] is not None
|
||||
|
||||
|
||||
def test_websocket_hello_and_live_tele(env):
|
||||
c, hub, sim, db = env
|
||||
with c.websocket_connect("/api/ws", headers={"X-Auth-User": "zhouyang"}) as ws:
|
||||
hello = ws.receive_json()
|
||||
assert hello["t"] == "hello" and hello["user"] == "zhouyang" and hello["boards"][0]["id"] == B
|
||||
sim.tick(now_ms())
|
||||
msg = ws.receive_json()
|
||||
while msg["t"] != "tele":
|
||||
msg = ws.receive_json()
|
||||
assert msg["board"] == B and len(msg["status"]["rails"]) == 5
|
||||
|
||||
|
||||
def test_spa_fallback_is_not_cached(env):
|
||||
c, hub, sim, db = env
|
||||
r = c.get("/some/deep/route")
|
||||
assert r.status_code == 200 and "rack" in r.text
|
||||
assert r.headers["cache-control"] == "no-cache"
|
||||
|
||||
|
||||
def test_a_real_websocket_library_is_installed():
|
||||
"""TestClient 自带 WS 实现，上面那条 WS 用例在没装 websockets 的机器上也是绿的；
|
||||
而真 uvicorn 缺了它会把 /api/ws 当普通 GET（落到 SPA 兜底，回 200 HTML）。
|
||||
deploy.sh 在服务器上跑这套测试，所以缺库会在换上去之前被拦住。"""
|
||||
from prrack.main import ws_library_ok
|
||||
assert ws_library_ok()
|
||||
@@ -0,0 +1,302 @@
|
||||
"""协议镜像 + Hub（经进程内假 broker 对接模拟板子）+ 自动恢复状态机。
|
||||
|
||||
和固件测试同一个口径：断言打在「板子那头真的收到 / 执行了什么」上（SimBoard.received、
|
||||
它的 rails 状态），不看 Hub 自己回了什么 —— 重传换了 seq 的现象恰恰是 Hub 这边一切正常、
|
||||
手机被按了两遍。"""
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from prrack import protocol
|
||||
from prrack.bus import MemBroker, MemBus, topic_matches
|
||||
from prrack.db import Db
|
||||
from prrack.hub import CmdTimeout, Hub
|
||||
from prrack.recovery import GAVE_UP, HARD, IDLE, SOFT, Params, PhoneRecovery, RecoveryManager
|
||||
from prrack.simboard import HUNG, NORMAL, OFF, SimBoard
|
||||
|
||||
B = "b81f3fc9c8bc"
|
||||
|
||||
|
||||
def run(coro):
|
||||
loop = asyncio.new_event_loop()
|
||||
try:
|
||||
return loop.run_until_complete(coro)
|
||||
finally: # 收掉 Hub 的后台任务（flusher / maint），免得退出时刷一屏 pending 警告
|
||||
tasks = asyncio.all_tasks(loop)
|
||||
for t in tasks:
|
||||
t.cancel()
|
||||
loop.run_until_complete(asyncio.gather(*tasks, return_exceptions=True))
|
||||
loop.close()
|
||||
|
||||
|
||||
async def settle():
|
||||
for _ in range(5):
|
||||
await asyncio.sleep(0)
|
||||
|
||||
|
||||
def rig(tmp_path, present=(True,) * 5, timeout=0.05):
|
||||
broker = MemBroker()
|
||||
db = Db(str(tmp_path / "t.db"))
|
||||
hub = Hub(db, MemBus(broker), attempt_timeout=timeout)
|
||||
sim = SimBoard(MemBus(broker), B, present=present)
|
||||
return hub, sim, db
|
||||
|
||||
|
||||
# ---- 协议镜像 -----------------------------------------------------------------
|
||||
|
||||
def test_build_command_whitelist_and_ranges():
|
||||
assert protocol.build_command({"cmd": "key", "rail": 3, "keys": "vdn+pwr", "ms": 2000}) == \
|
||||
{"cmd": "key", "rail": 3, "keys": "vdn+pwr", "ms": 2000}
|
||||
bad = [
|
||||
{"cmd": "wifi", "ssid": "x"}, # 只收 USB 的命令不给界面
|
||||
{"cmd": "events"}, # 服务器自己同步
|
||||
{"cmd": "set_voltage", "rail": 1, "mv": 4011}, # SR-13 之上
|
||||
{"cmd": "set_voltage", "rail": 1, "mv": "3900"},
|
||||
{"cmd": "key", "rail": 1, "keys": "pwr+home", "ms": 100},
|
||||
{"cmd": "key", "rail": 1, "keys": "pwr", "ms": 15001},
|
||||
{"cmd": "power", "rail": 6, "on": True},
|
||||
{"cmd": "power", "rail": True, "on": True}, # bool 不是 int
|
||||
{"cmd": "vbus", "rail": 1, "on": 1},
|
||||
{"cmd": "fan", "duty": 1001},
|
||||
]
|
||||
for b in bad:
|
||||
with pytest.raises(protocol.CmdError):
|
||||
protocol.build_command(b)
|
||||
|
||||
|
||||
def test_seq_is_strictly_increasing_and_in_range():
|
||||
t = [1700000000.0]
|
||||
g = protocol.SeqGen(clock=lambda: t[0])
|
||||
a = [g.next() for _ in range(5)]
|
||||
assert a == sorted(set(a)) and all(1 <= x <= 0xFFFFFFFF for x in a)
|
||||
t[0] -= 100 # 时钟往回跳也不许重复
|
||||
assert g.next() > a[-1] or g.next() == 1
|
||||
|
||||
|
||||
def test_event_table_matches_firmware_numbers():
|
||||
# firmware/main/evlog.h 冻结的数字（抽几个钉住）
|
||||
assert protocol.EVENT_TYPES[3][0] == "OVER_VOLT"
|
||||
assert protocol.EVENT_TYPES[12][0] == "REBOOT"
|
||||
assert protocol.EVENT_TYPES[13][0] == "CMD_REJECT"
|
||||
assert protocol.EVENT_TYPES[16][0] == "PD_LIMIT"
|
||||
assert protocol.CMD_NAMES[9] == "power" and protocol.CMD_NAMES[12] == "net"
|
||||
|
||||
|
||||
def test_topic_matching():
|
||||
assert topic_matches("phonerail/+/rsp", "phonerail/%s/rsp" % B)
|
||||
assert not topic_matches("phonerail/+/rsp", "phonerail/%s/tele" % B)
|
||||
assert topic_matches("phonerail/#", "phonerail/x/y")
|
||||
assert protocol.parse_topic("phonerail/%s/tele" % B) == (B, "tele")
|
||||
assert protocol.parse_topic("evse/123/state") is None
|
||||
|
||||
|
||||
# ---- Hub ---------------------------------------------------------------------
|
||||
|
||||
def test_tele_is_ingested_and_board_discovered(tmp_path):
|
||||
hub, sim, db = rig(tmp_path)
|
||||
|
||||
async def go():
|
||||
await hub.start()
|
||||
sim.start()
|
||||
sim.tick(1000)
|
||||
await settle()
|
||||
hub.flush()
|
||||
|
||||
run(go())
|
||||
snap = hub.snapshot(B)
|
||||
assert snap["online"] and snap["fw"] == "0.1.0-sim"
|
||||
assert snap["status"]["rails"][0]["en"] is True
|
||||
assert db.q("SELECT COUNT(*) c FROM tele")[0]["c"] == 5
|
||||
assert len(snap["phones"]) == 5 and snap["phones"][0]["auto_recover"] == 0
|
||||
|
||||
|
||||
def test_command_round_trip_and_audit(tmp_path):
|
||||
hub, sim, db = rig(tmp_path)
|
||||
|
||||
async def go():
|
||||
await hub.start()
|
||||
sim.start()
|
||||
return await hub.call(B, {"cmd": "power", "rail": 2, "on": False}, "alice")
|
||||
|
||||
rsp = run(go())
|
||||
assert rsp["ok"] is True
|
||||
assert sim.rails[1].en is False
|
||||
row = db.q("SELECT * FROM audit")[0]
|
||||
assert row["user"] == "alice" and row["rail"] == 2 and row["ok"] == 1
|
||||
|
||||
|
||||
def test_retransmit_reuses_the_same_seq_and_executes_once(tmp_path):
|
||||
"""⭐ 回应丢了一次：Hub 用**同一个 seq** 重发，板子判重，只执行一次。"""
|
||||
hub, sim, db = rig(tmp_path)
|
||||
sim.drop_next_replies = 1
|
||||
|
||||
async def go():
|
||||
await hub.start()
|
||||
sim.start()
|
||||
return await hub.call(B, {"cmd": "power_cycle", "rail": 1}, "bob")
|
||||
|
||||
rsp = run(go())
|
||||
assert rsp["ok"] is True and rsp.get("dup") is True
|
||||
sent = [r for r in sim.received if r.get("cmd") == "power_cycle"]
|
||||
assert len(sent) == 2 and sent[0]["seq"] == sent[1]["seq"]
|
||||
assert [e[3] for e in sim.events].count(12) == 1 # 重启事件只记了一次
|
||||
|
||||
|
||||
def test_timeout_is_audited_and_raised(tmp_path):
|
||||
hub, sim, db = rig(tmp_path)
|
||||
sim.drop_next_replies = 99
|
||||
|
||||
async def go():
|
||||
await hub.start()
|
||||
sim.start()
|
||||
with pytest.raises(CmdTimeout):
|
||||
await hub.call(B, {"cmd": "power", "rail": 1, "on": True}, "carol")
|
||||
|
||||
run(go())
|
||||
row = db.q("SELECT * FROM audit")[0]
|
||||
assert row["ok"] == 0 and json.loads(row["response"])["err"] == "timeout"
|
||||
assert len({r["seq"] for r in sim.received}) == 1 # 三次都是同一个 seq
|
||||
|
||||
|
||||
def test_events_are_synced_incrementally(tmp_path):
|
||||
hub, sim, db = rig(tmp_path)
|
||||
for i in range(19):
|
||||
sim._ev(1, 8, i) # 加上 BOOT 共 20 条：要分 3 页
|
||||
|
||||
async def go():
|
||||
await hub.start()
|
||||
sim.start()
|
||||
sim.tick(1000)
|
||||
for _ in range(20):
|
||||
await settle()
|
||||
sim._ev(2, 3, 0, 4200, 1) # 又来一条过压
|
||||
sim.tick(2000)
|
||||
for _ in range(20):
|
||||
await settle()
|
||||
|
||||
run(go())
|
||||
seqs = [r["seq"] for r in db.q("SELECT seq FROM events ORDER BY seq")]
|
||||
assert seqs == list(range(21))
|
||||
assert db.q("SELECT ev_cursor FROM boards")[0]["ev_cursor"] == 21
|
||||
|
||||
|
||||
def test_empty_slot_reply_passes_through(tmp_path):
|
||||
hub, sim, db = rig(tmp_path, present=(True, True, True, False, True))
|
||||
|
||||
async def go():
|
||||
await hub.start()
|
||||
sim.start()
|
||||
return await hub.call(B, {"cmd": "power", "rail": 4, "on": True}, "dave")
|
||||
|
||||
assert run(go()) == {"seq": sim.received[-1]["seq"], "ok": False, "err": "empty"}
|
||||
|
||||
|
||||
# ---- 自动恢复：状态机 ------------------------------------------------------------
|
||||
|
||||
P = Params(window_s=60, flat_ua=5000, low_ua=30000, settle_s=30, max_per_hour=3)
|
||||
|
||||
|
||||
def feed(ph, t0, secs, ua_fn):
|
||||
out = []
|
||||
for s in range(secs):
|
||||
st, act = ph.feed(t0 + s * 1000, ua_fn(s), 3)
|
||||
if st or act:
|
||||
out.append((t0 + s * 1000, st, act))
|
||||
return out
|
||||
|
||||
|
||||
def test_flat_current_escalates_soft_then_hard_then_gives_up():
|
||||
ph = PhoneRecovery(P)
|
||||
r = feed(ph, 0, 61, lambda s: 412000)
|
||||
assert r[-1][1] == SOFT and r[-1][2][1] == {"cmd": "key", "rail": 3, "keys": "pwr", "ms": 10000}
|
||||
t = r[-1][0]
|
||||
# settle 期间的样本一概不看（哪怕是平线）
|
||||
assert feed(ph, t + 1000, 29, lambda s: 412000) == []
|
||||
r = feed(ph, t + 30000, 62, lambda s: 412000)
|
||||
assert r[-1][1] == HARD and r[-1][2][1] == {"cmd": "power_cycle", "rail": 3}
|
||||
t = r[-1][0]
|
||||
r = feed(ph, t + 30000, 62, lambda s: 412000)
|
||||
assert r[-1][1] == GAVE_UP and r[-1][2] is None
|
||||
assert feed(ph, t + 200000, 100, lambda s: 412000) == [] # 放弃之后再也不动
|
||||
|
||||
|
||||
def test_recovers_after_soft_when_current_moves_again():
|
||||
ph = PhoneRecovery(P)
|
||||
t = feed(ph, 0, 61, lambda s: 412000)[-1][0]
|
||||
r = feed(ph, t + 30000, 70, lambda s: 300000 + (s % 7) * 40000)
|
||||
assert r[-1][1] == IDLE and ph.reason == "recovered"
|
||||
|
||||
|
||||
def test_low_current_is_detected_and_normal_is_not():
|
||||
ph = PhoneRecovery(P)
|
||||
assert feed(ph, 0, 200, lambda s: 300000 + (s % 5) * 50000) == []
|
||||
ph2 = PhoneRecovery(P)
|
||||
assert feed(ph2, 0, 61, lambda s: 800 + s % 3)[-1][2][0] == "low"
|
||||
|
||||
|
||||
def test_rate_limit_gives_up():
|
||||
ph = PhoneRecovery(Params(window_s=60, settle_s=30, max_per_hour=2))
|
||||
t = 0
|
||||
for _ in range(2): # 两次软重启都「恢复」了，但每次很快又卡
|
||||
assert feed(ph, t, 65, lambda s: 412000)[-1][1] == SOFT
|
||||
t += 65000 # 下一段接着上一段喂完的时刻往后走（时间不倒退）
|
||||
# 前 30 s 还在 settle 里不算数，之后要攒满一个 60 s 窗口才判「恢复」
|
||||
assert feed(ph, t, 100, lambda s: 300000 + (s % 7) * 40000)[-1][1] == IDLE
|
||||
t += 100000
|
||||
r = feed(ph, t, 65, lambda s: 412000)
|
||||
assert r[-1][1] == GAVE_UP and ph.reason == "rate_limit"
|
||||
|
||||
|
||||
# ---- 自动恢复：接到 Hub + 模拟板上 -------------------------------------------------
|
||||
|
||||
def test_auto_recovery_end_to_end_with_sim(tmp_path):
|
||||
"""⭐ 模拟板上第 3 台卡死：开了自动恢复 → 5 分钟后下长按电源 → 板子执行 → 恢复。
|
||||
没开自动恢复的第 2 台（也卡死）一个命令都不收到。"""
|
||||
hub, sim, db = rig(tmp_path)
|
||||
clock = [1_000_000]
|
||||
rec = RecoveryManager(hub, lambda: clock[0])
|
||||
sim.set_phone(3, HUNG)
|
||||
sim.set_phone(2, HUNG)
|
||||
|
||||
async def go():
|
||||
await hub.start()
|
||||
sim.start()
|
||||
await settle() # 等 Hub 处理完上线消息、建好这块板的 phones 行
|
||||
db.x("UPDATE phones SET auto_recover=1 WHERE rail=3")
|
||||
rec.reload()
|
||||
for s in range(300 + 130 + 320):
|
||||
clock[0] += 1000
|
||||
sim.tick(clock[0])
|
||||
await settle()
|
||||
|
||||
run(go())
|
||||
keys = [r for r in sim.received if r.get("cmd") == "key"]
|
||||
assert len(keys) == 1 and keys[0]["rail"] == 3 and keys[0]["ms"] == 10000
|
||||
assert not [r for r in sim.received if r.get("rail") == 2]
|
||||
assert sim.rails[2].phone == NORMAL
|
||||
states = [r["state"] for r in db.q("SELECT state FROM recovery_log ORDER BY id")]
|
||||
assert states == [SOFT, IDLE]
|
||||
assert db.q("SELECT user FROM audit")[0]["user"] == "auto-recovery"
|
||||
|
||||
|
||||
def test_auto_recovery_ignores_rails_that_are_off(tmp_path):
|
||||
hub, sim, db = rig(tmp_path)
|
||||
clock = [0]
|
||||
rec = RecoveryManager(hub, lambda: clock[0])
|
||||
sim.set_phone(1, OFF)
|
||||
sim.rails[0].en = False # 这一路本来就关着：电流 0 不是故障
|
||||
|
||||
async def go():
|
||||
await hub.start()
|
||||
sim.start()
|
||||
await settle()
|
||||
db.x("UPDATE phones SET auto_recover=1")
|
||||
rec.reload()
|
||||
for s in range(400):
|
||||
clock[0] += 1000
|
||||
sim.tick(clock[0])
|
||||
await settle()
|
||||
|
||||
run(go())
|
||||
assert not [r for r in sim.received if r.get("cmd") in ("key", "power_cycle")]
|
||||
在新工单中引用
屏蔽一个用户