rack：主板 / 底板功率与功率曲线；底板曲线加电压电流、开机重按事件
- 每路主板功率（V×I）：总览槽位、单板卡片、单路曲线新增「功率」一张 - 底板：单板页新增功率仪表（底板输入 / 五路主板合计 / 损耗），底板曲线加 功率图（输入 vs 主板合计）；board_history 每点多回一列 rails_mw - 底板曲线改为纵向 N 张（温度 / 风扇 / 输入电压 / 输入电流 / 功率），断档断线 - LOG_EV_BOOT_CHECK（19）事件命名，只有「没开起来」标红 Co-Authored-By: Claude Opus 5.5 <[email protected]>
这个提交包含在：
zhou yang
+187
-155
@@ -1,155 +1,187 @@
|
||||
"""HTTP / WebSocket 接口：用 TestClient 起完整的应用，Hub 经进程内假 broker 连模拟板子。"""
|
||||
import time
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from prrack.api import create_app
|
||||
from prrack.bus import MemBroker, MemBus
|
||||
from prrack.db import Db, now_ms
|
||||
from prrack.hub import Hub
|
||||
from prrack.recovery import RecoveryManager
|
||||
from prrack.simboard import SimBoard
|
||||
|
||||
B = "b81f3fc9c8bc"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def env(tmp_path):
|
||||
broker = MemBroker()
|
||||
db = Db(str(tmp_path / "a.db"))
|
||||
hub = Hub(db, MemBus(broker), attempt_timeout=0.2)
|
||||
rec = RecoveryManager(hub, now_ms)
|
||||
sim = SimBoard(MemBus(broker), B, present=(True, True, True, False, True))
|
||||
dist = tmp_path / "dist"
|
||||
(dist / "assets").mkdir(parents=True)
|
||||
(dist / "index.html").write_text("<html>rack</html>", encoding="utf-8")
|
||||
async def up():
|
||||
await hub.start()
|
||||
sim.start()
|
||||
sim.tick(now_ms())
|
||||
|
||||
app = create_app(hub, rec, str(dist), startup=[up], shutdown=[hub.stop])
|
||||
with TestClient(app) as c:
|
||||
deadline = time.time() + 2
|
||||
while B not in hub.latest and time.time() < deadline:
|
||||
time.sleep(0.01)
|
||||
yield c, hub, sim, db
|
||||
|
||||
|
||||
def test_boards_and_snapshot(env):
|
||||
c, hub, sim, db = env
|
||||
boards = c.get("/api/boards").json()
|
||||
assert [b["id"] for b in boards] == [B]
|
||||
assert boards[0]["online"] is True and len(boards[0]["phones"]) == 5
|
||||
assert c.get("/api/boards/nope").status_code == 404
|
||||
|
||||
|
||||
def test_command_records_the_portal_user(env):
|
||||
c, hub, sim, db = env
|
||||
r = c.post("/api/boards/%s/cmd" % B, json={"cmd": "power", "rail": 1, "on": False},
|
||||
headers={"X-Auth-User": "zhouyang"})
|
||||
assert r.status_code == 200 and r.json()["ok"] is True
|
||||
assert sim.rails[0].en is False
|
||||
a = c.get("/api/audit").json()[0]
|
||||
assert a["user"] == "zhouyang" and a["request"]["cmd"] == "power" and a["ok"] == 1
|
||||
# 没有门户头（服务器本机直接 curl）记作 local
|
||||
c.post("/api/boards/%s/cmd" % B, json={"cmd": "power", "rail": 1, "on": True})
|
||||
assert c.get("/api/audit").json()[0]["user"] == "local"
|
||||
|
||||
|
||||
def test_bad_commands_never_reach_the_board(env):
|
||||
c, hub, sim, db = env
|
||||
n = len(sim.received)
|
||||
for body in ({"cmd": "wifi", "ssid": "x"}, {"cmd": "set_voltage", "rail": 1, "mv": 4300},
|
||||
{"cmd": "key", "rail": 1, "keys": "pwr", "ms": 60000}, {"cmd": "rm -rf"}):
|
||||
r = c.post("/api/boards/%s/cmd" % B, json=body)
|
||||
assert r.status_code == 400, body
|
||||
assert len(sim.received) == n
|
||||
|
||||
|
||||
def test_board_rejection_is_passed_through(env):
|
||||
c, hub, sim, db = env
|
||||
r = c.post("/api/boards/%s/cmd" % B, json={"cmd": "power_cycle", "rail": 4})
|
||||
assert r.status_code == 200 and r.json()["err"] == "empty"
|
||||
|
||||
|
||||
def test_timeout_is_504(env):
|
||||
c, hub, sim, db = env
|
||||
sim.drop_next_replies = 99
|
||||
r = c.post("/api/boards/%s/cmd" % B, json={"cmd": "power", "rail": 2, "on": True})
|
||||
assert r.status_code == 504
|
||||
|
||||
|
||||
def test_phone_settings_toggle_auto_recovery(env):
|
||||
c, hub, sim, db = env
|
||||
r = c.patch("/api/boards/%s/rails/3" % B, json={"label": "A-03", "auto_recover": True})
|
||||
assert r.status_code == 200
|
||||
ph = [p for p in r.json()["phones"] if p["rail"] == 3][0]
|
||||
assert ph["label"] == "A-03" and ph["auto_recover"] == 1
|
||||
rec = c.get("/api/recovery").json()
|
||||
assert rec["params"]["window_s"] == 300
|
||||
r = c.put("/api/recovery/params", json={"window_s": 120, "flat_ua": -5, "bogus": 1})
|
||||
assert r.json()["window_s"] == 120 and r.json()["flat_ua"] == 5000 # 负数不收
|
||||
|
||||
|
||||
def test_history_after_telemetry(env):
|
||||
c, hub, sim, db = env
|
||||
for _ in range(3):
|
||||
sim.tick(now_ms())
|
||||
time.sleep(0.1)
|
||||
hub.flush()
|
||||
h = c.get("/api/boards/%s/rails/1/history" % B, params={"start": now_ms() - 60000}).json()
|
||||
assert h["resolution"] == "raw" and len(h["points"]) >= 1
|
||||
assert h["points"][0][2] is not None
|
||||
|
||||
|
||||
def test_websocket_hello_and_live_tele(env):
|
||||
c, hub, sim, db = env
|
||||
with c.websocket_connect("/api/ws", headers={"X-Auth-User": "zhouyang"}) as ws:
|
||||
hello = ws.receive_json()
|
||||
assert hello["t"] == "hello" and hello["user"] == "zhouyang" and hello["boards"][0]["id"] == B
|
||||
sim.tick(now_ms())
|
||||
msg = ws.receive_json()
|
||||
while msg["t"] != "tele":
|
||||
msg = ws.receive_json()
|
||||
assert msg["board"] == B and len(msg["status"]["rails"]) == 5
|
||||
|
||||
|
||||
def test_spa_fallback_is_not_cached(env):
|
||||
c, hub, sim, db = env
|
||||
r = c.get("/some/deep/route")
|
||||
assert r.status_code == 200 and "rack" in r.text
|
||||
assert r.headers["cache-control"] == "no-cache"
|
||||
|
||||
|
||||
def test_a_real_websocket_library_is_installed():
|
||||
"""TestClient 自带 WS 实现，上面那条 WS 用例在没装 websockets 的机器上也是绿的；
|
||||
而真 uvicorn 缺了它会把 /api/ws 当普通 GET（落到 SPA 兜底，回 200 HTML）。
|
||||
deploy.sh 在服务器上跑这套测试，所以缺库会在换上去之前被拦住。"""
|
||||
from prrack.main import ws_library_ok
|
||||
assert ws_library_ok()
|
||||
|
||||
|
||||
def test_board_history_raw_and_minute(env):
|
||||
c, hub, sim, db = env
|
||||
base = now_ms() // 60000 * 60000 - 3 * 60000
|
||||
rows = [(base + i * 1000, B, 12000, 1500000, 35000 + i, 40000 - i, 300) for i in range(120)]
|
||||
db.many("INSERT INTO board_tele VALUES (?,?,?,?,?,?,?)", rows)
|
||||
h = c.get("/api/boards/%s/history" % B, params={"start": base, "end": base + 120000}).json()
|
||||
assert h["resolution"] == "raw" and len(h["points"]) >= 100
|
||||
p = h["points"][0]
|
||||
assert p[1] == 35000 and p[4] == 40000 and p[7] == 300 and p[8] == 12000
|
||||
# 分钟聚合：两个整分钟，每分钟 min/max 就是那一分钟里的首尾
|
||||
assert db.downsample(base + 180000) > 0
|
||||
m = db.board_history(B, base, base + 120000 + 1) # 跨度小也能强制走聚合：t0 早于原始保留期时
|
||||
rows_min = db.q("SELECT * FROM board_tele_min WHERE board_id=? ORDER BY ts", (B,))
|
||||
assert [r["t0_min"] for r in rows_min][:2] == [35000, 35060]
|
||||
assert [r["t0_max"] for r in rows_min][:2] == [35059, 35119]
|
||||
assert rows_min[0]["t1_min"] == 40000 - 59 and rows_min[0]["fan_avg"] == 300
|
||||
assert m["resolution"] == "raw"
|
||||
old = db.board_history(B, base - 10 * 86400000, base + 120000)
|
||||
# 10 天跨度 ⇒ 桶约 9 分钟，两分钟落进同一桶：min 取两分钟的最小、max 取最大
|
||||
assert old["resolution"] == "minute" and old["points"][0][2] == 35000
|
||||
assert max(p[3] for p in old["points"]) == 35119
|
||||
assert c.get("/api/boards/nope/history", params={"start": 0}).status_code == 404
|
||||
"""HTTP / WebSocket 接口：用 TestClient 起完整的应用，Hub 经进程内假 broker 连模拟板子。"""
|
||||
import time
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from prrack.api import create_app
|
||||
from prrack.bus import MemBroker, MemBus
|
||||
from prrack.db import Db, now_ms
|
||||
from prrack.hub import Hub
|
||||
from prrack.recovery import RecoveryManager
|
||||
from prrack.simboard import SimBoard
|
||||
|
||||
B = "b81f3fc9c8bc"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def env(tmp_path):
|
||||
broker = MemBroker()
|
||||
db = Db(str(tmp_path / "a.db"))
|
||||
hub = Hub(db, MemBus(broker), attempt_timeout=0.2)
|
||||
rec = RecoveryManager(hub, now_ms)
|
||||
sim = SimBoard(MemBus(broker), B, present=(True, True, True, False, True))
|
||||
dist = tmp_path / "dist"
|
||||
(dist / "assets").mkdir(parents=True)
|
||||
(dist / "index.html").write_text("<html>rack</html>", encoding="utf-8")
|
||||
async def up():
|
||||
await hub.start()
|
||||
sim.start()
|
||||
sim.tick(now_ms())
|
||||
|
||||
app = create_app(hub, rec, str(dist), startup=[up], shutdown=[hub.stop])
|
||||
with TestClient(app) as c:
|
||||
deadline = time.time() + 2
|
||||
while B not in hub.latest and time.time() < deadline:
|
||||
time.sleep(0.01)
|
||||
yield c, hub, sim, db
|
||||
|
||||
|
||||
def test_boards_and_snapshot(env):
|
||||
c, hub, sim, db = env
|
||||
boards = c.get("/api/boards").json()
|
||||
assert [b["id"] for b in boards] == [B]
|
||||
assert boards[0]["online"] is True and len(boards[0]["phones"]) == 5
|
||||
assert c.get("/api/boards/nope").status_code == 404
|
||||
|
||||
|
||||
def test_command_records_the_portal_user(env):
|
||||
c, hub, sim, db = env
|
||||
r = c.post("/api/boards/%s/cmd" % B, json={"cmd": "power", "rail": 1, "on": False},
|
||||
headers={"X-Auth-User": "zhouyang"})
|
||||
assert r.status_code == 200 and r.json()["ok"] is True
|
||||
assert sim.rails[0].en is False
|
||||
a = c.get("/api/audit").json()[0]
|
||||
assert a["user"] == "zhouyang" and a["request"]["cmd"] == "power" and a["ok"] == 1
|
||||
# 没有门户头（服务器本机直接 curl）记作 local
|
||||
c.post("/api/boards/%s/cmd" % B, json={"cmd": "power", "rail": 1, "on": True})
|
||||
assert c.get("/api/audit").json()[0]["user"] == "local"
|
||||
|
||||
|
||||
def test_bad_commands_never_reach_the_board(env):
|
||||
c, hub, sim, db = env
|
||||
n = len(sim.received)
|
||||
for body in ({"cmd": "wifi", "ssid": "x"}, {"cmd": "set_voltage", "rail": 1, "mv": 4300},
|
||||
{"cmd": "key", "rail": 1, "keys": "pwr", "ms": 60000}, {"cmd": "rm -rf"}):
|
||||
r = c.post("/api/boards/%s/cmd" % B, json=body)
|
||||
assert r.status_code == 400, body
|
||||
assert len(sim.received) == n
|
||||
|
||||
|
||||
def test_board_rejection_is_passed_through(env):
|
||||
c, hub, sim, db = env
|
||||
r = c.post("/api/boards/%s/cmd" % B, json={"cmd": "power_cycle", "rail": 4})
|
||||
assert r.status_code == 200 and r.json()["err"] == "empty"
|
||||
|
||||
|
||||
def test_timeout_is_504(env):
|
||||
c, hub, sim, db = env
|
||||
sim.drop_next_replies = 99
|
||||
r = c.post("/api/boards/%s/cmd" % B, json={"cmd": "power", "rail": 2, "on": True})
|
||||
assert r.status_code == 504
|
||||
|
||||
|
||||
def test_phone_settings_toggle_auto_recovery(env):
|
||||
c, hub, sim, db = env
|
||||
r = c.patch("/api/boards/%s/rails/3" % B, json={"label": "A-03", "auto_recover": True})
|
||||
assert r.status_code == 200
|
||||
ph = [p for p in r.json()["phones"] if p["rail"] == 3][0]
|
||||
assert ph["label"] == "A-03" and ph["auto_recover"] == 1
|
||||
rec = c.get("/api/recovery").json()
|
||||
assert rec["params"]["window_s"] == 300
|
||||
r = c.put("/api/recovery/params", json={"window_s": 120, "flat_ua": -5, "bogus": 1})
|
||||
assert r.json()["window_s"] == 120 and r.json()["flat_ua"] == 5000 # 负数不收
|
||||
|
||||
|
||||
def test_history_after_telemetry(env):
|
||||
c, hub, sim, db = env
|
||||
for _ in range(3):
|
||||
sim.tick(now_ms())
|
||||
time.sleep(0.1)
|
||||
hub.flush()
|
||||
h = c.get("/api/boards/%s/rails/1/history" % B, params={"start": now_ms() - 60000}).json()
|
||||
assert h["resolution"] == "raw" and len(h["points"]) >= 1
|
||||
assert h["points"][0][2] is not None
|
||||
|
||||
|
||||
def test_websocket_hello_and_live_tele(env):
|
||||
c, hub, sim, db = env
|
||||
with c.websocket_connect("/api/ws", headers={"X-Auth-User": "zhouyang"}) as ws:
|
||||
hello = ws.receive_json()
|
||||
assert hello["t"] == "hello" and hello["user"] == "zhouyang" and hello["boards"][0]["id"] == B
|
||||
sim.tick(now_ms())
|
||||
msg = ws.receive_json()
|
||||
while msg["t"] != "tele":
|
||||
msg = ws.receive_json()
|
||||
assert msg["board"] == B and len(msg["status"]["rails"]) == 5
|
||||
|
||||
|
||||
def test_spa_fallback_is_not_cached(env):
|
||||
c, hub, sim, db = env
|
||||
r = c.get("/some/deep/route")
|
||||
assert r.status_code == 200 and "rack" in r.text
|
||||
assert r.headers["cache-control"] == "no-cache"
|
||||
|
||||
|
||||
def test_a_real_websocket_library_is_installed():
|
||||
"""TestClient 自带 WS 实现，上面那条 WS 用例在没装 websockets 的机器上也是绿的；
|
||||
而真 uvicorn 缺了它会把 /api/ws 当普通 GET（落到 SPA 兜底，回 200 HTML）。
|
||||
deploy.sh 在服务器上跑这套测试，所以缺库会在换上去之前被拦住。"""
|
||||
from prrack.main import ws_library_ok
|
||||
assert ws_library_ok()
|
||||
|
||||
|
||||
def test_board_history_raw_and_minute(env):
|
||||
c, hub, sim, db = env
|
||||
base = now_ms() // 60000 * 60000 - 3 * 60000
|
||||
rows = [(base + i * 1000, B, 12000, 1500000, 35000 + i, 40000 - i, 300) for i in range(120)]
|
||||
db.many("INSERT INTO board_tele VALUES (?,?,?,?,?,?,?)", rows)
|
||||
h = c.get("/api/boards/%s/history" % B, params={"start": base, "end": base + 120000}).json()
|
||||
assert h["resolution"] == "raw" and len(h["points"]) >= 100
|
||||
p = h["points"][0]
|
||||
assert p[1] == 35000 and p[4] == 40000 and p[7] == 300 and p[8] == 12000
|
||||
# 分钟聚合：两个整分钟，每分钟 min/max 就是那一分钟里的首尾
|
||||
assert db.downsample(base + 180000) > 0
|
||||
m = db.board_history(B, base, base + 120000 + 1) # 跨度小也能强制走聚合：t0 早于原始保留期时
|
||||
rows_min = db.q("SELECT * FROM board_tele_min WHERE board_id=? ORDER BY ts", (B,))
|
||||
assert [r["t0_min"] for r in rows_min][:2] == [35000, 35060]
|
||||
assert [r["t0_max"] for r in rows_min][:2] == [35059, 35119]
|
||||
assert rows_min[0]["t1_min"] == 40000 - 59 and rows_min[0]["fan_avg"] == 300
|
||||
assert m["resolution"] == "raw"
|
||||
old = db.board_history(B, base - 10 * 86400000, base + 120000)
|
||||
# 10 天跨度 ⇒ 桶约 9 分钟，两分钟落进同一桶：min 取两分钟的最小、max 取最大
|
||||
assert old["resolution"] == "minute" and old["points"][0][2] == 35000
|
||||
assert max(p[3] for p in old["points"]) == 35119
|
||||
assert c.get("/api/boards/nope/history", params={"start": 0}).status_code == 404
|
||||
|
||||
|
||||
def test_boot_check_event_is_named_and_only_the_failure_is_severe(env):
|
||||
# 固件 0.1.2 LOG_EV_BOOT_CHECK：arg 低 4 位 = 按了几次，bit7 = 最终没开起来
|
||||
c, hub, sim, db = env
|
||||
now = now_ms()
|
||||
db.many("INSERT INTO events (board_id, seq, t_ms, rail, type, arg, mv, ua, received) VALUES (?,?,?,?,?,?,?,?,?)",
|
||||
[(B, 9001, 1000, 1, 19, 2, 3931, 210000, now),
|
||||
(B, 9002, 2000, 1, 19, 0x83, 3931, 1500, now + 1)])
|
||||
ev = {e["seq"]: e for e in c.get("/api/events", params={"board": B}).json()}
|
||||
assert ev[9001]["code"] == "BOOT_CHECK" and ev[9001]["severe"] is False
|
||||
assert "按了 2 次，开起来了" in ev[9001]["name"]
|
||||
assert ev[9002]["severe"] is True and "按了 3 次，没开起来" in ev[9002]["name"]
|
||||
|
||||
|
||||
def test_board_history_sums_rail_power(env):
|
||||
# 五路主板合计功率：同一时刻各路 mv×ua 求和再按桶平均；没有主板遥测的时刻是 None
|
||||
c, hub, sim, db = env
|
||||
base = now_ms() // 60000 * 60000 - 3 * 60000
|
||||
db.many("INSERT INTO board_tele VALUES (?,?,?,?,?,?,?)",
|
||||
[(base + i * 1000, B, 12000, 1000000, 35000, 35000, 0) for i in range(120)])
|
||||
# 前 60 s 有两路：4.0 V × 500 mA + 3.9 V × 1 A = 5.9 W；后 60 s 没有主板数据
|
||||
db.many("INSERT INTO tele VALUES (?,?,?,?,?,?,?,?,?,?)",
|
||||
[(base + i * 1000, B, r, mv, ua, 1, 1, 1, 1, 0)
|
||||
for i in range(60) for r, mv, ua in ((1, 4000, 500000), (2, 3900, 1000000))])
|
||||
h = db.board_history(B, base, base + 120000)
|
||||
assert h["resolution"] == "raw"
|
||||
assert {p[10] for p in h["points"] if p[0] < base + 60000} == {5900}
|
||||
assert {p[10] for p in h["points"] if p[0] >= base + 60000} == {None}
|
||||
assert db.downsample(base + 180000) > 0
|
||||
old = db.board_history(B, base - 10 * 86400000, base + 120000)
|
||||
assert old["resolution"] == "minute" and old["points"][0][10] == 5900
|
||||
|
||||
+304
-303
@@ -1,303 +1,304 @@
|
||||
"""协议镜像 + Hub（经进程内假 broker 对接模拟板子）+ 自动恢复状态机。
|
||||
|
||||
和固件测试同一个口径：断言打在「板子那头真的收到 / 执行了什么」上（SimBoard.received、
|
||||
它的 rails 状态），不看 Hub 自己回了什么 —— 重传换了 seq 的现象恰恰是 Hub 这边一切正常、
|
||||
手机被按了两遍。"""
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from prrack import protocol
|
||||
from prrack.bus import MemBroker, MemBus, topic_matches
|
||||
from prrack.db import Db
|
||||
from prrack.hub import CmdTimeout, Hub
|
||||
from prrack.recovery import GAVE_UP, HARD, IDLE, SOFT, Params, PhoneRecovery, RecoveryManager
|
||||
from prrack.simboard import HUNG, NORMAL, OFF, SimBoard
|
||||
|
||||
B = "b81f3fc9c8bc"
|
||||
|
||||
|
||||
def run(coro):
|
||||
loop = asyncio.new_event_loop()
|
||||
try:
|
||||
return loop.run_until_complete(coro)
|
||||
finally: # 收掉 Hub 的后台任务（flusher / maint），免得退出时刷一屏 pending 警告
|
||||
tasks = asyncio.all_tasks(loop)
|
||||
for t in tasks:
|
||||
t.cancel()
|
||||
loop.run_until_complete(asyncio.gather(*tasks, return_exceptions=True))
|
||||
loop.close()
|
||||
|
||||
|
||||
async def settle():
|
||||
for _ in range(5):
|
||||
await asyncio.sleep(0)
|
||||
|
||||
|
||||
def rig(tmp_path, present=(True,) * 5, timeout=0.05):
|
||||
broker = MemBroker()
|
||||
db = Db(str(tmp_path / "t.db"))
|
||||
hub = Hub(db, MemBus(broker), attempt_timeout=timeout)
|
||||
sim = SimBoard(MemBus(broker), B, present=present)
|
||||
return hub, sim, db
|
||||
|
||||
|
||||
# ---- 协议镜像 -----------------------------------------------------------------
|
||||
|
||||
def test_build_command_whitelist_and_ranges():
|
||||
assert protocol.build_command({"cmd": "key", "rail": 3, "keys": "vdn+pwr", "ms": 2000}) == \
|
||||
{"cmd": "key", "rail": 3, "keys": "vdn+pwr", "ms": 2000}
|
||||
bad = [
|
||||
{"cmd": "wifi", "ssid": "x"}, # 只收 USB 的命令不给界面
|
||||
{"cmd": "events"}, # 服务器自己同步
|
||||
{"cmd": "set_voltage", "rail": 1, "mv": 4011}, # SR-13 之上
|
||||
{"cmd": "set_voltage", "rail": 1, "mv": "3900"},
|
||||
{"cmd": "key", "rail": 1, "keys": "pwr+home", "ms": 100},
|
||||
{"cmd": "key", "rail": 1, "keys": "pwr", "ms": 15001},
|
||||
{"cmd": "power", "rail": 6, "on": True},
|
||||
{"cmd": "power", "rail": True, "on": True}, # bool 不是 int
|
||||
{"cmd": "vbus", "rail": 1, "on": 1},
|
||||
{"cmd": "fan", "duty": 1001},
|
||||
]
|
||||
for b in bad:
|
||||
with pytest.raises(protocol.CmdError):
|
||||
protocol.build_command(b)
|
||||
|
||||
|
||||
def test_seq_is_strictly_increasing_and_in_range():
|
||||
t = [1700000000.0]
|
||||
g = protocol.SeqGen(clock=lambda: t[0])
|
||||
a = [g.next() for _ in range(5)]
|
||||
assert a == sorted(set(a)) and all(1 <= x <= 0xFFFFFFFF for x in a)
|
||||
t[0] -= 100 # 时钟往回跳也不许重复
|
||||
assert g.next() > a[-1] or g.next() == 1
|
||||
|
||||
|
||||
def test_event_table_matches_firmware_numbers():
|
||||
# firmware/main/evlog.h 冻结的数字（抽几个钉住）
|
||||
assert protocol.EVENT_TYPES[3][0] == "OVER_VOLT"
|
||||
assert protocol.EVENT_TYPES[12][0] == "REBOOT"
|
||||
assert protocol.EVENT_TYPES[13][0] == "CMD_REJECT"
|
||||
assert protocol.EVENT_TYPES[16][0] == "PD_LIMIT"
|
||||
assert protocol.CMD_NAMES[9] == "power" and protocol.CMD_NAMES[12] == "net"
|
||||
assert protocol.EVENT_TYPES[18][0] == "OTA" and protocol.CMD_NAMES[13] == "ota" # C2
|
||||
|
||||
|
||||
def test_topic_matching():
|
||||
assert topic_matches("phonerail/+/rsp", "phonerail/%s/rsp" % B)
|
||||
assert not topic_matches("phonerail/+/rsp", "phonerail/%s/tele" % B)
|
||||
assert topic_matches("phonerail/#", "phonerail/x/y")
|
||||
assert protocol.parse_topic("phonerail/%s/tele" % B) == (B, "tele")
|
||||
assert protocol.parse_topic("evse/123/state") is None
|
||||
|
||||
|
||||
# ---- Hub ---------------------------------------------------------------------
|
||||
|
||||
def test_tele_is_ingested_and_board_discovered(tmp_path):
|
||||
hub, sim, db = rig(tmp_path)
|
||||
|
||||
async def go():
|
||||
await hub.start()
|
||||
sim.start()
|
||||
sim.tick(1000)
|
||||
await settle()
|
||||
hub.flush()
|
||||
|
||||
run(go())
|
||||
snap = hub.snapshot(B)
|
||||
assert snap["online"] and snap["fw"] == "0.1.0-sim"
|
||||
assert snap["status"]["rails"][0]["en"] is True
|
||||
assert db.q("SELECT COUNT(*) c FROM tele")[0]["c"] == 5
|
||||
assert len(snap["phones"]) == 5 and snap["phones"][0]["auto_recover"] == 0
|
||||
|
||||
|
||||
def test_command_round_trip_and_audit(tmp_path):
|
||||
hub, sim, db = rig(tmp_path)
|
||||
|
||||
async def go():
|
||||
await hub.start()
|
||||
sim.start()
|
||||
return await hub.call(B, {"cmd": "power", "rail": 2, "on": False}, "alice")
|
||||
|
||||
rsp = run(go())
|
||||
assert rsp["ok"] is True
|
||||
assert sim.rails[1].en is False
|
||||
row = db.q("SELECT * FROM audit")[0]
|
||||
assert row["user"] == "alice" and row["rail"] == 2 and row["ok"] == 1
|
||||
|
||||
|
||||
def test_retransmit_reuses_the_same_seq_and_executes_once(tmp_path):
|
||||
"""⭐ 回应丢了一次：Hub 用**同一个 seq** 重发，板子判重，只执行一次。"""
|
||||
hub, sim, db = rig(tmp_path)
|
||||
sim.drop_next_replies = 1
|
||||
|
||||
async def go():
|
||||
await hub.start()
|
||||
sim.start()
|
||||
return await hub.call(B, {"cmd": "power_cycle", "rail": 1}, "bob")
|
||||
|
||||
rsp = run(go())
|
||||
assert rsp["ok"] is True and rsp.get("dup") is True
|
||||
sent = [r for r in sim.received if r.get("cmd") == "power_cycle"]
|
||||
assert len(sent) == 2 and sent[0]["seq"] == sent[1]["seq"]
|
||||
assert [e[3] for e in sim.events].count(12) == 1 # 重启事件只记了一次
|
||||
|
||||
|
||||
def test_timeout_is_audited_and_raised(tmp_path):
|
||||
hub, sim, db = rig(tmp_path)
|
||||
sim.drop_next_replies = 99
|
||||
|
||||
async def go():
|
||||
await hub.start()
|
||||
sim.start()
|
||||
with pytest.raises(CmdTimeout):
|
||||
await hub.call(B, {"cmd": "power", "rail": 1, "on": True}, "carol")
|
||||
|
||||
run(go())
|
||||
row = db.q("SELECT * FROM audit")[0]
|
||||
assert row["ok"] == 0 and json.loads(row["response"])["err"] == "timeout"
|
||||
assert len({r["seq"] for r in sim.received}) == 1 # 三次都是同一个 seq
|
||||
|
||||
|
||||
def test_events_are_synced_incrementally(tmp_path):
|
||||
hub, sim, db = rig(tmp_path)
|
||||
for i in range(19):
|
||||
sim._ev(1, 8, i) # 加上 BOOT 共 20 条：要分 3 页
|
||||
|
||||
async def go():
|
||||
await hub.start()
|
||||
sim.start()
|
||||
sim.tick(1000)
|
||||
for _ in range(20):
|
||||
await settle()
|
||||
sim._ev(2, 3, 0, 4200, 1) # 又来一条过压
|
||||
sim.tick(2000)
|
||||
for _ in range(20):
|
||||
await settle()
|
||||
|
||||
run(go())
|
||||
seqs = [r["seq"] for r in db.q("SELECT seq FROM events ORDER BY seq")]
|
||||
assert seqs == list(range(21))
|
||||
assert db.q("SELECT ev_cursor FROM boards")[0]["ev_cursor"] == 21
|
||||
|
||||
|
||||
def test_empty_slot_reply_passes_through(tmp_path):
|
||||
hub, sim, db = rig(tmp_path, present=(True, True, True, False, True))
|
||||
|
||||
async def go():
|
||||
await hub.start()
|
||||
sim.start()
|
||||
return await hub.call(B, {"cmd": "power", "rail": 4, "on": True}, "dave")
|
||||
|
||||
assert run(go()) == {"seq": sim.received[-1]["seq"], "ok": False, "err": "empty"}
|
||||
|
||||
|
||||
# ---- 自动恢复：状态机 ------------------------------------------------------------
|
||||
|
||||
P = Params(window_s=60, flat_ua=5000, low_ua=30000, settle_s=30, max_per_hour=3)
|
||||
|
||||
|
||||
def feed(ph, t0, secs, ua_fn):
|
||||
out = []
|
||||
for s in range(secs):
|
||||
st, act = ph.feed(t0 + s * 1000, ua_fn(s), 3)
|
||||
if st or act:
|
||||
out.append((t0 + s * 1000, st, act))
|
||||
return out
|
||||
|
||||
|
||||
def test_flat_current_escalates_soft_then_hard_then_gives_up():
|
||||
ph = PhoneRecovery(P)
|
||||
r = feed(ph, 0, 61, lambda s: 412000)
|
||||
assert r[-1][1] == SOFT and r[-1][2][1] == {"cmd": "key", "rail": 3, "keys": "pwr", "ms": 10000}
|
||||
t = r[-1][0]
|
||||
# settle 期间的样本一概不看（哪怕是平线）
|
||||
assert feed(ph, t + 1000, 29, lambda s: 412000) == []
|
||||
r = feed(ph, t + 30000, 62, lambda s: 412000)
|
||||
assert r[-1][1] == HARD and r[-1][2][1] == {"cmd": "power_cycle", "rail": 3}
|
||||
t = r[-1][0]
|
||||
r = feed(ph, t + 30000, 62, lambda s: 412000)
|
||||
assert r[-1][1] == GAVE_UP and r[-1][2] is None
|
||||
assert feed(ph, t + 200000, 100, lambda s: 412000) == [] # 放弃之后再也不动
|
||||
|
||||
|
||||
def test_recovers_after_soft_when_current_moves_again():
|
||||
ph = PhoneRecovery(P)
|
||||
t = feed(ph, 0, 61, lambda s: 412000)[-1][0]
|
||||
r = feed(ph, t + 30000, 70, lambda s: 300000 + (s % 7) * 40000)
|
||||
assert r[-1][1] == IDLE and ph.reason == "recovered"
|
||||
|
||||
|
||||
def test_low_current_is_detected_and_normal_is_not():
|
||||
ph = PhoneRecovery(P)
|
||||
assert feed(ph, 0, 200, lambda s: 300000 + (s % 5) * 50000) == []
|
||||
ph2 = PhoneRecovery(P)
|
||||
assert feed(ph2, 0, 61, lambda s: 800 + s % 3)[-1][2][0] == "low"
|
||||
|
||||
|
||||
def test_rate_limit_gives_up():
|
||||
ph = PhoneRecovery(Params(window_s=60, settle_s=30, max_per_hour=2))
|
||||
t = 0
|
||||
for _ in range(2): # 两次软重启都「恢复」了，但每次很快又卡
|
||||
assert feed(ph, t, 65, lambda s: 412000)[-1][1] == SOFT
|
||||
t += 65000 # 下一段接着上一段喂完的时刻往后走（时间不倒退）
|
||||
# 前 30 s 还在 settle 里不算数，之后要攒满一个 60 s 窗口才判「恢复」
|
||||
assert feed(ph, t, 100, lambda s: 300000 + (s % 7) * 40000)[-1][1] == IDLE
|
||||
t += 100000
|
||||
r = feed(ph, t, 65, lambda s: 412000)
|
||||
assert r[-1][1] == GAVE_UP and ph.reason == "rate_limit"
|
||||
|
||||
|
||||
# ---- 自动恢复：接到 Hub + 模拟板上 -------------------------------------------------
|
||||
|
||||
def test_auto_recovery_end_to_end_with_sim(tmp_path):
|
||||
"""⭐ 模拟板上第 3 台卡死：开了自动恢复 → 5 分钟后下长按电源 → 板子执行 → 恢复。
|
||||
没开自动恢复的第 2 台（也卡死）一个命令都不收到。"""
|
||||
hub, sim, db = rig(tmp_path)
|
||||
clock = [1_000_000]
|
||||
rec = RecoveryManager(hub, lambda: clock[0])
|
||||
sim.set_phone(3, HUNG)
|
||||
sim.set_phone(2, HUNG)
|
||||
|
||||
async def go():
|
||||
await hub.start()
|
||||
sim.start()
|
||||
await settle() # 等 Hub 处理完上线消息、建好这块板的 phones 行
|
||||
db.x("UPDATE phones SET auto_recover=1 WHERE rail=3")
|
||||
rec.reload()
|
||||
for s in range(300 + 130 + 320):
|
||||
clock[0] += 1000
|
||||
sim.tick(clock[0])
|
||||
await settle()
|
||||
|
||||
run(go())
|
||||
keys = [r for r in sim.received if r.get("cmd") == "key"]
|
||||
assert len(keys) == 1 and keys[0]["rail"] == 3 and keys[0]["ms"] == 10000
|
||||
assert not [r for r in sim.received if r.get("rail") == 2]
|
||||
assert sim.rails[2].phone == NORMAL
|
||||
states = [r["state"] for r in db.q("SELECT state FROM recovery_log ORDER BY id")]
|
||||
assert states == [SOFT, IDLE]
|
||||
assert db.q("SELECT user FROM audit")[0]["user"] == "auto-recovery"
|
||||
|
||||
|
||||
def test_auto_recovery_ignores_rails_that_are_off(tmp_path):
|
||||
hub, sim, db = rig(tmp_path)
|
||||
clock = [0]
|
||||
rec = RecoveryManager(hub, lambda: clock[0])
|
||||
sim.set_phone(1, OFF)
|
||||
sim.rails[0].en = False # 这一路本来就关着：电流 0 不是故障
|
||||
|
||||
async def go():
|
||||
await hub.start()
|
||||
sim.start()
|
||||
await settle()
|
||||
db.x("UPDATE phones SET auto_recover=1")
|
||||
rec.reload()
|
||||
for s in range(400):
|
||||
clock[0] += 1000
|
||||
sim.tick(clock[0])
|
||||
await settle()
|
||||
|
||||
run(go())
|
||||
assert not [r for r in sim.received if r.get("cmd") in ("key", "power_cycle")]
|
||||
"""协议镜像 + Hub（经进程内假 broker 对接模拟板子）+ 自动恢复状态机。
|
||||
|
||||
和固件测试同一个口径：断言打在「板子那头真的收到 / 执行了什么」上（SimBoard.received、
|
||||
它的 rails 状态），不看 Hub 自己回了什么 —— 重传换了 seq 的现象恰恰是 Hub 这边一切正常、
|
||||
手机被按了两遍。"""
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from prrack import protocol
|
||||
from prrack.bus import MemBroker, MemBus, topic_matches
|
||||
from prrack.db import Db
|
||||
from prrack.hub import CmdTimeout, Hub
|
||||
from prrack.recovery import GAVE_UP, HARD, IDLE, SOFT, Params, PhoneRecovery, RecoveryManager
|
||||
from prrack.simboard import HUNG, NORMAL, OFF, SimBoard
|
||||
|
||||
B = "b81f3fc9c8bc"
|
||||
|
||||
|
||||
def run(coro):
|
||||
loop = asyncio.new_event_loop()
|
||||
try:
|
||||
return loop.run_until_complete(coro)
|
||||
finally: # 收掉 Hub 的后台任务（flusher / maint），免得退出时刷一屏 pending 警告
|
||||
tasks = asyncio.all_tasks(loop)
|
||||
for t in tasks:
|
||||
t.cancel()
|
||||
loop.run_until_complete(asyncio.gather(*tasks, return_exceptions=True))
|
||||
loop.close()
|
||||
|
||||
|
||||
async def settle():
|
||||
for _ in range(5):
|
||||
await asyncio.sleep(0)
|
||||
|
||||
|
||||
def rig(tmp_path, present=(True,) * 5, timeout=0.05):
|
||||
broker = MemBroker()
|
||||
db = Db(str(tmp_path / "t.db"))
|
||||
hub = Hub(db, MemBus(broker), attempt_timeout=timeout)
|
||||
sim = SimBoard(MemBus(broker), B, present=present)
|
||||
return hub, sim, db
|
||||
|
||||
|
||||
# ---- 协议镜像 -----------------------------------------------------------------
|
||||
|
||||
def test_build_command_whitelist_and_ranges():
|
||||
assert protocol.build_command({"cmd": "key", "rail": 3, "keys": "vdn+pwr", "ms": 2000}) == \
|
||||
{"cmd": "key", "rail": 3, "keys": "vdn+pwr", "ms": 2000}
|
||||
bad = [
|
||||
{"cmd": "wifi", "ssid": "x"}, # 只收 USB 的命令不给界面
|
||||
{"cmd": "events"}, # 服务器自己同步
|
||||
{"cmd": "set_voltage", "rail": 1, "mv": 4011}, # SR-13 之上
|
||||
{"cmd": "set_voltage", "rail": 1, "mv": "3900"},
|
||||
{"cmd": "key", "rail": 1, "keys": "pwr+home", "ms": 100},
|
||||
{"cmd": "key", "rail": 1, "keys": "pwr", "ms": 15001},
|
||||
{"cmd": "power", "rail": 6, "on": True},
|
||||
{"cmd": "power", "rail": True, "on": True}, # bool 不是 int
|
||||
{"cmd": "vbus", "rail": 1, "on": 1},
|
||||
{"cmd": "fan", "duty": 1001},
|
||||
]
|
||||
for b in bad:
|
||||
with pytest.raises(protocol.CmdError):
|
||||
protocol.build_command(b)
|
||||
|
||||
|
||||
def test_seq_is_strictly_increasing_and_in_range():
|
||||
t = [1700000000.0]
|
||||
g = protocol.SeqGen(clock=lambda: t[0])
|
||||
a = [g.next() for _ in range(5)]
|
||||
assert a == sorted(set(a)) and all(1 <= x <= 0xFFFFFFFF for x in a)
|
||||
t[0] -= 100 # 时钟往回跳也不许重复
|
||||
assert g.next() > a[-1] or g.next() == 1
|
||||
|
||||
|
||||
def test_event_table_matches_firmware_numbers():
|
||||
# firmware/main/evlog.h 冻结的数字（抽几个钉住）
|
||||
assert protocol.EVENT_TYPES[3][0] == "OVER_VOLT"
|
||||
assert protocol.EVENT_TYPES[12][0] == "REBOOT"
|
||||
assert protocol.EVENT_TYPES[13][0] == "CMD_REJECT"
|
||||
assert protocol.EVENT_TYPES[16][0] == "PD_LIMIT"
|
||||
assert protocol.CMD_NAMES[9] == "power" and protocol.CMD_NAMES[12] == "net"
|
||||
assert protocol.EVENT_TYPES[18][0] == "OTA" and protocol.CMD_NAMES[13] == "ota" # C2
|
||||
assert protocol.EVENT_TYPES[19][0] == "BOOT_CHECK" and protocol.BOOT_CHECK_FAILED == 0x80 # 0.1.2
|
||||
|
||||
|
||||
def test_topic_matching():
|
||||
assert topic_matches("phonerail/+/rsp", "phonerail/%s/rsp" % B)
|
||||
assert not topic_matches("phonerail/+/rsp", "phonerail/%s/tele" % B)
|
||||
assert topic_matches("phonerail/#", "phonerail/x/y")
|
||||
assert protocol.parse_topic("phonerail/%s/tele" % B) == (B, "tele")
|
||||
assert protocol.parse_topic("evse/123/state") is None
|
||||
|
||||
|
||||
# ---- Hub ---------------------------------------------------------------------
|
||||
|
||||
def test_tele_is_ingested_and_board_discovered(tmp_path):
|
||||
hub, sim, db = rig(tmp_path)
|
||||
|
||||
async def go():
|
||||
await hub.start()
|
||||
sim.start()
|
||||
sim.tick(1000)
|
||||
await settle()
|
||||
hub.flush()
|
||||
|
||||
run(go())
|
||||
snap = hub.snapshot(B)
|
||||
assert snap["online"] and snap["fw"] == "0.1.0-sim"
|
||||
assert snap["status"]["rails"][0]["en"] is True
|
||||
assert db.q("SELECT COUNT(*) c FROM tele")[0]["c"] == 5
|
||||
assert len(snap["phones"]) == 5 and snap["phones"][0]["auto_recover"] == 0
|
||||
|
||||
|
||||
def test_command_round_trip_and_audit(tmp_path):
|
||||
hub, sim, db = rig(tmp_path)
|
||||
|
||||
async def go():
|
||||
await hub.start()
|
||||
sim.start()
|
||||
return await hub.call(B, {"cmd": "power", "rail": 2, "on": False}, "alice")
|
||||
|
||||
rsp = run(go())
|
||||
assert rsp["ok"] is True
|
||||
assert sim.rails[1].en is False
|
||||
row = db.q("SELECT * FROM audit")[0]
|
||||
assert row["user"] == "alice" and row["rail"] == 2 and row["ok"] == 1
|
||||
|
||||
|
||||
def test_retransmit_reuses_the_same_seq_and_executes_once(tmp_path):
|
||||
"""⭐ 回应丢了一次：Hub 用**同一个 seq** 重发，板子判重，只执行一次。"""
|
||||
hub, sim, db = rig(tmp_path)
|
||||
sim.drop_next_replies = 1
|
||||
|
||||
async def go():
|
||||
await hub.start()
|
||||
sim.start()
|
||||
return await hub.call(B, {"cmd": "power_cycle", "rail": 1}, "bob")
|
||||
|
||||
rsp = run(go())
|
||||
assert rsp["ok"] is True and rsp.get("dup") is True
|
||||
sent = [r for r in sim.received if r.get("cmd") == "power_cycle"]
|
||||
assert len(sent) == 2 and sent[0]["seq"] == sent[1]["seq"]
|
||||
assert [e[3] for e in sim.events].count(12) == 1 # 重启事件只记了一次
|
||||
|
||||
|
||||
def test_timeout_is_audited_and_raised(tmp_path):
|
||||
hub, sim, db = rig(tmp_path)
|
||||
sim.drop_next_replies = 99
|
||||
|
||||
async def go():
|
||||
await hub.start()
|
||||
sim.start()
|
||||
with pytest.raises(CmdTimeout):
|
||||
await hub.call(B, {"cmd": "power", "rail": 1, "on": True}, "carol")
|
||||
|
||||
run(go())
|
||||
row = db.q("SELECT * FROM audit")[0]
|
||||
assert row["ok"] == 0 and json.loads(row["response"])["err"] == "timeout"
|
||||
assert len({r["seq"] for r in sim.received}) == 1 # 三次都是同一个 seq
|
||||
|
||||
|
||||
def test_events_are_synced_incrementally(tmp_path):
|
||||
hub, sim, db = rig(tmp_path)
|
||||
for i in range(19):
|
||||
sim._ev(1, 8, i) # 加上 BOOT 共 20 条：要分 3 页
|
||||
|
||||
async def go():
|
||||
await hub.start()
|
||||
sim.start()
|
||||
sim.tick(1000)
|
||||
for _ in range(20):
|
||||
await settle()
|
||||
sim._ev(2, 3, 0, 4200, 1) # 又来一条过压
|
||||
sim.tick(2000)
|
||||
for _ in range(20):
|
||||
await settle()
|
||||
|
||||
run(go())
|
||||
seqs = [r["seq"] for r in db.q("SELECT seq FROM events ORDER BY seq")]
|
||||
assert seqs == list(range(21))
|
||||
assert db.q("SELECT ev_cursor FROM boards")[0]["ev_cursor"] == 21
|
||||
|
||||
|
||||
def test_empty_slot_reply_passes_through(tmp_path):
|
||||
hub, sim, db = rig(tmp_path, present=(True, True, True, False, True))
|
||||
|
||||
async def go():
|
||||
await hub.start()
|
||||
sim.start()
|
||||
return await hub.call(B, {"cmd": "power", "rail": 4, "on": True}, "dave")
|
||||
|
||||
assert run(go()) == {"seq": sim.received[-1]["seq"], "ok": False, "err": "empty"}
|
||||
|
||||
|
||||
# ---- 自动恢复：状态机 ------------------------------------------------------------
|
||||
|
||||
P = Params(window_s=60, flat_ua=5000, low_ua=30000, settle_s=30, max_per_hour=3)
|
||||
|
||||
|
||||
def feed(ph, t0, secs, ua_fn):
|
||||
out = []
|
||||
for s in range(secs):
|
||||
st, act = ph.feed(t0 + s * 1000, ua_fn(s), 3)
|
||||
if st or act:
|
||||
out.append((t0 + s * 1000, st, act))
|
||||
return out
|
||||
|
||||
|
||||
def test_flat_current_escalates_soft_then_hard_then_gives_up():
|
||||
ph = PhoneRecovery(P)
|
||||
r = feed(ph, 0, 61, lambda s: 412000)
|
||||
assert r[-1][1] == SOFT and r[-1][2][1] == {"cmd": "key", "rail": 3, "keys": "pwr", "ms": 10000}
|
||||
t = r[-1][0]
|
||||
# settle 期间的样本一概不看（哪怕是平线）
|
||||
assert feed(ph, t + 1000, 29, lambda s: 412000) == []
|
||||
r = feed(ph, t + 30000, 62, lambda s: 412000)
|
||||
assert r[-1][1] == HARD and r[-1][2][1] == {"cmd": "power_cycle", "rail": 3}
|
||||
t = r[-1][0]
|
||||
r = feed(ph, t + 30000, 62, lambda s: 412000)
|
||||
assert r[-1][1] == GAVE_UP and r[-1][2] is None
|
||||
assert feed(ph, t + 200000, 100, lambda s: 412000) == [] # 放弃之后再也不动
|
||||
|
||||
|
||||
def test_recovers_after_soft_when_current_moves_again():
|
||||
ph = PhoneRecovery(P)
|
||||
t = feed(ph, 0, 61, lambda s: 412000)[-1][0]
|
||||
r = feed(ph, t + 30000, 70, lambda s: 300000 + (s % 7) * 40000)
|
||||
assert r[-1][1] == IDLE and ph.reason == "recovered"
|
||||
|
||||
|
||||
def test_low_current_is_detected_and_normal_is_not():
|
||||
ph = PhoneRecovery(P)
|
||||
assert feed(ph, 0, 200, lambda s: 300000 + (s % 5) * 50000) == []
|
||||
ph2 = PhoneRecovery(P)
|
||||
assert feed(ph2, 0, 61, lambda s: 800 + s % 3)[-1][2][0] == "low"
|
||||
|
||||
|
||||
def test_rate_limit_gives_up():
|
||||
ph = PhoneRecovery(Params(window_s=60, settle_s=30, max_per_hour=2))
|
||||
t = 0
|
||||
for _ in range(2): # 两次软重启都「恢复」了，但每次很快又卡
|
||||
assert feed(ph, t, 65, lambda s: 412000)[-1][1] == SOFT
|
||||
t += 65000 # 下一段接着上一段喂完的时刻往后走（时间不倒退）
|
||||
# 前 30 s 还在 settle 里不算数，之后要攒满一个 60 s 窗口才判「恢复」
|
||||
assert feed(ph, t, 100, lambda s: 300000 + (s % 7) * 40000)[-1][1] == IDLE
|
||||
t += 100000
|
||||
r = feed(ph, t, 65, lambda s: 412000)
|
||||
assert r[-1][1] == GAVE_UP and ph.reason == "rate_limit"
|
||||
|
||||
|
||||
# ---- 自动恢复：接到 Hub + 模拟板上 -------------------------------------------------
|
||||
|
||||
def test_auto_recovery_end_to_end_with_sim(tmp_path):
|
||||
"""⭐ 模拟板上第 3 台卡死：开了自动恢复 → 5 分钟后下长按电源 → 板子执行 → 恢复。
|
||||
没开自动恢复的第 2 台（也卡死）一个命令都不收到。"""
|
||||
hub, sim, db = rig(tmp_path)
|
||||
clock = [1_000_000]
|
||||
rec = RecoveryManager(hub, lambda: clock[0])
|
||||
sim.set_phone(3, HUNG)
|
||||
sim.set_phone(2, HUNG)
|
||||
|
||||
async def go():
|
||||
await hub.start()
|
||||
sim.start()
|
||||
await settle() # 等 Hub 处理完上线消息、建好这块板的 phones 行
|
||||
db.x("UPDATE phones SET auto_recover=1 WHERE rail=3")
|
||||
rec.reload()
|
||||
for s in range(300 + 130 + 320):
|
||||
clock[0] += 1000
|
||||
sim.tick(clock[0])
|
||||
await settle()
|
||||
|
||||
run(go())
|
||||
keys = [r for r in sim.received if r.get("cmd") == "key"]
|
||||
assert len(keys) == 1 and keys[0]["rail"] == 3 and keys[0]["ms"] == 10000
|
||||
assert not [r for r in sim.received if r.get("rail") == 2]
|
||||
assert sim.rails[2].phone == NORMAL
|
||||
states = [r["state"] for r in db.q("SELECT state FROM recovery_log ORDER BY id")]
|
||||
assert states == [SOFT, IDLE]
|
||||
assert db.q("SELECT user FROM audit")[0]["user"] == "auto-recovery"
|
||||
|
||||
|
||||
def test_auto_recovery_ignores_rails_that_are_off(tmp_path):
|
||||
hub, sim, db = rig(tmp_path)
|
||||
clock = [0]
|
||||
rec = RecoveryManager(hub, lambda: clock[0])
|
||||
sim.set_phone(1, OFF)
|
||||
sim.rails[0].en = False # 这一路本来就关着：电流 0 不是故障
|
||||
|
||||
async def go():
|
||||
await hub.start()
|
||||
sim.start()
|
||||
await settle()
|
||||
db.x("UPDATE phones SET auto_recover=1")
|
||||
rec.reload()
|
||||
for s in range(400):
|
||||
clock[0] += 1000
|
||||
sim.tick(clock[0])
|
||||
await settle()
|
||||
|
||||
run(go())
|
||||
assert not [r for r in sim.received if r.get("cmd") in ("key", "power_cycle")]
|
||||
|
||||
在新工单中引用
屏蔽一个用户