test(scripts): 36 个 E2E 与 6 个机械校验工具（1548 条断言）+ 一键交付自检
新增：check-api-parity（契约 121 接口 ↔ 控制器，自带「必须恰好 121」自检）、 check-entity-columns（基类要求的列 vs 迁移建表列，F-35 那个坑踩了四次之后上的工具）、 check-manual（手册引用的错误码与提示文案对得上代码）、check-test-profile（test profile 真起一次）、 e2e-prodguard、load-test（四场景、按端点分别统计）、verify-all（一条命令跑完交付自检）、 dev-seed-audit（界面验收用的演示数据），以及 M1~M9 各轮 E2E。 压测抓到一个真缺陷并修掉（F-92）：门禁 IP 限流原来计「所有请求」，于是一个班连同一个 WiFi 时 60 个不同学生同时核验只过了 20 个 → 改为只计失败 + 单独的成功上限，20/60 → 60/60。 度量工具的偏差会伪装成产品缺陷（F-60/F-77/F-97），所以每个脚本都断言「动作自己的结论字段」 （submitted / created / canExport），而不是 HTTP 200 + code 0。 验证：verify-all.py --quick 5/5；--with-e2e 见 docs/09 §8。
This commit is contained in:
@@ -0,0 +1,206 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""交付自检：一条命令把"这个仓库现在到底对不对"跑一遍。
|
||||
|
||||
## 为什么需要它
|
||||
|
||||
这个项目的验证手段有六类，散在六个地方：
|
||||
|
||||
| 类别 | 命令 | 需要什么 |
|
||||
|---|---|---|
|
||||
| 契约 ↔ 路由 | `check-api-parity.py` | 无 |
|
||||
| 实体 ↔ 建表 | `check-entity-columns.py` | 无 |
|
||||
| 文档自身一致 | `verify-docs.py` | 无 |
|
||||
| 手册 ↔ 代码文案 | `check-manual.py` | 无 |
|
||||
| 接口行为（1548 条断言） | `scripts/e2e-*.py` | **后端在跑 + PG/Redis** |
|
||||
| 生产配置拦截 | `e2e-prodguard.py` | **能起 jar** |
|
||||
| CI 那个 profile 能不能起 | `check-test-profile.py` | 后端 jar + PG/Redis |
|
||||
|
||||
**要记住"该跑哪几个、哪些需要前置"本身就是一种负担**，而漏跑一个的后果是
|
||||
"以为过了"。所以这里合成一个入口，并且**如实说明哪些因为前置不足而跳过**
|
||||
（不是静默跳过 —— 那与 F-85 是同一类问题）。
|
||||
|
||||
## 用法
|
||||
|
||||
```bash
|
||||
python scripts/verify-all.py # 全部（跳过需要外部前置的，并说明）
|
||||
python scripts/verify-all.py --with-e2e # 连 1548 条接口断言一起跑（需要后端在跑）
|
||||
python scripts/verify-all.py --quick # 只跑不需要任何前置的五项
|
||||
```
|
||||
|
||||
退出码：0 = 全过；1 = 有失败（**跳过不算失败**，但会明确列出来）。
|
||||
"""
|
||||
import argparse
|
||||
import glob
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
sys.stdout.reconfigure(encoding="utf-8")
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
os.chdir(ROOT)
|
||||
|
||||
GREEN, RED, YELLOW, DIM, RESET = "\033[32m", "\033[31m", "\033[33m", "\033[2m", "\033[0m"
|
||||
if not sys.stdout.isatty():
|
||||
GREEN = RED = YELLOW = DIM = RESET = ""
|
||||
|
||||
|
||||
def run(cmd, timeout=3600):
|
||||
"""跑一条命令，返回 (退出码, 输出)。"""
|
||||
try:
|
||||
p = subprocess.run(cmd, shell=True, capture_output=True, text=True,
|
||||
encoding="utf-8", errors="replace", timeout=timeout)
|
||||
return p.returncode, (p.stdout or "") + (p.stderr or "")
|
||||
except subprocess.TimeoutExpired:
|
||||
return -9, "超时"
|
||||
|
||||
|
||||
def backend_up():
|
||||
try:
|
||||
with urllib.request.urlopen("http://localhost:8080/api/v1/auth/captcha", timeout=5):
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description="交付自检")
|
||||
ap.add_argument("--with-e2e", action="store_true", help="连 e2e 一起跑（需要后端在跑）")
|
||||
ap.add_argument("--quick", action="store_true", help="只跑不需要前置的五项")
|
||||
args = ap.parse_args()
|
||||
|
||||
results = [] # (名称, 结果, 摘要)
|
||||
|
||||
print("=" * 92)
|
||||
print(" 交付自检 仓库：%s" % ROOT)
|
||||
print("=" * 92)
|
||||
|
||||
# ── 一、不需要任何前置的机械校验 ──
|
||||
print("\n【一】机械校验（不需要后端/数据库）")
|
||||
checks = [
|
||||
("契约 ↔ 路由", "python scripts/check-api-parity.py --quiet",
|
||||
r"合计：契约 (\d+) ／ 已实现 (\d+) ／ 缺口 (\d+)"),
|
||||
("实体 ↔ 建表", "python scripts/check-entity-columns.py --quiet",
|
||||
r"(\d+) 个实体全部与迁移一致（(\d+) 张表）"),
|
||||
("文档自身一致", "python scripts/verify-docs.py", r"结果: (\d+) 项全部通过"),
|
||||
("手册 ↔ 代码文案", "python scripts/check-manual.py", r"结果：.*"),
|
||||
]
|
||||
for name, cmd, pat in checks:
|
||||
code, out = run(cmd, timeout=300)
|
||||
line = ""
|
||||
m = re.search(pat, out)
|
||||
if m:
|
||||
line = m.group(0).strip()
|
||||
else:
|
||||
tail = [x for x in out.strip().split("\n") if x.strip()]
|
||||
line = tail[-1][:80] if tail else "（无输出）"
|
||||
ok = code == 0
|
||||
results.append((name, ok, line))
|
||||
print(" %s%-14s%s %s" % (GREEN if ok else RED, name, RESET, line))
|
||||
|
||||
# 脚本语法（改脚本时最容易踩：中文串里嵌半角引号）
|
||||
bad = []
|
||||
for fn in sorted(glob.glob("scripts/*.py")):
|
||||
try:
|
||||
compile(open(fn, encoding="utf-8").read(), fn, "exec")
|
||||
except SyntaxError as e:
|
||||
bad.append("%s:%s" % (os.path.basename(fn), e.lineno))
|
||||
ok = not bad
|
||||
results.append(("脚本语法", ok, "全部可编译" if ok else "有语法错：%s" % ", ".join(bad)))
|
||||
print(" %s%-14s%s %s" % (GREEN if ok else RED, "脚本语法", RESET,
|
||||
"全部可编译" if ok else "有语法错：%s" % ", ".join(bad)))
|
||||
|
||||
if args.quick:
|
||||
return summary(results)
|
||||
|
||||
# ── 二、需要后端在跑 / 能起 jar 的 ──
|
||||
print("\n【二】带前置的检查")
|
||||
up = backend_up()
|
||||
if not up:
|
||||
msg = "**跳过**：后端 8080 没在跑（先 `cd backend && java -jar ... --spring.profiles.active=dev`）"
|
||||
results.append(("接口断言（E2E）", None, msg))
|
||||
print(" %s%-14s%s %s" % (YELLOW, "接口断言", RESET, msg))
|
||||
elif not args.with_e2e:
|
||||
msg = "**跳过**：要跑就加 `--with-e2e`（36 个脚本 / 1500+ 条断言，约 10 分钟）"
|
||||
results.append(("接口断言（E2E）", None, msg))
|
||||
print(" %s%-14s%s %s" % (YELLOW, "接口断言", RESET, msg))
|
||||
else:
|
||||
t0 = time.time()
|
||||
files = sorted(glob.glob("scripts/e2e-*.py"))
|
||||
total = fails = 0
|
||||
failed_names = []
|
||||
for f in files:
|
||||
code, out = run("python %s" % f, timeout=900)
|
||||
m = re.search(r"通过 (\d+) 失败 (\d+) 共 (\d+)", out)
|
||||
if m:
|
||||
total += int(m.group(1))
|
||||
if int(m.group(2)) > 0:
|
||||
fails += int(m.group(2))
|
||||
failed_names.append(os.path.basename(f))
|
||||
else:
|
||||
m2 = re.search(r"结果: (\d+)/(\d+)", out)
|
||||
if m2:
|
||||
total += int(m2.group(2))
|
||||
if m2.group(1) != m2.group(2):
|
||||
fails += 1
|
||||
failed_names.append(os.path.basename(f))
|
||||
else:
|
||||
fails += 1
|
||||
failed_names.append(os.path.basename(f) + "(无汇总)")
|
||||
ok = fails == 0
|
||||
msg = "%d 个脚本 / %d 条断言 / 失败 %d%s（%.0fs）" % (
|
||||
len(files), total, fails,
|
||||
"" if ok else "：" + ", ".join(failed_names[:5]), time.time() - t0)
|
||||
results.append(("接口断言（E2E）", ok, msg))
|
||||
print(" %s%-14s%s %s" % (GREEN if ok else RED, "接口断言", RESET, msg))
|
||||
|
||||
# 生产自检：需要 jar
|
||||
jar = glob.glob("backend/xt-server/target/xt-server-*.jar")
|
||||
if not jar:
|
||||
msg = "**跳过**：没有 jar（先 `cd backend && mvn -DskipTests package`）"
|
||||
results.append(("生产配置拦截", None, msg))
|
||||
print(" %s%-14s%s %s" % (YELLOW, "生产配置拦截", RESET, msg))
|
||||
else:
|
||||
# 起 jar 的检查会撞端口，所以只在没别的 java 在跑时做 —— 如实说明
|
||||
code, out = run("python scripts/e2e-prodguard.py", timeout=900)
|
||||
m = re.search(r"通过 (\d+) 失败 (\d+) 共 (\d+)", out)
|
||||
ok = code == 0
|
||||
msg = m.group(0) if m else ("通过" if ok else "失败")
|
||||
results.append(("生产配置拦截", ok, msg))
|
||||
print(" %s%-14s%s %s" % (GREEN if ok else RED, "生产配置拦截", RESET, msg))
|
||||
|
||||
if up:
|
||||
code, out = run("python scripts/check-test-profile.py", timeout=600)
|
||||
m = re.search(r"结果：(.*)", out)
|
||||
ok = code == 0
|
||||
msg = (m.group(1).strip() if m else ("通过" if ok else "失败"))[:70]
|
||||
results.append(("test profile 能起", ok, msg))
|
||||
print(" %s%-14s%s %s" % (GREEN if ok else RED, "test profile 能起", RESET, msg))
|
||||
else:
|
||||
results.append(("test profile 能起", None, "**跳过**：后端没在跑"))
|
||||
print(" %s%-14s%s **跳过**：后端没在跑" % (YELLOW, "test profile 能起", RESET))
|
||||
|
||||
return summary(results)
|
||||
|
||||
|
||||
def summary(results):
|
||||
print("\n" + "=" * 92)
|
||||
passed = [r for r in results if r[1] is True]
|
||||
failed = [r for r in results if r[1] is False]
|
||||
skipped = [r for r in results if r[1] is None]
|
||||
print(" 通过 %d 失败 %d 跳过 %d" % (len(passed), len(failed), len(skipped)))
|
||||
for name, ok, msg in failed:
|
||||
print(" %s✗ %s%s：%s" % (RED, name, RESET, msg[:110]))
|
||||
for name, ok, msg in skipped:
|
||||
# ⚠️ 跳过要**明确列出来**，不能静默 —— 否则"以为过了"（与 F-85 同类）
|
||||
print(" %s— %s 跳过：%s%s" % (YELLOW, name, msg[:100], RESET))
|
||||
print("=" * 92)
|
||||
sys.stdout.flush()
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user