Files
xiaotutongxue/scripts/e2e-m7-cleanup.py
T
lovife 30c3714d35 test(scripts): 36 个 E2E 与 6 个机械校验工具(1548 条断言)+ 一键交付自检
新增:check-api-parity(契约 121 接口 ↔ 控制器,自带「必须恰好 121」自检)、
check-entity-columns(基类要求的列 vs 迁移建表列,F-35 那个坑踩了四次之后上的工具)、
check-manual(手册引用的错误码与提示文案对得上代码)、check-test-profile(test profile 真起一次)、
e2e-prodguard、load-test(四场景、按端点分别统计)、verify-all(一条命令跑完交付自检)、
dev-seed-audit(界面验收用的演示数据),以及 M1~M9 各轮 E2E。

压测抓到一个真缺陷并修掉(F-92):门禁 IP 限流原来计「所有请求」,于是一个班连同一个
WiFi 时 60 个不同学生同时核验只过了 20 个 → 改为只计失败 + 单独的成功上限,20/60 → 60/60。

度量工具的偏差会伪装成产品缺陷(F-60/F-77/F-97),所以每个脚本都断言「动作自己的结论字段」
(submitted / created / canExport),而不是 HTTP 200 + code 0。

验证:verify-all.py --quick 5/5;--with-e2e 见 docs/09 §8。
2026-09-30 09:02:57 +08:00

435 lines
20 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# -*- coding: utf-8 -*-
"""历史数据整理器 E2E(《03-接口设计》§13.2)。"""
import argparse
import base64
import importlib.util
import io as _io
import json
import os
import re
import subprocess
import sys
import time
import zipfile
sys.stdout.reconfigure(encoding="utf-8")
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
PASSED = []
FAILED = []
CREATED_TASK_IDS = []
CREATED_JOB_IDS = []
TMP_ROSTER = os.path.join(ROOT, "scripts", "_e2e-cleanup-roster.xlsx")
def _load(module_name, filename):
path = os.path.join(ROOT, "scripts", filename)
spec = importlib.util.spec_from_file_location(module_name, path)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod
R = _load("e2e_m2_roster", "e2e-m2-roster.py")
BASE = "http://localhost:8080"
S_A = ("整理甲", R.with_checksum("36102920080101071"))
S_B = ("整理乙", R.with_checksum("36102920080202072"))
S_C = ("整理丙", R.with_checksum("36102920080303073"))
WRONG_ID = R.with_checksum("44052419900101099")
def ok(name):
PASSED.append(name)
print(" [OK] " + name)
def fail(name, detail):
FAILED.append((name, detail))
print(" [FAIL] " + name + "\n " + detail)
def check(name, cond, detail=""):
if cond:
ok(name)
else:
fail(name, detail or "断言不成立")
return bool(cond)
def sql(stmt):
_, out = R.sql(stmt)
return out
def body_of(method, path, token=None, payload=None):
_, b = R.call(method, BASE + path, token, payload)
return b
def wait_job(job_id, token, tries=60):
d = {}
for _ in range(tries):
d = body_of("GET", "/api/v1/cleanup/jobs/%d" % job_id, token).get("data") or {}
if d.get("status") in (3, 4):
return d
time.sleep(1)
return d
def make_zip(files):
bos = _io.BytesIO()
with zipfile.ZipFile(bos, "w", zipfile.ZIP_DEFLATED) as z:
for name, data in files.items():
z.writestr(name, data)
return bos.getvalue()
def tiny(tag):
return tag.encode("utf-8") * 40
def main():
ap = argparse.ArgumentParser(description="历史整理器 E2E")
ap.parse_args()
print("=" * 74)
print(" 历史数据整理器 E2E(§13.2)")
print("=" * 74)
_, adm = R.login(BASE, "admin", "Admin@123", "demo")
if adm.get("code") != 0:
print(" x 管理员登录失败:" + str(adm))
return 2
at = adm["data"]["token"]
print("\n[0] 摆夹具(正式任务 + 名册)")
suffix = os.urandom(3).hex()
_, b = R.call("POST", BASE + "/api/v1/tasks", at, {
"taskName": "E2E整理名册 " + suffix, "batchNo": "202609", "majorName": "数控加工",
"collectMode": 1, "certType": 1, "startTime": "2026-01-01", "endTime": "2026-12-31"
})
if b.get("code") != 0:
print(" x 建任务失败:" + str(b))
return 2
task_id = b["data"]["id"]
CREATED_TASK_IDS.append(task_id)
_, b = R.call("GET", BASE + "/api/v1/orgs/tree", at)
classes = []
def walk(nodes):
for n in nodes or []:
if n.get("orgType") == 3:
classes.append(n)
walk(n.get("children"))
walk(b.get("data") or [])
org = classes[0]
R.call("PUT", BASE + "/api/v1/tasks/%d/orgs" % task_id, at,
{"orgs": [{"orgId": org["id"], "expectedCount": 3}]})
R.call("POST", BASE + "/api/v1/tasks/%d/materials/apply-template" % task_id, at,
{"templateCode": "TPL_SCHOOL_ROLL"})
_build_roster([S_A, S_B, S_C])
body = R.upload_roster(BASE, at, task_id, TMP_ROSTER,
extra={"dryRun": "false", "mode": "APPEND", "orgId": org["id"]})
check("正式任务的名册就绪(3 人)", body.get("code") == 0,
json.dumps(body, ensure_ascii=False)[:200])
print("\n[1] 造一个形态真实的历史包")
cls = "24中技数控1班"
files = {
cls + "/整理甲/身份证.jpg.jpg": tiny("a1"),
cls + "/整理甲-" + S_A[1] + "/一寸照片.png": tiny("a2"),
cls + "/整理甲/学籍表(1).JPG": tiny("a3"),
cls + "/整理乙/身份证-" + WRONG_ID + ".jpg": tiny("b1"),
cls + "/整理乙/工作证明-" + WRONG_ID + ".png": tiny("b2"),
cls + "/新建文件夹/IMG_2034.jpg": tiny("c1"),
"__MACOSX/" + cls + "/._身份证.jpg": tiny("d1"),
cls + "/.DS_Store": tiny("d2"),
}
zip_bytes = make_zip(files)
check("历史包已生成", len(zip_bytes) > 100, "%d 字节" % len(zip_bytes))
print("\n[2] 解析预览(调规则时看每一步)")
zp = os.path.join(ROOT, "scripts", "_e2e-cleanup.zip")
with open(zp, "wb") as f:
f.write(zip_bytes)
r = subprocess.run(["curl", "-s", "-X", "POST",
BASE + "/api/v1/cleanup/jobs/parse-preview",
"-H", "Authorization: Bearer " + at,
"-F", "file=@" + zp + ";filename=history.zip;type=application/zip"],
capture_output=True, text=True, encoding="utf-8", timeout=120)
try:
pv = json.loads(r.stdout)
except Exception:
pv = {"code": -1, "msg": (r.stdout or "")[:200]}
check("解析预览可用", pv.get("code") == 0, json.dumps(pv, ensure_ascii=False)[:250])
plist = pv.get("data") or []
check("macOS 元数据被跳过(不然报告会虚高)", len(plist) == 6,
"%d 条:%s" % (len(plist), [x.get("path") for x in plist]))
by_path = {}
for x in plist:
by_path[x.get("path")] = x
k1 = cls + "/整理甲/身份证.jpg.jpg"
check("从目录名解析出姓名", by_path.get(k1, {}).get("studentName") == "整理甲",
json.dumps(by_path.get(k1), ensure_ascii=False))
check("解析出材料码 IDCARD", by_path.get(k1, {}).get("materialCode") == "IDCARD",
json.dumps(by_path.get(k1), ensure_ascii=False))
k2 = cls + "/整理甲-" + S_A[1] + "/一寸照片.png"
check("从「姓名-证件号」目录名里解析出号码(且校验位合法)",
by_path.get(k2, {}).get("idCard") is not None
and by_path.get(k2, {}).get("idCardValid") is True,
json.dumps(by_path.get(k2), ensure_ascii=False))
check("解析结果里的证件号是脱敏值(不能把明文回给前端)",
"*" in (by_path.get(k2, {}).get("idCard") or ""),
str(by_path.get(k2, {}).get("idCard")))
k3 = cls + "/新建文件夹/IMG_2034.jpg"
check("认不出的文件不硬塞分类(姓名与材料码都为空)",
by_path.get(k3, {}).get("studentName") is None
and by_path.get(k3, {}).get("materialCode") is None,
json.dumps(by_path.get(k3), ensure_ascii=False))
print("\n[3] 创建整理任务(带 rosterTaskId)")
c = body_of("POST", "/api/v1/cleanup/jobs/json", at, {
"jobName": "E2E历史整理 " + suffix,
"sourceType": 1,
"rosterTaskId": task_id,
"zipBase64": base64.b64encode(zip_bytes).decode()
})
check("创建成功", c.get("code") == 0, json.dumps(c, ensure_ascii=False)[:250])
job_id = (c.get("data") or {}).get("jobId")
CREATED_JOB_IDS.append(job_id)
check("返回 jobId 与估时",
isinstance(job_id, int) and bool((c.get("data") or {}).get("estimatedSeconds")),
json.dumps(c.get("data"), ensure_ascii=False))
d = wait_job(job_id, at)
check("整理完成(status=3)", d.get("status") == 3,
"status=%s err=%s" % (d.get("status"), d.get("errorMsg")))
check("进度 100", d.get("progress") == 100, str(d.get("progress")))
check("总文件数 6(元数据已排除)", d.get("totalFiles") == 6, str(d.get("totalFiles")))
check("解析出 2 个学员", d.get("parsedStudents") == 2, str(d.get("parsedStudents")))
check("解析出材料种类 >= 3", (d.get("parsedMaterials") or 0) >= 3,
str(d.get("parsedMaterials")))
# ⚠️ 断言不能只是 `"%" in compressRatio` —— 那对 `-568.3%` 也成立,
# 而负数压缩率是**没有意义**的(包里的文件本来就压过、或总量太小,
# 重新打 zip 的包头开销比省下来的还多)。服务端对这种情况回「—」。
# 所以这里断言"要么是个非负百分比,要么是 —"。
_ratio = str(d.get("compressRatio"))
check("压缩率要么形如 12.3%(非负),要么是「—」" + "(不可是负百分比)",
_ratio == "—" or (re.fullmatch(r"\d+(\.\d+)?%", _ratio) is not None),
"compressRatio=" + _ratio)
check("结果包路径非空", bool(d.get("resultPath")), str(d.get("resultPath")))
print("\n[4] 体检报告(契约里最重要的一句话)")
rep = body_of("GET", "/api/v1/cleanup/jobs/%d/report" % job_id, at).get("data") or {}
s = rep.get("summary") or {}
check("报告可用", s.get("totalFiles") == 6, json.dumps(s, ensure_ascii=False))
check("算出 autoHandled / needManual(服务端算,不让前端减)",
rep.get("autoHandled") is not None and rep.get("needManual") is not None,
json.dumps({"autoHandled": rep.get("autoHandled"),
"needManual": rep.get("needManual")}))
check("autoHandled + needManual = 总文件数(不多不少)",
(rep.get("autoHandled") or 0) + (rep.get("needManual") or 0) == 6,
"%s + %s != 6" % (rep.get("autoHandled"), rep.get("needManual")))
check("检测出疑似串档 1 例(乙的两项材料号都不是名册里那个,且彼此一致)",
s.get("crossFileSuspected") == 1, json.dumps(s, ensure_ascii=False))
check("串档进了 highlights 且级别是 3(必须人工)",
any(h.get("type") == "CROSS_FILE" and h.get("level") == 3
for h in rep.get("highlights") or []),
json.dumps(rep.get("highlights"), ensure_ascii=False)[:300])
check("无法归位的记进 orphanImages(不硬塞分类)",
(s.get("orphanImages") or 0) >= 1, json.dumps(s, ensure_ascii=False))
check("改名样例给了(客户要看到系统到底做了什么)",
len(rep.get("nameFixedSamples") or []) >= 1,
json.dumps(rep.get("nameFixedSamples"), ensure_ascii=False)[:250])
check("改名样例里的规范名带姓名",
any("整理甲" in (x.get("fixed") or "") for x in rep.get("nameFixedSamples") or []),
json.dumps(rep.get("nameFixedSamples"), ensure_ascii=False)[:300])
conf = rep.get("idcardConflicts") or []
check("证件号冲突有明细", len(conf) >= 1, json.dumps(conf, ensure_ascii=False)[:250])
check("没有 OCR 时判定一律是 NEED_CONFIRM(不假装能自动修)",
all(x.get("verdict") == "NEED_CONFIRM" for x in conf),
json.dumps([x.get("verdict") for x in conf], ensure_ascii=False))
check("冲突明细里的证件号都是脱敏值(报告不外泄明文)",
all("*" in (x.get("fileNameValue") or "") or "*" in (x.get("rosterValue") or "")
for x in conf),
json.dumps(conf, ensure_ascii=False)[:300])
print("\n[5] 整理结果包(客户拿到能直接用)")
rp = os.path.join(ROOT, "scripts", "_e2e-cleanup-result.zip")
code = os.system('curl -sf -o "%s" -H "Authorization: Bearer %s" "%s/api/v1/cleanup/jobs/%d/result"'
% (rp, at, BASE, job_id))
size = os.path.getsize(rp) if os.path.exists(rp) else 0
check("结果包能下载", size > 100, "exit=%s size=%s" % (code, size))
if size > 100:
with zipfile.ZipFile(rp) as z:
names = z.namelist()
check("结果包是合法 ZIP", z.testzip() is None)
check("目录结构是班级/姓名/材料/规范文件名",
any(n.startswith(cls + "/整理甲/") and n.count("/") >= 3 for n in names),
str(names[:4]))
check("规范文件名已修好(不再是 .jpg.jpg 双扩展名)",
not any(n.endswith(".jpg.jpg") for n in names),
str([n for n in names if n.endswith(".jpg.jpg")][:2]))
check("结果包里带了「待归位」目录(认不出的那些)",
any("/待归位/" in n for n in names),
str([n for n in names if "待归位" in n][:2]))
os.remove(rp)
print("\n[6] 导入为正式任务(幂等)")
imp = body_of("POST", "/api/v1/cleanup/jobs/%d/import" % job_id, at, {
"taskName": "E2E整理导入 " + suffix, "collectMode": 1, "batchNo": "202609"
})
check("导入成功", imp.get("code") == 0, json.dumps(imp, ensure_ascii=False)[:250])
new_task = (imp.get("data") or {}).get("taskId")
if isinstance(new_task, int):
CREATED_TASK_IDS.append(new_task)
check("返回新任务 id", isinstance(new_task, int), str(new_task))
check("名册建了 2 人(甲、乙各 1 行;丙在名册里但材料里没出现)",
(imp.get("data") or {}).get("rosterCount") == 2,
json.dumps(imp.get("data"), ensure_ascii=False))
check("新任务的名册行数与返回一致",
sql("select count(*) from collect_roster where task_id=%d and deleted=0;"
% new_task).strip() == "2",
sql("select real_name, id_card_mask from collect_roster where task_id=%d;"
% new_task))
check("名册里的证件号三列都写了(密文 + hash + 脱敏)",
sql("select count(*) from collect_roster where task_id=%d "
"and id_card_no is not null and id_card_hash is not null "
"and id_card_mask is not null;" % new_task).strip() == "2",
sql("select id_card_no is not null, id_card_hash is not null, id_card_mask "
"from collect_roster where task_id=%d;" % new_task))
check("新任务是草稿状态(让用户先核对再发布)",
sql("select status from collect_task where id=%d;" % new_task).strip() == "0",
sql("select status from collect_task where id=%d;" % new_task))
imp2 = body_of("POST", "/api/v1/cleanup/jobs/%d/import" % job_id, at, {
"taskName": "E2E整理导入-第二次 " + suffix, "collectMode": 1, "batchNo": "202609"
})
check("重复导入不新建任务(客户手滑点两次是常事)",
(imp2.get("data") or {}).get("taskId") == new_task,
json.dumps(imp2, ensure_ascii=False)[:250])
check("库里没有多出第二个任务",
sql("select count(*) from collect_task where task_name like 'E2E整理导入%' "
"and deleted=0;").strip() == "1",
sql("select id, task_name from collect_task where task_name like 'E2E整理导入%';"))
print("\n[7] 做不到的要如实拒绝(不安静忽略)")
imp3 = body_of("POST", "/api/v1/cleanup/jobs/%d/import" % job_id, at, {
"taskName": "E2E不该建出来", "collectMode": 1, "batchNo": "202609", "importFiles": True
})
check("importFiles=true 明确拒绝(一期还不能自动挂文件)",
imp3.get("code") == 40044, "code=%s msg=%s" % (imp3.get("code"), imp3.get("msg")))
check("拒绝时没有偷偷建任务",
sql("select count(*) from collect_task where task_name='E2E不该建出来';").strip() == "0")
cloud = body_of("POST", "/api/v1/cleanup/jobs/json", at, {
"jobName": "云盘链接", "sourceType": 3, "rosterTaskId": task_id,
"zipBase64": base64.b64encode(zip_bytes).decode()
})
check("sourceType=3(云盘)返回 40044,且提示改成 zip",
cloud.get("code") == 40044 and "zip" in (cloud.get("msg") or ""),
"code=%s msg=%s" % (cloud.get("code"), cloud.get("msg")))
folder = body_of("POST", "/api/v1/cleanup/jobs/json", at, {
"jobName": "文件夹", "sourceType": 2, "rosterTaskId": task_id,
"zipBase64": base64.b64encode(zip_bytes).decode()
})
check("sourceType=2(文件夹)返回 40044", folder.get("code") == 40044,
"code=%s msg=%s" % (folder.get("code"), folder.get("msg")))
no_roster = body_of("POST", "/api/v1/cleanup/jobs/json", at, {
"jobName": "无名单", "sourceType": 1,
"zipBase64": base64.b64encode(zip_bytes).decode()
})
check("不传 rosterTaskId 也能跑(只做文件名规范化)", no_roster.get("code") == 0,
json.dumps(no_roster, ensure_ascii=False)[:200])
if no_roster.get("code") == 0:
jid2 = no_roster["data"]["jobId"]
CREATED_JOB_IDS.append(jid2)
d2 = wait_job(jid2, at)
check("无名单时也能完成", d2.get("status") == 3, "status=%s" % d2.get("status"))
rep2 = body_of("GET", "/api/v1/cleanup/jobs/%d/report" % jid2, at).get("data") or {}
check("无名单时串档检测恒为 0(没有基准就无从比对,不是没检测出来)",
(rep2.get("summary") or {}).get("crossFileSuspected") == 0,
json.dumps(rep2.get("summary"), ensure_ascii=False))
bad = body_of("POST", "/api/v1/cleanup/jobs/json", at, {
"jobName": "坏包", "sourceType": 1, "zipBase64": base64.b64encode(b"not a zip").decode()
})
if bad.get("code") == 0:
jid3 = bad["data"]["jobId"]
CREATED_JOB_IDS.append(jid3)
d3 = wait_job(jid3, at)
check("坏包最终是「失败」且给了可读原因(不是挂在那里不动)",
d3.get("status") == 4 and bool(d3.get("errorMsg")),
"status=%s err=%s" % (d3.get("status"), d3.get("errorMsg")))
else:
check("坏包在创建时就被拒(形态在入口就拦)", bad.get("code") != 0,
json.dumps(bad, ensure_ascii=False)[:200])
print("\n[8] 列表与权限")
lst = body_of("GET", "/api/v1/cleanup/jobs?pageSize=5", at).get("data") or {}
check("列表可用", len(lst.get("list") or []) >= 1, json.dumps(lst, ensure_ascii=False)[:200])
check("列表项带状态文案", all(x.get("statusText") for x in lst.get("list") or []),
json.dumps((lst.get("list") or [])[:1], ensure_ascii=False)[:200])
cfg = R.read_dev_config()
stok = R.mint_mp_token(cfg["jwt_secret"], cfg["issuer"], 993801, 1, ["STUDENT"], 4)
b = body_of("GET", "/api/v1/cleanup/jobs", stok)
check("学生看整理任务返回 403", b.get("code") in (40101, 40301, 403),
"code=%s" % b.get("code"))
b = body_of("GET", "/api/v1/cleanup/jobs")
check("未登录返回 40101", b.get("code") == 40101, "code=%s" % b.get("code"))
cleanup()
print("\n" + "=" * 74)
print("通过 %d 失败 %d 共 %d" % (len(PASSED), len(FAILED), len(PASSED) + len(FAILED)))
print("=" * 74)
for n, det in FAILED:
print(" x " + n + "\n " + det)
return 1 if FAILED else 0
def _build_roster(rows):
from openpyxl import Workbook
wb = Workbook()
ws = wb.active
ws.title = "整理班"
header = ["序号", "姓名", "身份证号", "学历", "联系电话", "工作单位",
"报考职业", "报考工种", "报考等级", "是否补考",
"补考理论", "补考实操", "补考综合", "是否在校生",
"在读学校", "所学专业", "入学时间"]
for c, t in enumerate(header, start=1):
ws.cell(row=1, column=c, value=t)
for i, (name, idc) in enumerate(rows):
vals = [i + 1, name, idc, "中专", "138001333%d" % i, "无",
"车工", "数控车工", "四级", "否", "", "", "", "是",
"示范职业学校", "数控加工", "2024-09-01"]
for c, v in enumerate(vals, start=1):
if v != "":
ws.cell(row=i + 2, column=c, value=v)
wb.save(TMP_ROSTER)
def cleanup():
for jid in CREATED_JOB_IDS:
R.sql("delete from collect_cleanup_job where id=%d;" % jid)
for tid in CREATED_TASK_IDS:
R.sql("delete from collect_roster where task_id=%d;" % tid)
R.sql("delete from collect_task_org where task_id=%d;" % tid)
R.sql("delete from collect_material_def where task_id=%d;" % tid)
R.sql("delete from collect_task where id=%d;" % tid)
R.sql("delete from collect_cleanup_job where job_name like 'E2E历史整理%' "
"or job_name in ('无名单','云盘链接','文件夹','坏包');")
R.sql("delete from collect_task where task_name like 'E2E整理导入%' "
"or task_name='E2E不该建出来';")
for p in (TMP_ROSTER, os.path.join(ROOT, "scripts", "_e2e-cleanup.zip"),
os.path.join(ROOT, "scripts", "_e2e-cleanup-result.zip")):
if os.path.exists(p):
os.remove(p)
print(" 已清理:整理任务 / 测试任务与名册 / 临时 zip")
if __name__ == "__main__":
raise SystemExit(main())