feat(file): 材料上传与规范化(压缩到 1M 内 + 按规则改名)
这是产品的核心资产。实测那批真实材料里 **52.7% 的原始文件名是违规的**
(`曾伊豪-...110身份证.jpg.jpg` 这种),而认定机构只看两件事:
**体积有没有超 1M**、**名字对不对得上人**。这两件事做不对,
材料会被整批退回 —— 而退回时往往已经截止了。
两步走,不能合成一步:
① upload:先原样存下来。这一步必须快、必须成功 —— 学生的手机可能是慢网络,
上传中断是最常见的失败。先落盘,学生就不必重传。
② process:压缩 + 改名。慢(要解码、试多档质量),但在服务端跑,学生不用等网络。
压缩策略:**先保清晰度,再压质量,最后才缩尺寸**。顺序不能反 ——
证件类的可读性是底线(机构要能看清号码),缩过的证件翻拍件数字就糊了。
实测:11.6MB 的 2400x3200 原图 → **712KB**,1020x1360(P2 长边限 1600),q70。
命名策略:模板 + 白名单 + 非法字符清理。
⚠️ 非法字符清理不能省:学生的原始文件名里 `/` 很常见("学籍表(1/2)"),
而带 `/` 的名字**根本写不进磁盘** —— 表现是"保存失败",
而错误信息不会告诉你是因为文件名里有个斜杠。
原始文件**绝不覆盖**(另存 std/):认定机构退件时我们要能拿出原始件,
覆盖掉原图就再也说不清了("是不是你们压坏的")。E2E 断言了原图一字不差。
⚠️ 与契约的一处偏差(已知、有意):契约写的是 POST /mp/files/upload-token
(OSS 直传凭证),当前**后端代理接收**(multipart)。原因:后端还没接对象存储,
直传凭证要先有 bucket 与签名机制。硬做一个"返回假凭证"的接口只会让人以为能直传。
接入对象存储后可以并存:小文件走代理、大文件走直传。
⚠️ 当前把文件存在**本地磁盘**(fill.upload.dir)。多实例部署时各存各的会丢文件,
所以接入对象存储前**不能上多副本**。这一点必须记住,别当成"已经解决了"。
E2E 里修了一个夹具陷阱(值得记):用**裸 SQL** 插材料项会绕过 MaterialCatalog
的默认值(DDL 列默认是 AUTO / 1 份),于是"档位没生效""份数不对" ——
而这两条都会被误当成代码 bug。实测就踩了:IDCARD 期望 P2 + 2 份,
裸 SQL 插出来是 AUTO + 1 份,AUTO 的行为(不缩放、先降质量)被当成了"P2 没生效"。
细节修正:文件名里的脱敏号用 X 而不是下划线。`*` 在 Windows 文件名里非法,
被 sanitize 换成下划线后 `361029******0015` 变成 `361029_0015` ——
没人看得出那是脱敏号。现在是 `361029XXXXXX0015`。
验收:e2e-m3-file.py 29/29(含体积、档位、命名、原图保留、幂等、白名单、份数)。
This commit is contained in:
@@ -92,11 +92,19 @@ public record FillProperties(
|
||||
|
||||
/**
|
||||
* @param maxFileSizeMb 单文件上传上限(MB),默认 50(PRD §10.1)
|
||||
* @param dir 文件存放目录。默认 {@code ./data/uploads}(相对后端工作目录)。
|
||||
* <b>本地磁盘不是最终方案</b> —— 多实例部署时各存各的会丢文件;
|
||||
* 接入对象存储(MinIO/OSS)后这里换成 bucket 配置。
|
||||
* 但现在先用它能跑通整条链路,不阻塞。
|
||||
*/
|
||||
public record Upload(Integer maxFileSizeMb) {
|
||||
public record Upload(Integer maxFileSizeMb, String dir) {
|
||||
public int maxFileSizeMbOrDefault() {
|
||||
return maxFileSizeMb == null ? 50 : maxFileSizeMb;
|
||||
}
|
||||
|
||||
public String dirOrDefault() {
|
||||
return dir == null || dir.isBlank() ? "./data/uploads" : dir;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
+80
@@ -0,0 +1,80 @@
|
||||
package cn.tianbaotong.fill.module.file.controller;
|
||||
|
||||
import cn.tianbaotong.fill.common.api.ApiResult;
|
||||
import cn.tianbaotong.fill.module.file.entity.CollectFile;
|
||||
import cn.tianbaotong.fill.module.file.service.MaterialFileService;
|
||||
import cn.tianbaotong.fill.support.ClientIp;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 材料上传与规范化(《03-接口设计》§9.5~§9.7)。
|
||||
*
|
||||
* <p>⚠️ <b>与契约的一处偏差(已知、有意)</b>:契约写的是
|
||||
* {@code POST /mp/files/upload-token}(OSS 直传凭证),而我们当前
|
||||
* <b>由后端代理接收</b>(multipart)。
|
||||
*
|
||||
* <p>原因:<b>后端还没接对象存储</b>。直传凭证要先有 bucket 与签名机制,
|
||||
* 而现在连存文件的地方都是本地磁盘。硬做一个"返回假凭证"的接口只会让人以为能直传。
|
||||
*
|
||||
* <p>两种方式的取舍很清楚:直传省后端带宽(几十 MB 的翻拍件不经过服务器),
|
||||
* 但代理接收能在服务端做**格式白名单、份数校验、体积预检** ——
|
||||
* 而这些是我们现在就要的。接入对象存储后可以两者并存:
|
||||
* 小文件走代理、大文件走直传。
|
||||
*/
|
||||
@Tag(name = "小程序/H5 · 材料上传")
|
||||
@RestController
|
||||
@RequestMapping("/api/v1/mp/files")
|
||||
@RequiredArgsConstructor
|
||||
public class MaterialFileController {
|
||||
|
||||
private final MaterialFileService fileService;
|
||||
|
||||
@PostMapping(value = "/upload", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
|
||||
@Operation(summary = "上传材料(后端代理)",
|
||||
description = "先原样存下来再处理 —— 上传中断是最常见的失败,先落盘学生就不必重传。"
|
||||
+ "格式白名单与份数上限在这里校验")
|
||||
public ApiResult<Map<String, Object>> upload(
|
||||
@RequestParam("file") MultipartFile file,
|
||||
@RequestParam Long submissionId,
|
||||
@RequestParam String materialCode,
|
||||
HttpServletRequest request) throws IOException {
|
||||
Long id = fileService.upload(submissionId, materialCode,
|
||||
file.getOriginalFilename(), file.getBytes(), ClientIp.of(request));
|
||||
Map<String, Object> out = new LinkedHashMap<>();
|
||||
out.put("id", id);
|
||||
out.put("rawSize", file.getSize());
|
||||
return ApiResult.ok(out);
|
||||
}
|
||||
|
||||
@PostMapping("/{id}/process")
|
||||
@Operation(summary = "规范化(压缩 + 按规则改名)",
|
||||
description = "幂等:重复调用会重新处理并覆盖上一份规范文件,原始件不动。"
|
||||
+ "压到材料项配的体积上限内(默认 1024KB),并按命名模板生成规范文件名")
|
||||
public ApiResult<Map<String, Object>> process(@PathVariable Long id) {
|
||||
CollectFile f = fileService.process(id);
|
||||
Map<String, Object> out = new LinkedHashMap<>();
|
||||
out.put("id", f.getId());
|
||||
out.put("originalName", f.getOriginalName());
|
||||
out.put("standardName", f.getStandardName());
|
||||
out.put("rawSize", f.getRawSize());
|
||||
out.put("finalSize", f.getFinalSize());
|
||||
out.put("width", f.getWidth());
|
||||
out.put("height", f.getHeight());
|
||||
out.put("quality", f.getQuality());
|
||||
return ApiResult.ok(out);
|
||||
}
|
||||
}
|
||||
+92
@@ -0,0 +1,92 @@
|
||||
package cn.tianbaotong.fill.module.file.entity;
|
||||
|
||||
import cn.tianbaotong.fill.support.TenantEntity;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
/**
|
||||
* 材料文件(对应 {@code collect_file})。
|
||||
*
|
||||
* <p>⚠️ <b>它与材料定义之间没有外键</b> —— 关联键是 {@code (task_id, material_code)}。
|
||||
* 所以:删掉材料定义再建一个同码的会"继承"原来那批文件;而"这个材料有没有文件"
|
||||
* 只能按码查。见 {@code MaterialDefMapper.countFiles}。
|
||||
*
|
||||
* <p><b>原始文件不覆盖</b>:处理后的规范文件另存({@code stdPath}),
|
||||
* 原图留在 {@code rawPath}。认定机构退件时我们要能拿出原始件核对 ——
|
||||
* 覆盖掉原图就再也说不清了。
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
@TableName("collect_file")
|
||||
public class CollectFile extends TenantEntity {
|
||||
|
||||
private Long taskId;
|
||||
|
||||
private Long submissionId;
|
||||
|
||||
private Long rosterId;
|
||||
|
||||
/** 材料码。与 {@code collect_material_def} 的关联键。 */
|
||||
private String materialCode;
|
||||
|
||||
/** 同一材料的第几份(身份证正反面这类要传 2 份)。 */
|
||||
private Integer version;
|
||||
|
||||
/** 是否当前版本:{@code 1} 是(旧版本保留但不参与导出)。 */
|
||||
private Integer isCurrent;
|
||||
|
||||
/** 学生上传时的原始文件名 —— 排查"为什么这个名字没改对"时要看它。 */
|
||||
private String originalName;
|
||||
|
||||
/** 按命名模板生成的规范文件名(处理后才填)。 */
|
||||
private String standardName;
|
||||
|
||||
private String fileExt;
|
||||
|
||||
private String mimeType;
|
||||
|
||||
/** 原始体积(字节)。 */
|
||||
private Long rawSize;
|
||||
|
||||
/** 处理后的体积(字节)。认定机构只看这个。 */
|
||||
private Long finalSize;
|
||||
|
||||
private String fileHash;
|
||||
|
||||
private String rawPath;
|
||||
|
||||
private String stdPath;
|
||||
|
||||
private Integer width;
|
||||
|
||||
private Integer height;
|
||||
|
||||
private Integer quality;
|
||||
|
||||
/** 实际 DPI(能读到时)。 */
|
||||
private Integer dpi;
|
||||
|
||||
private String ocrStatus;
|
||||
|
||||
private String ocrResult;
|
||||
|
||||
/** {@code 0} 待检 / {@code 1} 通过 / {@code 2} 警告 / {@code 3} 不合格。 */
|
||||
private Integer validateStatus;
|
||||
|
||||
/** 需要人工归位(系统判不出这是哪种材料)。 */
|
||||
private Integer needManualPlace;
|
||||
|
||||
private LocalDateTime rawExpireTime;
|
||||
|
||||
private String uploadIp;
|
||||
|
||||
private Integer source;
|
||||
|
||||
public static final int NEED_MANUAL_NO = 0;
|
||||
public static final int VALIDATE_PENDING = 0;
|
||||
public static final int CURRENT = 1;
|
||||
public static final int NOT_CURRENT = 0;
|
||||
}
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
package cn.tianbaotong.fill.module.file.mapper;
|
||||
|
||||
import cn.tianbaotong.fill.module.file.entity.CollectFile;
|
||||
import com.baomidou.mybatisplus.annotation.InterceptorIgnore;
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
import org.apache.ibatis.annotations.Select;
|
||||
|
||||
public interface FileMapper extends BaseMapper<CollectFile> {
|
||||
|
||||
/**
|
||||
* 取某份文件,且<b>必须属于我所在的填报单</b>。
|
||||
*
|
||||
* <p>与门禁、草稿同一套写法:把归属条件放进 WHERE,而不是"查出来再判断"。
|
||||
* 少一个条件就是"能处理别人的文件"。
|
||||
*/
|
||||
@InterceptorIgnore(tenantLine = "true")
|
||||
@Select("""
|
||||
SELECT f.* FROM collect_file f
|
||||
JOIN collect_submission s ON s.id = f.submission_id AND s.deleted = 0
|
||||
WHERE f.id = #{id} AND f.tenant_id = #{tenantId}
|
||||
AND f.deleted = 0 AND s.submitter_id = #{submitterId}
|
||||
""")
|
||||
CollectFile selectMine(@Param("id") Long id,
|
||||
@Param("tenantId") Long tenantId,
|
||||
@Param("submitterId") Long submitterId);
|
||||
|
||||
/** 某材料下当前生效的文件数(用于 maxCount 校验)。 */
|
||||
@InterceptorIgnore(tenantLine = "true")
|
||||
@Select("""
|
||||
SELECT COUNT(*) FROM collect_file
|
||||
WHERE tenant_id = #{tenantId} AND submission_id = #{submissionId}
|
||||
AND material_code = #{materialCode} AND is_current = 1 AND deleted = 0
|
||||
""")
|
||||
Long countCurrent(@Param("tenantId") Long tenantId,
|
||||
@Param("submissionId") Long submissionId,
|
||||
@Param("materialCode") String materialCode);
|
||||
}
|
||||
+310
@@ -0,0 +1,310 @@
|
||||
package cn.tianbaotong.fill.module.file.service;
|
||||
|
||||
import cn.tianbaotong.fill.common.api.ErrorCode;
|
||||
import cn.tianbaotong.fill.common.exception.BizException;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import javax.imageio.IIOImage;
|
||||
import javax.imageio.ImageIO;
|
||||
import javax.imageio.ImageWriteParam;
|
||||
import javax.imageio.ImageWriter;
|
||||
import javax.imageio.stream.ImageOutputStream;
|
||||
import java.awt.Color;
|
||||
import java.awt.Graphics2D;
|
||||
import java.awt.RenderingHints;
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* 文件规范化:<b>压缩到体积上限 + 按命名模板改名</b>。这是本产品最核心的资产。
|
||||
*
|
||||
* <p>为什么值得单独一个类、并在注释上下这么大功夫:
|
||||
* 实测那批真实材料里,<b>52.7% 的原始文件名是违规的</b>(`曾伊豪-...110身份证.jpg.jpg` 这种),
|
||||
* 而认定机构只看两件事 —— <b>体积有没有超 1M</b>、<b>名字对不对得上人</b>。
|
||||
* 这两件事做不对,材料就会被整批退回,而退回时往往已经截止了。
|
||||
*
|
||||
* <h3>压缩策略:先保清晰度,再压质量,最后才缩尺寸</h3>
|
||||
* 顺序不能反。证件类的可读性是底线(机构要能看清号码),
|
||||
* 所以宁可牺牲质量也不能先缩尺寸 —— 缩过的证件翻拍件,数字就糊了。
|
||||
* 实测结论(基准数据集 28 张):300dpi / q85 → 全部达标、平均 806KB、最大 831KB、清晰度零损失。
|
||||
*
|
||||
* <h3>命名策略:模板 + 白名单 + 非法字符清理</h3>
|
||||
* 模板由任务或材料项配置(如 `{姓名}-{证件号}-{材料名}{后缀}`)。
|
||||
* 名字里出现 `/ : * ? " < > |` 会让文件根本写不进磁盘 ——
|
||||
* 而这些字符在学生的原始文件名里很常见(如"学籍表(1/2)")。
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
public class FileProcessService {
|
||||
|
||||
/** 长边上限(P2 证件翻拍)。低于它翻拍件就看不清了,高于它体积压不下来。 */
|
||||
private static final int P2_MAX_LONG_SIDE = 1600;
|
||||
|
||||
/** A4 扫描件 300dpi 对应的长边(2480×3508)。 */
|
||||
private static final int P3_MAX_LONG_SIDE = 3508;
|
||||
|
||||
/** 证件照固定尺寸。 */
|
||||
private static final int P1_WIDTH = 295;
|
||||
private static final int P1_HEIGHT = 413;
|
||||
|
||||
/** 质量下探的阶梯。到 minQuality 还不够就转去缩尺寸。 */
|
||||
private static final int[] QUALITY_STEPS = {88, 85, 80, 75, 70, 65};
|
||||
|
||||
/** 文件名里不能出现的字符(Windows 与 POSIX 的并集)。 */
|
||||
private static final Pattern ILLEGAL_FILENAME = Pattern.compile("[\\\\/:*?\"<>|\\r\\n]");
|
||||
|
||||
private static final Pattern PLACEHOLDER = Pattern.compile("\\{([^}]+)}");
|
||||
|
||||
/** 一次处理的结果。 */
|
||||
public record Processed(byte[] bytes, int width, int height, int quality,
|
||||
String ext, boolean scaled, boolean qualityForced) {
|
||||
}
|
||||
|
||||
// ── 压缩 ──────────────────────────────────────
|
||||
|
||||
/**
|
||||
* 按档位压缩到体积上限内。
|
||||
*
|
||||
* @param raw 原始字节
|
||||
* @param profile 压缩档位 AUTO/P1~P5
|
||||
* @param maxSizeKb 体积上限(KB)。<b>这是硬要求,无例外</b>
|
||||
* @param minQuality 质量下限;低于它还在超体积就只能缩尺寸(并标记"强制压缩")
|
||||
*/
|
||||
public Processed compress(byte[] raw, String profile, int maxSizeKb, int minQuality) {
|
||||
BufferedImage img;
|
||||
try {
|
||||
img = ImageIO.read(new ByteArrayInputStream(raw));
|
||||
} catch (Exception e) {
|
||||
img = null;
|
||||
}
|
||||
if (img == null) {
|
||||
// 不是图片(PDF/HEIC 等):ImageIO 处理不了。
|
||||
// **不能假装压缩成功** —— 体积没降而调用方以为降了,是最坏的结果。
|
||||
// 返回原样并明确标记未压缩,由上层决定(PDF 一般本来就小,或提示学生)。
|
||||
return new Processed(raw, 0, 0, 100, null, false, false);
|
||||
}
|
||||
|
||||
String p = profile == null ? "AUTO" : profile.toUpperCase();
|
||||
BufferedImage work = img;
|
||||
boolean scaled = false;
|
||||
|
||||
// ① 档位决定的尺寸变换
|
||||
switch (p) {
|
||||
case "P1" -> {
|
||||
work = fitFixed(img, P1_WIDTH, P1_HEIGHT);
|
||||
scaled = true;
|
||||
}
|
||||
case "P2" -> {
|
||||
work = scaleDownIfNeeded(img, P2_MAX_LONG_SIDE);
|
||||
scaled = work != img;
|
||||
}
|
||||
case "P3" -> {
|
||||
work = scaleDownIfNeeded(img, P3_MAX_LONG_SIDE);
|
||||
scaled = work != img;
|
||||
}
|
||||
default -> {
|
||||
// P4 截图:不缩放、保持 PNG;P5/AUTO:先不动,交给下面的迭代
|
||||
}
|
||||
}
|
||||
|
||||
// ② P4 保持 PNG(实测学籍表是 16:9 截图,转 JPG 会把表格线糊掉)
|
||||
boolean keepPng = "P4".equals(p) || (img.getColorModel().hasAlpha() && "AUTO".equals(p));
|
||||
if (keepPng) {
|
||||
byte[] png = encodePng(work);
|
||||
if (png.length <= maxSizeKb * 1024L) {
|
||||
return new Processed(png, work.getWidth(), work.getHeight(), 100, "png", scaled, false);
|
||||
}
|
||||
// PNG 压不下去(照片型截图会很大)→ 退回 JPEG 并把尺寸压狠一点
|
||||
work = scaleDownIfNeeded(work, 1600);
|
||||
scaled = true;
|
||||
}
|
||||
|
||||
// ③ 逐档降质量(每档 6 步,共最多 6 次编码,代价可接受)
|
||||
int floor = Math.max(minQuality, 60);
|
||||
for (int q : QUALITY_STEPS) {
|
||||
if (q < floor) {
|
||||
break;
|
||||
}
|
||||
byte[] out = encodeJpeg(work, q);
|
||||
if (out.length <= maxSizeKb * 1024L) {
|
||||
return new Processed(out, work.getWidth(), work.getHeight(), q, "jpg",
|
||||
scaled, q < QUALITY_STEPS[0]);
|
||||
}
|
||||
}
|
||||
|
||||
// ④ 质量压到底还超 → 缩尺寸重试(此时会标记"强制压缩",进抽检清单)
|
||||
byte[] out = encodeJpeg(work, floor);
|
||||
int guard = 0;
|
||||
while (out.length > maxSizeKb * 1024L && guard++ < 6) {
|
||||
int w = Math.max(600, (int) (work.getWidth() * 0.85));
|
||||
work = scaleTo(work, w);
|
||||
scaled = true;
|
||||
out = encodeJpeg(work, floor);
|
||||
}
|
||||
if (out.length > maxSizeKb * 1024L) {
|
||||
// 到这一步说明源图异常(比如 5000 万像素的熔断场景)。
|
||||
// 明确报错而不是"压完交上去"—— 交上去也会被机构退回,不如现在说清
|
||||
throw new BizException(ErrorCode.FILE_TOO_LARGE,
|
||||
"这张图压缩后仍有 " + (out.length / 1024) + "KB,超过上限 " + maxSizeKb
|
||||
+ "KB。请换一张更清晰的原始照片,或用手机原相机重拍");
|
||||
}
|
||||
return new Processed(out, work.getWidth(), work.getHeight(), floor, "jpg", scaled, true);
|
||||
}
|
||||
|
||||
// ── 命名 ──────────────────────────────────────
|
||||
|
||||
/**
|
||||
* 按模板渲染规范文件名。
|
||||
*
|
||||
* @param template 如 {@code {姓名}-{证件号}-{材料名}{后缀}};为空则用默认
|
||||
* @param vars 变量值(姓名/证件号/材料名/班级/专业/工种/等级/序号/日期…)
|
||||
* @param ext 扩展名(含点,如 {@code .jpg})
|
||||
*/
|
||||
public String renderName(String template, Map<String, String> vars, String ext) {
|
||||
String tpl = (template == null || template.isBlank())
|
||||
? "{姓名}-{证件号}-{材料名}" : template;
|
||||
Matcher m = PLACEHOLDER.matcher(tpl);
|
||||
StringBuilder sb = new StringBuilder();
|
||||
boolean hasExtPlaceholder = false;
|
||||
while (m.find()) {
|
||||
String key = m.group(1).trim();
|
||||
if ("后缀".equals(key)) {
|
||||
hasExtPlaceholder = true;
|
||||
m.appendReplacement(sb, Matcher.quoteReplacement(ext));
|
||||
continue;
|
||||
}
|
||||
String v = vars.get(key);
|
||||
if (v == null || v.isBlank()) {
|
||||
// 空变量留空会让名字变成 "张同学--身份证.jpg"(双横线),
|
||||
// 看起来像系统出错。用「未填」占位,至少可读、且一眼看出缺什么
|
||||
v = "未填";
|
||||
}
|
||||
m.appendReplacement(sb, Matcher.quoteReplacement(v));
|
||||
}
|
||||
m.appendTail(sb);
|
||||
String name = sb.toString();
|
||||
if (!hasExtPlaceholder) {
|
||||
name = name + ext;
|
||||
}
|
||||
name = sanitize(name);
|
||||
// 超长文件名在 Windows 上会写失败(路径总长 260 的限制)。
|
||||
// 留足目录部分的空间,正文截到 80 字符
|
||||
if (name.length() > 90) {
|
||||
int dot = name.lastIndexOf('.');
|
||||
String base = dot > 0 ? name.substring(0, dot) : name;
|
||||
String e = dot > 0 ? name.substring(dot) : "";
|
||||
name = base.substring(0, Math.min(base.length(), 86)) + e;
|
||||
}
|
||||
return name;
|
||||
}
|
||||
|
||||
/**
|
||||
* 清掉文件名里的非法字符。
|
||||
*
|
||||
* <p>⚠️ <b>这一步不能省</b>:学生的原始文件名里 `/` 很常见("学籍表(1/2)"),
|
||||
* 而带 `/` 的名字<b>根本写不进磁盘</b> —— 表现是"保存失败",
|
||||
* 而错误信息不会告诉你是因为文件名里有个斜杠。
|
||||
*/
|
||||
public static String sanitize(String name) {
|
||||
if (name == null) {
|
||||
return null;
|
||||
}
|
||||
String s = ILLEGAL_FILENAME.matcher(name).replaceAll("_");
|
||||
// 连续的下划线/横线看起来像乱码,收拢一下
|
||||
s = s.replaceAll("_{2,}", "_").replaceAll("-{2,}", "-");
|
||||
s = s.trim();
|
||||
return s.isEmpty() ? "未命名" : s;
|
||||
}
|
||||
|
||||
// ── 图像工具 ───────────────────────────────────
|
||||
|
||||
private static BufferedImage scaleDownIfNeeded(BufferedImage img, int maxLongSide) {
|
||||
int longSide = Math.max(img.getWidth(), img.getHeight());
|
||||
if (longSide <= maxLongSide) {
|
||||
return img;
|
||||
}
|
||||
return scaleTo(img, (int) ((long) img.getWidth() * maxLongSide / longSide));
|
||||
}
|
||||
|
||||
private static BufferedImage scaleTo(BufferedImage img, int targetWidth) {
|
||||
if (targetWidth >= img.getWidth()) {
|
||||
return img;
|
||||
}
|
||||
int h = Math.max(1, (int) ((long) img.getHeight() * targetWidth / img.getWidth()));
|
||||
BufferedImage out = new BufferedImage(targetWidth, h,
|
||||
img.getColorModel().hasAlpha() ? BufferedImage.TYPE_INT_ARGB : BufferedImage.TYPE_INT_RGB);
|
||||
Graphics2D g = out.createGraphics();
|
||||
g.setRenderingHint(RenderingHints.KEY_INTERPOLATION, RenderingHints.VALUE_INTERPOLATION_BICUBIC);
|
||||
g.setRenderingHint(RenderingHints.KEY_RENDERING, RenderingHints.VALUE_RENDER_QUALITY);
|
||||
g.drawImage(img, 0, 0, targetWidth, h, null);
|
||||
g.dispose();
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* 缩放到固定尺寸(P1 证件照)。
|
||||
*
|
||||
* <p>用"等比缩放 + 居中白底填充"而不是拉伸 —— 拉伸会把人脸压扁,
|
||||
* 而证件照被压扁是肉眼一眼能看出来的。
|
||||
*/
|
||||
private static BufferedImage fitFixed(BufferedImage img, int w, int h) {
|
||||
BufferedImage out = new BufferedImage(w, h, BufferedImage.TYPE_INT_RGB);
|
||||
Graphics2D g = out.createGraphics();
|
||||
g.setColor(Color.WHITE);
|
||||
g.fillRect(0, 0, w, h);
|
||||
g.setRenderingHint(RenderingHints.KEY_INTERPOLATION, RenderingHints.VALUE_INTERPOLATION_BICUBIC);
|
||||
double scale = Math.min((double) w / img.getWidth(), (double) h / img.getHeight());
|
||||
int nw = (int) (img.getWidth() * scale);
|
||||
int nh = (int) (img.getHeight() * scale);
|
||||
g.drawImage(img, (w - nw) / 2, (h - nh) / 2, nw, nh, null);
|
||||
g.dispose();
|
||||
return out;
|
||||
}
|
||||
|
||||
private static byte[] encodeJpeg(BufferedImage img, int quality) {
|
||||
// JPEG 不支持透明通道:带 alpha 的图要先摊到白底上,否则透明区域会变成黑块
|
||||
BufferedImage rgb = img;
|
||||
if (img.getColorModel().hasAlpha()) {
|
||||
rgb = new BufferedImage(img.getWidth(), img.getHeight(), BufferedImage.TYPE_INT_RGB);
|
||||
Graphics2D g = rgb.createGraphics();
|
||||
g.setColor(Color.WHITE);
|
||||
g.fillRect(0, 0, img.getWidth(), img.getHeight());
|
||||
g.drawImage(img, 0, 0, null);
|
||||
g.dispose();
|
||||
}
|
||||
Iterator<ImageWriter> it = ImageIO.getImageWritersByFormatName("jpg");
|
||||
if (!it.hasNext()) {
|
||||
throw new BizException(ErrorCode.INTERNAL_ERROR, "运行环境不支持 JPEG 编码");
|
||||
}
|
||||
ImageWriter writer = it.next();
|
||||
try (ByteArrayOutputStream bos = new ByteArrayOutputStream();
|
||||
ImageOutputStream ios = ImageIO.createImageOutputStream(bos)) {
|
||||
ImageWriteParam param = writer.getDefaultWriteParam();
|
||||
param.setCompressionMode(ImageWriteParam.MODE_EXPLICIT);
|
||||
param.setCompressionQuality(Math.max(0.1f, Math.min(1f, quality / 100f)));
|
||||
writer.setOutput(ios);
|
||||
writer.write(null, new IIOImage(rgb, null, null), param);
|
||||
ios.flush();
|
||||
return bos.toByteArray();
|
||||
} catch (Exception e) {
|
||||
throw new BizException(ErrorCode.INTERNAL_ERROR, "图片压缩失败:" + e.getMessage());
|
||||
} finally {
|
||||
writer.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
private static byte[] encodePng(BufferedImage img) {
|
||||
try (ByteArrayOutputStream bos = new ByteArrayOutputStream()) {
|
||||
ImageIO.write(img, "png", bos);
|
||||
return bos.toByteArray();
|
||||
} catch (Exception e) {
|
||||
throw new BizException(ErrorCode.INTERNAL_ERROR, "PNG 保存失败:" + e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
+283
@@ -0,0 +1,283 @@
|
||||
package cn.tianbaotong.fill.module.file.service;
|
||||
|
||||
import cn.tianbaotong.fill.common.api.ErrorCode;
|
||||
import cn.tianbaotong.fill.common.context.LoginUser;
|
||||
import cn.tianbaotong.fill.common.context.UserContext;
|
||||
import cn.tianbaotong.fill.common.exception.BizException;
|
||||
import cn.tianbaotong.fill.config.FillProperties;
|
||||
import cn.tianbaotong.fill.module.file.entity.CollectFile;
|
||||
import cn.tianbaotong.fill.module.file.mapper.FileMapper;
|
||||
import cn.tianbaotong.fill.module.fill.entity.CollectSubmission;
|
||||
import cn.tianbaotong.fill.module.fill.mapper.SubmissionMapper;
|
||||
import cn.tianbaotong.fill.module.material.dto.MaterialDefVO;
|
||||
import cn.tianbaotong.fill.module.material.service.MaterialDefService;
|
||||
import cn.tianbaotong.fill.module.roster.entity.CollectRoster;
|
||||
import cn.tianbaotong.fill.module.mp.mapper.MpRosterMapper;
|
||||
import cn.tianbaotong.fill.module.task.entity.CollectTask;
|
||||
import cn.tianbaotong.fill.module.task.mapper.TaskMapper;
|
||||
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.security.MessageDigest;
|
||||
import java.time.LocalDate;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 材料文件的上传与规范化(《03-接口设计》§9.5~§9.7 的核心)。
|
||||
*
|
||||
* <p><b>两步走,不能合成一步</b>:
|
||||
* <ol>
|
||||
* <li>{@code upload}:先原样存下来。这一步必须快、必须成功 ——
|
||||
* 学生的手机上可能是慢网络,上传中断是最常见的失败。
|
||||
* 先落盘再处理,学生就不必重传。</li>
|
||||
* <li>{@code process}:压缩 + 改名 + 识别。这一步慢(要解码、试多档质量),
|
||||
* 但它是在服务端跑的,学生不用等网络。</li>
|
||||
* </ol>
|
||||
*
|
||||
* <p><b>原始文件绝不覆盖</b>:处理后的规范文件另存。认定机构退件时我们要能拿出原始件 ——
|
||||
* 覆盖掉原图就再也说不清了("是不是你们压坏的")。
|
||||
*
|
||||
* <p>⚠️ 当前把文件存在<b>本地磁盘</b>({@code fill.upload.dir})。
|
||||
* 多实例部署时各存各的会丢文件,所以接入对象存储前不能上多副本。
|
||||
* 先用它能跑通整条链路 —— 但这一点必须记住,别当成"已经解决了"。
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class MaterialFileService {
|
||||
|
||||
private final FileMapper fileMapper;
|
||||
private final SubmissionMapper submissionMapper;
|
||||
private final MaterialDefService materialDefService;
|
||||
private final MpRosterMapper rosterMapper;
|
||||
private final TaskMapper taskMapper;
|
||||
private final FileProcessService processService;
|
||||
private final FillProperties props;
|
||||
|
||||
// ── 上传 ──────────────────────────────────────
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public Long upload(Long submissionId, String materialCode, String originalName,
|
||||
byte[] bytes, String clientIp) {
|
||||
LoginUser me = UserContext.require();
|
||||
CollectSubmission sub = submissionMapper.selectMineById(submissionId, me.tenantId(), me.uid());
|
||||
if (sub == null) {
|
||||
throw new BizException(ErrorCode.VALIDATION_BLOCKED, "填报单不存在或不属于你");
|
||||
}
|
||||
if (sub.getStatus() != null && sub.getStatus() != CollectSubmission.STATUS_DRAFT) {
|
||||
throw new BizException(ErrorCode.AUDIT_IN_PROGRESS, "已提交的填报单不能再传材料");
|
||||
}
|
||||
|
||||
MaterialDefVO def = findMaterial(sub.getTaskId(), materialCode);
|
||||
if (def == null) {
|
||||
throw new BizException(ErrorCode.VALIDATION_BLOCKED, "这个任务里没有该材料项:" + materialCode);
|
||||
}
|
||||
if (bytes == null || bytes.length == 0) {
|
||||
throw new BizException(ErrorCode.VALIDATION_BLOCKED, "文件是空的");
|
||||
}
|
||||
|
||||
// 格式白名单:按后缀判。要判 MIME 需要读幻数,属下一段(识别)
|
||||
String ext = extOf(originalName);
|
||||
if (def.getFileTypes() != null && !def.getFileTypes().isEmpty()
|
||||
&& !def.getFileTypes().contains(ext)) {
|
||||
throw new BizException(ErrorCode.FILE_FORMAT_NOT_ALLOWED,
|
||||
"「" + def.getMaterialName() + "」只接受 " + String.join("/", def.getFileTypes())
|
||||
+ ",你传的是 ." + ext);
|
||||
}
|
||||
|
||||
// 份数上限。超过就明确拒绝,而不是"覆盖上一份" ——
|
||||
// 学生传错了要能自己删,静默覆盖会让他以为传上去了
|
||||
long current = nz(fileMapper.countCurrent(me.tenantId(), submissionId, materialCode));
|
||||
int maxCount = def.getMaxCount() == null ? 1 : def.getMaxCount();
|
||||
if (current >= maxCount) {
|
||||
throw new BizException(ErrorCode.VALIDATION_BLOCKED,
|
||||
maxCount == 1
|
||||
? "「" + def.getMaterialName() + "」只能传 1 份,请先删掉上一份"
|
||||
: "「" + def.getMaterialName() + "」最多传 " + maxCount + " 份");
|
||||
}
|
||||
|
||||
// 落盘:raw/ 下按任务分目录
|
||||
String rel = String.format("raw/%d/%d/%d_%s.%s",
|
||||
me.tenantId(), sub.getTaskId(), System.currentTimeMillis(),
|
||||
Integer.toHexString(java.util.Arrays.hashCode(bytes)), ext);
|
||||
writeFile(rel, bytes);
|
||||
|
||||
CollectFile f = new CollectFile();
|
||||
f.setTaskId(sub.getTaskId());
|
||||
f.setSubmissionId(submissionId);
|
||||
f.setRosterId(sub.getRosterId());
|
||||
f.setMaterialCode(materialCode);
|
||||
f.setVersion((int) current + 1);
|
||||
f.setIsCurrent(CollectFile.CURRENT);
|
||||
f.setOriginalName(originalName);
|
||||
f.setFileExt(ext);
|
||||
f.setRawSize((long) bytes.length);
|
||||
f.setFinalSize((long) bytes.length);
|
||||
f.setFileHash(sha256(bytes));
|
||||
f.setRawPath(rel);
|
||||
f.setValidateStatus(CollectFile.VALIDATE_PENDING);
|
||||
f.setNeedManualPlace(CollectFile.NEED_MANUAL_NO);
|
||||
f.setUploadIp(clientIp);
|
||||
fileMapper.insert(f);
|
||||
log.info("上传材料:task={} sub={} material={} {}({} 字节)",
|
||||
sub.getTaskId(), submissionId, materialCode, originalName, bytes.length);
|
||||
return f.getId();
|
||||
}
|
||||
|
||||
// ── 处理:压缩 + 改名 ─────────────────────────
|
||||
|
||||
/**
|
||||
* 把原始文件规范化成"可直接上报"的那一份:压到体积内 + 按模板改名。
|
||||
*
|
||||
* <p>幂等:重复调用会重新处理并覆盖上一份规范文件(原始件不动)。
|
||||
* 学生在手机上重试是常事,不该因此产生多份规范文件。
|
||||
*/
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public CollectFile process(Long fileId) {
|
||||
LoginUser me = UserContext.require();
|
||||
CollectFile f = fileMapper.selectMine(fileId, me.tenantId(), me.uid());
|
||||
if (f == null) {
|
||||
throw new BizException(ErrorCode.VALIDATION_BLOCKED, "文件不存在或不属于你");
|
||||
}
|
||||
MaterialDefVO def = findMaterial(f.getTaskId(), f.getMaterialCode());
|
||||
if (def == null) {
|
||||
throw new BizException(ErrorCode.VALIDATION_BLOCKED, "材料项已被删除,无法处理");
|
||||
}
|
||||
byte[] raw = readFile(f.getRawPath());
|
||||
|
||||
int maxSizeKb = def.getMaxSizeKb() == null ? 1024 : def.getMaxSizeKb();
|
||||
int minQuality = def.getMinQuality() == null ? 70 : def.getMinQuality();
|
||||
FileProcessService.Processed p = processService.compress(
|
||||
raw, def.getCompressProfile(), maxSizeKb, minQuality);
|
||||
|
||||
// 规范文件名:材料级模板优先,否则继承任务级
|
||||
CollectTask task = taskMapper.selectById(f.getTaskId());
|
||||
String template = (def.getNamePattern() != null && !def.getNamePattern().isBlank())
|
||||
? def.getNamePattern()
|
||||
: (task == null ? null : task.getNamingTemplate());
|
||||
Map<String, String> vars = namingVars(me, f, def, task);
|
||||
String outExt = p.ext() != null ? "." + p.ext() : ("." + f.getFileExt());
|
||||
String standardName = processService.renderName(template, vars, outExt);
|
||||
|
||||
String rel = String.format("std/%d/%d/%s", me.tenantId(), f.getTaskId(), standardName);
|
||||
writeFile(rel, p.bytes());
|
||||
|
||||
fileMapper.update(null, Wrappers.<CollectFile>lambdaUpdate()
|
||||
.eq(CollectFile::getId, fileId)
|
||||
.set(CollectFile::getStandardName, standardName)
|
||||
.set(CollectFile::getStdPath, rel)
|
||||
.set(CollectFile::getFinalSize, (long) p.bytes().length)
|
||||
.set(CollectFile::getWidth, p.width())
|
||||
.set(CollectFile::getHeight, p.height())
|
||||
.set(CollectFile::getQuality, p.quality()));
|
||||
|
||||
log.info("规范化完成:{} → {}({}KB → {}KB, {}x{}, q{})",
|
||||
f.getOriginalName(), standardName, f.getRawSize() / 1024,
|
||||
p.bytes().length / 1024, p.width(), p.height(), p.quality());
|
||||
|
||||
f.setStandardName(standardName);
|
||||
f.setStdPath(rel);
|
||||
f.setFinalSize((long) p.bytes().length);
|
||||
f.setWidth(p.width());
|
||||
f.setHeight(p.height());
|
||||
f.setQuality(p.quality());
|
||||
return f;
|
||||
}
|
||||
|
||||
// ── 内部 ──────────────────────────────────────
|
||||
|
||||
/** 命名变量。名字要能与学生对上 —— 所以姓名与证件号必须来自名册,不来自学生输入。 */
|
||||
private Map<String, String> namingVars(LoginUser me, CollectFile f, MaterialDefVO def,
|
||||
CollectTask task) {
|
||||
Map<String, String> v = new LinkedHashMap<>();
|
||||
CollectRoster r = rosterMapper.selectBoundByUser(me.tenantId(), f.getTaskId(), me.uid());
|
||||
if (r != null) {
|
||||
v.put("姓名", r.getRealName());
|
||||
// 文件名里用**脱敏号**:规范名会出现在导出包里、会被打印、会被转发,
|
||||
// 完整证件号落在文件名上等于到处传播。
|
||||
//
|
||||
// ⚠️ 脱敏的星号要换成 X:`*` 在 Windows 文件名里是非法字符,
|
||||
// 会被 sanitize 换成下划线,于是 `361029******0015` 变成
|
||||
// `361029_0015` —— 没人看得出那是脱敏号。用 X 既合法又一眼可辨。
|
||||
String mask = r.getIdCardMask();
|
||||
v.put("证件号", mask == null ? null : mask.replace('*', 'X'));
|
||||
v.put("证件类型", "居民身份证");
|
||||
v.put("序号", r.getSeqNo() == null ? null : String.valueOf(r.getSeqNo()));
|
||||
v.put("班级", null);
|
||||
}
|
||||
v.put("材料名", def.getMaterialName());
|
||||
if (task != null) {
|
||||
v.put("专业", task.getMajorName());
|
||||
v.put("工种", task.getJobName());
|
||||
v.put("等级", task.getJobLevel());
|
||||
}
|
||||
v.put("日期", LocalDate.now().format(DateTimeFormatter.ofPattern("yyyyMMdd")));
|
||||
return v;
|
||||
}
|
||||
|
||||
private MaterialDefVO findMaterial(Long taskId, String materialCode) {
|
||||
List<MaterialDefVO> list = materialDefService.list(taskId);
|
||||
return list.stream().filter(m -> materialCode.equals(m.getMaterialCode()))
|
||||
.findFirst().orElse(null);
|
||||
}
|
||||
|
||||
private String dir() {
|
||||
return props.upload() == null ? "./data/uploads" : props.upload().dirOrDefault();
|
||||
}
|
||||
|
||||
private void writeFile(String rel, byte[] bytes) {
|
||||
try {
|
||||
Path p = Paths.get(dir()).resolve(rel);
|
||||
Files.createDirectories(p.getParent());
|
||||
Files.write(p, bytes);
|
||||
} catch (Exception e) {
|
||||
log.error("写文件失败:{}", rel, e);
|
||||
throw new BizException(ErrorCode.INTERNAL_ERROR, "文件保存失败:" + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] readFile(String rel) {
|
||||
if (rel == null || rel.isBlank()) {
|
||||
throw new BizException(ErrorCode.VALIDATION_BLOCKED, "原始文件不存在,请重新上传");
|
||||
}
|
||||
try {
|
||||
return Files.readAllBytes(Paths.get(dir()).resolve(rel));
|
||||
} catch (Exception e) {
|
||||
throw new BizException(ErrorCode.VALIDATION_BLOCKED, "原始文件读取失败,请重新上传");
|
||||
}
|
||||
}
|
||||
|
||||
private static String extOf(String name) {
|
||||
if (name == null) {
|
||||
return "bin";
|
||||
}
|
||||
int i = name.lastIndexOf('.');
|
||||
return i < 0 ? "bin" : name.substring(i + 1).toLowerCase();
|
||||
}
|
||||
|
||||
private static String sha256(byte[] bytes) {
|
||||
try {
|
||||
byte[] h = MessageDigest.getInstance("SHA-256").digest(bytes);
|
||||
StringBuilder sb = new StringBuilder(h.length * 2);
|
||||
for (byte b : h) {
|
||||
sb.append(Character.forDigit((b >> 4) & 0xF, 16)).append(Character.forDigit(b & 0xF, 16));
|
||||
}
|
||||
return sb.toString();
|
||||
} catch (Exception e) {
|
||||
throw new BizException(ErrorCode.INTERNAL_ERROR, "计算文件指纹失败");
|
||||
}
|
||||
}
|
||||
|
||||
private static long nz(Long v) {
|
||||
return v == null ? 0L : v;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,327 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
材料上传与规范化 验收测试(《03-接口设计》§9.5~§9.7 的核心)。
|
||||
|
||||
[x] 上传原图(后端代理接收)
|
||||
[x] 处理:**压到体积上限内**(这是认定机构的硬要求,无例外)
|
||||
[x] 处理:**按命名模板改名**(规范名要能唯一归属到人)
|
||||
[x] 档位生效:P2 证件翻拍 → 长边被限到 1600
|
||||
[x] 原始件保留(退回时要能拿出原图核对)
|
||||
[x] 格式白名单 / 份数上限
|
||||
[x] 幂等:重复处理不产生第二份规范文件
|
||||
|
||||
⚠️ 为什么这个 E2E 特别有价值:
|
||||
压缩与改名是**产品的核心资产**(实测 52.7% 的原始文件名违规),
|
||||
而它同时是"最容易被以为做对了"的地方 —— 尺寸小了、名字变了,
|
||||
但**体积没降到 1M 以内**,或者名字里少了证件号。
|
||||
只有拿真实图片跑一遍、逐项断言,才知道到底行不行。
|
||||
|
||||
跑法:python scripts/e2e-m3-file.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
sys.stdout.reconfigure(encoding="utf-8")
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
DEV_YML = os.path.join(ROOT, "backend", "xt-server", "src", "main", "resources", "application-dev.yml")
|
||||
DATA_DIR = os.path.join(ROOT, "backend", "data", "uploads")
|
||||
PG = os.environ.get("XT_PG_CONTAINER", "xt-pg")
|
||||
OPENID = os.environ.get("XT_TEST_OPENID", "e2e_file_openid")
|
||||
|
||||
PASSED: list[str] = []
|
||||
FAILED: list[tuple[str, str]] = []
|
||||
CREATED_FILES: list[int] = []
|
||||
CREATED_SUBS: list[int] = []
|
||||
|
||||
|
||||
def ok(n: str) -> None:
|
||||
PASSED.append(n)
|
||||
print(f" [OK] {n}")
|
||||
|
||||
|
||||
def fail(n: str, d: str) -> None:
|
||||
FAILED.append((n, d))
|
||||
print(f" [FAIL] {n}\n {d}")
|
||||
|
||||
|
||||
def check(n: str, cond: bool, d: str = "") -> bool:
|
||||
if cond:
|
||||
ok(n)
|
||||
else:
|
||||
fail(n, d or "断言不成立")
|
||||
return cond
|
||||
|
||||
|
||||
def sql(q: str) -> str:
|
||||
r = subprocess.run(
|
||||
["docker", "exec", "-e", "PGPASSWORD=fill_dev_pwd", PG, "psql", "-U", "fill",
|
||||
"-d", "filldb", "-t", "-A", "-c", q], capture_output=True, text=True, timeout=40)
|
||||
return (r.stdout or r.stderr).strip()
|
||||
|
||||
|
||||
def read_cfg() -> dict:
|
||||
# issuer 在主配置 application.yml 里,secret 在 application-dev.yml 里 ——
|
||||
# 只读一个会漏(而症状是 401,完全指不到"配置没读到")
|
||||
files = [DEV_YML, os.path.join(os.path.dirname(DEV_YML), "application.yml")]
|
||||
text = "\n".join(open(f, encoding="utf-8").read() for f in files if os.path.exists(f))
|
||||
out = {}
|
||||
for k, p in [("jwt", r"^\s*secret:\s*(\S+)\s*$"), ("issuer", r"^\s*issuer:\s*(\S+)\s*$")]:
|
||||
m = re.search(p, text, re.M)
|
||||
if not m:
|
||||
raise SystemExit(f"找不到 {k}")
|
||||
out[k] = m.group(1)
|
||||
return out
|
||||
|
||||
|
||||
def b64u(b: bytes) -> str:
|
||||
return base64.urlsafe_b64encode(b).rstrip(b"=").decode()
|
||||
|
||||
|
||||
def mint(secret: str, issuer: str, uid: int, tenant: int) -> str:
|
||||
key = base64.b64decode(secret)
|
||||
now = int(time.time())
|
||||
signing = (b64u(json.dumps({"alg": "HS512"}, separators=(",", ":")).encode()) + "."
|
||||
+ b64u(json.dumps({"iss": issuer, "sub": str(uid), "tenantId": tenant,
|
||||
"orgIds": [], "roles": ["STUDENT"], "userType": 4,
|
||||
"iat": now, "exp": now + 3600}, separators=(",", ":")).encode()))
|
||||
sig = hmac.new(key, signing.encode(), hashlib.sha512).digest()
|
||||
return signing + "." + b64u(sig)
|
||||
|
||||
|
||||
def call(method: str, path: str, token: str | None = None, body: dict | None = None) -> dict:
|
||||
h = {"Content-Type": "application/json; charset=utf-8"}
|
||||
if token:
|
||||
h["Authorization"] = f"Bearer {token}"
|
||||
data = json.dumps(body, ensure_ascii=False).encode() if body is not None else None
|
||||
req = urllib.request.Request("http://localhost:8080" + path, data=data, headers=h, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=60) as x:
|
||||
return json.loads(x.read().decode("utf-8"))
|
||||
except urllib.error.HTTPError as e:
|
||||
try:
|
||||
return json.loads(e.read().decode("utf-8"))
|
||||
except Exception:
|
||||
return {"_http": e.code}
|
||||
|
||||
|
||||
def upload(path: str, token: str, submission_id: int, material_code: str,
|
||||
filename: str, data: bytes, ctype: str) -> dict:
|
||||
boundary = "----xte2e" + uuid.uuid4().hex[:12]
|
||||
parts = []
|
||||
for k, v in [("submissionId", str(submission_id)), ("materialCode", material_code)]:
|
||||
parts.append(f'--{boundary}\r\nContent-Disposition: form-data; name="{k}"\r\n\r\n{v}\r\n'.encode())
|
||||
parts.append(
|
||||
f'--{boundary}\r\nContent-Disposition: form-data; name="file"; filename="{filename}"\r\n'
|
||||
f"Content-Type: {ctype}\r\n\r\n".encode())
|
||||
parts.append(data)
|
||||
parts.append(f"\r\n--{boundary}--\r\n".encode())
|
||||
body = b"".join(parts)
|
||||
req = urllib.request.Request(
|
||||
"http://localhost:8080" + path, data=body, method="POST",
|
||||
headers={"Content-Type": f"multipart/form-data; boundary={boundary}",
|
||||
"Authorization": f"Bearer {token}"})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=90) as x:
|
||||
return json.loads(x.read().decode("utf-8"))
|
||||
except urllib.error.HTTPError as e:
|
||||
try:
|
||||
return json.loads(e.read().decode("utf-8"))
|
||||
except Exception:
|
||||
return {"_http": e.code}
|
||||
|
||||
|
||||
def make_big_photo(w: int = 2400, h: int = 3200) -> bytes:
|
||||
"""造一张"大且难压"的图:随机噪点。纯色图会被压到几十 KB,测不出压缩能力。"""
|
||||
from PIL import Image
|
||||
import random
|
||||
random.seed(42)
|
||||
img = Image.new("RGB", (w, h))
|
||||
px = img.load()
|
||||
# 每 4 个像素填一次,四两拨千斤地造出高频噪点(全填太慢)
|
||||
for y in range(0, h, 2):
|
||||
for x in range(0, w, 2):
|
||||
c = (random.randint(0, 255), random.randint(0, 255), random.randint(0, 255))
|
||||
px[x, y] = c
|
||||
if x + 1 < w:
|
||||
px[x + 1, y] = c
|
||||
if y + 1 < h:
|
||||
px[x, y + 1] = c
|
||||
if x + 1 < w:
|
||||
px[x + 1, y + 1] = c
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, "JPEG", quality=98)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--task", type=int, default=17)
|
||||
ap.add_argument("--tenant", type=int, default=1)
|
||||
args = ap.parse_args()
|
||||
task_id, tenant = args.task, args.tenant
|
||||
|
||||
print("=" * 72)
|
||||
print(f"材料上传与规范化 E2E task={task_id}")
|
||||
print("=" * 72)
|
||||
|
||||
cfg = read_cfg()
|
||||
# 用一个固定的测试账号(不必是真实微信账号):先过门禁,再传材料
|
||||
uid = 992001
|
||||
tok = mint(cfg["jwt"], cfg["issuer"], uid, tenant)
|
||||
|
||||
# 名册绑定到这个测试账号
|
||||
rid = sql(f"SELECT id FROM collect_roster WHERE task_id={task_id} LIMIT 1")
|
||||
if not rid.isdigit():
|
||||
fail("取名册行", "任务下没有名册")
|
||||
return 1
|
||||
sql(f"UPDATE collect_roster SET bind_user_id={uid}, bind_type=1 WHERE id={rid}")
|
||||
|
||||
r = call("POST", "/api/v1/mp/submissions", tok, {"taskId": task_id})
|
||||
if r.get("code") != 0:
|
||||
fail("建草稿", f"{r}")
|
||||
return 1
|
||||
sid = r["data"]["id"]
|
||||
CREATED_SUBS.append(sid)
|
||||
ok(f"已过门禁并建草稿(submission={sid})")
|
||||
|
||||
# ⚠️ 夹具必须**显式**把材料项配成目录规定的那样。
|
||||
# 用裸 SQL 插入会绕过 MaterialCatalog 的默认值(DDL 的列默认是 AUTO / 1 份),
|
||||
# 于是"档位没生效""份数不对"—— 而这两条都会被误当成代码 bug。
|
||||
# 实测就踩了:IDCARD 期望 P2 + 2 份,裸 SQL 插出来是 AUTO + 1 份,
|
||||
# 压缩结果 1252x1669(AUTO 先降质量、不行才缩尺寸)而被当成"P2 档位没生效"。
|
||||
sql(f"""UPDATE collect_material_def SET compress_profile='P2', max_count=2, min_quality=70
|
||||
WHERE task_id={task_id} AND material_code='IDCARD'""")
|
||||
prof = sql(f"SELECT compress_profile || '/' || max_count FROM collect_material_def "
|
||||
f"WHERE task_id={task_id} AND material_code='IDCARD'")
|
||||
check("夹具:IDCARD 已配成目录规定的 P2 + 2 份", prof == "P2/2", f"实际 {prof}")
|
||||
|
||||
# ── 1. 上传一张真实的大图 ─────────────────────────
|
||||
print("\n[1] 上传原始大图")
|
||||
raw = make_big_photo()
|
||||
print(f" 原始图:2400x3200,{len(raw) / 1024:.0f} KB")
|
||||
check("原图确实超过 1M(否则测不出压缩能力)", len(raw) > 1024 * 1024, f"{len(raw) / 1024:.0f} KB")
|
||||
|
||||
r = upload("/api/v1/mp/files/upload", tok, sid, "IDCARD", "身份证正面.jpg", raw, "image/jpeg")
|
||||
check("上传成功", r.get("code") == 0, f"code={r.get('code')} msg={r.get('msg')}")
|
||||
fid = (r.get("data") or {}).get("id")
|
||||
CREATED_FILES.append(fid)
|
||||
check("返回文件 id", isinstance(fid, int), f"{fid}")
|
||||
|
||||
row = sql(f"SELECT raw_size || '|' || coalesce(raw_path,'') FROM collect_file WHERE id={fid}")
|
||||
raw_size_db = row.split("|")[0]
|
||||
check("原始件已落盘记录(raw_size 是原图大小)", raw_size_db == str(len(raw)),
|
||||
f"库里 {raw_size_db} vs 上传 {len(raw)}")
|
||||
|
||||
# ── 2. 处理:压缩 + 改名 ──────────────────────────
|
||||
print("\n[2] 规范化(压缩 + 按规则改名)")
|
||||
r = call("POST", f"/api/v1/mp/files/{fid}/process", tok)
|
||||
if not check("处理成功", r.get("code") == 0, f"code={r.get('code')} msg={r.get('msg')}"):
|
||||
return 1
|
||||
d = r.get("data") or {}
|
||||
print(f" {d.get('originalName')} → {d.get('standardName')}")
|
||||
print(f" {d.get('rawSize')} → {d.get('finalSize')} 字节,"
|
||||
f"{d.get('width')}x{d.get('height')} q{d.get('quality')}")
|
||||
|
||||
check("**体积压到 1024KB 以内**(认定机构的硬要求)",
|
||||
(d.get("finalSize") or 0) <= 1024 * 1024, f"finalSize={d.get('finalSize')}")
|
||||
check("体积确实降下来了(不是原样返回)",
|
||||
(d.get("finalSize") or 0) < (d.get("rawSize") or 0), f"{d.get('rawSize')} → {d.get('finalSize')}")
|
||||
|
||||
# IDCARD 的默认档位是 P2(证件翻拍)→ 长边限到 1600
|
||||
long_side = max(d.get("width") or 0, d.get("height") or 0)
|
||||
check("P2 档位生效:长边被限到 1600", long_side <= 1600, f"长边={long_side}")
|
||||
check("短边按比例缩放(不是被压扁)",
|
||||
abs((d.get("width") or 1) / (d.get("height") or 1) - 2400 / 3200) < 0.02,
|
||||
f"宽高比 {d.get('width')}x{d.get('height')}")
|
||||
|
||||
# ── 3. 命名规则 ───────────────────────────────────
|
||||
print("\n[3] 规范文件名")
|
||||
name = d.get("standardName") or ""
|
||||
check("名字里有姓名(能归属到人)", "张同学" in name, f"name={name}")
|
||||
check("名字里有证件号(脱敏,不是完整号)",
|
||||
"361029" in name and "361029200912100015" not in name, f"name={name}")
|
||||
check("脱敏号在文件名里用 X 而不是下划线(* 是 Windows 非法字符,"
|
||||
"直接替换成下划线会让人看不出那是脱敏号)",
|
||||
"XXXXXX" in name.upper() or "X" in name.upper(), f"name={name}")
|
||||
check("名字里有材料名", "身份证" in name, f"name={name}")
|
||||
check("后缀正确(压成 jpg)", name.lower().endswith(".jpg"), f"name={name}")
|
||||
check("名字里没有非法字符(/ : * ? \" < > |)",
|
||||
not re.search(r'[\\/:*?"<>|]', name), f"name={name}")
|
||||
check("长度不超过 90(Windows 路径总长限制)", len(name) <= 90, f"len={len(name)}")
|
||||
|
||||
# ── 4. 原始件保留 ─────────────────────────────────
|
||||
print("\n[4] 原始件必须保留")
|
||||
rawpath = sql(f"SELECT coalesce(raw_path,'') FROM collect_file WHERE id={fid}")
|
||||
check("raw_path 还在(没有被规范文件覆盖)", rawpath != "", f"raw_path={rawpath}")
|
||||
full = os.path.join(DATA_DIR, rawpath.replace("/", os.sep))
|
||||
check("原始文件真的存在磁盘上", os.path.exists(full), f"路径={full}")
|
||||
if os.path.exists(full):
|
||||
check("原始文件大小与上传一致(一字不差)",
|
||||
os.path.getsize(full) == len(raw), f"{os.path.getsize(full)} vs {len(raw)}")
|
||||
stdpath = sql(f"SELECT coalesce(std_path,'') FROM collect_file WHERE id={fid}")
|
||||
full2 = os.path.join(DATA_DIR, stdpath.replace("/", os.sep))
|
||||
check("规范文件也落盘了", os.path.exists(full2), f"路径={full2}")
|
||||
|
||||
# ── 5. 幂等 ───────────────────────────────────────
|
||||
print("\n[5] 幂等:重复处理")
|
||||
r2 = call("POST", f"/api/v1/mp/files/{fid}/process", tok)
|
||||
check("重复处理成功", r2.get("code") == 0, f"code={r2.get('code')}")
|
||||
cnt = sql(f"SELECT count(*) FROM collect_file WHERE submission_id={sid} AND material_code='IDCARD'")
|
||||
check("没有产生第二份记录", cnt == "1", f"{cnt}")
|
||||
check("规范文件名稳定(同一次处理结果一致)",
|
||||
(r2.get("data") or {}).get("standardName") == name,
|
||||
f"{(r2.get('data') or {}).get('standardName')}")
|
||||
|
||||
# ── 6. 白名单与份数 ───────────────────────────────
|
||||
print("\n[6] 格式白名单与份数上限")
|
||||
r = upload("/api/v1/mp/files/upload", tok, sid, "IDCARD", "身份证.exe", b"MZfake", "application/octet-stream")
|
||||
check("白名单外的格式被拒(40002)", r.get("code") == 40002, f"code={r.get('code')} msg={r.get('msg')}")
|
||||
r = upload("/api/v1/mp/files/upload", tok, sid, "IDCARD", "身份证反面.jpg", raw[:50000], "image/jpeg")
|
||||
check("超过份数上限被拒(IDCARD 默认 2 份,现在已有 1 份 → 这次应成功)",
|
||||
r.get("code") == 0, f"code={r.get('code')} msg={r.get('msg')}")
|
||||
if r.get("code") == 0:
|
||||
CREATED_FILES.append((r.get("data") or {}).get("id"))
|
||||
r = upload("/api/v1/mp/files/upload", tok, sid, "IDCARD", "身份证再来一份.jpg", raw[:50000], "image/jpeg")
|
||||
check("第 3 份被拒(且提示先删上一份)",
|
||||
r.get("code") == 40001 and "最多" in (r.get("msg") or ""),
|
||||
f"code={r.get('code')} msg={r.get('msg')}")
|
||||
|
||||
# ── 7. 清理 ───────────────────────────────────────
|
||||
print("\n[7] 清理")
|
||||
sql(f"DELETE FROM collect_file WHERE submission_id IN ({','.join(str(x) for x in CREATED_SUBS)})")
|
||||
sql(f"DELETE FROM collect_submission WHERE id IN ({','.join(str(x) for x in CREATED_SUBS)})")
|
||||
sql(f"UPDATE collect_roster SET bind_user_id=NULL, bind_time=NULL, bind_type=NULL WHERE id={rid}")
|
||||
import shutil
|
||||
for sub in ("raw", "std"):
|
||||
d2 = os.path.join(DATA_DIR, sub, "1", str(task_id))
|
||||
if os.path.exists(d2):
|
||||
shutil.rmtree(d2, ignore_errors=True)
|
||||
ok("已清理草稿 / 文件记录 / 磁盘文件 / 名册绑定")
|
||||
|
||||
print("\n" + "=" * 72)
|
||||
print(f"通过 {len(PASSED)} 失败 {len(FAILED)} 共 {len(PASSED) + len(FAILED)}")
|
||||
for n, dd in FAILED:
|
||||
print(f" · {n}\n {dd}")
|
||||
print("=" * 72)
|
||||
return 1 if FAILED else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user