From fcd3a426d6a612d857c1fd62ac11f3d536057486 Mon Sep 17 00:00:00 2001 From: cangtianhuang <1903374751@qq.com> Date: Tue, 14 Jul 2026 12:42:29 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E2=9C=A8=20Add=20generic=20configs,=20fix?= =?UTF-8?q?=20breakdown?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- run.py | 34 ++++++++++- .../V4_configs/run_cpu_0size_full.yaml | 19 ------ .../V4_configs/run_dsv4_0size_accuracy.yaml | 16 ----- .../V4_configs/run_dsv4_0size_paddleonly.yaml | 18 ------ .../V4_configs/run_dsv4_1M_accuracy.yaml | 15 ----- .../V4_configs/run_gpu_0size_full.yaml | 18 ------ .../V4_configs/run_gpu_accuracy_full.yaml | 21 ------- .../V4_configs/run_gpu_bigtensor_full.yaml | 19 ------ .../V4_configs/run_v2_0size_accuracy.yaml | 15 ----- .../V4_configs/run_v2_0size_paddleonly.yaml | 18 ------ .../V4_configs/run_v2_1M_accuracy.yaml | 15 ----- .../V4_configs/run_v2_1M_paddleonly.yaml | 20 ------- test_pipeline/generic_configs/README.md | 60 +++++++++++++++++++ .../generic_configs/run_0size_accuracy.yaml | 17 ++++++ .../generic_configs/run_0size_paddleonly.yaml | 20 +++++++ .../generic_configs/run_1M_accuracy.yaml | 17 ++++++ .../run_1M_paddleonly.yaml} | 10 ++-- .../generic_configs/run_4096_accuracy.yaml | 17 ++++++ .../generic_configs/run_4096_paddleonly.yaml | 22 +++++++ .../run_accuracy_compatible.yaml | 20 +++++++ .../run_accuracy_manual_threshold.yaml | 21 +++++++ tester/api_config/log_writer.py | 14 ++--- 22 files changed, 238 insertions(+), 208 deletions(-) delete mode 100644 test_pipeline/V4_configs/run_cpu_0size_full.yaml delete mode 100644 test_pipeline/V4_configs/run_dsv4_0size_accuracy.yaml delete mode 100644 test_pipeline/V4_configs/run_dsv4_0size_paddleonly.yaml delete mode 100644 test_pipeline/V4_configs/run_dsv4_1M_accuracy.yaml delete mode 100644 test_pipeline/V4_configs/run_gpu_0size_full.yaml delete mode 100644 test_pipeline/V4_configs/run_gpu_accuracy_full.yaml delete mode 100644 test_pipeline/V4_configs/run_gpu_bigtensor_full.yaml delete mode 100644 test_pipeline/V4_configs/run_v2_0size_accuracy.yaml delete mode 100644 test_pipeline/V4_configs/run_v2_0size_paddleonly.yaml delete mode 100644 test_pipeline/V4_configs/run_v2_1M_accuracy.yaml delete mode 100644 test_pipeline/V4_configs/run_v2_1M_paddleonly.yaml create mode 100644 test_pipeline/generic_configs/README.md create mode 100644 test_pipeline/generic_configs/run_0size_accuracy.yaml create mode 100644 test_pipeline/generic_configs/run_0size_paddleonly.yaml create mode 100644 test_pipeline/generic_configs/run_1M_accuracy.yaml rename test_pipeline/{V4_configs/run_dsv4_1M_paddleonly.yaml => generic_configs/run_1M_paddleonly.yaml} (50%) create mode 100644 test_pipeline/generic_configs/run_4096_accuracy.yaml create mode 100644 test_pipeline/generic_configs/run_4096_paddleonly.yaml create mode 100644 test_pipeline/generic_configs/run_accuracy_compatible.yaml create mode 100644 test_pipeline/generic_configs/run_accuracy_manual_threshold.yaml diff --git a/run.py b/run.py index 33f3ad40..ba3a4ba4 100644 --- a/run.py +++ b/run.py @@ -168,12 +168,28 @@ def validate_yaml_config(config: dict[str, Any]) -> None: raise ValueError("engine_args.custom_device_vs_gpu_mode 仅支持 upload 或 download") +def expand_env_vars(value: Any) -> Any: + """递归展开字符串中的 ${VAR} / $VAR 环境变量引用，非字符串原样返回。 + + 用于支持通用 pipeline 配置（如 generic_configs/）中以 + ${JELLY_APITEST_MODEL} 等环境变量占位模型名，从而仅需切换环境变量即可 + 复用同一份配置文件。未设置的变量保持原样，不报错。 + """ + if isinstance(value, str): + return os.path.expandvars(value) + if isinstance(value, dict): + return {key: expand_env_vars(item) for key, item in value.items()} + if isinstance(value, list): + return [expand_env_vars(item) for item in value] + return value + + def load_yaml(path: Path) -> dict[str, Any]: with path.open("r", encoding="utf-8") as config_file: config = yaml.safe_load(config_file) or {} if not isinstance(config, dict): raise ValueError(f"配置文件必须是 YAML mapping: {path}") - return config + return expand_env_vars(config) def parse_key_value(value: str, option_name: str) -> tuple[str, str]: @@ -634,9 +650,21 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--background", action="store_true", help="后台运行") parser.add_argument("--engine", choices=["engineV2", "engineV4"], help="覆盖 runner.engine") parser.add_argument("--api-config", help="覆盖 input.api_config") - parser.add_argument("--api-config-file", help="覆盖 input.api_config_file") + parser.add_argument( + "-i", + "--input", + "--api-config-file", + dest="api_config_file", + help="覆盖 input.api_config_file（-i/--input 为别名）", + ) parser.add_argument("--api-config-file-pattern", help="覆盖 input.api_config_file_pattern") - parser.add_argument("--log-dir", help="覆盖 output.log_dir") + parser.add_argument( + "-o", + "--output", + "--log-dir", + dest="log_dir", + help="覆盖 output.log_dir（-o/--output 为别名）", + ) parser.add_argument("--timeout", type=int, help="覆盖 engine_args.timeout") parser.add_argument("--num-gpus", type=int, help="覆盖 engine_args.num_gpus") parser.add_argument( diff --git a/test_pipeline/V4_configs/run_cpu_0size_full.yaml b/test_pipeline/V4_configs/run_cpu_0size_full.yaml deleted file mode 100644 index 53be47e7..00000000 --- a/test_pipeline/V4_configs/run_cpu_0size_full.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# yaml-language-server: $schema=../run_config.schema.json -name: run_cpu_0size_full -runner: - engine: engineV4 - foreground: true -env: - FLAGS_use_system_allocator: "true" - FLAGS_check_cuda_error: "true" -input: - api_config_file_pattern: tester/api_config/monitor_config/0_size/CPU/monitoring_configs*.txt -output: - log_dir: tester/api_config/test_log_cpu_0size_full -engine_args: - accuracy: true - test_cpu: true - num_gpus: -1 - num_workers_per_gpu: 10 - timeout: 600 - random_seed: 2025 diff --git a/test_pipeline/V4_configs/run_dsv4_0size_accuracy.yaml b/test_pipeline/V4_configs/run_dsv4_0size_accuracy.yaml deleted file mode 100644 index 9e3168d2..00000000 --- a/test_pipeline/V4_configs/run_dsv4_0size_accuracy.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# yaml-language-server: $schema=../run_config.schema.json -name: run_dsv4_0size_accuracy -runner: - engine: engineV4 - foreground: true -input: - api_config_file: tester/api_config/monitor_config/dsv4_v2/dsv4_0size.txt -output: - log_dir: tester/api_config/test_log_dsv4_0size_accuracy -engine_args: - accuracy_stable: true - use_cached_numpy: true - num_gpus: -1 - num_workers_per_gpu: 4 - timeout: 600 - diff --git a/test_pipeline/V4_configs/run_dsv4_0size_paddleonly.yaml b/test_pipeline/V4_configs/run_dsv4_0size_paddleonly.yaml deleted file mode 100644 index de851938..00000000 --- a/test_pipeline/V4_configs/run_dsv4_0size_paddleonly.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# yaml-language-server: $schema=../run_config.schema.json -name: run_dsv4_0size_paddleonly -runner: - engine: engineV4 - foreground: true -env: - FLAGS_use_system_allocator: "true" - FLAGS_check_cuda_error: "true" -input: - api_config_file: tester/api_config/monitor_config/dsv4_v2/dsv4_0size.txt -output: - log_dir: tester/api_config/test_log_dsv4_0size_paddleonly -engine_args: - paddle_only: true - num_gpus: -1 - num_workers_per_gpu: 1 - timeout: 600 - use_compute_sanitizer: true diff --git a/test_pipeline/V4_configs/run_dsv4_1M_accuracy.yaml b/test_pipeline/V4_configs/run_dsv4_1M_accuracy.yaml deleted file mode 100644 index 00ef4f4f..00000000 --- a/test_pipeline/V4_configs/run_dsv4_1M_accuracy.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# yaml-language-server: $schema=../run_config.schema.json -name: run_dsv4_1M_accuracy -runner: - engine: engineV4 - foreground: true -input: - api_config_file: tester/api_config/monitor_config/dsv4_v2/dsv4_1M.txt -output: - log_dir: tester/api_config/test_log_dsv4_1M_accuracy -engine_args: - accuracy_stable: true - use_cached_numpy: true - num_gpus: -1 - num_workers_per_gpu: 1 - timeout: 1200 diff --git a/test_pipeline/V4_configs/run_gpu_0size_full.yaml b/test_pipeline/V4_configs/run_gpu_0size_full.yaml deleted file mode 100644 index 0ee2dc60..00000000 --- a/test_pipeline/V4_configs/run_gpu_0size_full.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# yaml-language-server: $schema=../run_config.schema.json -name: run_gpu_0size_full -runner: - engine: engineV4 - foreground: true -env: - FLAGS_use_system_allocator: "true" - FLAGS_check_cuda_error: "true" -input: - api_config_file_pattern: tester/api_config/monitor_config/0_size/GPU/monitoring_configs*.txt -output: - log_dir: tester/api_config/test_log_gpu_0size_full -engine_args: - accuracy: true - num_gpus: -1 - num_workers_per_gpu: 10 - timeout: 300 - random_seed: 2025 diff --git a/test_pipeline/V4_configs/run_gpu_accuracy_full.yaml b/test_pipeline/V4_configs/run_gpu_accuracy_full.yaml deleted file mode 100644 index de3b7951..00000000 --- a/test_pipeline/V4_configs/run_gpu_accuracy_full.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# yaml-language-server: $schema=../run_config.schema.json -name: run_gpu_accuracy_full -runner: - engine: engineV4 - foreground: true -env: - FLAGS_use_system_allocator: "true" - FLAGS_check_cuda_error: "true" - FLAGS_alloc_fill_value: "255" - FLAGS_check_nan_inf: "true" -input: - api_config_file_pattern: tester/api_config/monitor_config/accuracy/GPU/monitoring_configs*.txt -output: - log_dir: tester/api_config/test_log_gpu_accuracy_full -engine_args: - accuracy: true - atol: 0.0 - rtol: 0.0 - bitwise_alignment: true - num_gpus: -1 - num_workers_per_gpu: 10 diff --git a/test_pipeline/V4_configs/run_gpu_bigtensor_full.yaml b/test_pipeline/V4_configs/run_gpu_bigtensor_full.yaml deleted file mode 100644 index 8297f817..00000000 --- a/test_pipeline/V4_configs/run_gpu_bigtensor_full.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# yaml-language-server: $schema=../run_config.schema.json -name: run_gpu_bigtensor_full -runner: - engine: engineV4 - foreground: true -env: - FLAGS_use_system_allocator: "true" - FLAGS_check_cuda_error: "true" - FLAGS_alloc_fill_value: "255" - FLAGS_check_nan_inf: "true" -input: - api_config_file_pattern: tester/api_config/monitor_config/big_tensor/GPU/monitoring_configs*.txt -output: - log_dir: tester/api_config/test_log_gpu_bigtensor_full -engine_args: - accuracy: true - use_cached_numpy: true - num_gpus: -1 - num_workers_per_gpu: 10 diff --git a/test_pipeline/V4_configs/run_v2_0size_accuracy.yaml b/test_pipeline/V4_configs/run_v2_0size_accuracy.yaml deleted file mode 100644 index f04ff7b4..00000000 --- a/test_pipeline/V4_configs/run_v2_0size_accuracy.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# yaml-language-server: $schema=../run_config.schema.json -name: run_v2_0size_accuracy -runner: - engine: engineV4 - foreground: true -input: - api_config_file: tester/api_config/monitor_config/dsv4_v2/v2_0size.txt -output: - log_dir: tester/api_config/test_log_v2_0size_accuracy -engine_args: - accuracy_stable: true - use_cached_numpy: true - num_gpus: -1 - num_workers_per_gpu: 4 - timeout: 600 diff --git a/test_pipeline/V4_configs/run_v2_0size_paddleonly.yaml b/test_pipeline/V4_configs/run_v2_0size_paddleonly.yaml deleted file mode 100644 index 9628c0b9..00000000 --- a/test_pipeline/V4_configs/run_v2_0size_paddleonly.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# yaml-language-server: $schema=../run_config.schema.json -name: run_v2_0size_paddleonly -runner: - engine: engineV4 - foreground: true -env: - FLAGS_use_system_allocator: "true" - FLAGS_check_cuda_error: "true" -input: - api_config_file: tester/api_config/monitor_config/dsv4_v2/v2_0size.txt -output: - log_dir: tester/api_config/test_log_v2_0size_paddleonly -engine_args: - paddle_only: true - num_gpus: -1 - num_workers_per_gpu: 1 - timeout: 600 - use_compute_sanitizer: true diff --git a/test_pipeline/V4_configs/run_v2_1M_accuracy.yaml b/test_pipeline/V4_configs/run_v2_1M_accuracy.yaml deleted file mode 100644 index 9e8308fb..00000000 --- a/test_pipeline/V4_configs/run_v2_1M_accuracy.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# yaml-language-server: $schema=../run_config.schema.json -name: run_v2_1M_accuracy -runner: - engine: engineV4 - foreground: true -input: - api_config_file: tester/api_config/monitor_config/dsv4_v2/v2_1M.txt -output: - log_dir: tester/api_config/test_log_v2_1M_accuracy -engine_args: - accuracy_stable: true - use_cached_numpy: true - num_gpus: -1 - num_workers_per_gpu: 1 - timeout: 1200 diff --git a/test_pipeline/V4_configs/run_v2_1M_paddleonly.yaml b/test_pipeline/V4_configs/run_v2_1M_paddleonly.yaml deleted file mode 100644 index 76ca8e1b..00000000 --- a/test_pipeline/V4_configs/run_v2_1M_paddleonly.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# yaml-language-server: $schema=../run_config.schema.json -name: run_v2_1M_paddleonly -runner: - engine: engineV4 - foreground: true -env: - FLAGS_use_system_allocator: "true" - FLAGS_check_cuda_error: "true" - FLAGS_alloc_fill_value: "255" - FLAGS_check_nan_inf: "true" -input: - api_config_file: tester/api_config/monitor_config/dsv4_v2/v2_1M.txt -output: - log_dir: tester/api_config/test_log_v2_1M_paddleonly -engine_args: - paddle_only: true - num_gpus: -1 - num_workers_per_gpu: 1 - timeout: 1200 - use_compute_sanitizer: true diff --git a/test_pipeline/generic_configs/README.md b/test_pipeline/generic_configs/README.md new file mode 100644 index 00000000..94858f4b --- /dev/null +++ b/test_pipeline/generic_configs/README.md @@ -0,0 +1,60 @@ +# generic_configs + +通用 CI 配置：6 种 `{0size, 1M, 4096} × {accuracy, paddleonly}` 组合， +特殊 accuracy 变体：2 种（`accuracy_compatible`、`accuracy_manual_threshold`）， +输入/输出路径用 `${APITEST_MODEL}` 占位，切换模型只需改这一个环境变量。 + +## 目录约定 + +需先把模型专属配置拷贝到 `PaddleAPITest/${APITEST_MODEL}/` 下： + +```text +${APITEST_MODEL}/ +├── accuracy_0size/0size.txt +├── accuracy/1M.txt +├── accuracy/4096.txt +├── paddleonly_0size/0size.txt +├── paddleonly/1M.txt +├── paddleonly/4096.txt +├── accuracy_compatible/accuracy_compatible.txt +└── accuracy_manual_threshold/ + ├── accuracy_manual_threshold.txt + └── accuracy_manual_threshold_config.yaml +``` + +## 用法 + +```bash +export APITEST_MODEL=eb5_1 +python run.py -c test_pipeline/generic_configs/run_0size_accuracy.yaml +python run.py -c test_pipeline/generic_configs/run_1M_accuracy.yaml +python run.py -c test_pipeline/generic_configs/run_4096_accuracy.yaml +python run.py -c test_pipeline/generic_configs/run_0size_paddleonly.yaml +python run.py -c test_pipeline/generic_configs/run_1M_paddleonly.yaml +python run.py -c test_pipeline/generic_configs/run_4096_paddleonly.yaml +python run.py -c test_pipeline/generic_configs/run_accuracy_compatible.yaml +python run.py -c test_pipeline/generic_configs/run_accuracy_manual_threshold.yaml +``` + +也可用 `-i/--input`、`-o/--output` 临时覆盖某次运行的输入文件和日志目录： + +```bash +python run.py -c test_pipeline/generic_configs/run_1M_accuracy.yaml \ + -i ${APITEST_MODEL}/accuracy/1M.txt -o test_${APITEST_MODEL}_log_1M_accuracy +``` + +## num_workers_per_gpu 配置 + +| 配置 | num_workers_per_gpu | +| --- | --- | +| 0size accuracy | 16 | +| 0size paddleonly | 16 | +| 4096 accuracy | 8 | +| 4096 paddleonly | 8 | +| 1M accuracy/paddleonly | 2 | +| accuracy_compatible / accuracy_manual_threshold | 2 | + +## 依赖的 run.py 能力 + +`run.py` 加载 YAML 后会对所有字符串字段执行 `os.path.expandvars`， +支持 `${VAR}`/`$VAR` 占位符；未设置的变量原样保留。 diff --git a/test_pipeline/generic_configs/run_0size_accuracy.yaml b/test_pipeline/generic_configs/run_0size_accuracy.yaml new file mode 100644 index 00000000..ded2a29c --- /dev/null +++ b/test_pipeline/generic_configs/run_0size_accuracy.yaml @@ -0,0 +1,17 @@ +# yaml-language-server: $schema=../run_config.schema.json +# 通用 0-size accuracy 配置：切换 ${APITEST_MODEL} 即可切换测试模型。 +# 依赖 CI 提前将 ${APITEST_MODEL}/accuracy_0size/0size.txt 拷贝到 PaddleAPITest 下。 +name: run_0size_accuracy +runner: + engine: engineV4 + foreground: true +input: + api_config_file: "${APITEST_MODEL}/accuracy_0size/0size.txt" +output: + log_dir: "test_${APITEST_MODEL}_log_0size_accuracy" +engine_args: + accuracy_stable: true + use_cached_numpy: true + num_gpus: -1 + num_workers_per_gpu: 16 + timeout: 600 diff --git a/test_pipeline/generic_configs/run_0size_paddleonly.yaml b/test_pipeline/generic_configs/run_0size_paddleonly.yaml new file mode 100644 index 00000000..11a01b2e --- /dev/null +++ b/test_pipeline/generic_configs/run_0size_paddleonly.yaml @@ -0,0 +1,20 @@ +# yaml-language-server: $schema=../run_config.schema.json +# 通用 0-size paddle_only 配置：切换 ${APITEST_MODEL} 即可切换测试模型。 +# 依赖 CI 提前将 ${APITEST_MODEL}/paddleonly_0size/0size.txt 拷贝到 PaddleAPITest 下。 +name: run_0size_paddleonly +runner: + engine: engineV4 + foreground: true +env: + FLAGS_use_system_allocator: "true" + FLAGS_check_cuda_error: "true" +input: + api_config_file: "${APITEST_MODEL}/paddleonly_0size/0size.txt" +output: + log_dir: "test_${APITEST_MODEL}_log_0size_paddleonly" +engine_args: + paddle_only: true + num_gpus: -1 + num_workers_per_gpu: 16 + timeout: 600 + use_compute_sanitizer: true diff --git a/test_pipeline/generic_configs/run_1M_accuracy.yaml b/test_pipeline/generic_configs/run_1M_accuracy.yaml new file mode 100644 index 00000000..42f4aa5b --- /dev/null +++ b/test_pipeline/generic_configs/run_1M_accuracy.yaml @@ -0,0 +1,17 @@ +# yaml-language-server: $schema=../run_config.schema.json +# 通用 1M/大 tensor accuracy 配置：切换 ${APITEST_MODEL} 即可切换测试模型。 +# 依赖 CI 提前将 ${APITEST_MODEL}/accuracy/1M.txt 拷贝到 PaddleAPITest 下。 +name: run_1M_accuracy +runner: + engine: engineV4 + foreground: true +input: + api_config_file: "${APITEST_MODEL}/accuracy/1M.txt" +output: + log_dir: "test_${APITEST_MODEL}_log_1M_accuracy" +engine_args: + accuracy_stable: true + use_cached_numpy: true + num_gpus: -1 + num_workers_per_gpu: 2 + timeout: 1200 diff --git a/test_pipeline/V4_configs/run_dsv4_1M_paddleonly.yaml b/test_pipeline/generic_configs/run_1M_paddleonly.yaml similarity index 50% rename from test_pipeline/V4_configs/run_dsv4_1M_paddleonly.yaml rename to test_pipeline/generic_configs/run_1M_paddleonly.yaml index 81650ee0..4c07516f 100644 --- a/test_pipeline/V4_configs/run_dsv4_1M_paddleonly.yaml +++ b/test_pipeline/generic_configs/run_1M_paddleonly.yaml @@ -1,5 +1,7 @@ # yaml-language-server: $schema=../run_config.schema.json -name: run_dsv4_1M_paddleonly +# 通用 1M/大 tensor paddle_only 配置：切换 ${APITEST_MODEL} 即可切换测试模型。 +# 依赖 CI 提前将 ${APITEST_MODEL}/paddleonly/1M.txt 拷贝到 PaddleAPITest 下。 +name: run_1M_paddleonly runner: engine: engineV4 foreground: true @@ -9,12 +11,12 @@ env: FLAGS_alloc_fill_value: "255" FLAGS_check_nan_inf: "true" input: - api_config_file: tester/api_config/monitor_config/dsv4_v2/dsv4_1M.txt + api_config_file: "${APITEST_MODEL}/paddleonly/1M.txt" output: - log_dir: tester/api_config/test_log_dsv4_1M_paddleonly + log_dir: "test_${APITEST_MODEL}_log_1M_paddleonly" engine_args: paddle_only: true num_gpus: -1 - num_workers_per_gpu: 1 + num_workers_per_gpu: 2 timeout: 1200 use_compute_sanitizer: true diff --git a/test_pipeline/generic_configs/run_4096_accuracy.yaml b/test_pipeline/generic_configs/run_4096_accuracy.yaml new file mode 100644 index 00000000..ae41a053 --- /dev/null +++ b/test_pipeline/generic_configs/run_4096_accuracy.yaml @@ -0,0 +1,17 @@ +# yaml-language-server: $schema=../run_config.schema.json +# 通用 4096（大 shape）accuracy 配置：切换 ${APITEST_MODEL} 即可切换测试模型。 +# 依赖 CI 提前将 ${APITEST_MODEL}/accuracy/4096.txt 拷贝到 PaddleAPITest 下。 +name: run_4096_accuracy +runner: + engine: engineV4 + foreground: true +input: + api_config_file: "${APITEST_MODEL}/accuracy/4096.txt" +output: + log_dir: "test_${APITEST_MODEL}_log_4096_accuracy" +engine_args: + accuracy_stable: true + use_cached_numpy: true + num_gpus: -1 + num_workers_per_gpu: 8 + timeout: 1200 diff --git a/test_pipeline/generic_configs/run_4096_paddleonly.yaml b/test_pipeline/generic_configs/run_4096_paddleonly.yaml new file mode 100644 index 00000000..6caba2f5 --- /dev/null +++ b/test_pipeline/generic_configs/run_4096_paddleonly.yaml @@ -0,0 +1,22 @@ +# yaml-language-server: $schema=../run_config.schema.json +# 通用 4096（大 shape）paddle_only 配置：切换 ${APITEST_MODEL} 即可切换测试模型。 +# 依赖 CI 提前将 ${APITEST_MODEL}/paddleonly/4096.txt 拷贝到 PaddleAPITest 下。 +name: run_4096_paddleonly +runner: + engine: engineV4 + foreground: true +env: + FLAGS_use_system_allocator: "true" + FLAGS_check_cuda_error: "true" + FLAGS_alloc_fill_value: "255" + FLAGS_check_nan_inf: "true" +input: + api_config_file: "${APITEST_MODEL}/paddleonly/4096.txt" +output: + log_dir: "test_${APITEST_MODEL}_log_4096_paddleonly" +engine_args: + paddle_only: true + num_gpus: -1 + num_workers_per_gpu: 8 + timeout: 1200 + use_compute_sanitizer: true diff --git a/test_pipeline/generic_configs/run_accuracy_compatible.yaml b/test_pipeline/generic_configs/run_accuracy_compatible.yaml new file mode 100644 index 00000000..626780e4 --- /dev/null +++ b/test_pipeline/generic_configs/run_accuracy_compatible.yaml @@ -0,0 +1,20 @@ +# yaml-language-server: $schema=../run_config.schema.json +# 通用 accuracy_compatible 配置：开启 FLAGS_use_accuracy_compatible_kernel 后的精度对比。 +# 切换 ${APITEST_MODEL} 即可切换测试模型。 +# 依赖 CI 提前将 ${APITEST_MODEL}/accuracy_compatible/accuracy_compatible.txt 拷贝到 PaddleAPITest 下。 +name: run_accuracy_compatible +runner: + engine: engineV4 + foreground: true +env: + FLAGS_use_accuracy_compatible_kernel: "1" +input: + api_config_file: "${APITEST_MODEL}/accuracy_compatible/accuracy_compatible.txt" +output: + log_dir: "test_${APITEST_MODEL}_log_accuracy_compatible" +engine_args: + accuracy_stable: true + use_cached_numpy: true + num_gpus: -1 + num_workers_per_gpu: 2 + timeout: 1200 diff --git a/test_pipeline/generic_configs/run_accuracy_manual_threshold.yaml b/test_pipeline/generic_configs/run_accuracy_manual_threshold.yaml new file mode 100644 index 00000000..4965d471 --- /dev/null +++ b/test_pipeline/generic_configs/run_accuracy_manual_threshold.yaml @@ -0,0 +1,21 @@ +# yaml-language-server: $schema=../run_config.schema.json +# 通用 accuracy_manual_threshold 配置：使用手动误差阈值配置文件做精度对比。 +# 切换 ${APITEST_MODEL} 即可切换测试模型。 +# 依赖 CI 提前将以下文件拷贝到 PaddleAPITest 下： +# ${APITEST_MODEL}/accuracy_manual_threshold/accuracy_manual_threshold.txt +# ${APITEST_MODEL}/accuracy_manual_threshold/accuracy_manual_threshold_config.yaml +name: run_accuracy_manual_threshold +runner: + engine: engineV4 + foreground: true +input: + api_config_file: "${APITEST_MODEL}/accuracy_manual_threshold/accuracy_manual_threshold.txt" +output: + log_dir: "test_${APITEST_MODEL}_log_accuracy_manual_threshold" +engine_args: + accuracy: true + manual_threshold_config_file: "${APITEST_MODEL}/accuracy_manual_threshold/accuracy_manual_threshold_config.yaml" + use_cached_numpy: true + num_gpus: -1 + num_workers_per_gpu: 2 + timeout: 1200 diff --git a/tester/api_config/log_writer.py b/tester/api_config/log_writer.py index 85df01d1..5daf558c 100644 --- a/tester/api_config/log_writer.py +++ b/tester/api_config/log_writer.py @@ -496,6 +496,7 @@ def print_log_info(all_case, log_counts=None): log_counts = {} test_case = log_counts.get("checkpoint", 0) pass_case = log_counts.get("pass", 0) + skip_case = log_counts.get("skip", 0) paddle_issue_case = sum( log_counts.get(log_type, 0) for log_type in [ @@ -506,8 +507,7 @@ def print_log_info(all_case, log_counts=None): "paddle_crash", ] ) - retest_case = sum(log_counts.get(log_type, 0) for log_type in ["oom", "timeout"]) - framework_blocked_case = sum( + test_issue_case = sum( log_counts.get(log_type, 0) for log_type in [ "torch_error", @@ -516,19 +516,19 @@ def print_log_info(all_case, log_counts=None): "config_convert", ] ) - skip_case = log_counts.get("skip", 0) + retest_case = sum(log_counts.get(log_type, 0) for log_type in ["oom", "timeout"]) # 打印统计信息 print("\n" + "=" * 50) print("Test Case Statistics".center(50)) print("=" * 50) - print(f"{'Total cases':<30}: {all_case}") + print(f"{'Pending cases':<30}: {all_case}") print(f"{'Tested cases':<30}: {test_case}") - print(f"{'Passed cases':<30}: {pass_case}") + print(f"{'Pass cases':<30}: {pass_case}") + print(f"{'Skip cases':<30}: {skip_case}") print(f"{'Paddle issue cases':<30}: {paddle_issue_case}") + print(f"{'Test issue cases':<30}: {test_issue_case}") print(f"{'Retest cases':<30}: {retest_case}") - print(f"{'Framework blocked cases':<30}: {framework_blocked_case}") - print(f"{'Skipped cases':<30}: {skip_case}") if log_counts: print("-" * 50) print("Log Type Breakdown:") From 8de87439d15825c1be2660b1b3430d9ef5bac877 Mon Sep 17 00:00:00 2001 From: cangtianhuang <1903374751@qq.com> Date: Tue, 14 Jul 2026 18:38:57 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=9A=B8=20Fix=20Test=20Case=20Statisti?= =?UTF-8?q?cs=20in=20engineV4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- engineV4.py | 68 +++++++++++++++++++++++++++------ tester/api_config/log_writer.py | 49 ++++++++++++++++++++---- tester/base.py | 2 - 3 files changed, 98 insertions(+), 21 deletions(-) diff --git a/engineV4.py b/engineV4.py index a98bb167..7ac07a15 100644 --- a/engineV4.py +++ b/engineV4.py @@ -47,6 +47,10 @@ os.environ["FLAGS_use_system_allocator"] = "1" os.environ["NVIDIA_TF32_OVERRIDE"] = "0" +FATAL_CUDA_EXIT_CODE = 99 +FATAL_OOM_EXIT_CODE = 98 +FATAL_TORCH_EXIT_CODE = 97 + VALID_TEST_ARGS = { "test_amp", "test_backward", @@ -1283,13 +1287,49 @@ def run_test_case(api_config_str, options): try: case.test() except Exception as err: - # if fatal error happens, subprocess need to exit with non-zero status - if "CUDA error" in str(err) or "memory corruption" in str(err): - os._exit(99) - if "CUDA out of memory" in str(err) or "Out of memory error" in str(err): - os._exit(98) - if "AssertionError" in str(err) or "Tensor-likes are not equal" in str(err): - os._exit(1) + err_msg = str(err).lower() + terminal_log_type = get_terminal_log_type(api_config_str) + oom_markers = ( + "cuda out of memory", + "out of memory error", + "resourceexhaustederror", + "out of memory", + "outofmemoryerror", + "cannot allocate memory", + "std::bad_alloc", + "bad allocation", + "memoryerror", + "cublas_status_alloc_failed", + ) + cuda_markers = ( + "cuda error", + "memory corruption", + "illegal memory access", + "invalid configuration argument", + "invalid resource handle", + ) + exit_code = None + if any(marker in err_msg for marker in oom_markers): + exit_code = FATAL_OOM_EXIT_CODE + elif terminal_log_type == "torch_error" and any( + marker in err_msg for marker in cuda_markers + ): + exit_code = FATAL_TORCH_EXIT_CODE + elif any(marker in err_msg for marker in cuda_markers): + exit_code = FATAL_CUDA_EXIT_CODE + if exit_code is not None: + if has_terminal_log(api_config_str): + write_checkpoint(api_config_str) + try: + close_process_files() + finally: + try: + restore_stdio() + finally: + os._exit(exit_code) + if has_terminal_log(api_config_str): + write_checkpoint(api_config_str) + return # if not fatal error, subprocess will be alive and report error print(f"[test error] {api_config_str}: {err}", flush=True) raise @@ -1959,16 +1999,22 @@ def cleanup_handler(*args): flush=True, ) elif msg_type == "crashed": - if exitcode == 99: + if exitcode == FATAL_CUDA_EXIT_CODE: write_to_log("paddle_cuda", config) print( - f"[error] CUDA error for {config}", + f"[paddle_cuda] {config}: worker exited with CUDA error", flush=True, ) - elif exitcode == 98: + elif exitcode == FATAL_OOM_EXIT_CODE: write_to_log("oom", config) print( - f"[error] CUDA out of memory for {config}", + f"[oom] {config}: worker exited with OOM", + flush=True, + ) + elif exitcode == FATAL_TORCH_EXIT_CODE: + write_to_log("torch_error", config) + print( + f"[torch_error] {config}: worker exited with torch CUDA error", flush=True, ) elif ( diff --git a/tester/api_config/log_writer.py b/tester/api_config/log_writer.py index 5daf558c..468211cc 100644 --- a/tester/api_config/log_writer.py +++ b/tester/api_config/log_writer.py @@ -487,6 +487,39 @@ def aggregate_logs(end=False, cleanup=False): f.writelines(f"{line}\n" for line in sorted(api_configs)) except Exception as err: print(f"Error writing to {incomplete_file}: {err}", flush=True) + + # === 终态日志互斥性检查 === + config_to_types: dict[str, list[str]] = {} + for log_type, prefix in LOG_PREFIXES.items(): + if log_type == "checkpoint": + continue + log_file = TEST_LOG_PATH / f"{prefix}.txt" + if not log_file.exists(): + continue + try: + with log_file.open("r") as f: + for raw_line in f: + line = raw_line.strip() + if line: + config_to_types.setdefault(line, []).append(log_type) + except Exception: + pass + + duplicates = {config: types for config, types in config_to_types.items() if len(types) > 1} + if duplicates: + print("\n" + "!" * 50) + print("INTEGRITY ERROR: configs found in multiple log types:") + for config, types in sorted(duplicates.items())[:20]: + print(f" {config}") + print(f" -> {', '.join(types)}") + if len(duplicates) > 20: + print(f" ... and {len(duplicates) - 20} more") + print("!" * 50 + "\n") + assert not duplicates, ( + f"Log integrity violation: {len(duplicates)} config(s) appear in " + f"multiple terminal log types. This indicates a classification bug." + ) + return log_counts @@ -522,18 +555,18 @@ def print_log_info(all_case, log_counts=None): print("\n" + "=" * 50) print("Test Case Statistics".center(50)) print("=" * 50) - print(f"{'Pending cases':<30}: {all_case}") - print(f"{'Tested cases':<30}: {test_case}") - print(f"{'Pass cases':<30}: {pass_case}") - print(f"{'Skip cases':<30}: {skip_case}") - print(f"{'Paddle issue cases':<30}: {paddle_issue_case}") - print(f"{'Test issue cases':<30}: {test_issue_case}") - print(f"{'Retest cases':<30}: {retest_case}") + print(f"{'Pending cases':<30}: {all_case:>8}") + print(f"{'Tested cases':<30}: {test_case:>8}") + print(f"{'Pass cases':<30}: {pass_case:>8}") + print(f"{'Skip cases':<30}: {skip_case:>8}") + print(f"{'Paddle issue cases':<30}: {paddle_issue_case:>8}") + print(f"{'Test issue cases':<30}: {test_issue_case:>8}") + print(f"{'Retest cases':<30}: {retest_case:>8}") if log_counts: print("-" * 50) print("Log Type Breakdown:") for log_type, count in log_counts.items(): - print(f" {log_type:<28}: {count}") + print(f" {log_type:<28}: {count:>8}") print("=" * 50 + "\n") diff --git a/tester/base.py b/tester/base.py index 42901b21..1753eeb9 100644 --- a/tester/base.py +++ b/tester/base.py @@ -234,8 +234,6 @@ def report_runtime_error(self, err, default_log_type, phase="", allow_ignore_pad return "pass", False if log_type is None: log_type = default_log_type - elif default_log_type == "torch_error" and log_type != "oom": - log_type = "torch_error" phase_text = f" phase={phase}" if phase else "" print( f"[{log_type}]{phase_text} {self.api_config.config}\n{err_msg}",