From 617f170d714039e9370d5c35d4059bd7b2817d43 Mon Sep 17 00:00:00 2001 From: Pandakingxbc Date: Thu, 22 Oct 2026 18:16:25 +0900 Subject: [PATCH] Release InfoNav for NeurIPS 2026 --- .github/workflows/quality.yml | 48 + .gitignore | 11 +- .gitmodules | 3 + CITATION.cff | 28 + README.md | 372 ++-- THIRD_PARTY.md | 21 + basic_utils/logging/log_manager.py | 4 +- config/README.md | 267 +++ config/default.yaml | 195 ++ config/habitat_eval_hm3dv1.yaml | 15 +- config/habitat_eval_hm3dv2.yaml | 15 +- config/habitat_eval_hm3dv2.yaml.backup | 64 - config/habitat_eval_mp3d.yaml | 54 +- config/logging_config.yaml | 2 +- docs/REPRODUCIBILITY.md | 140 ++ habitat2ros/pyproject.toml | 8 +- habitat_evaluation.py | 724 ++++++- habitat_manual_control.py | 2 - infonav_environment.yaml | 2 +- llm/client/deepseek_answer.py | 27 +- llm/client/ollama_answer.py | 20 +- llm/prompt/value_map_hypothesis.py | 18 +- llm/prompt/vlm_environment_awareness.py | 2 +- llm/semantic_expansion_service.py | 2 - llm/utils/http_llm_client.py | 2 - llm/utils/semantic_prompt_expansion.py | 2 - pyproject.toml | 26 +- scripts/__init__.py | 2 + scripts/check_hypothesis_coverage.py | 121 ++ scripts/collect_panorama_views.py | 481 +++++ scripts/compare_hm3dv2_value_wtrp.py | 85 + scripts/llm_hypothesis_analyzer.py | 1806 +++++++++++++++++ scripts/probe_cloud_models.py | 94 + scripts/report_hm3dv2_greedy1000.py | 251 +++ scripts/report_hm3dv2_value_wtrp_pair100.py | 248 +++ scripts/run_evaluation_with_restart.sh | 4 +- scripts/run_hm3dv2_greedy1000.sh | 845 ++++++++ scripts/run_hm3dv2_value_wtrp_pair100.sh | 148 ++ scripts/run_retry_missing.sh | 17 +- scripts/run_with_periodic_restart.sh | 2 +- scripts/set_logging_env.sh | 4 +- scripts/start_vlm_servers.sh | 3 +- scripts/system_monitor.sh | 4 +- scripts/test_api_services.py | 204 ++ scripts/verify_panorama_views.py | 208 ++ scripts/vlm_environment_analyzer.py | 748 +++++++ .../exploration_manager/exploration_manager.h | 10 +- .../exploration_manager/launch/algorithm.xml | 35 +- .../launch/exploration.launch | 22 + src/planner/exploration_manager/package.xml | 4 +- .../src/exploration_fsm.cpp | 42 +- .../src/exploration_manager.cpp | 161 +- src/planner/path_searching/package.xml | 2 +- src/planner/plan_env/CMakeLists.txt | 4 + .../plan_env/include/plan_env/ig_value_map.h | 12 +- .../plan_env/include/plan_env/map_ros.h | 7 +- .../include/plan_env/multi_source_value_map.h | 11 +- src/planner/plan_env/package.xml | 2 +- .../plan_env/scripts/multi_source_itm_node.py | 3 - .../scripts/qwen_vlm_validation_node.py | 3 - .../plan_env/scripts/vlm_approach_verifier.py | 3 - src/planner/plan_env/src/frontier_map2d.cpp | 2 - src/planner/plan_env/src/ig_value_map.cpp | 44 +- src/planner/plan_env/src/map_ros.cpp | 59 +- .../plan_env/src/multi_source_value_map.cpp | 53 +- src/planner/plan_env/src/object_map2d.cpp | 2 - src/planner/plan_env/src/sdf_map2d.cpp | 2 - src/planner/plan_env/src/value_map2d.cpp | 2 - src/planner/utils/lkh_mtsp_solver/package.xml | 4 +- src/planner/utils/vis_utils/package.xml | 2 +- .../.github/ISSUE_TEMPLATE/bug_report.md | 38 - .../.github/ISSUE_TEMPLATE/feature_request.md | 20 - .../D-FINE.backup/.github/workflows/is.yml | 63 - .../.github/workflows/pre-commit.yml | 15 - vlm/detector/D-FINE.backup/.gitignore | 6 - .../D-FINE.backup/.pre-commit-config.yaml | 68 - vlm/detector/D-FINE.backup/Dockerfile | 48 - vlm/detector/D-FINE.backup/LICENSE | 201 -- vlm/detector/D-FINE.backup/README.md | 700 ------- vlm/detector/D-FINE.backup/README_cn.md | 673 ------ vlm/detector/D-FINE.backup/README_ja.md | 698 ------- .../configs/dataset/coco_detection.yml | 41 - .../configs/dataset/crowdhuman_detection.yml | 41 - .../configs/dataset/custom_detection.yml | 41 - .../configs/dataset/obj365_detection.yml | 41 - .../configs/dataset/voc_detection.yml | 40 - .../dfine/crowdhuman/dfine_hgnetv2_l_ch.yml | 44 - .../dfine/crowdhuman/dfine_hgnetv2_m_ch.yml | 60 - .../dfine/crowdhuman/dfine_hgnetv2_n_ch.yml | 82 - .../dfine/crowdhuman/dfine_hgnetv2_s_ch.yml | 65 - .../dfine/crowdhuman/dfine_hgnetv2_x_ch.yml | 55 - .../dfine/custom/dfine_hgnetv2_l_custom.yml | 44 - .../dfine/custom/dfine_hgnetv2_m_custom.yml | 60 - .../dfine/custom/dfine_hgnetv2_n_custom.yml | 82 - .../dfine/custom/dfine_hgnetv2_s_custom.yml | 65 - .../dfine/custom/dfine_hgnetv2_x_custom.yml | 55 - .../objects365/dfine_hgnetv2_l_obj2custom.yml | 53 - .../objects365/dfine_hgnetv2_m_obj2custom.yml | 66 - .../objects365/dfine_hgnetv2_s_obj2custom.yml | 67 - .../objects365/dfine_hgnetv2_x_obj2custom.yml | 62 - .../configs/dfine/dfine_hgnetv2_l_coco.yml | 44 - .../configs/dfine/dfine_hgnetv2_m_coco.yml | 60 - .../configs/dfine/dfine_hgnetv2_n_coco.yml | 82 - .../configs/dfine/dfine_hgnetv2_s_coco.yml | 61 - .../configs/dfine/dfine_hgnetv2_x_coco.yml | 56 - .../configs/dfine/include/dataloader.yml | 39 - .../configs/dfine/include/dfine_hgnetv2.yml | 82 - .../configs/dfine/include/optimizer.yml | 36 - .../objects365/dfine_hgnetv2_l_obj2coco.yml | 52 - .../objects365/dfine_hgnetv2_l_obj365.yml | 49 - .../objects365/dfine_hgnetv2_m_obj2coco.yml | 65 - .../objects365/dfine_hgnetv2_m_obj365.yml | 62 - .../objects365/dfine_hgnetv2_n_obj2coco.yml | 88 - .../objects365/dfine_hgnetv2_n_obj365.yml | 84 - .../objects365/dfine_hgnetv2_s_obj2coco.yml | 66 - .../objects365/dfine_hgnetv2_s_obj365.yml | 63 - .../objects365/dfine_hgnetv2_x_obj2coco.yml | 61 - .../objects365/dfine_hgnetv2_x_obj365.yml | 58 - .../D-FINE.backup/configs/runtime.yml | 24 - .../D-FINE.backup/reference/convert_weight.py | 30 - .../D-FINE.backup/reference/safe_training.sh | 97 - vlm/detector/D-FINE.backup/requirements.txt | 9 - vlm/detector/D-FINE.backup/src/__init__.py | 6 - .../D-FINE.backup/src/core/__init__.py | 9 - .../D-FINE.backup/src/core/_config.py | 299 --- .../D-FINE.backup/src/core/workspace.py | 178 -- .../D-FINE.backup/src/core/yaml_config.py | 187 -- .../D-FINE.backup/src/core/yaml_utils.py | 126 -- .../D-FINE.backup/src/misc/__init__.py | 9 - .../D-FINE.backup/src/misc/box_ops.py | 106 - .../D-FINE.backup/src/misc/dist_utils.py | 281 --- .../D-FINE.backup/src/misc/lazy_loader.py | 70 - vlm/detector/D-FINE.backup/src/misc/logger.py | 255 --- .../D-FINE.backup/src/misc/profiler_utils.py | 30 - .../D-FINE.backup/src/misc/visualizer.py | 121 -- vlm/detector/D-FINE.backup/src/nn/__init__.py | 16 - .../D-FINE.backup/src/nn/arch/__init__.py | 7 - .../src/nn/arch/classification.py | 45 - .../D-FINE.backup/src/nn/arch/yolo.py | 42 - .../D-FINE.backup/src/nn/backbone/__init__.py | 17 - .../D-FINE.backup/src/nn/backbone/common.py | 117 -- .../src/nn/backbone/csp_darknet.py | 203 -- .../src/nn/backbone/csp_resnet.py | 302 --- .../D-FINE.backup/src/nn/backbone/hgnetv2.py | 579 ------ .../D-FINE.backup/src/nn/backbone/presnet.py | 263 --- .../src/nn/backbone/test_resnet.py | 83 - .../src/nn/backbone/timm_model.py | 66 - .../src/nn/backbone/torchvision_model.py | 50 - .../D-FINE.backup/src/nn/backbone/utils.py | 56 - .../src/nn/criterion/__init__.py | 11 - .../src/nn/criterion/det_criterion.py | 188 -- .../src/nn/postprocessor/__init__.py | 6 - .../src/nn/postprocessor/box_revert.py | 66 - .../nn/postprocessor/detr_postprocessor.py | 86 - .../src/nn/postprocessor/nms_postprocessor.py | 86 - .../D-FINE.backup/src/optim/__init__.py | 9 - vlm/detector/D-FINE.backup/src/optim/amp.py | 12 - vlm/detector/D-FINE.backup/src/optim/ema.py | 108 - vlm/detector/D-FINE.backup/src/optim/optim.py | 22 - .../D-FINE.backup/src/optim/warmup.py | 56 - .../D-FINE.backup/src/solver/__init__.py | 15 - .../D-FINE.backup/src/solver/_solver.py | 783 ------- .../D-FINE.backup/src/solver/clas_engine.py | 74 - .../D-FINE.backup/src/solver/clas_solver.py | 75 - .../D-FINE.backup/src/solver/det_engine.py | 259 --- .../D-FINE.backup/src/solver/det_solver.py | 228 --- .../D-FINE.backup/src/solver/validator.py | 347 ---- .../D-FINE.backup/src/zoo/__init__.py | 6 - .../D-FINE.backup/src/zoo/dfine/__init__.py | 11 - .../D-FINE.backup/src/zoo/dfine/blog.md | 90 - .../D-FINE.backup/src/zoo/dfine/blog_cn.md | 90 - .../D-FINE.backup/src/zoo/dfine/box_ops.py | 93 - .../D-FINE.backup/src/zoo/dfine/denoising.py | 121 -- .../D-FINE.backup/src/zoo/dfine/dfine.py | 47 - .../src/zoo/dfine/dfine_criterion.py | 525 ----- .../src/zoo/dfine/dfine_decoder.py | 959 --------- .../src/zoo/dfine/dfine_utils.py | 169 -- .../src/zoo/dfine/hybrid_encoder.py | 488 ----- .../D-FINE.backup/src/zoo/dfine/matcher.py | 160 -- .../src/zoo/dfine/postprocessor.py | 93 - .../D-FINE.backup/src/zoo/dfine/utils.py | 182 -- .../D-FINE.backup/tools/benchmark/dataset.py | 113 -- .../D-FINE.backup/tools/benchmark/get_info.py | 60 - .../tools/benchmark/requirements.txt | 6 - .../tools/benchmark/trt_benchmark.py | 214 -- .../D-FINE.backup/tools/benchmark/utils.py | 95 - .../tools/dataset/remap_obj365.py | 153 -- .../tools/dataset/resize_obj365.py | 153 -- .../tools/deployment/export_onnx.py | 128 -- .../tools/deployment/export_yolo_w_nms.py | 75 - .../inference/cppExample/onnx/CMakeLists.txt | 38 - .../inference/cppExample/onnx/onnxExample.cpp | 331 --- .../cppExample/openvino/CMakeLists.txt | 31 - .../cppExample/openvino/openvinoExample.cpp | 42 - .../inference/cppExample/trt/CMakeLists.txt | 35 - .../inference/cppExample/trt/trtExample.cpp | 346 ---- .../D-FINE.backup/tools/inference/onnx_inf.py | 163 -- .../tools/inference/openvino_inf.py | 101 - .../tools/inference/requirements.txt | 3 - .../tools/inference/torch_inf.py | 166 -- .../D-FINE.backup/tools/inference/trt_inf.py | 244 --- .../tools/visualization/fiftyone_vis.py | 406 ---- vlm/detector/D-FINE.backup/train.py | 113 -- vlm/detector/__init__.py | 2 +- vlm/detector/dfine.py | 26 +- vlm/utils/get_itm_message.py | 223 +- vlm/utils/get_object_utils.py | 32 +- vlm/utils/http_vlm_client.py | 2 - vlm_environment_analyzer.py | 2 +- 209 files changed, 7484 insertions(+), 17509 deletions(-) create mode 100644 .github/workflows/quality.yml create mode 100644 .gitmodules create mode 100644 CITATION.cff create mode 100644 THIRD_PARTY.md create mode 100644 config/README.md create mode 100644 config/default.yaml delete mode 100644 config/habitat_eval_hm3dv2.yaml.backup create mode 100644 docs/REPRODUCIBILITY.md create mode 100644 scripts/__init__.py create mode 100644 scripts/check_hypothesis_coverage.py create mode 100644 scripts/collect_panorama_views.py create mode 100755 scripts/compare_hm3dv2_value_wtrp.py create mode 100644 scripts/llm_hypothesis_analyzer.py create mode 100644 scripts/probe_cloud_models.py create mode 100755 scripts/report_hm3dv2_greedy1000.py create mode 100755 scripts/report_hm3dv2_value_wtrp_pair100.py create mode 100755 scripts/run_hm3dv2_greedy1000.sh create mode 100755 scripts/run_hm3dv2_value_wtrp_pair100.sh create mode 100644 scripts/test_api_services.py create mode 100644 scripts/verify_panorama_views.py create mode 100644 scripts/vlm_environment_analyzer.py delete mode 100644 vlm/detector/D-FINE.backup/.github/ISSUE_TEMPLATE/bug_report.md delete mode 100644 vlm/detector/D-FINE.backup/.github/ISSUE_TEMPLATE/feature_request.md delete mode 100644 vlm/detector/D-FINE.backup/.github/workflows/is.yml delete mode 100644 vlm/detector/D-FINE.backup/.github/workflows/pre-commit.yml delete mode 100644 vlm/detector/D-FINE.backup/.gitignore delete mode 100644 vlm/detector/D-FINE.backup/.pre-commit-config.yaml delete mode 100644 vlm/detector/D-FINE.backup/Dockerfile delete mode 100644 vlm/detector/D-FINE.backup/LICENSE delete mode 100644 vlm/detector/D-FINE.backup/README.md delete mode 100644 vlm/detector/D-FINE.backup/README_cn.md delete mode 100644 vlm/detector/D-FINE.backup/README_ja.md delete mode 100644 vlm/detector/D-FINE.backup/configs/dataset/coco_detection.yml delete mode 100644 vlm/detector/D-FINE.backup/configs/dataset/crowdhuman_detection.yml delete mode 100644 vlm/detector/D-FINE.backup/configs/dataset/custom_detection.yml delete mode 100644 vlm/detector/D-FINE.backup/configs/dataset/obj365_detection.yml delete mode 100644 vlm/detector/D-FINE.backup/configs/dataset/voc_detection.yml delete mode 100644 vlm/detector/D-FINE.backup/configs/dfine/crowdhuman/dfine_hgnetv2_l_ch.yml delete mode 100644 vlm/detector/D-FINE.backup/configs/dfine/crowdhuman/dfine_hgnetv2_m_ch.yml delete mode 100644 vlm/detector/D-FINE.backup/configs/dfine/crowdhuman/dfine_hgnetv2_n_ch.yml delete mode 100644 vlm/detector/D-FINE.backup/configs/dfine/crowdhuman/dfine_hgnetv2_s_ch.yml delete mode 100644 vlm/detector/D-FINE.backup/configs/dfine/crowdhuman/dfine_hgnetv2_x_ch.yml delete mode 100644 vlm/detector/D-FINE.backup/configs/dfine/custom/dfine_hgnetv2_l_custom.yml delete mode 100644 vlm/detector/D-FINE.backup/configs/dfine/custom/dfine_hgnetv2_m_custom.yml delete mode 100644 vlm/detector/D-FINE.backup/configs/dfine/custom/dfine_hgnetv2_n_custom.yml delete mode 100644 vlm/detector/D-FINE.backup/configs/dfine/custom/dfine_hgnetv2_s_custom.yml delete mode 100644 vlm/detector/D-FINE.backup/configs/dfine/custom/dfine_hgnetv2_x_custom.yml delete mode 100644 vlm/detector/D-FINE.backup/configs/dfine/custom/objects365/dfine_hgnetv2_l_obj2custom.yml delete mode 100644 vlm/detector/D-FINE.backup/configs/dfine/custom/objects365/dfine_hgnetv2_m_obj2custom.yml delete mode 100644 vlm/detector/D-FINE.backup/configs/dfine/custom/objects365/dfine_hgnetv2_s_obj2custom.yml delete mode 100644 vlm/detector/D-FINE.backup/configs/dfine/custom/objects365/dfine_hgnetv2_x_obj2custom.yml delete mode 100644 vlm/detector/D-FINE.backup/configs/dfine/dfine_hgnetv2_l_coco.yml delete mode 100644 vlm/detector/D-FINE.backup/configs/dfine/dfine_hgnetv2_m_coco.yml delete mode 100644 vlm/detector/D-FINE.backup/configs/dfine/dfine_hgnetv2_n_coco.yml delete mode 100644 vlm/detector/D-FINE.backup/configs/dfine/dfine_hgnetv2_s_coco.yml delete mode 100644 vlm/detector/D-FINE.backup/configs/dfine/dfine_hgnetv2_x_coco.yml delete mode 100644 vlm/detector/D-FINE.backup/configs/dfine/include/dataloader.yml delete mode 100644 vlm/detector/D-FINE.backup/configs/dfine/include/dfine_hgnetv2.yml delete mode 100644 vlm/detector/D-FINE.backup/configs/dfine/include/optimizer.yml delete mode 100644 vlm/detector/D-FINE.backup/configs/dfine/objects365/dfine_hgnetv2_l_obj2coco.yml delete mode 100644 vlm/detector/D-FINE.backup/configs/dfine/objects365/dfine_hgnetv2_l_obj365.yml delete mode 100644 vlm/detector/D-FINE.backup/configs/dfine/objects365/dfine_hgnetv2_m_obj2coco.yml delete mode 100644 vlm/detector/D-FINE.backup/configs/dfine/objects365/dfine_hgnetv2_m_obj365.yml delete mode 100644 vlm/detector/D-FINE.backup/configs/dfine/objects365/dfine_hgnetv2_n_obj2coco.yml delete mode 100644 vlm/detector/D-FINE.backup/configs/dfine/objects365/dfine_hgnetv2_n_obj365.yml delete mode 100644 vlm/detector/D-FINE.backup/configs/dfine/objects365/dfine_hgnetv2_s_obj2coco.yml delete mode 100644 vlm/detector/D-FINE.backup/configs/dfine/objects365/dfine_hgnetv2_s_obj365.yml delete mode 100644 vlm/detector/D-FINE.backup/configs/dfine/objects365/dfine_hgnetv2_x_obj2coco.yml delete mode 100644 vlm/detector/D-FINE.backup/configs/dfine/objects365/dfine_hgnetv2_x_obj365.yml delete mode 100644 vlm/detector/D-FINE.backup/configs/runtime.yml delete mode 100644 vlm/detector/D-FINE.backup/reference/convert_weight.py delete mode 100644 vlm/detector/D-FINE.backup/reference/safe_training.sh delete mode 100644 vlm/detector/D-FINE.backup/requirements.txt delete mode 100644 vlm/detector/D-FINE.backup/src/__init__.py delete mode 100644 vlm/detector/D-FINE.backup/src/core/__init__.py delete mode 100644 vlm/detector/D-FINE.backup/src/core/_config.py delete mode 100644 vlm/detector/D-FINE.backup/src/core/workspace.py delete mode 100644 vlm/detector/D-FINE.backup/src/core/yaml_config.py delete mode 100644 vlm/detector/D-FINE.backup/src/core/yaml_utils.py delete mode 100644 vlm/detector/D-FINE.backup/src/misc/__init__.py delete mode 100644 vlm/detector/D-FINE.backup/src/misc/box_ops.py delete mode 100644 vlm/detector/D-FINE.backup/src/misc/dist_utils.py delete mode 100644 vlm/detector/D-FINE.backup/src/misc/lazy_loader.py delete mode 100644 vlm/detector/D-FINE.backup/src/misc/logger.py delete mode 100644 vlm/detector/D-FINE.backup/src/misc/profiler_utils.py delete mode 100644 vlm/detector/D-FINE.backup/src/misc/visualizer.py delete mode 100644 vlm/detector/D-FINE.backup/src/nn/__init__.py delete mode 100644 vlm/detector/D-FINE.backup/src/nn/arch/__init__.py delete mode 100644 vlm/detector/D-FINE.backup/src/nn/arch/classification.py delete mode 100644 vlm/detector/D-FINE.backup/src/nn/arch/yolo.py delete mode 100644 vlm/detector/D-FINE.backup/src/nn/backbone/__init__.py delete mode 100644 vlm/detector/D-FINE.backup/src/nn/backbone/common.py delete mode 100644 vlm/detector/D-FINE.backup/src/nn/backbone/csp_darknet.py delete mode 100644 vlm/detector/D-FINE.backup/src/nn/backbone/csp_resnet.py delete mode 100644 vlm/detector/D-FINE.backup/src/nn/backbone/hgnetv2.py delete mode 100644 vlm/detector/D-FINE.backup/src/nn/backbone/presnet.py delete mode 100644 vlm/detector/D-FINE.backup/src/nn/backbone/test_resnet.py delete mode 100644 vlm/detector/D-FINE.backup/src/nn/backbone/timm_model.py delete mode 100644 vlm/detector/D-FINE.backup/src/nn/backbone/torchvision_model.py delete mode 100644 vlm/detector/D-FINE.backup/src/nn/backbone/utils.py delete mode 100644 vlm/detector/D-FINE.backup/src/nn/criterion/__init__.py delete mode 100644 vlm/detector/D-FINE.backup/src/nn/criterion/det_criterion.py delete mode 100644 vlm/detector/D-FINE.backup/src/nn/postprocessor/__init__.py delete mode 100644 vlm/detector/D-FINE.backup/src/nn/postprocessor/box_revert.py delete mode 100644 vlm/detector/D-FINE.backup/src/nn/postprocessor/detr_postprocessor.py delete mode 100644 vlm/detector/D-FINE.backup/src/nn/postprocessor/nms_postprocessor.py delete mode 100644 vlm/detector/D-FINE.backup/src/optim/__init__.py delete mode 100644 vlm/detector/D-FINE.backup/src/optim/amp.py delete mode 100644 vlm/detector/D-FINE.backup/src/optim/ema.py delete mode 100644 vlm/detector/D-FINE.backup/src/optim/optim.py delete mode 100644 vlm/detector/D-FINE.backup/src/optim/warmup.py delete mode 100644 vlm/detector/D-FINE.backup/src/solver/__init__.py delete mode 100644 vlm/detector/D-FINE.backup/src/solver/_solver.py delete mode 100644 vlm/detector/D-FINE.backup/src/solver/clas_engine.py delete mode 100644 vlm/detector/D-FINE.backup/src/solver/clas_solver.py delete mode 100644 vlm/detector/D-FINE.backup/src/solver/det_engine.py delete mode 100644 vlm/detector/D-FINE.backup/src/solver/det_solver.py delete mode 100644 vlm/detector/D-FINE.backup/src/solver/validator.py delete mode 100644 vlm/detector/D-FINE.backup/src/zoo/__init__.py delete mode 100644 vlm/detector/D-FINE.backup/src/zoo/dfine/__init__.py delete mode 100644 vlm/detector/D-FINE.backup/src/zoo/dfine/blog.md delete mode 100644 vlm/detector/D-FINE.backup/src/zoo/dfine/blog_cn.md delete mode 100644 vlm/detector/D-FINE.backup/src/zoo/dfine/box_ops.py delete mode 100644 vlm/detector/D-FINE.backup/src/zoo/dfine/denoising.py delete mode 100644 vlm/detector/D-FINE.backup/src/zoo/dfine/dfine.py delete mode 100644 vlm/detector/D-FINE.backup/src/zoo/dfine/dfine_criterion.py delete mode 100644 vlm/detector/D-FINE.backup/src/zoo/dfine/dfine_decoder.py delete mode 100644 vlm/detector/D-FINE.backup/src/zoo/dfine/dfine_utils.py delete mode 100644 vlm/detector/D-FINE.backup/src/zoo/dfine/hybrid_encoder.py delete mode 100644 vlm/detector/D-FINE.backup/src/zoo/dfine/matcher.py delete mode 100644 vlm/detector/D-FINE.backup/src/zoo/dfine/postprocessor.py delete mode 100644 vlm/detector/D-FINE.backup/src/zoo/dfine/utils.py delete mode 100644 vlm/detector/D-FINE.backup/tools/benchmark/dataset.py delete mode 100644 vlm/detector/D-FINE.backup/tools/benchmark/get_info.py delete mode 100644 vlm/detector/D-FINE.backup/tools/benchmark/requirements.txt delete mode 100644 vlm/detector/D-FINE.backup/tools/benchmark/trt_benchmark.py delete mode 100644 vlm/detector/D-FINE.backup/tools/benchmark/utils.py delete mode 100644 vlm/detector/D-FINE.backup/tools/dataset/remap_obj365.py delete mode 100644 vlm/detector/D-FINE.backup/tools/dataset/resize_obj365.py delete mode 100644 vlm/detector/D-FINE.backup/tools/deployment/export_onnx.py delete mode 100644 vlm/detector/D-FINE.backup/tools/deployment/export_yolo_w_nms.py delete mode 100644 vlm/detector/D-FINE.backup/tools/inference/cppExample/onnx/CMakeLists.txt delete mode 100644 vlm/detector/D-FINE.backup/tools/inference/cppExample/onnx/onnxExample.cpp delete mode 100644 vlm/detector/D-FINE.backup/tools/inference/cppExample/openvino/CMakeLists.txt delete mode 100644 vlm/detector/D-FINE.backup/tools/inference/cppExample/openvino/openvinoExample.cpp delete mode 100644 vlm/detector/D-FINE.backup/tools/inference/cppExample/trt/CMakeLists.txt delete mode 100644 vlm/detector/D-FINE.backup/tools/inference/cppExample/trt/trtExample.cpp delete mode 100644 vlm/detector/D-FINE.backup/tools/inference/onnx_inf.py delete mode 100644 vlm/detector/D-FINE.backup/tools/inference/openvino_inf.py delete mode 100644 vlm/detector/D-FINE.backup/tools/inference/requirements.txt delete mode 100644 vlm/detector/D-FINE.backup/tools/inference/torch_inf.py delete mode 100644 vlm/detector/D-FINE.backup/tools/inference/trt_inf.py delete mode 100644 vlm/detector/D-FINE.backup/tools/visualization/fiftyone_vis.py delete mode 100644 vlm/detector/D-FINE.backup/train.py diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml new file mode 100644 index 0000000..25a35f9 --- /dev/null +++ b/.github/workflows/quality.yml @@ -0,0 +1,48 @@ +name: Lightweight quality checks + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +jobs: + syntax-and-metadata: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: + submodules: false + - uses: actions/setup-python@v6 + with: + python-version: "3.9" + - name: Install metadata validators + run: python -m pip install --disable-pip-version-check PyYAML==6.0.2 + - name: Compile project Python sources + run: | + git ls-files -z '*.py' \ + | while IFS= read -r -d '' file; do python -m py_compile "$file"; done + - name: Check shell syntax + run: | + git ls-files -z 'scripts/*.sh' \ + | while IFS= read -r -d '' file; do bash -n "$file"; done + - name: Parse YAML metadata and configuration + run: | + python - <<'PY' + from pathlib import Path + import yaml + + paths = [Path("CITATION.cff"), Path("infonav_environment.yaml")] + paths.extend(sorted(Path("config").glob("*.yaml"))) + paths.extend(sorted(Path(".github/workflows").glob("*.yml"))) + for path in paths: + with path.open(encoding="utf-8") as stream: + yaml.safe_load(stream) + print(f"parsed {path}") + PY + - name: Reject release-blocking placeholders + run: | + ! grep -RInE 'Under Review|Anonymous Submission|author=\{\}|journal=\{\}' \ + README.md CITATION.cff docs config pyproject.toml diff --git a/.gitignore b/.gitignore index 38ffeed..3006bf0 100644 --- a/.gitignore +++ b/.gitignore @@ -6,12 +6,15 @@ data debug/ outputs/ outputs +.live_demo_logs/ +.run_state/ # Python cache __pycache__/ *.egg-info *.pyc *.pyo +*.backup # Build artifacts build/ @@ -27,6 +30,7 @@ lockfiles/ yolov7/ GroundingDINO/ habitat-lab/ +vlm/detector/D-FINE.backup/ # Model weights and checkpoints traced_model.pt @@ -48,6 +52,7 @@ bag/ wandb/ logs/ tmp/ +*.log # Python virtual environment env/ @@ -64,10 +69,14 @@ videos/ # Test results and generated files *.png +!assets/*.png test_*_results.png # Habitat environment data env/ # PDF and documentation artifacts -*.pdf \ No newline at end of file +*.pdf + +# Internal review and rebuttal notes +rebuttal/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..8cfdaf4 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "vlm/detector/D-FINE"] + path = vlm/detector/D-FINE + url = https://github.com/Peterande/D-FINE.git diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000..c3755ae --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,28 @@ +cff-version: 1.2.0 +message: "If you use InfoNav, please cite the NeurIPS 2026 paper." +title: "InfoNav" +type: software +authors: + - family-names: Yang + given-names: Zhi + - family-names: Yu + given-names: Chengpu + - family-names: Ferrer + given-names: Gonzalo +repository-code: "https://github.com/Pandakingxbc/InfoNav" +url: "https://openreview.net/forum?id=lzhhlAlmjK" +license: GPL-3.0-only +date-released: 2026-09-25 +preferred-citation: + type: conference-paper + title: "InfoNav: A Unified Value Framework Integrating Semantic Relevance and Information Gain for Zero-Shot Object Goal Navigation" + authors: + - family-names: Yang + given-names: Zhi + - family-names: Yu + given-names: Chengpu + - family-names: Ferrer + given-names: Gonzalo + collection-title: "Advances in Neural Information Processing Systems" + year: 2026 + url: "https://openreview.net/forum?id=lzhhlAlmjK" diff --git a/README.md b/README.md index 115aea1..13e49c7 100644 --- a/README.md +++ b/README.md @@ -1,323 +1,253 @@ # InfoNav -**InfoNav: A Unified Value Framework Integrating Semantics Value and Information Gain for Zero-Shot Navigation** +**A Unified Value Framework Integrating Semantic Relevance and Information Gain for Zero-Shot Object Goal Navigation** -> **Paper:** Coming soon +[![NeurIPS 2026](https://img.shields.io/badge/NeurIPS-2026-7B2CBF)](https://neurips.cc/virtual/2026/poster/149947) +[![OpenReview](https://img.shields.io/badge/paper-OpenReview-8C1B13)](https://openreview.net/forum?id=lzhhlAlmjK) +[![License: GPL v3](https://img.shields.io/badge/license-GPLv3-blue)](LICENSE) +[![Python 3.9](https://img.shields.io/badge/python-3.9-3776AB)](pyproject.toml) +[![ROS Noetic](https://img.shields.io/badge/ROS-Noetic-22314E)](https://wiki.ros.org/noetic) -## ๐ŸŽฅ Demo Video +InfoNav was accepted to **NeurIPS 2026**. It formulates zero-shot ObjectNav as unified value estimation over frontier candidates by combining a hierarchical semantic value map, visual-semantic information gain, and memory-augmented verification. -**[Watch the Demo Video](https://github.com/Pandakingxbc/InfoNav/releases/download/v1.0.0-demo/InfoNav.mp4)** - See InfoNav in action! +**Authors:** [Zhi Yang](https://github.com/Pandakingxbc), Chengpu Yu, and Gonzalo Ferrer. -[![Demo Video](https://img.shields.io/badge/Demo-Video-blue?style=for-the-badge&logo=youtube)](https://github.com/Pandakingxbc/InfoNav/releases/download/v1.0.0-demo/InfoNav.mp4) +[[Paper / OpenReview](https://openreview.net/forum?id=lzhhlAlmjK)] +[[NeurIPS page](https://neurips.cc/virtual/2026/poster/149947)] +[[Demo video](https://github.com/Pandakingxbc/InfoNav/releases/download/v1.0.0-demo/InfoNav.mp4)] +[[Reproducibility guide](docs/REPRODUCIBILITY.md)] -## Architecture +![InfoNav architecture](assets/architecture.png) -![Architecture Overview](assets/architecture.png) +## Results -## Overview +The NeurIPS 2026 paper reports the following zero-shot ObjectNav results: -InfoNav is an intelligent navigation system for embodied agents in indoor environments, combining Vision-Language Models (VLM) and Large Language Models (LLM) for semantic-aware object navigation tasks. The system proposes a unified value framework that integrates semantic value and information gain for zero-shot navigation. +| Benchmark | Success rate | +| --- | ---: | +| HM3D-v1 | 62.0% | +| HM3D-v2 | 81.5% | +| MP3D | 41.8% | -InfoNav is built on top of [Habitat-Sim](https://github.com/facebookresearch/habitat-sim) and uses ROS for inter-module communication. The system integrates: +Please consult the paper for the complete protocol, SPL, ablations, and comparisons. The repository records the exact runtime configuration and Git revision alongside long evaluation runs; see the [reproducibility guide](docs/REPRODUCIBILITY.md). -- **VLM-based Object Detection**: GroundingDINO, BLIP2 for open-vocabulary detection and image-text matching -- **LLM-based Semantic Reasoning**: Room-object association and hypothesis generation -- **Unified Value Framework**: Combining semantic value with information gain -- **Hybrid Exploration Strategy**: Frontier-based exploration with semantic value maps -- **Robust Navigation Planning**: A* path planning with TSP optimization for multi-target scenarios +## Method -## System Requirements +InfoNav integrates: -- **OS**: Ubuntu 20.04 -- **GPU**: NVIDIA GPU with CUDA support and 8GB+ VRAM (tested on RTX 4070 Ti) -- **Python**: 3.9+ -- **ROS**: ROS Noetic +- a hierarchical semantic value map (HSVM) that models spatially weighted object, part, co-occurrence, and room cues; +- an information-gain estimator grounded in visual-semantic connectivity; +- adaptive fusion of semantic relevance and information gain; +- weighted traveling repairman (WTRP) frontier scheduling; and +- memory-augmented verification with time-varying confidence thresholds. + +The system uses Habitat for simulation and ROS for communication between the simulator, perception services, mapping, and exploration planner. + +## Requirements + +- Ubuntu 20.04 +- ROS Noetic +- Python 3.9 (the provided environment is pinned to Python 3.9) +- NVIDIA GPU with CUDA support; the authors tested on an RTX 4070 Ti +- `conda`, `git`, `git-lfs`, `tmux`, and `catkin_tools` + +Scene datasets and model weights are not redistributed in this repository. Review their upstream licenses and access conditions before downloading them. ## Installation -### 1. Prerequisites +### 1. Clone InfoNav and its pinned D-FINE submodule -#### 1.1 System Dependencies ```bash -sudo apt update -sudo apt-get install libarmadillo-dev libompl-dev +git clone --recurse-submodules https://github.com/Pandakingxbc/InfoNav.git +cd InfoNav ``` -#### 1.2 External Code Dependencies +If the repository was cloned without submodules: + ```bash -git clone git@github.com:WongKinYiu/yolov7.git # yolov7 -git clone https://github.com/IDEA-Research/GroundingDINO.git # GroundingDINO +git submodule update --init --recursive ``` -#### 1.3 Model Weights Download - -Download the following model weights and place them in the `data/` directory: -- `mobile_sam.pt`: https://github.com/ChaoningZhang/MobileSAM/tree/master/weights/mobile_sam.pt -- `groundingdino_swint_ogc.pth`: - ```bash - wget -O data/groundingdino_swint_ogc.pth https://github.com/IDEA-Research/GroundingDINO/releases/download/v0.1.0-alpha/groundingdino_swint_ogc.pth - ``` -- `yolov7-e6e.pt`: - ```bash - wget -O data/yolov7-e6e.pt https://github.com/WongKinYiu/yolov7/releases/download/v0.1/yolov7-e6e.pt - ``` - -### 2. Setup Python Environment +### 2. Install system dependencies -#### 2.1 Clone Repository ```bash -git clone git@github.com:Pandakingxbc/InfoNav.git -cd InfoNav +sudo apt update +sudo apt install -y git-lfs tmux libarmadillo-dev libompl-dev ``` -#### 2.2 Create Conda Environment +Install [ROS Noetic](https://wiki.ros.org/noetic/Installation/Ubuntu) and `catkin_tools` before building the workspace. + +### 3. Create the Python environment + ```bash -conda env create -f infonav_environment.yaml -y +conda env create -f infonav_environment.yaml conda activate infonav ``` -#### 2.3 PyTorch +Install the PyTorch build matching your CUDA runtime. For example, for CUDA 12.1: + ```bash -# You can use 'nvcc --version' to check your CUDA version. -# CUDA 11.8 -pip install torch==2.5.0 torchvision==0.20.0 torchaudio==2.5.0 --index-url https://download.pytorch.org/whl/cu118 -# CUDA 12.1 -pip install torch==2.5.0 torchvision==0.20.0 torchaudio==2.5.0 --index-url https://download.pytorch.org/whl/cu121 -# CUDA 12.4 -pip install torch==2.5.0 torchvision==0.20.0 torchaudio==2.5.0 --index-url https://download.pytorch.org/whl/cu124 +pip install torch==2.5.0 torchvision==0.20.0 torchaudio==2.5.0 \ + --index-url https://download.pytorch.org/whl/cu121 ``` -#### 2.4 Habitat Simulator -> We recommend using habitat-lab v0.3.1 +CUDA 11.8 and 12.4 wheels can be selected by replacing `cu121` with `cu118` or `cu124`. + +### 4. Install Habitat and external perception code + ```bash -# habitat-lab v0.3.1 git clone https://github.com/facebookresearch/habitat-lab.git -cd habitat-lab; git checkout tags/v0.3.1; -pip install -e habitat-lab +git -C habitat-lab checkout v0.3.1 +pip install -e habitat-lab/habitat-lab +pip install -e habitat-lab/habitat-baselines -# habitat-baselines v0.3.1 -pip install -e habitat-baselines -``` +git clone https://github.com/IDEA-Research/GroundingDINO.git +git -C GroundingDINO checkout 856dde20aee659246248e20734ef9ba5214f5e44 -**Note:** Any numpy-related errors will not affect subsequent operations, as long as `numpy==1.23.5` and `numba==0.60.0` are correctly installed. +git clone https://github.com/WongKinYiu/yolov7.git +git -C yolov7 checkout a207844b1ce82d204ab36d87d496728d3d2348e7 -#### 2.5 Others -```bash +pip install -r vlm/detector/D-FINE/requirements.txt pip install salesforce-lavis==1.0.2 -cd .. # Return to InfoNav directory pip install -e . ``` -**Note:** Any numpy-related errors will not affect subsequent operations, as long as `numpy==1.23.5` and `numba==0.60.0` are correctly installed. +### 5. Download checkpoints -### 3. Environment Variables +Create the expected directories: -Set up the DeepSeek API key for LLM hypothesis analysis: ```bash -export DEEPSEEK_API_KEY="your_api_key_here" +mkdir -p data pretrained_weights ``` -You can add this to your `~/.bashrc` to make it persistent: -```bash -echo 'export DEEPSEEK_API_KEY="your_api_key_here"' >> ~/.bashrc -source ~/.bashrc -``` +| File | Destination | Source | +| --- | --- | --- | +| MobileSAM `mobile_sam.pt` | `data/mobile_sam.pt` | [MobileSAM weights](https://github.com/ChaoningZhang/MobileSAM/tree/master/weights) | +| GroundingDINO Swin-T OGC | `data/groundingdino_swint_ogc.pth` | [GroundingDINO release](https://github.com/IDEA-Research/GroundingDINO/releases/tag/v0.1.0-alpha) | +| YOLOv7-E6E (legacy detector mode) | `data/yolov7-e6e.pt` | [YOLOv7 release](https://github.com/WongKinYiu/yolov7/releases/tag/v0.1) | +| D-FINE-X COCO | `pretrained_weights/dfine_x_coco.pth` | [D-FINE model zoo](https://github.com/Peterande/D-FINE#model-zoo) | -> You can skip this and use our pre-generated LLM output results in `llm/answers/` instead. +The default InfoNav detector mode uses D-FINE together with GroundingDINO. YOLOv7 is retained for the legacy comparison mode. -### 4. Build ROS Packages +### 6. Build the ROS workspace ```bash -# Install ROS Noetic first: http://wiki.ros.org/noetic/Installation/Ubuntu catkin init catkin config --extend /opt/ros/noetic catkin build source devel/setup.bash ``` -## ๐Ÿ“ฅ Datasets Download -> Official Reference: https://github.com/facebookresearch/habitat-lab/blob/main/DATASETS.md - -### ๐Ÿ  Scene Datasets -**Note:** Both HM3D and MP3D scene datasets require applying for official permission first. - -#### HM3D Scene Dataset -1. Apply for permission at https://matterport.com/habitat-matterport-3d-research-dataset. -2. Download https://api.matterport.com/resources/habitat/hm3d-val-habitat-v0.2.tar. -3. Save `hm3d-val-habitat-v0.2.tar` to the `InfoNav/` directory, then run: -```bash -mkdir -p data/scene_datasets/hm3d/val -mv hm3d-val-habitat-v0.2.tar data/scene_datasets/hm3d/val/ -cd data/scene_datasets/hm3d/val -tar -xvf hm3d-val-habitat-v0.2.tar -rm hm3d-val-habitat-v0.2.tar -cd ../.. -ln -s hm3d hm3d_v0.2 # Create a symbolic link for hm3d_v0.2 -``` - -#### MP3D Scene Dataset -1. Apply for download access at https://niessner.github.io/Matterport/. -2. After successful application, you will receive a `download_mp.py` script, which should be run with `python2.7` to download the dataset. -3. After downloading, place the files in `InfoNav/data/scene_datasets`. - -### ๐ŸŽฏ Task Datasets -```bash -# Create necessary directory structure -mkdir -p data/datasets/objectnav/hm3d -mkdir -p data/datasets/objectnav/mp3d - -# HM3D-v0.1 -wget -O data/datasets/objectnav/hm3d/v1.zip https://dl.fbaipublicfiles.com/habitat/data/datasets/objectnav/hm3d/v1/objectnav_hm3d_v1.zip -unzip data/datasets/objectnav/hm3d/v1.zip -d data/datasets/objectnav/hm3d && mv data/datasets/objectnav/hm3d/objectnav_hm3d_v1 data/datasets/objectnav/hm3d/v1 && rm data/datasets/objectnav/hm3d/v1.zip - -# HM3D-v0.2 -wget -O data/datasets/objectnav/hm3d/v2.zip https://dl.fbaipublicfiles.com/habitat/data/datasets/objectnav/hm3d/v2/objectnav_hm3d_v2.zip -unzip data/datasets/objectnav/hm3d/v2.zip -d data/datasets/objectnav/hm3d && mv data/datasets/objectnav/hm3d/objectnav_hm3d_v2 data/datasets/objectnav/hm3d/v2 && rm data/datasets/objectnav/hm3d/v2.zip +## Datasets -# MP3D -wget -O data/datasets/objectnav/mp3d/v1.zip https://dl.fbaipublicfiles.com/habitat/data/datasets/objectnav/m3d/v1/objectnav_mp3d_v1.zip -unzip data/datasets/objectnav/mp3d/v1.zip -d data/datasets/objectnav/mp3d/v1 && rm data/datasets/objectnav/mp3d/v1.zip -``` +InfoNav evaluates on HM3D-v1, HM3D-v2, and MP3D ObjectNav. Follow the official [Habitat dataset instructions](https://github.com/facebookresearch/habitat-lab/blob/v0.3.1/DATASETS.md) and obtain the required licenses or access approvals from the dataset owners. -
-Make sure that the folder data has the following structure: +Expected layout: -``` -data -โ”œโ”€โ”€ datasets -โ”‚ โ””โ”€โ”€ objectnav -โ”‚ โ”œโ”€โ”€ hm3d -โ”‚ โ”‚ โ”œโ”€โ”€ v1 -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ train -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ val -โ”‚ โ”‚ โ”‚ โ””โ”€โ”€ val_mini -โ”‚ โ”‚ โ””โ”€โ”€ v2 -โ”‚ โ”‚ โ”œโ”€โ”€ train -โ”‚ โ”‚ โ”œโ”€โ”€ val -โ”‚ โ”‚ โ””โ”€โ”€ val_mini -โ”‚ โ””โ”€โ”€ mp3d -โ”‚ โ””โ”€โ”€ v1 -โ”‚ โ”œโ”€โ”€ train -โ”‚ โ”œโ”€โ”€ val -โ”‚ โ””โ”€โ”€ val_mini -โ”œโ”€โ”€ scene_datasets -โ”‚ โ”œโ”€โ”€ hm3d -โ”‚ โ”‚ โ””โ”€โ”€ val -โ”‚ โ”‚ โ”œโ”€โ”€ 00800-TEEsavR23oF -โ”‚ โ”‚ โ”œโ”€โ”€ 00801-HaxA7YrQdEC -โ”‚ โ”‚ โ””โ”€โ”€ ..... -โ”‚ โ”œโ”€โ”€ hm3d_v0.2 -> hm3d -โ”‚ โ””โ”€โ”€ mp3d -โ”‚ โ”œโ”€โ”€ 17DRP5sb8fy -โ”‚ โ”œโ”€โ”€ 1LXtFkjw3qL -โ”‚ โ””โ”€โ”€ ..... -โ”œโ”€โ”€ groundingdino_swint_ogc.pth -โ”œโ”€โ”€ mobile_sam.pt -โ””โ”€โ”€ yolov7-e6e.pt +```text +data/ +โ”œโ”€โ”€ datasets/objectnav/ +โ”‚ โ”œโ”€โ”€ hm3d/v1/{train,val,val_mini} +โ”‚ โ”œโ”€โ”€ hm3d/v2/{train,val,val_mini} +โ”‚ โ””โ”€โ”€ mp3d/v1/{train,val,val_mini} +โ””โ”€โ”€ scene_datasets/ + โ”œโ”€โ”€ hm3d/val/ + โ”œโ”€โ”€ hm3d_v0.2 -> hm3d + โ””โ”€โ”€ mp3d/ ``` -Note that `train` and `val_mini` are not required and you can choose to delete them. -
+Dataset paths can be changed in `config/habitat_eval_*.yaml`. -## Usage +## Running InfoNav -### Quick Start +Use separate terminals from the repository root. Activate the `infonav` environment in each terminal. -1. **Start VLM Servers** +Start ROS: -The system relies on several VLM servers (GroundingDINO, BLIP2-ITM, SAM, YOLOv7, D-FINE). Start them all at once using the provided script: ```bash -# Requires tmux: sudo apt-get install tmux -chmod +x scripts/start_vlm_servers.sh -./scripts/start_vlm_servers.sh start infonav +roscore ``` -Wait until all servers finish loading (monitor with `tmux attach -t vlm_servers`). Server URLs: -- GroundingDINO: `http://localhost:12181` -- BLIP2-ITM: `http://localhost:12182` -- SAM: `http://localhost:12183` -- YOLOv7: `http://localhost:12184` -- D-FINE: `http://localhost:12185` +Start the local perception services: -To stop servers: `./scripts/start_vlm_servers.sh stop` - -2. **Start ROS Core** ```bash -roscore +./scripts/start_vlm_servers.sh restart infonav +./scripts/start_vlm_servers.sh status ``` -3. **Launch Exploration Node** +Launch the full InfoNav planner. The explicit arguments below match the release defaults and make the experiment configuration visible in the command: + ```bash source devel/setup.bash -roslaunch exploration_manager exploration.launch +roslaunch exploration_manager exploration.launch \ + dataset:=hm3dv2 \ + exploration_policy:=4 \ + use_multi_source:=true \ + fusion_alpha:=0.8 \ + use_ig_fusion:=true \ + use_adaptive_alpha:=true \ + alpha_base:=0.8 \ + tau_conf:=0.8 \ + top_k_value:=8 ``` -4. **Run Evaluation** +Run the evaluator: + ```bash -python habitat_evaluation.py --config config/habitat_eval_hm3dv2.yaml +python habitat_evaluation.py --dataset hm3dv2 --detector-mode 2 ``` -### Configuration - -Main configuration files: -- `config/habitat_eval_hm3dv1.yaml` - HM3D-v1 dataset config -- `config/habitat_eval_hm3dv2.yaml` - HM3D-v2 dataset config -- `config/habitat_eval_mp3d.yaml` - MP3D dataset config -- `src/planner/exploration_manager/config/algorithm.xml` - Algorithm parameters +Use `hm3dv1`, `hm3dv2`, or `mp3d` for `--dataset`. To generate scene hypotheses online, set `DEEPSEEK_API_KEY` and `DASHSCOPE_API_KEY`, then add `--realtime-hypothesis`. Without that flag, the checked-in responses under `llm/answers/` are used where supported. -### Manual Control +Optional visualization: -For testing and debugging: ```bash -python habitat_manual_control.py +source devel/setup.bash +roslaunch exploration_manager rviz.launch ``` -## Project Structure +## Configuration -``` +- `config/habitat_eval_hm3dv1.yaml`: HM3D-v1 evaluation +- `config/habitat_eval_hm3dv2.yaml`: HM3D-v2 evaluation +- `config/habitat_eval_mp3d.yaml`: MP3D evaluation +- `config/default.yaml`: documented system defaults +- `src/planner/exploration_manager/launch/algorithm.xml`: ROS planner parameters + +See [config/README.md](config/README.md) for the full parameter reference. + +## Repository layout + +```text InfoNav/ -โ”œโ”€โ”€ habitat_evaluation.py # Main evaluation loop -โ”œโ”€โ”€ params.py # Global constants -โ”œโ”€โ”€ src/planner/ # C++ ROS planning modules -โ”‚ โ”œโ”€โ”€ exploration_manager/ # FSM and exploration logic -โ”‚ โ”œโ”€โ”€ plan_env/ # Map representations -โ”‚ โ””โ”€โ”€ path_searching/ # A* path planning -โ”œโ”€โ”€ vlm/ # Vision-Language Models -โ”œโ”€โ”€ llm/ # Large Language Models -โ”œโ”€โ”€ basic_utils/ # Utility functions -โ”œโ”€โ”€ habitat2ros/ # Habitat-ROS bridge -โ””โ”€โ”€ config/ # Configuration files +โ”œโ”€โ”€ habitat_evaluation.py # Habitat evaluation loop +โ”œโ”€โ”€ config/ # Dataset and runtime configuration +โ”œโ”€โ”€ src/planner/ # C++ ROS mapping and planning packages +โ”œโ”€โ”€ vlm/ # Perception and image-text services +โ”œโ”€โ”€ llm/ # Semantic hypothesis generation and cached answers +โ”œโ”€โ”€ habitat2ros/ # Habitat-to-ROS bridge +โ”œโ”€โ”€ scripts/ # Evaluation and reporting utilities +โ””โ”€โ”€ docs/REPRODUCIBILITY.md # Exact protocol and artifact notes ``` ## Citation -If you find this work useful, please cite: - ```bibtex -@article{infonav2025, - title={InfoNav: A Unified Value Framework Integrating Semantics Value and Information Gain for Zero-Shot Navigation}, - author={}, - journal={}, - year={2025} +@inproceedings{yang2026infonav, + title = {InfoNav: A Unified Value Framework Integrating Semantic Relevance and Information Gain for Zero-Shot Object Goal Navigation}, + author = {Yang, Zhi and Yu, Chengpu and Ferrer, Gonzalo}, + booktitle = {Advances in Neural Information Processing Systems}, + year = {2026}, + url = {https://openreview.net/forum?id=lzhhlAlmjK} } ``` -## License +## License and third-party components -This project is released under the MIT License. +The original InfoNav code is released under [GPL-3.0](LICENSE). Third-party code, models, and datasets remain subject to their own licenses and terms. In particular, the bundled LKH source is marked by its author for research use with rights reserved; consult [THIRD_PARTY.md](THIRD_PARTY.md) before redistribution or non-research use. ## Acknowledgements -This project builds upon several excellent open-source projects: -- [Habitat-Sim](https://github.com/facebookresearch/habitat-sim) -- [GroundingDINO](https://github.com/IDEA-Research/GroundingDINO) -- [BLIP2](https://github.com/salesforce/LAVIS) - -## Contact - -For questions or issues, please open an issue on GitHub. - -## TODO +InfoNav builds on [Habitat](https://github.com/facebookresearch/habitat-lab), [GroundingDINO](https://github.com/IDEA-Research/GroundingDINO), [BLIP-2/LAVIS](https://github.com/salesforce/LAVIS), [MobileSAM](https://github.com/ChaoningZhang/MobileSAM), [D-FINE](https://github.com/Peterande/D-FINE), [YOLOv7](https://github.com/WongKinYiu/yolov7), and [LKH-3](https://webhotel4.ruc.dk/~keld/research/LKH-3/). -- [ ] Release real-world deployment code -- [ ] Release ROS2 support version +For questions or reproducibility reports, please open a GitHub issue. diff --git a/THIRD_PARTY.md b/THIRD_PARTY.md new file mode 100644 index 0000000..b4246d6 --- /dev/null +++ b/THIRD_PARTY.md @@ -0,0 +1,21 @@ +# Third-party software and assets + +InfoNav depends on software, model checkpoints, and datasets maintained by other projects. The top-level GPL-3.0 license applies only to original InfoNav code unless a file or component states otherwise. + +| Component | Location or use | Upstream terms | +| --- | --- | --- | +| D-FINE | `vlm/detector/D-FINE` submodule | Apache-2.0; see the submodule `LICENSE` | +| LKH-3.0.6 | `src/planner/utils/lkh_mtsp_solver` | The included upstream README says research use and that the author reserves all rights. This is not GPL-3.0. Review the upstream terms before redistribution or non-research use. | +| Habitat-Lab / Habitat-Sim | external dependency | See the [Habitat repository](https://github.com/facebookresearch/habitat-lab) | +| GroundingDINO | external dependency and checkpoint | See the [GroundingDINO repository](https://github.com/IDEA-Research/GroundingDINO) | +| BLIP-2 / LAVIS | external dependency and checkpoint | See the [LAVIS repository](https://github.com/salesforce/LAVIS) | +| MobileSAM | external dependency and checkpoint | See the [MobileSAM repository](https://github.com/ChaoningZhang/MobileSAM) | +| YOLOv7 | external dependency and checkpoint | See the [YOLOv7 repository](https://github.com/WongKinYiu/yolov7) | +| HM3D | scene and ObjectNav datasets | Registration/access terms apply; see [Habitat datasets](https://github.com/facebookresearch/habitat-lab/blob/v0.3.1/DATASETS.md) | +| MP3D | scene and ObjectNav datasets | Matterport access terms apply; see [Habitat datasets](https://github.com/facebookresearch/habitat-lab/blob/v0.3.1/DATASETS.md) | + +Model weights and datasets are intentionally excluded from Git. Downloading a file does not change its upstream license. When publishing derived artifacts, cite the original projects and record the exact version or checkpoint used. + +## Release note + +Before creating an archival release, the maintainers should obtain legal confirmation that redistributing the bundled LKH source is permitted. If not, remove that source from the release and provide a user-side download/setup step that preserves the upstream terms. diff --git a/basic_utils/logging/log_manager.py b/basic_utils/logging/log_manager.py index 546f7f7..597e2bd 100644 --- a/basic_utils/logging/log_manager.py +++ b/basic_utils/logging/log_manager.py @@ -1,5 +1,5 @@ """ -Logging Manager for ApexNav +Logging Manager for InfoNav Provides centralized logging control with environment variable support and configuration file integration. @@ -24,7 +24,7 @@ class LogManager: - """Centralized logging manager for ApexNav""" + """Centralized logging manager for InfoNav.""" # Log levels VERBOSE = "VERBOSE" diff --git a/config/README.md b/config/README.md new file mode 100644 index 0000000..afb361f --- /dev/null +++ b/config/README.md @@ -0,0 +1,267 @@ +# InfoNav Configuration System + +This directory contains configuration files for the InfoNav object navigation system. + +## Configuration Files + +### 1. `default.yaml` - Unified Configuration +**NEW in Phase 6: Configuration Management** + +Centralizes all hardcoded parameters for: +- VLM server ports (BLIP2, GroundingDINO, SAM, YOLOv7, D-FINE) +- Exploration parameters (alpha_base, tau_conf, fusion_alpha) +- Patience-aware navigation thresholds (tau_high, tau_low, T_max) +- Object detection defaults +- Path configurations + +**Usage:** +```python +import yaml +from pathlib import Path + +config_path = Path(__file__).parent / "config" / "default.yaml" +if config_path.exists(): + with open(config_path) as f: + CONFIG = yaml.safe_load(f) +else: + CONFIG = {} # Use hardcoded defaults + +# Access parameters with fallback +blip2_port = CONFIG.get('vlm', {}).get('blip2_itm', {}).get('port', 12182) +alpha_base = CONFIG.get('exploration', {}).get('adaptive_alpha', {}).get('alpha_base', 0.8) +``` + +**Backward Compatibility:** +If `default.yaml` is missing, the system uses hardcoded defaults. This ensures existing deployments continue to work without modification. + +### 2. `habitat_eval_*.yaml` - Dataset-Specific Configs +Hydra configuration files for different datasets: +- `habitat_eval_hm3dv1.yaml` - HM3D-v1 dataset +- `habitat_eval_hm3dv2.yaml` - HM3D-v2 dataset (default) +- `habitat_eval_mp3d.yaml` - MP3D dataset + +These files configure: +- Habitat simulator settings +- Dataset paths +- Detector mode (YOLOv7 vs D-FINE) +- LLM client settings +- Episode constraints + +### 3. `logging_config.yaml` - Logging Configuration +Controls verbosity levels across Python and ROS components. + +**Log Levels:** +- `VERBOSE`: All debug info, timing breakdowns, VLM details +- `NORMAL`: Basic progress info, episode summaries (default) +- `MINIMAL`: Only episode results and errors +- `SILENT`: Errors only + +**Environment Variable Overrides:** +```bash +export APEXNAV_LOG_LEVEL=MINIMAL +export APEXNAV_VLM_DETAIL=false +export APEXNAV_TIMING_DETAIL=false +``` + +## Configuration Hierarchy + +Parameters are loaded in the following priority order: + +1. **Environment variables** (highest priority) + - `USE_PARALLEL_BLIP2`: Enable parallel BLIP2 servers + - `APEXNAV_LOG_LEVEL`: Override logging level + +2. **Hydra config files** (`habitat_eval_*.yaml`) + - Dataset-specific settings + - CLI overrides: `python habitat_evaluation.py key=value` + +3. **Unified config** (`default.yaml`) + - VLM server ports + - Exploration parameters + - Patience thresholds + +4. **Hardcoded defaults** (lowest priority) + - Used when config file is missing + - Ensures backward compatibility + +## Key Parameters Reference + +### VLM Server Ports +```yaml +vlm: + blip2_itm: + port: 12182 # BLIP2-ITM server + grounding_dino: + port: 12181 # GroundingDINO server + sam: + port: 12183 # SAM segmentation server + yolov7: + port: 12184 # YOLOv7 detector (Mode 1) + dfine: + port: 12185 # D-FINE detector (Mode 2) +``` + +### Exploration Parameters +```yaml +exploration: + policy: 4 # 0: distance, 1: value(greedy), 2: TSP, 4: WTRP + fusion_alpha: 0.8 # V_total = alpha * V_sem + (1-alpha) * V_ig + adaptive_alpha: + enabled: true + alpha_base: 0.8 # Base alpha when no semantic cues + tau_conf: 0.8 # Confidence threshold for adaptation +``` + +### Patience-Aware Navigation +```yaml +patience_nav: + T_max: 450 # Maximum episode steps + tau_high: 0.4 # Starting threshold (strict) + tau_low: 0.2 # Ending threshold (relaxed) + + # Target-specific thresholds by object size + threshold_by_size: + large: # bed, couch, bathtub + tau_high: 0.85 + tau_low: 0.55 + medium_large: # tv, counter, cabinet + tau_high: 0.85 + tau_low: 0.5 + medium: # chair, table, sink + tau_high: 0.75 + tau_low: 0.5 + small: # plant, pillow, towel + tau_high: 0.65 + tau_low: 0.4 +``` + +## Testing Configuration + +The lightweight GitHub Actions workflow parses all YAML configuration files on every push and pull request. Locally, the same check can be run with: + +```bash +python - <<'PY' +from pathlib import Path +import yaml + +for path in sorted(Path("config").glob("*.yaml")): + yaml.safe_load(path.read_text(encoding="utf-8")) + print(f"parsed {path}") +PY +``` + +## Migration Guide + +### For Developers +If you previously used hardcoded values: + +**Before (hardcoded):** +```python +BLIP2_PORT = 12182 +alpha_base = 0.8 +tau_high = 0.4 +``` + +**After (config-based):** +```python +# In module initialization +from pathlib import Path +import yaml + +CONFIG_PATH = Path(__file__).parent / "config" / "default.yaml" +if CONFIG_PATH.exists(): + with open(CONFIG_PATH) as f: + CONFIG = yaml.safe_load(f) +else: + CONFIG = {} + +# In code +BLIP2_PORT = CONFIG.get('vlm', {}).get('blip2_itm', {}).get('port', 12182) +alpha_base = CONFIG.get('exploration', {}).get('adaptive_alpha', {}).get('alpha_base', 0.8) +tau_high = CONFIG.get('patience_nav', {}).get('tau_high', 0.4) +``` + +### For System Administrators +To customize parameters without modifying code: + +1. Edit `/path/to/InfoNav/config/default.yaml` +2. Modify desired parameters +3. Restart the system + +Example customization: +```yaml +# Increase BLIP2 timeout for slower systems +vlm: + blip2_itm: + port: 12182 + timeout: 120 # seconds + +# Adjust exploration aggressiveness +exploration: + fusion_alpha: 0.9 # More semantic-driven (less exploration) + +# Relax patience thresholds for difficult environments +patience_nav: + tau_high: 0.35 # Start more relaxed + tau_low: 0.15 # End more relaxed +``` + +## Phase 6 Implementation Summary + +**Goal:** Create unified configuration system for centralized parameter management. + +**Changes:** +1. Created `config/default.yaml` with all hardcoded parameters +2. Modified `habitat_evaluation.py` to load config on startup +3. Updated VLM client modules (`get_object_utils.py`, `get_itm_message.py`) +4. Maintained backward compatibility (config file optional) +5. Added automated YAML parsing in `.github/workflows/quality.yml` + +**Benefits:** +- โœ“ Centralized parameter management +- โœ“ Easy tuning without code modification +- โœ“ Backward compatible (optional config file) +- โœ“ Self-documenting (YAML comments) +- โœ“ Environment variable overrides supported + +## Troubleshooting + +### Config file not loaded +**Problem:** System uses hardcoded defaults instead of config file. + +**Solution:** +1. Verify file exists: `ls -la /path/to/InfoNav/config/default.yaml` +2. Check file permissions: `chmod 644 config/default.yaml` +3. Validate YAML syntax: `python -c "import yaml; yaml.safe_load(open('config/default.yaml'))"` + +### YAML parsing error +**Problem:** `yaml.scanner.ScannerError` when loading config. + +**Solution:** +1. Check YAML syntax (indentation, colons, quotes) +2. Use online YAML validator +3. Check for special characters in values + +### Parameter not applied +**Problem:** Changed config value but system still uses old value. + +**Solution:** +1. Restart Python process (config loaded on import) +2. Check parameter path is correct (nested dict access) +3. Verify fallback default matches expected value +4. Check if environment variable is overriding config + +## Related Documentation + +- [Main README.md](../README.md) - Project overview +- [Logging Documentation](logging_config.yaml) - Logging system details +- [Habitat Config](habitat_eval_hm3dv2.yaml) - Dataset configuration +- [ROS Launch Files](../src/planner/exploration_manager/launch/) - ROS parameter server config + +## Support + +For questions or issues with configuration: +1. Run the YAML parsing command in the testing section above. +2. Check log output for "[Config]" messages +3. Review this README for parameter reference +4. Submit issue with config file and error message diff --git a/config/default.yaml b/config/default.yaml new file mode 100644 index 0000000..c00f58c --- /dev/null +++ b/config/default.yaml @@ -0,0 +1,195 @@ +# ============================================================ +# InfoNav Unified Configuration File +# ============================================================ +# This configuration file centralizes all hardcoded parameters +# for VLM servers, exploration, and patience-aware navigation. +# +# Usage: Parameters are loaded in habitat_evaluation.py +# If this file is missing, hardcoded defaults will be used (backward compatible). +# ============================================================ + +# ============================================================ +# VLM Server Configuration +# ============================================================ +vlm: + # BLIP2-ITM server (Image-Text Matching for semantic scoring) + blip2_itm: + host: "localhost" + port: 12182 + + # Parallel BLIP2 configuration (when USE_PARALLEL_BLIP2=true) + parallel: + enabled: false + ports: [12182, 12192] + + # GroundingDINO server (open-vocabulary object detection) + grounding_dino: + host: "localhost" + port: 12181 + + # SAM server (Segment Anything Model for object segmentation) + sam: + host: "localhost" + port: 12183 + + # YOLOv7 server (Mode 1: ApexNav-style detection) + yolov7: + host: "localhost" + port: 12184 + + # D-FINE server (Mode 2: InfoNav detection) + dfine: + host: "localhost" + port: 12185 + + # VLM inference thresholds + thresholds: + # Minimum confidence for VLM-based semantic scoring + min_confidence: 0.3 + # ITM score threshold for positive matches + itm_threshold: 0.5 + +# ============================================================ +# Exploration Parameters +# ============================================================ +exploration: + # Policy selection + # 0: distance, 1: value(greedy), 2: value(TSP top-k), 3: distance(TSP all), 4: WTRP(top-k) + policy: 4 + + # Top-k frontiers for value-based exploration + top_k_value: 8 + + # Dual-Value Fusion: V_total = alpha * V_sem + (1-alpha) * V_ig + fusion_alpha: 0.8 + use_ig_fusion: true + + # Adaptive Alpha: alpha(t) = alpha_base + (1-alpha_base) * max_fi(V_HSVM(fi)) / tau_conf + # When strong semantic cues detected at frontiers, favor exploitation; otherwise explore + adaptive_alpha: + enabled: true + alpha_base: 0.8 # Base alpha when no strong semantic cues (default 0.8) + tau_conf: 0.8 # Confidence threshold for alpha adaptation (default 0.8) + + # WTRP: Weighted Traveling Repairman Problem (min weighted arrival time) + wtrp: + temperature: 0.3 + hysteresis_ratio: 0.8 + max_brute_force: 8 + +# ============================================================ +# Frontier Detection Parameters +# ============================================================ +frontier: + cluster_min: 2 + cluster_size_xy: 0.65 + min_view_finish_fraction: 0.2 + min_contain_unknown: 30 + +# ============================================================ +# Object Detection and Fusion +# ============================================================ +object: + min_observation_num: 2 + # 0: no_fusion, 1: ours_fusion, 2: max_fusion + fusion_type: 1 + use_observation: true + vis_cloud: true + + # Default detection confidence threshold (used when VLM is disabled) + default_detection_confidence: 0.5 + +# ============================================================ +# Patience-Aware Navigation (Paper Section 3.5) +# ============================================================ +# Formula: ฯ„(t, obj) = tau_high - (t/T_max) ร— (tau_high - tau_low) +# This implements time-decay threshold adaptation for approach verification. +# ============================================================ +patience_nav: + # Maximum episode steps for threshold decay + T_max: 450 + + # Minimum detection confidence for approach trigger + min_detection_confidence: 0.2 + + # Global fallback thresholds (for unknown objects) + tau_high: 0.4 + tau_low: 0.2 + + # Target-specific thresholds by object size + threshold_by_size: + # Large objects: bed, couch, sofa, bathtub, fireplace, gym equipment + large: + tau_high: 0.85 + tau_low: 0.55 + + # Medium-large objects: tv, tv_monitor, counter, cabinet, shower, seating + medium_large: + tau_high: 0.85 + tau_low: 0.5 + + # Medium objects: chair, sink, table, nightstand, stool + medium: + tau_high: 0.75 + tau_low: 0.5 + + # Toilet: separate category for fine-tuning + toilet: + tau_high: 0.75 + tau_low: 0.45 + + # Small objects: plant, potted plant, pillow, towel, clothes + small: + tau_high: 0.65 + tau_low: 0.4 + +# ============================================================ +# VLM Validation (Multi-view Verification) +# ============================================================ +vlm_validation: + enabled: false + timeout: 90.0 # Timeout for VLM service call (seconds) + num_views: 4 # Number of views for multi-view validation (AND logic) + +# ============================================================ +# Path Configuration +# ============================================================ +paths: + # TSP solver directory + tsp_dir: "$(find lkh_mtsp_solver)/resource" + + # LLM hypothesis analysis paths + llm: + env_base_dir: "env" + hypothesis_file: "llm_hypothesis_analysis.json" + +# ============================================================ +# LLM Hypothesis Hierarchy Configuration (Ablation Study) +# ============================================================ +# Controls which hierarchical hypothesis levels to use in HSVM +# L1: target_object only +# L2: target_object + room_type +# L3: target_object + room_type + co_occurrence +# L4: target_object + room_type + co_occurrence + object_part (Full) +# ============================================================ +llm: + # Maximum hierarchy level to use (1-4) + # Default: 4 (use all hypothesis levels) + # Override with: python habitat_evaluation.py llm.max_hierarchy_level=2 + max_hierarchy_level: 4 + + # Debug output paths + debug: + base_dir: "debug" + stats_file_pattern: "evaluation_stats_{timestamp}.txt" + +# ============================================================ +# Logging Configuration Reference +# ============================================================ +# For detailed logging configuration, see config/logging_config.yaml +# Environment variables override logging settings: +# - APEXNAV_LOG_LEVEL: VERBOSE, NORMAL, MINIMAL, SILENT +# - APEXNAV_VLM_DETAIL: true/false +# - APEXNAV_TIMING_DETAIL: true/false +# - APEXNAV_SEMANTIC_SCORES: true/false +# ============================================================ diff --git a/config/habitat_eval_hm3dv1.yaml b/config/habitat_eval_hm3dv1.yaml index 4140e4e..669b247 100644 --- a/config/habitat_eval_hm3dv1.yaml +++ b/config/habitat_eval_hm3dv1.yaml @@ -24,7 +24,7 @@ habitat: dimensionality: 3 measurements: success: - success_distance: 0.2 + success_distance: 1.0 simulator: habitat_sim_v0: allow_sliding: false @@ -68,6 +68,19 @@ llm: llm_client: llm_client: ollama # deepseek, ollama or none ollama: "qwen3:8b" + # Hierarchical hypothesis level for ablation study (1-4) + # L1: target_object only + # L2: L1 + room_type + # L3: L2 + co_occurrence + # L4: L3 + object_part (Full) + max_hierarchy_level: 4 + +# Exploration parameters (override ROS launch defaults for ablation study) +exploration: + # Dual-Value Fusion: V_total = alpha * V_sem + (1-alpha) * V_ig + # Set to 1.0 for pure semantic ablation (disable IG) + fusion_alpha: 0.8 + use_ig_fusion: true record_file_name: "record.txt" continue_file_name: "continue.txt" diff --git a/config/habitat_eval_hm3dv2.yaml b/config/habitat_eval_hm3dv2.yaml index a64db74..34e50b4 100644 --- a/config/habitat_eval_hm3dv2.yaml +++ b/config/habitat_eval_hm3dv2.yaml @@ -23,7 +23,7 @@ habitat: dimensionality: 3 measurements: success: - success_distance: 0.2 + success_distance: 1.0 simulator: habitat_sim_v0: allow_sliding: false @@ -68,6 +68,19 @@ llm: llm_client: llm_client: ollama ollama: qwen3:8b + # Hierarchical hypothesis level for ablation study (1-4) + # L1: target_object only + # L2: L1 + room_type + # L3: L2 + co_occurrence + # L4: L3 + object_part (Full) + max_hierarchy_level: 4 + +# Exploration parameters (override ROS launch defaults for ablation study) +exploration: + # Dual-Value Fusion: V_total = alpha * V_sem + (1-alpha) * V_ig + # Set to 1.0 for pure semantic ablation (disable IG) + fusion_alpha: 0.8 + use_ig_fusion: true record_file_name: record.txt continue_file_name: continue.txt need_video: false diff --git a/config/habitat_eval_hm3dv2.yaml.backup b/config/habitat_eval_hm3dv2.yaml.backup deleted file mode 100644 index 9a7b21b..0000000 --- a/config/habitat_eval_hm3dv2.yaml.backup +++ /dev/null @@ -1,64 +0,0 @@ -defaults: - - /benchmark/nav/objectnav: - - objectnav_hm3d - - _self_ - -habitat_baselines: - evaluate: True - force_torch_single_threaded: True - -habitat: - dataset: - split: val - data_path: data/datasets/objectnav/hm3d/v2/{split}/{split}.json.gz # HM3D-v2 dataset - environment: - max_episode_steps: 500 - iterator_options: - cycle: false - shuffle: false - max_scene_repeat_steps: 10000000 - task: - lab_sensors: - gps_sensor: - type: GPSSensor - dimensionality: 3 - measurements: - success: - success_distance: 0.2 - simulator: - habitat_sim_v0: - allow_sliding: false - forward_step_size: 0.25 - turn_angle: 30 - agents: - main_agent: - radius: 0.18 - sim_sensors: - depth_sensor: - min_depth: 0.0 - max_depth: 5.0 - normalize_depth: true - -detector: - yolo: - confidence_threshold_yolo: 0.3 - iou_threshold_yolo: 0.5 - agnostic_nms: True - - groundingDINO: - confidence_threshold_dino: 0.40 - text_threshold: 0.25 - -llm: - llm_answer_path: llm/answers/llm_answer_hm3d.txt - llm_response_path: llm/answers/llm_response_list.txt - llm_client: - llm_client: ollama # deepseek, ollama or none - ollama: "qwen3:8b" - -record_file_name: "record.txt" -continue_file_name: "continue.txt" -need_video: false -video_output_path: "videos/test_hm3dv2_{split}" -test_epi_num: -1 # Deprecated: use test_task_num instead -test_task_num: -1 # Test specific task index (e.g., 50 for task50), -1 to run all diff --git a/config/habitat_eval_mp3d.yaml b/config/habitat_eval_mp3d.yaml index bb7d3da..f114f27 100644 --- a/config/habitat_eval_mp3d.yaml +++ b/config/habitat_eval_mp3d.yaml @@ -24,7 +24,7 @@ habitat: dimensionality: 3 measurements: success: - success_distance: 0.2 + success_distance: 1.0 simulator: habitat_sim_v0: allow_sliding: false @@ -68,9 +68,59 @@ llm: llm_client: llm_client: ollama # deepseek, ollama, or none ollama: 'qwen3:8b' + # Hierarchical hypothesis level for ablation study (1-4) + # L1: target_object only + # L2: L1 + room_type + # L3: L2 + co_occurrence + # L4: L3 + object_part (Full) + max_hierarchy_level: 4 + +# Exploration parameters (override ROS launch defaults for ablation study) +exploration: + # Dual-Value Fusion: V_total = alpha * V_sem + (1-alpha) * V_ig + # Set to 1.0 for pure semantic ablation (disable IG) + fusion_alpha: 0.8 + use_ig_fusion: true record_file_name: "record.txt" continue_file_name: "continue.txt" need_video: false video_output_path: "videos/test_mp3d_{split}" -test_epi_num: -1 \ No newline at end of file +test_epi_num: -1 + +# ============================================================ +# MP3D-specific Patience-Aware Navigation Thresholds +# ============================================================ +# Lower thresholds to match ApexNav-style behavior (0.25~0.35 range) +# Formula: ฯ„(t, obj) = tau_high - (t/T_max) ร— (tau_high - tau_low) +# ============================================================ +patience_nav: + T_max: 450 + min_detection_confidence: 0.2 + + # Global fallback thresholds (unified low range like ApexNav) + tau_high: 0.4 + tau_low: 0.2 + + # Target-specific thresholds (all lowered for MP3D) + threshold_by_size: + # Large objects: bed, couch, sofa, bathtub, fireplace, gym equipment + large: + tau_high: 0.4 + tau_low: 0.2 + # Medium-large objects: tv, tv_monitor, counter, cabinet, shower, seating + medium_large: + tau_high: 0.4 + tau_low: 0.2 + # Medium objects: chair, sink, table, nightstand, stool + medium: + tau_high: 0.4 + tau_low: 0.2 + # Toilet: separate category + toilet: + tau_high: 0.4 + tau_low: 0.2 + # Small objects: plant, potted plant, pillow, towel, clothes + small: + tau_high: 0.4 + tau_low: 0.2 \ No newline at end of file diff --git a/config/logging_config.yaml b/config/logging_config.yaml index e6668e8..0d6b1f0 100644 --- a/config/logging_config.yaml +++ b/config/logging_config.yaml @@ -1,4 +1,4 @@ -# Logging Configuration for ApexNav +# Logging Configuration for InfoNav # Controls verbosity levels across Python and ROS components # # Usage: Set environment variables before running evaluation diff --git a/docs/REPRODUCIBILITY.md b/docs/REPRODUCIBILITY.md new file mode 100644 index 0000000..184f86f --- /dev/null +++ b/docs/REPRODUCIBILITY.md @@ -0,0 +1,140 @@ +# Reproducing InfoNav experiments + +This document records the public evaluation protocol for the NeurIPS 2026 release. The paper remains the authoritative source for the full experimental design and reported tables. + +## Supported benchmarks + +| Dataset flag | Habitat episodes | Scene dataset | +| --- | --- | --- | +| `hm3dv1` | HM3D ObjectNav v1 validation | HM3D v0.1-compatible scenes | +| `hm3dv2` | HM3D ObjectNav v2 validation | HM3D v0.2 scenes | +| `mp3d` | MP3D ObjectNav validation | Matterport3D scenes | + +Dataset files are not redistributed. Follow the access and license instructions linked from the main README. + +## Release protocol + +The full InfoNav planner uses: + +```text +detector_mode=2 # D-FINE + GroundingDINO +dfine_num_top_queries=15 # applied by vlm/detector/dfine.py +exploration_policy=4 # WTRP over top-k fused-value frontiers +use_multi_source=true # hierarchical semantic sources +fusion_alpha=0.8 +use_ig_fusion=true +use_adaptive_alpha=true +alpha_base=0.8 +tau_conf=0.8 +top_k_value=8 +wtrp_temperature=0.3 +wtrp_hysteresis_ratio=0.8 +wtrp_max_brute_force=8 +``` + +The checked-in dataset configurations additionally fix the episode iterator, maximum step count, success distance, camera geometry, and detector thresholds. Record any CLI or ROS override with the result. + +## Run one evaluation + +Start the following processes from the repository root in separate terminals. + +1. ROS master: + + ```bash + roscore + ``` + +2. Perception services: + + ```bash + conda activate infonav + ./scripts/start_vlm_servers.sh restart infonav + ``` + +3. Planner: + + ```bash + conda activate infonav + source /opt/ros/noetic/setup.bash + source devel/setup.bash + roslaunch exploration_manager exploration.launch \ + dataset:=hm3dv2 \ + exploration_policy:=4 \ + use_multi_source:=true \ + fusion_alpha:=0.8 \ + use_ig_fusion:=true \ + use_adaptive_alpha:=true \ + alpha_base:=0.8 \ + tau_conf:=0.8 \ + top_k_value:=8 \ + wtrp_temperature:=0.3 \ + wtrp_hysteresis_ratio:=0.8 \ + wtrp_max_brute_force:=8 + ``` + +4. Evaluator: + + ```bash + conda activate infonav + source /opt/ros/noetic/setup.bash + source devel/setup.bash + python habitat_evaluation.py \ + --dataset hm3dv2 \ + --detector-mode 2 \ + --control-rate-hz 10 + ``` + +Replace `hm3dv2` consistently in the planner and evaluator commands for the other benchmarks. + +## Cached versus online semantic hypotheses + +The default command uses checked-in semantic responses under `llm/answers/` where supported. For online generation, export credentials locally and add `--realtime-hypothesis`: + +```bash +export DASHSCOPE_API_KEY="..." +export DEEPSEEK_API_KEY="..." +python habitat_evaluation.py \ + --dataset hm3dv2 \ + --detector-mode 2 \ + --realtime-hypothesis \ + --vlm-api-type dashscope \ + --vlm-model qwen-vl-max \ + --llm-model deepseek-chat +``` + +Never commit API keys, generated credentials, or private dataset URLs. + +## Outputs and provenance + +The evaluator writes per-episode logs and JSON metrics below the configured `video_output_path`. Long-run scripts also write a protocol file containing the Git commit, dirty-worktree state, seed, model endpoints, and ROS parameters. Preserve these files with any reported aggregate. + +For a result to be comparable, archive at least: + +- the Git commit and whether tracked files were modified; +- dataset version, split, ordered episode range, and seed; +- detector and checkpoint names; +- LLM/VLM model identifiers and whether cached responses were used; +- success distance, step budget, and control rate; +- all exploration and fusion parameters; and +- `episode_metrics.jsonl` plus the aggregate report. + +## Ablations + +Use explicit planner arguments so the changed factor is visible: + +| Experiment | Override | +| --- | --- | +| Distance-greedy | `exploration_policy:=0` | +| Value-greedy | `exploration_policy:=1` | +| No information-gain fusion | `use_ig_fusion:=false` | +| Fixed fusion weight | `use_adaptive_alpha:=false fusion_alpha:=0.8` | +| Single-source semantic map | `use_multi_source:=false` | + +Do not describe value-greedy as simply โ€œwithout WTRPโ€: it is a replacement frontier-selection policy. Keep the episode order and every other parameter fixed for paired comparisons. + +## Known release constraints + +- Full evaluation requires Ubuntu 20.04/ROS Noetic and a CUDA-capable GPU; it is not exercised by the lightweight GitHub Actions checks. +- HM3D and MP3D require external access approval and substantial local storage. +- Online hypotheses depend on external APIs and can change as hosted models change. Cached responses are preferable for deterministic comparisons. +- The historical long-run scripts predate this release cleanup. Inspect their generated protocol file before treating a run as paper-equivalent. diff --git a/habitat2ros/pyproject.toml b/habitat2ros/pyproject.toml index 45bbd61..1ffda11 100644 --- a/habitat2ros/pyproject.toml +++ b/habitat2ros/pyproject.toml @@ -4,7 +4,9 @@ build-backend = "setuptools.build_meta" [project] name = "habitat2ros" -version = "0.1.0" +version = "1.0.0" authors = [ - {name = "Zager-Zhang"}, -] \ No newline at end of file + {name = "Zhi Yang"}, + {name = "Chengpu Yu"}, + {name = "Gonzalo Ferrer"}, +] diff --git a/habitat_evaluation.py b/habitat_evaluation.py index 7b8d4a8..ae95715 100644 --- a/habitat_evaluation.py +++ b/habitat_evaluation.py @@ -19,8 +19,6 @@ # Test specific episode python habitat_evaluation.py --dataset hm3dv2 test_epi_num=10 - -Author: Zhi Yang -modified from ApexNav """ # Standard library imports @@ -33,11 +31,13 @@ import subprocess import time from copy import deepcopy +from pathlib import Path # Third-party library imports from hydra import initialize, compose import numpy as np import rospy +import yaml from geometry_msgs.msg import PoseStamped from omegaconf import DictConfig from prettytable import PrettyTable @@ -77,7 +77,30 @@ from params import HABITAT_STATE, ROS_STATE, ACTION, RESULT_TYPES from vlm.Labels import MP3D_ID_TO_NAME from vlm.utils.get_object_utils import get_object -from pathlib import Path + +# Real-time analysis imports (for autonomous hypothesis generation) +try: + import cv2 + from scripts.vlm_environment_analyzer import VLMEnvironmentAnalyzer + from scripts.llm_hypothesis_analyzer import LLMHypothesisAnalyzer, AnalysisReportWriter + REALTIME_ANALYSIS_AVAILABLE = True +except ImportError as e: + print(f"[Warning] Real-time analysis modules not available: {e}") + REALTIME_ANALYSIS_AVAILABLE = False + +# ============================================================ +# Configuration Loading +# ============================================================ +# Load unified configuration from config/default.yaml +# If file does not exist, use hardcoded defaults (backward compatible) +CONFIG_PATH = Path(__file__).parent / "config" / "default.yaml" +if CONFIG_PATH.exists(): + with open(CONFIG_PATH, encoding='utf-8') as f: + CONFIG = yaml.safe_load(f) + print(f"[Config] Loaded configuration from {CONFIG_PATH}") +else: + CONFIG = {} + print(f"[Config] Configuration file not found at {CONFIG_PATH}, using hardcoded defaults") # VLM (BLIP2) imports - will be loaded conditionally based on --disable-vlm flag # These are set in main() after parsing arguments @@ -92,8 +115,8 @@ global_action = None ros_state = None # NOTE: fusion_threshold ๅทฒ็งป้™ค๏ผŒ็Žฐๅœจๅœจ src/exploration_manager ไธญ้…็ฝฎ -# ไฝฟ็”จๅ›บๅฎš็š„้ป˜่ฎคๆฃ€ๆต‹็ฝฎไฟกๅบฆ้˜ˆๅ€ผ -DEFAULT_DETECTION_CONFIDENCE = 0.5 +# ไฝฟ็”จๅ›บๅฎš็š„้ป˜่ฎคๆฃ€ๆต‹็ฝฎไฟกๅบฆ้˜ˆๅ€ผ (ไปŽ้…็ฝฎๆ–‡ไปถ่ฏปๅ–๏ผŒๅ›ž้€€ๅˆฐ 0.5) +DEFAULT_DETECTION_CONFIDENCE = CONFIG.get('object', {}).get('default_detection_confidence', 0.5) ros_pub = None trigger_pub = None obj_point_cloud_pub = None @@ -129,19 +152,22 @@ def publish_float32_array(publisher, data_list): publisher.publish(msg) -def load_llm_hypotheses(env_base_dir, dataset, episode_id): +def load_llm_hypotheses(env_base_dir, dataset, episode_id, max_hierarchy_level=4): """ - ๅŠ ่ฝฝLLMๅ‡่ฏดๅˆ†ๆž็ป“ๆžœ + ๅŠ ่ฝฝLLMๅ‡่ฏดๅˆ†ๆž็ป“ๆžœ๏ผŒๆ”ฏๆŒๅฑ‚็บง่ฟ‡ๆปค Args: - env_base_dir: ็Žฏๅขƒๆ•ฐๆฎๆ น็›ฎๅฝ• (ๅฆ‚ "env") + env_base_dir: ็Žฏๅขƒๆ•ฐๆฎๆ น็›ฎๅฝ• (ๅฆ‚ "env")๏ผŒๅฏไปŽ้…็ฝฎๆ–‡ไปถ่ฏปๅ– dataset: ๆ•ฐๆฎ้›†ๅ็งฐ (ๅฆ‚ "hm3dv2") episode_id: Episode ID + max_hierarchy_level: ๆœ€ๅคงๅฑ‚็บง (1-4)๏ผŒไปŽ Hydra ้…็ฝฎไผ ๅ…ฅ Returns: - List[dict]: ๅ‡่ฏดๅˆ—่กจ๏ผŒๅฆ‚ๆžœๆ–‡ไปถไธๅญ˜ๅœจๅˆ™่ฟ”ๅ›ž็ฉบๅˆ—่กจ + List[dict]: ๅ‡่ฏดๅˆ—่กจ๏ผˆๅทฒๆŒ‰ๅฑ‚็บง่ฟ‡ๆปค๏ผ‰๏ผŒๅฆ‚ๆžœๆ–‡ไปถไธๅญ˜ๅœจๅˆ™่ฟ”ๅ›ž็ฉบๅˆ—่กจ """ - hypothesis_file = Path(env_base_dir) / f"env_{dataset}" / f"task{episode_id}" / "llm_hypothesis_analysis.json" + # Get hypothesis file name from config (with fallback) + hypothesis_filename = CONFIG.get('paths', {}).get('llm', {}).get('hypothesis_file', 'llm_hypothesis_analysis.json') + hypothesis_file = Path(env_base_dir) / f"env_{dataset}" / f"task{episode_id}" / hypothesis_filename if hypothesis_file.exists(): try: @@ -149,8 +175,41 @@ def load_llm_hypotheses(env_base_dir, dataset, episode_id): data = json.load(f) hypotheses = data.get('semantic_hypotheses', []) - print(f"[LLM Hypothesis] Loaded {len(hypotheses)} hypotheses from {hypothesis_file}") - return hypotheses + + # ไฝฟ็”จไผ ๅ…ฅ็š„ๅฑ‚็บง่ฟ‡ๆปคๅ‚ๆ•ฐ + max_level = max_hierarchy_level + + # ๅฎšไน‰ๅฑ‚็บงๆ˜ ๅฐ„๏ผˆๆ นๆฎๆถˆ่žๅฎž้ชŒ้œ€ๆฑ‚ไฟฎๆญฃ๏ผ‰ + # L1: ไป…็›ฎๆ ‡็‰ฉไฝ“ + # L2: ็›ฎๆ ‡็‰ฉไฝ“ + ๆˆฟ้—ด็ฑปๅž‹ + # L3: ็›ฎๆ ‡็‰ฉไฝ“ + ๆˆฟ้—ด็ฑปๅž‹ + ๅ…ฑ็Žฐ็‰ฉไฝ“ + # L4: ็›ฎๆ ‡็‰ฉไฝ“ + ๆˆฟ้—ด็ฑปๅž‹ + ๅ…ฑ็Žฐ็‰ฉไฝ“ + ็‰ฉไฝ“้ƒจไปถ (Full) + level_map = { + "target_object": 1, # Level 1: ๆ ธๅฟƒ็›ฎๆ ‡ + "room_type": 2, # Level 2: ็ฉบ้—ดไธŠไธ‹ๆ–‡ + "co_occurrence": 3, # Level 3: ๅ…ฑ็Žฐ็‰ฉไฝ“็บฟ็ดข + "object_part": 4 # Level 4: ็ป†็ฒ’ๅบฆ้ƒจไปถ + } + + # ่ฟ‡ๆปคๅ‡่ฎพ๏ผšๅชไฟ็•™ level <= max_level ็š„ๅ‡่ฎพ + filtered_hypotheses = [ + h for h in hypotheses + if level_map.get(h.get('hypothesis_type', 'unknown'), 999) <= max_level + ] + + # ๆ‰“ๅฐ่ฟ‡ๆปคไฟกๆฏ + if len(filtered_hypotheses) < len(hypotheses): + print(f"[LLM Hypothesis] Hierarchy filtering: {len(hypotheses)} โ†’ {len(filtered_hypotheses)} " + f"hypotheses (max_level={max_level})") + # ๆ˜พ็คบ่ขซ่ฟ‡ๆปคๆŽ‰็š„็ฑปๅž‹ + filtered_types = set(h.get('hypothesis_type') for h in hypotheses) - \ + set(h.get('hypothesis_type') for h in filtered_hypotheses) + if filtered_types: + print(f"[LLM Hypothesis] Filtered out types: {', '.join(filtered_types)}") + else: + print(f"[LLM Hypothesis] Loaded {len(hypotheses)} hypotheses from {hypothesis_file} (max_level={max_level}, no filtering)") + + return filtered_hypotheses except Exception as e: print(f"[LLM Hypothesis] Failed to load {hypothesis_file}: {e}") @@ -160,6 +219,368 @@ def load_llm_hypotheses(env_base_dir, dataset, episode_id): return [] +# ============================================================ +# Real-time Panorama Collection and Hypothesis Generation +# ============================================================ +# These functions enable autonomous hypothesis generation at the start +# of each episode, replacing pre-computed hypotheses with real-time analysis. + +# Configuration constants for panorama collection +TURN_ANGLE = 30 # Turn angle per action (degrees) +TURNS_PER_90_DEG = 3 # Number of turns needed for 90 degrees (90/30=3) +VIEW_DIRECTIONS = {0: "front", 1: "left", 2: "back", 3: "right"} + + +def collect_panorama_views_realtime(env, observations) -> list: + """ + Collect 4 panorama views by rotating the agent in place. + + This function rotates the agent 90ยฐ three times to capture views at: + - View 0: Front (0ยฐ) + - View 1: Left (90ยฐ) + - View 2: Back (180ยฐ) + - View 3: Right (270ยฐ) + + Args: + env: Habitat environment + observations: Current observations dict + + Returns: + List of 4 RGB images (numpy arrays) + """ + views = [] + + # Capture front view (current position) + rgb_image = observations["rgb"].copy() + views.append(rgb_image) + print(f" [Panorama] Captured view 0 (front)") + + # Rotate and capture remaining 3 views + for view_idx in range(1, 4): + # Rotate left 90 degrees (3 turns ร— 30ยฐ) + for _ in range(TURNS_PER_90_DEG): + observations = env.step(HabitatSimActions.turn_left) + + rgb_image = observations["rgb"].copy() + views.append(rgb_image) + direction = VIEW_DIRECTIONS[view_idx] + print(f" [Panorama] Captured view {view_idx} ({direction})") + + # Rotate back to original orientation (one more 90ยฐ turn) + for _ in range(TURNS_PER_90_DEG): + observations = env.step(HabitatSimActions.turn_left) + + return views + + +def save_panorama_views(views: list, output_dir: Path, task_info: dict) -> bool: + """ + Save panorama views and task info to disk. + + Args: + views: List of 4 RGB images + output_dir: Directory to save views + task_info: Dictionary containing task_index, episode_id, target_object, scene_id + + Returns: + True if successful + """ + if not REALTIME_ANALYSIS_AVAILABLE: + print("[Error] cv2 not available for saving images") + return False + + output_dir.mkdir(parents=True, exist_ok=True) + + # Save each view as PNG + for i, rgb_image in enumerate(views): + view_path = output_dir / f"view{i}.png" + # Convert RGB to BGR for OpenCV + bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) + cv2.imwrite(str(view_path), bgr_image) + + # Save task_info.txt + info_path = output_dir / "task_info.txt" + with open(info_path, 'w', encoding='utf-8') as f: + f.write(f"Task Index: {task_info.get('task_index', 'N/A')}\n") + f.write(f"Episode ID: {task_info.get('episode_id', 'N/A')}\n") + f.write(f"Target Object: {task_info.get('target_object', 'unknown')}\n") + f.write(f"Scene ID: {task_info.get('scene_id', 'N/A')}\n") + + print(f" [Panorama] Saved 4 views and task_info to {output_dir}") + return True + + +def run_realtime_vlm_analysis(task_dir: Path, target_object: str, + vlm_config: dict) -> tuple: + """ + Run VLM environment analysis on collected panorama views. + + Args: + task_dir: Directory containing view*.png files + target_object: Target object name + vlm_config: VLM configuration dict with api_url, model, api_type, timeout + + Returns: + (success, vlm_content, error_msg) + """ + if not REALTIME_ANALYSIS_AVAILABLE: + return False, None, "VLMEnvironmentAnalyzer not available" + + try: + analyzer = VLMEnvironmentAnalyzer( + api_url=vlm_config.get('api_url', 'http://localhost:20004/api/chat'), + model=vlm_config.get('model', 'qwen3-vl:32b'), + env_base_dir=str(task_dir.parent.parent), # env directory + timeout=vlm_config.get('timeout', 300), + api_type=vlm_config.get('api_type', 'ollama') + ) + + # Extract env_name and task_num from task_dir path + # task_dir format: env/env_{dataset}/task{num} + env_name = task_dir.parent.name.replace('env_', '') + task_num = int(task_dir.name.replace('task', '')) + + success, content, duration, error_msg = analyzer.analyze_single( + env_name, task_num, + temperature=vlm_config.get('temperature', 0.2) + ) + + if success: + print(f" [VLM] Analysis completed in {duration:.2f}s") + return True, content, None + else: + return False, None, error_msg + + except Exception as e: + return False, None, str(e) + + +def run_realtime_llm_hypothesis(task_dir: Path, target_object: str, + llm_config: dict, generate_ig: bool = True) -> tuple: + """ + Run LLM hypothesis generation (HSVM + IG) based on VLM analysis. + + Args: + task_dir: Directory containing vlm_analysis.txt + target_object: Target object name + llm_config: LLM configuration dict with api_url, model, temperature + generate_ig: Whether to generate IG hypotheses + + Returns: + (success, hypotheses_list, ig_prompts_list, error_msg) + """ + if not REALTIME_ANALYSIS_AVAILABLE: + return False, [], [], "LLMHypothesisAnalyzer not available" + + try: + analyzer = LLMHypothesisAnalyzer( + api_url=llm_config.get('api_url', 'https://api.deepseek.com/chat/completions'), + model=llm_config.get('model', 'deepseek-chat'), + env_base_dir=str(task_dir.parent.parent) + ) + + # Extract env_name and task_num from task_dir path + env_name = task_dir.parent.name.replace('env_', '') + task_num = int(task_dir.name.replace('task', '')) + + # Run analysis with IG generation + if generate_ig: + success, analysis, error_msg = analyzer.analyze_single_with_ig( + env_name, task_num, + temperature=llm_config.get('temperature', 0.3), + generate_ig=True + ) + else: + success, analysis, error_msg = analyzer.analyze_single( + env_name, task_num, + temperature=llm_config.get('temperature', 0.3) + ) + + if success and analysis: + # Save JSON result + writer = AnalysisReportWriter() + json_file = task_dir / "llm_hypothesis_analysis.json" + writer.write_json(json_file, analysis) + print(f" [LLM] Hypotheses saved to {json_file}") + + # Extract hypotheses for return + hypotheses = [h.to_habitat_dict() for h in analysis.semantic_hypotheses] + + # Extract IG prompts + ig_prompts = [] + if analysis.ig_hypotheses: + ig_prompts = [h.prompt for h in analysis.ig_hypotheses if h.prompt] + + print(f" [LLM] Generated {len(hypotheses)} HSVM hypotheses, {len(ig_prompts)} IG prompts") + return True, hypotheses, ig_prompts, None + else: + return False, [], [], error_msg + + except Exception as e: + return False, [], [], str(e) + + +def generate_hypotheses_realtime(env, observations, dataset: str, task_index: int, + target_object: str, episode_id: str, scene_id: str, + env_base_dir: str, vlm_config: dict, llm_config: dict, + skip_existing: bool = True) -> tuple: + """ + Complete real-time hypothesis generation pipeline: + 1. Collect 4 panorama views by rotating the agent + 2. Run VLM environment analysis + 3. Generate HSVM + IG hypotheses + + Args: + env: Habitat environment + observations: Current observations + dataset: Dataset name (hm3dv2, mp3d, etc.) + task_index: Task index number + target_object: Target object name + episode_id: Episode ID + scene_id: Scene ID + env_base_dir: Base directory for env data + vlm_config: VLM configuration dict + llm_config: LLM configuration dict + skip_existing: Skip if hypotheses already exist + + Returns: + (observations, hypotheses_list, ig_prompts_list, success) + - observations: Updated observations after rotation + - hypotheses_list: List of HSVM hypothesis dicts + - ig_prompts_list: List of IG prompt strings + - success: Whether generation was successful + """ + task_dir = Path(env_base_dir) / f"env_{dataset}" / f"task{task_index}" + json_file = task_dir / "llm_hypothesis_analysis.json" + + # Check if hypotheses already exist + if skip_existing and json_file.exists(): + print(f"[Real-time Hypothesis] Existing hypotheses found at {json_file}, loading...") + # Load existing hypotheses + hypotheses = load_llm_hypotheses(env_base_dir, dataset, task_index, max_hierarchy_level=4) + ig_prompts = load_ig_hypotheses(env_base_dir, dataset, task_index) + return observations, hypotheses, ig_prompts, True + + print(f"\n{'='*60}") + print(f"[Real-time Hypothesis Generation] Task {task_index}") + print(f"{'='*60}") + print(f"Target: {target_object}") + print(f"Scene: {scene_id}") + + # Default fallback values + DEFAULT_IG_PROMPTS = [ + "This view shows a corridor or hallway leading to other areas", + "There is a doorway or opening that leads to another room", + "This passage connects to multiple rooms or spaces" + ] + + if not REALTIME_ANALYSIS_AVAILABLE: + print("[Warning] Real-time analysis not available, using defaults") + return observations, [], DEFAULT_IG_PROMPTS, False + + # Step 1: Collect panorama views + print(f"\n[Step 1/3] Collecting panorama views...") + views = collect_panorama_views_realtime(env, observations) + + # Save views + task_info = { + 'task_index': task_index, + 'episode_id': episode_id, + 'target_object': target_object, + 'scene_id': scene_id + } + save_panorama_views(views, task_dir, task_info) + + # Step 2: Run VLM analysis + print(f"\n[Step 2/3] Running VLM environment analysis...") + vlm_success, vlm_content, vlm_error = run_realtime_vlm_analysis( + task_dir, target_object, vlm_config + ) + + if not vlm_success: + print(f"[Warning] VLM analysis failed: {vlm_error}") + print("[Warning] Using default IG prompts") + return observations, [], DEFAULT_IG_PROMPTS, False + + # Step 3: Generate hypotheses + print(f"\n[Step 3/3] Generating HSVM + IG hypotheses...") + llm_success, hypotheses, ig_prompts, llm_error = run_realtime_llm_hypothesis( + task_dir, target_object, llm_config, generate_ig=True + ) + + if not llm_success: + print(f"[Warning] LLM hypothesis generation failed: {llm_error}") + print("[Warning] Using default IG prompts") + return observations, [], DEFAULT_IG_PROMPTS, False + + # Use default IG prompts if none were generated + if not ig_prompts: + print("[Warning] No IG prompts generated, using defaults") + ig_prompts = DEFAULT_IG_PROMPTS + + print(f"\n{'='*60}") + print(f"[Real-time Hypothesis] Complete!") + print(f" HSVM Hypotheses: {len(hypotheses)}") + print(f" IG Prompts: {len(ig_prompts)}") + print(f"{'='*60}\n") + + return observations, hypotheses, ig_prompts, True + + +def load_ig_hypotheses(env_base_dir, dataset, episode_id): + """ + ๅŠ ่ฝฝLLM็”Ÿๆˆ็š„IGๅ‡่ฎพ๏ผˆInformation Gainๅ‡่ฎพ๏ผ‰ + + IGๅ‡่ฎพๆ˜ฏ็›ฎๆ ‡ๆ— ๅ…ณ็š„๏ผŒ็”จไบŽไผฐ่ฎก่ง†ๅ›พ็š„ๆŽข็ดขๆฝœๅŠ›ใ€‚ + ไธŽHSVMๅ‡่ฎพไธๅŒ๏ผŒIGๅ‡่ฎพๆ่ฟฐ็ฉบ้—ด่ฟž้€šๆ€ง็ป“ๆž„๏ผˆ่ตฐๅปŠใ€้—จๆดžใ€้€š้“็ญ‰๏ผ‰ใ€‚ + + Args: + env_base_dir: ็Žฏๅขƒๆ•ฐๆฎๆ น็›ฎๅฝ• (ๅฆ‚ "env") + dataset: ๆ•ฐๆฎ้›†ๅ็งฐ (ๅฆ‚ "hm3dv2") + episode_id: Episode ID + + Returns: + List[str]: IG promptsๅˆ—่กจ๏ผŒๅฆ‚ๆžœๆ–‡ไปถไธๅญ˜ๅœจๅˆ™่ฟ”ๅ›ž้ป˜่ฎคprompts + """ + # ้ป˜่ฎค็š„IG prompts๏ผˆๅ›บๅฎš็‰ˆๆœฌ๏ผŒไฝœไธบfallback๏ผ‰ + DEFAULT_IG_PROMPTS = [ + "This view shows a corridor or hallway leading to other areas", + "There is a doorway or opening that leads to another room", + "This passage connects to multiple rooms or spaces" + ] + + # Get hypothesis file name from config (with fallback) + hypothesis_filename = CONFIG.get('paths', {}).get('llm', {}).get('hypothesis_file', 'llm_hypothesis_analysis.json') + hypothesis_file = Path(env_base_dir) / f"env_{dataset}" / f"task{episode_id}" / hypothesis_filename + + if hypothesis_file.exists(): + try: + with open(hypothesis_file, 'r', encoding='utf-8') as f: + data = json.load(f) + + ig_hypotheses = data.get('ig_hypotheses', []) + + if ig_hypotheses: + # ไปŽIGๅ‡่ฎพไธญๆๅ–prompts + ig_prompts = [h.get('prompt', '') for h in ig_hypotheses if h.get('prompt')] + if ig_prompts: + print(f"[IG Hypothesis] Loaded {len(ig_prompts)} dynamic IG prompts from {hypothesis_file}") + return ig_prompts + else: + print(f"[IG Hypothesis] No valid prompts in ig_hypotheses, using default") + return DEFAULT_IG_PROMPTS + else: + print(f"[IG Hypothesis] No ig_hypotheses found in {hypothesis_file}, using default") + return DEFAULT_IG_PROMPTS + + except Exception as e: + print(f"[IG Hypothesis] Failed to load {hypothesis_file}: {e}, using default") + return DEFAULT_IG_PROMPTS + else: + print(f"[IG Hypothesis] File not found: {hypothesis_file}, using default") + return DEFAULT_IG_PROMPTS + + def signal_handler(sig, frame): """Handle Ctrl+C signal for graceful shutdown""" print("Ctrl+C detected! Shutting down...") @@ -218,7 +639,9 @@ def _write_evaluation_stats(episode_stats_list, video_output_path): video_output_path: Path to video output directory (used to determine debug path) """ # Create debug directory if it doesn't exist - debug_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "debug") + # Keep each arm's summaries inside its own output directory. A global + # debug directory mixes concurrently or sequentially evaluated protocols. + debug_dir = os.path.join(video_output_path, "debug") os.makedirs(debug_dir, exist_ok=True) # Generate timestamp for filename @@ -277,9 +700,45 @@ def _write_evaluation_stats(episode_stats_list, video_output_path): f.write("=" * 80 + "\n") print(f"[Stats] Evaluation statistics written to: {stats_file}") + + # Also save JSON format for per-category analysis + json_file = os.path.join(debug_dir, f"evaluation_stats_{timestamp}.json") + with open(json_file, 'w', encoding='utf-8') as f: + json.dump(episode_stats_list, f, indent=2, ensure_ascii=False) + print(f"[Stats] JSON data written to: {json_file}") + return stats_file +def _append_episode_metrics(video_output_path, episode_stats): + """Persist one comparison-ready row, rejecting conflicting duplicates.""" + metrics_path = os.path.join(video_output_path, "episode_metrics.jsonl") + episode_key = ( + str(episode_stats["scene_id"]), + str(episode_stats["episode_id"]), + ) + if os.path.exists(metrics_path): + with open(metrics_path, "r", encoding="utf-8") as stream: + for line in stream: + if not line.strip(): + continue + existing = json.loads(line) + existing_key = ( + str(existing["scene_id"]), + str(existing["episode_id"]), + ) + if existing_key == episode_key: + if int(existing["episode_num"]) != int(episode_stats["episode_num"]): + raise RuntimeError( + f"episode key {episode_key} has conflicting sequence numbers" + ) + return + with open(metrics_path, "a", encoding="utf-8") as stream: + stream.write(json.dumps(episode_stats, ensure_ascii=False) + "\n") + stream.flush() + os.fsync(stream.fileno()) + + def _parse_dataset_arg(): """Parse CLI to choose dataset, detection mode, and capture remaining Hydra overrides.""" parser = argparse.ArgumentParser( @@ -308,29 +767,121 @@ def _parse_dataset_arg(): "When enabled, skips semantic scoring and IG estimation. " "Use this with exploration/policy=0 (distance greedy) for pure frontier exploration baseline.", ) + # Real-time hypothesis generation options + parser.add_argument( + "--realtime-hypothesis", + action="store_true", + default=False, + help="Enable real-time hypothesis generation. At the start of each episode, " + "the agent will: (1) collect 4 panorama views by rotating, " + "(2) run VLM environment analysis, (3) generate HSVM+IG hypotheses. " + "This replaces loading pre-computed hypotheses from disk.", + ) + parser.add_argument( + "--vlm-api-url", + type=str, + default="https://dashscope.aliyuncs.com/compatible-mode/v1", + help="VLM API URL for real-time analysis (default: DashScope API)", + ) + parser.add_argument( + "--vlm-model", + type=str, + default="qwen-vl-max", + help="VLM model name (default: qwen-vl-max for DashScope)", + ) + parser.add_argument( + "--vlm-api-type", + type=str, + choices=["ollama", "dashscope"], + default="dashscope", + help="VLM API type: ollama (local) or dashscope (Alibaba Cloud, default)", + ) + parser.add_argument( + "--llm-api-url", + type=str, + default="https://api.deepseek.com/chat/completions", + help="LLM API URL for hypothesis generation (default: DeepSeek API)", + ) + parser.add_argument( + "--llm-model", + type=str, + default="deepseek-chat", + help="LLM model name (default: deepseek-chat)", + ) + parser.add_argument( + "--skip-existing-hypothesis", + action="store_true", + default=True, + help="Skip hypothesis generation if JSON file already exists (default: True)", + ) + parser.add_argument( + "--force-regenerate", + action="store_true", + default=False, + help="Force regenerate hypotheses even if they exist", + ) + parser.add_argument( + "--control-rate-hz", + type=float, + default=10.0, + help="Habitat action/control loop frequency (default: 10 Hz)", + ) + parser.add_argument( + "--max-episodes-per-process", + type=int, + default=-1, + help="Exit cleanly after this many episodes; -1 runs all remaining episodes", + ) # Keep unknown so users can still pass Hydra-style overrides (e.g., key=value) args, unknown = parser.parse_known_args() - return args.dataset, args.detector_mode, args.disable_vlm, unknown + + # Build config dicts for real-time analysis + vlm_config = { + 'api_url': args.vlm_api_url, + 'model': args.vlm_model, + 'api_type': args.vlm_api_type, + 'timeout': 300, + 'temperature': 0.2 + } + llm_config = { + 'api_url': args.llm_api_url, + 'model': args.llm_model, + 'temperature': 0.3 + } + + return (args.dataset, args.detector_mode, args.disable_vlm, + args.realtime_hypothesis, vlm_config, llm_config, + not args.force_regenerate, args.control_rate_hz, + args.max_episodes_per_process, unknown) -def main(cfg: DictConfig, dataset: str = "hm3dv2", disable_vlm: bool = False) -> None: +def main(cfg: DictConfig, dataset: str = "hm3dv2", disable_vlm: bool = False, + realtime_hypothesis: bool = False, vlm_config: dict = None, + llm_config: dict = None, skip_existing: bool = True, + control_rate_hz: float = 10.0, + max_episodes_per_process: int = -1) -> None: global msg_observations, global_action, ros_state global ros_pub, trigger_pub, obj_point_cloud_pub, confidence_threshold_pub global final_state, expl_result global vlm_verify_count_this_episode - global get_multi_source_cosine_with_ig, VLM_DISABLED + global get_multi_source_cosine_with_ig, get_multi_source_cosine_with_dynamic_ig, VLM_DISABLED # Set VLM disabled flag VLM_DISABLED = disable_vlm # Conditionally import VLM module if not disable_vlm: - from vlm.utils.get_itm_message import get_multi_source_cosine_with_ig as _get_multi_source_cosine_with_ig + from vlm.utils.get_itm_message import ( + get_multi_source_cosine_with_ig as _get_multi_source_cosine_with_ig, + get_multi_source_cosine_with_dynamic_ig as _get_multi_source_cosine_with_dynamic_ig + ) get_multi_source_cosine_with_ig = _get_multi_source_cosine_with_ig - print("[VLM] BLIP2 ITM client initialized") + get_multi_source_cosine_with_dynamic_ig = _get_multi_source_cosine_with_dynamic_ig + print("[VLM] BLIP2 ITM client initialized (with dynamic IG support)") else: print("[VLM] DISABLED - Skipping BLIP2 initialization (ablation mode)") get_multi_source_cosine_with_ig = None + get_multi_source_cosine_with_dynamic_ig = None # Load MP3D validation data for object category mapping with gzip.open( @@ -425,6 +976,23 @@ def main(cfg: DictConfig, dataset: str = "hm3dv2", disable_vlm: bool = False) -> llm_client = llm_cfg.llm_client llm_answer_path = llm_cfg.llm_answer_path llm_response_path = llm_cfg.llm_response_path + # Hierarchical hypothesis level for ablation study (1-4) + max_hierarchy_level = llm_cfg.get('max_hierarchy_level', 4) + print(f"[LLM Hypothesis] Using max_hierarchy_level={max_hierarchy_level}") + + # ==================== Publish Exploration Parameters to ROS ==================== + # Override ROS launch defaults with Hydra config values (for ablation study) + # NOTE: Parameters must be set in the exploration_node's private namespace + # because launch file defines them inside tag + if hasattr(cfg, 'exploration'): + expl_cfg = cfg.exploration + if hasattr(expl_cfg, 'fusion_alpha'): + # Set in node's private namespace (where C++ reads from) + rospy.set_param('/exploration_node/exploration/fusion_alpha', float(expl_cfg.fusion_alpha)) + print(f"[Exploration] fusion_alpha={expl_cfg.fusion_alpha} (V_total = ฮฑ*V_sem + (1-ฮฑ)*V_ig)") + if hasattr(expl_cfg, 'use_ig_fusion'): + rospy.set_param('/exploration_node/exploration/use_ig_fusion', bool(expl_cfg.use_ig_fusion)) + print(f"[Exploration] use_ig_fusion={expl_cfg.use_ig_fusion}") # Single test parameters env_num_once = cfg.test_epi_num # Which episode to test for single run (deprecated) @@ -537,7 +1105,14 @@ def main(cfg: DictConfig, dataset: str = "hm3dv2", disable_vlm: bool = False) -> episode_stats_list = [] stats_written = False # Flag to track if stats have been written - for epi in range(number_of_episodes - num_total): + process_episode_count = number_of_episodes - num_total + if max_episodes_per_process > 0: + process_episode_count = min(process_episode_count, max_episodes_per_process) + print( + f"[Runtime] control_rate={control_rate_hz:g}Hz, " + f"episodes_this_process={process_episode_count}" + ) + for epi in range(process_episode_count): # Publish progress information publish_int32_array(progress_pub, [num_total, number_of_episodes]) @@ -591,12 +1166,53 @@ def main(cfg: DictConfig, dataset: str = "hm3dv2", disable_vlm: bool = False) -> llm_answer_path, llm_response_path, label, llm_client ) - # Load LLM semantic hypotheses for HSVM (Hierarchical Semantic Value Map) + # Load or generate LLM semantic hypotheses for HSVM (Hierarchical Semantic Value Map) # HSVM ๅทฒ็ปๅŒ…ๅซไบ†ๅคšๅฑ‚็บงๅ‡่ฏด: room_type, target_object, co_occurrence, part_attribute episode_id = env.current_episode.episode_id + scene_id = env.current_episode.scene_id task_index = num_total # Use task index instead of episode_id for hypothesis loading - current_hypotheses = load_llm_hypotheses("env", dataset, task_index) - print(f"[Task {task_index}, Episode {episode_id}] Target: {label}, Hypotheses: {len(current_hypotheses)}") + # Get env_base_dir from config (with fallback to "env") + env_base_dir = CONFIG.get('paths', {}).get('llm', {}).get('env_base_dir', 'env') + + # Real-time hypothesis generation mode + if realtime_hypothesis and REALTIME_ANALYSIS_AVAILABLE: + print(f"\n[Real-time Mode] Generating hypotheses for Task {task_index}...") + # Use generate_hypotheses_realtime which handles: + # 1. Panorama view collection (rotates agent) + # 2. VLM environment analysis + # 3. HSVM + IG hypothesis generation + observations, current_hypotheses, current_ig_prompts, gen_success = generate_hypotheses_realtime( + env=env, + observations=observations, + dataset=dataset, + task_index=task_index, + target_object=label, + episode_id=episode_id, + scene_id=scene_id, + env_base_dir=env_base_dir, + vlm_config=vlm_config or {}, + llm_config=llm_config or {}, + skip_existing=skip_existing + ) + + # Update msg_observations after rotation + observations["camera_pitch"] = camera_pitch + msg_observations = deepcopy(observations) + del observations["camera_pitch"] + + if not gen_success: + print("[Warning] Real-time hypothesis generation failed, falling back to file-based loading") + current_hypotheses = load_llm_hypotheses(env_base_dir, dataset, task_index, max_hierarchy_level) + current_ig_prompts = load_ig_hypotheses(env_base_dir, dataset, task_index) + else: + # Load from pre-computed files (original behavior) + current_hypotheses = load_llm_hypotheses(env_base_dir, dataset, task_index, max_hierarchy_level) + # Load IG hypotheses (target-agnostic connectivity hypotheses) + # IGๅ‡่ฎพ็”จไบŽไผฐ่ฎก่ง†ๅ›พ็š„ๆŽข็ดขๆฝœๅŠ›๏ผŒๆ่ฟฐ่ตฐๅปŠใ€้—จๆดžใ€้€š้“็ญ‰็ฉบ้—ด่ฟž้€šๆ€ง็ป“ๆž„ + current_ig_prompts = load_ig_hypotheses(env_base_dir, dataset, task_index) + + print(f"[Task {task_index}, Episode {episode_id}] Target: {label}, " + f"HSVM Hypotheses: {len(current_hypotheses)}, IG Prompts: {len(current_ig_prompts)}") # Initialize video frame collection vis_frames = [] @@ -636,7 +1252,7 @@ def publish_odom_and_confidence(event): odom_timer = rospy.Timer(rospy.Duration(0.2), publish_odom_and_confidence) - rate = rospy.Rate(10) + rate = rospy.Rate(control_rate_hz) while not rospy.is_shutdown() and not env.episode_over: # Skip episode if target is not on the same floor is_feasible = 0 @@ -699,12 +1315,12 @@ def publish_odom_and_confidence(event): t0 = time_module.time() if not VLM_DISABLED: - # Normal mode: run VLM inference - hypotheses_data, ig_data = get_multi_source_cosine_with_ig( - observations["rgb"], label, current_hypotheses + # Normal mode: run VLM inference with dynamic IG hypotheses + hypotheses_data, ig_data = get_multi_source_cosine_with_dynamic_ig( + observations["rgb"], label, current_hypotheses, current_ig_prompts ) timing_breakdown['vlm_inference'] = time_module.time() - t0 - timing_breakdown['vlm_num_prompts'] = len(hypotheses_data) + 3 # +3 for IG prompts + timing_breakdown['vlm_num_prompts'] = len(hypotheses_data) + len(current_ig_prompts) # Build and publish SemanticScores message t0 = time_module.time() @@ -725,12 +1341,20 @@ def publish_odom_and_confidence(event): semantic_scores_pub.publish(semantic_scores_msg) # Build and publish IG score message + # NOTE: ๆ”ฏๆŒๅŠจๆ€IG prompts๏ผŒๅ‘ๅŽๅ…ผๅฎนๅ›บๅฎš3ไธชๅญ—ๆฎต ig_score_msg = IGScore() ig_score_msg.header.stamp = rospy.Time.now() ig_score_msg.ig_score = ig_data["ig_score"] - ig_score_msg.corridor_score = ig_data["corridor_score"] - ig_score_msg.doorway_score = ig_data["doorway_score"] - ig_score_msg.passage_score = ig_data["passage_score"] + # ๅ‘ๅŽๅ…ผๅฎน๏ผšๅฆ‚ๆžœไฝฟ็”จๅŠจๆ€prompts๏ผŒ็”จๅนณๅ‡ๅˆ†ๅกซๅ……ๅ›บๅฎšๅญ—ๆฎต + if ig_data.get("is_dynamic", False): + ig_avg = ig_data["ig_score"] + ig_score_msg.corridor_score = ig_avg + ig_score_msg.doorway_score = ig_avg + ig_score_msg.passage_score = ig_avg + else: + ig_score_msg.corridor_score = ig_data.get("corridor_score", ig_data["ig_score"]) + ig_score_msg.doorway_score = ig_data.get("doorway_score", ig_data["ig_score"]) + ig_score_msg.passage_score = ig_data.get("passage_score", ig_data["ig_score"]) ig_score_pub.publish(ig_score_msg) timing_breakdown['semantic_msg_build'] = time_module.time() - t0 @@ -739,10 +1363,18 @@ def publish_odom_and_confidence(event): print(f"Multi-source semantic scores:") for i, h in enumerate(hypotheses_data): print(f" [{i}] {h['type']:15s} | Score: {h['score']:.3f} | Weight: {h['weight']:.3f} | {h['prompt'][:50]}") - print(f"Information Gain (IG) score: {ig_data['ig_score']:.3f} " - f"[corridor: {ig_data['corridor_score']:.3f}, " - f"doorway: {ig_data['doorway_score']:.3f}, " - f"passage: {ig_data['passage_score']:.3f}]") + # ๅŠจๆ€IGไฟกๆฏ + is_dynamic = ig_data.get("is_dynamic", False) + individual_scores = ig_data.get("individual_scores", []) + if is_dynamic and individual_scores: + scores_str = ", ".join(f"{s:.3f}" for s in individual_scores) + print(f"Information Gain (IG) score: {ig_data['ig_score']:.3f} " + f"[dynamic, {len(individual_scores)} prompts: {scores_str}]") + else: + print(f"Information Gain (IG) score: {ig_data['ig_score']:.3f} " + f"[corridor: {ig_data.get('corridor_score', 0):.3f}, " + f"doorway: {ig_data.get('doorway_score', 0):.3f}, " + f"passage: {ig_data.get('passage_score', 0):.3f}]") else: # Ablation mode: skip VLM inference entirely timing_breakdown['vlm_inference'] = 0.0 @@ -972,15 +1604,20 @@ def publish_odom_and_confidence(event): episode_stats = { "episode_num": num_total, "episode_id": episode_id, + "scene_id": scene_id, "target": label, + "target_category": env.current_episode.object_category, "steps": count_steps, "time_seconds": episode_time, "vlm_calls": vlm_verify_count_this_episode, "success": success, "spl": spl, + "soft_spl": soft_spl, + "distance_to_goal": distance_to_goal, "result": result_text, } episode_stats_list.append(episode_stats) + _append_episode_metrics(video_output_path, episode_stats) print(f"[Stats] Episode {num_total}: steps={count_steps}, time={episode_time:.1f}s, vlm_calls={vlm_verify_count_this_episode}, success={success}") # Write stats at 100 episodes milestone (but don't stop the test) @@ -1030,7 +1667,11 @@ def publish_odom_and_confidence(event): rospy.init_node("habitat_eval_node", anonymous=True) try: - dataset, detector_mode, disable_vlm, overrides = _parse_dataset_arg() + (dataset, detector_mode, disable_vlm, + realtime_hypothesis, vlm_config, llm_config, + skip_existing, control_rate_hz, + max_episodes_per_process, overrides) = _parse_dataset_arg() + cfg_name = f"habitat_eval_{dataset}" # Compose the chosen config and pass through extra Hydra overrides with initialize(version_base=None, config_path="config"): @@ -1049,7 +1690,18 @@ def publish_odom_and_confidence(event): if disable_vlm: print(f"[Ablation Mode] VLM (BLIP2) DISABLED - Pure frontier exploration baseline") - main(cfg, dataset, disable_vlm) + # Print real-time hypothesis mode status + if realtime_hypothesis: + print(f"\n{'='*60}") + print(f"[Real-time Hypothesis Mode] ENABLED") + print(f" VLM: {vlm_config['model']} @ {vlm_config['api_url']}") + print(f" LLM: {llm_config['model']} @ {llm_config['api_url']}") + print(f" Skip existing: {skip_existing}") + print(f"{'='*60}\n") + + main(cfg, dataset, disable_vlm, + realtime_hypothesis, vlm_config, llm_config, skip_existing, + control_rate_hz, max_episodes_per_process) except Exception as e: print(f"Unexpected error occurred: {e}") rospy.signal_shutdown("Shutdown due to error") diff --git a/habitat_manual_control.py b/habitat_manual_control.py index 8c6f391..af6cf9a 100644 --- a/habitat_manual_control.py +++ b/habitat_manual_control.py @@ -16,8 +16,6 @@ # Test specific episode python habitat_manual_control.py --dataset hm3dv2 test_epi_num=10 - -Author: Zager-Zhang """ # Standard library imports diff --git a/infonav_environment.yaml b/infonav_environment.yaml index f136e1e..9c8bf9a 100644 --- a/infonav_environment.yaml +++ b/infonav_environment.yaml @@ -1,4 +1,4 @@ -name: apexnav +name: infonav channels: - aihabitat - conda-forge diff --git a/llm/client/deepseek_answer.py b/llm/client/deepseek_answer.py index 7741fc8..cf30d1a 100644 --- a/llm/client/deepseek_answer.py +++ b/llm/client/deepseek_answer.py @@ -1,15 +1,24 @@ from llm.utils.get_sys_prompt import get_similar_answer_prompt from openai import OpenAI -import httpx - -# ๅˆ›ๅปบไธไฝฟ็”จไปฃ็†็š„ HTTP ๅฎขๆˆท็ซฏ -http_client = httpx.Client(proxy=None) import os -client = OpenAI( - api_key=os.getenv("DEEPSEEK_API_KEY", ""), - base_url="https://api.deepseek.com", - http_client=http_client -) + +# Temporarily disable proxy environment variables to avoid SOCKS proxy issues +old_proxies = {} +proxy_vars = ['http_proxy', 'https_proxy', 'HTTP_PROXY', 'HTTPS_PROXY', 'all_proxy', 'ALL_PROXY'] +for var in proxy_vars: + if var in os.environ: + old_proxies[var] = os.environ[var] + del os.environ[var] + +try: + client = OpenAI( + api_key=os.getenv("DEEPSEEK_API_KEY", ""), + base_url="https://api.deepseek.com" + ) +finally: + # Restore proxy environment variables + for var, value in old_proxies.items(): + os.environ[var] = value def deepseek_respond(prompt): system_prompts = get_similar_answer_prompt() diff --git a/llm/client/ollama_answer.py b/llm/client/ollama_answer.py index 412297b..4cec2f3 100644 --- a/llm/client/ollama_answer.py +++ b/llm/client/ollama_answer.py @@ -1,5 +1,21 @@ -from ollama import chat -from ollama import ChatResponse +import os + +# Temporarily disable proxy environment variables to avoid SOCKS proxy issues +old_proxies = {} +proxy_vars = ['http_proxy', 'https_proxy', 'HTTP_PROXY', 'HTTPS_PROXY', 'all_proxy', 'ALL_PROXY'] +for var in proxy_vars: + if var in os.environ: + old_proxies[var] = os.environ[var] + del os.environ[var] + +try: + from ollama import chat + from ollama import ChatResponse +finally: + # Restore proxy environment variables + for var, value in old_proxies.items(): + os.environ[var] = value + from llm.utils.get_sys_prompt import get_similar_answer_prompt def ollama_respond(model, prompt): diff --git a/llm/prompt/value_map_hypothesis.py b/llm/prompt/value_map_hypothesis.py index 2491072..7d87f7d 100644 --- a/llm/prompt/value_map_hypothesis.py +++ b/llm/prompt/value_map_hypothesis.py @@ -1,7 +1,7 @@ import json from typing import List, Dict, Any -# yangzhi_12_24: ๆ‹Ÿ้‡‡็”จ็š„LLMๅผ•ๅฏผ่ฏpromptๆจกๆฟ๏ผŒ้œ€่ฆ่พ“ๅ…ฅtarget object ่ฟ˜ๆœ‰vlm_perception +# LLM hypothesis prompt template, requires target_object and vlm_perception prompt = f""" You are an indoor navigation agent searching for **{target_object}**. @@ -40,17 +40,17 @@ ## Scoring: - ### Confidence (็ฝฎไฟกๅบฆ): 0.0-1.0 - How reliable is this hypothesis based on current evidence? + ### Confidence (C_l, ็ฝฎไฟกๅบฆ): 0.0-1.0 + Co-occurrence likelihood: How likely is target A given semantic cue B_l? P(A | B_l) - ### Navigation Value (ๅฏผ่ˆชไปทๅ€ผ): 0.0-1.0 - How large is the spatial scope this hypothesis guides to? + ### Spatial Footprint (a_l, ็ฉบ้—ดๅ ๆฏ”): 0.0-1.0 + Area ratio: Relative physical extent of this entity. Larger entities occupy more sampling space. | Score | Scope | Example | |-------|-------|---------| - | 0.8-1.0 | Room-level | "living room" - large area | - | 0.5-0.7 | Zone-level | "sofa area" - medium area | - | 0.2-0.4 | Point-level | "black screen" - specific spot | + | 0.8-1.0 | Room-level footprint | "living room" - large area | + | 0.5-0.7 | Zone-level footprint | "sofa area" - medium area | + | 0.2-0.4 | Point-level footprint | "black screen" - specific spot | --- @@ -73,7 +73,7 @@ "description": "Text description for CLIP matching", "reasoning": "Why this relates to {target_object}", "confidence": <0.0-1.0>, - "navigation_value": <0.0-1.0> + "spatial_footprint": <0.0-1.0> }} ], diff --git a/llm/prompt/vlm_environment_awareness.py b/llm/prompt/vlm_environment_awareness.py index 47e3b77..20b5b29 100644 --- a/llm/prompt/vlm_environment_awareness.py +++ b/llm/prompt/vlm_environment_awareness.py @@ -17,7 +17,7 @@ import json from typing import List, Dict, Any -# yangzhi_12_23: ๆ‹Ÿ้‡‡็”จ็š„promotๆจกๆฟ +# VLM environment awareness prompt template prompt = f""" You are searching for **{target_object}** in an indoor environment. Four images show the same location:Image 0=front, Image 1=left, Image 2=back, Image 3=right. diff --git a/llm/semantic_expansion_service.py b/llm/semantic_expansion_service.py index bb3a536..2976ca2 100644 --- a/llm/semantic_expansion_service.py +++ b/llm/semantic_expansion_service.py @@ -8,8 +8,6 @@ Service: /semantic_expansion Request: target_object (string) Response: prompts (string[]), weights (float64[]) - -Author: Zager-Zhang """ import rospy diff --git a/llm/utils/http_llm_client.py b/llm/utils/http_llm_client.py index c860cab..2b3d734 100644 --- a/llm/utils/http_llm_client.py +++ b/llm/utils/http_llm_client.py @@ -3,8 +3,6 @@ Connects to local LLM service at http://localhost:20006 Compatible with OpenAI chat completion format. - -Author: Zager-Zhang """ import requests diff --git a/llm/utils/semantic_prompt_expansion.py b/llm/utils/semantic_prompt_expansion.py index de06b95..bac3079 100644 --- a/llm/utils/semantic_prompt_expansion.py +++ b/llm/utils/semantic_prompt_expansion.py @@ -9,8 +9,6 @@ - Structural/layout priors Reference: Value_Map_Construction_and_Fusion.md - Semantic Prompt Expansion - -Author: Zager-Zhang """ from typing import List, Tuple, Dict diff --git a/pyproject.toml b/pyproject.toml index 9691cf4..04e186a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,12 +3,14 @@ requires = ["setuptools>=64"] build-backend = "setuptools.build_meta" [project] -name = "ApexNav" -version = "0.1" +name = "infonav" +version = "1.0.0" authors = [ - {name = "Zager-Zhang"}, + {name = "Zhi Yang"}, + {name = "Chengpu Yu"}, + {name = "Gonzalo Ferrer"}, ] -description = "ApexNav" +description = "InfoNav: unified semantic relevance and information gain for zero-shot ObjectNav" readme = "README.md" requires-python = ">=3.9" dependencies = [ @@ -23,15 +25,14 @@ dependencies = [ "prettytable >= 3.12.0", "opencv-python == 4.6.0.66", "moviepy == 1.0.3", - "mobile_sam @ git+https://github.com/ChaoningZhang/MobileSAM.git", - "groundingdino @ git+https://github.com/IDEA-Research/GroundingDINO.git", + "mobile_sam @ git+https://github.com/ChaoningZhang/MobileSAM.git@f706ad9c4eb7f219c00d9050e46328518ffb65d2", + "groundingdino @ git+https://github.com/IDEA-Research/GroundingDINO.git@856dde20aee659246248e20734ef9ba5214f5e44", "ultralytics==8.3.39", "fastapi==0.111.1", "uvicorn==0.30.3", "modelscope==1.16.1", "accelerate==0.32.1", "openai>=1.58.1", - "habitat2ros @ file:./habitat2ros", "ollama", ] @@ -40,4 +41,13 @@ dev = [ "pre-commit >= 3.1.1", "pytest >= 7.2.1", "pytest-cov >= 4.0.0", -] \ No newline at end of file +] + +[tool.setuptools.packages.find] +where = ["."] +include = ["basic_utils*", "habitat2ros*", "llm*", "scripts*", "vlm*"] +exclude = ["vlm.detector.D-FINE*", "vlm.detector.D-FINE.backup*"] +namespaces = true + +[tool.setuptools.package-data] +llm = ["answers/*.txt"] diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..f9ba4aa --- /dev/null +++ b/scripts/__init__.py @@ -0,0 +1,2 @@ +# Scripts package for InfoNav +# Contains analysis tools: vlm_environment_analyzer, llm_hypothesis_analyzer diff --git a/scripts/check_hypothesis_coverage.py b/scripts/check_hypothesis_coverage.py new file mode 100644 index 0000000..c0c6b60 --- /dev/null +++ b/scripts/check_hypothesis_coverage.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +"""Validate the offline per-episode hypotheses used by InfoNav evaluation.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Iterable + + +EXPECTED_TYPES = {"room_type", "target_object", "co_occurrence", "object_part"} + + +def compact_ranges(values: Iterable[int]) -> str: + numbers = sorted(set(values)) + if not numbers: + return "none" + + ranges: list[str] = [] + start = previous = numbers[0] + for number in numbers[1:]: + if number == previous + 1: + previous = number + continue + ranges.append(str(start) if start == previous else f"{start}-{previous}") + start = previous = number + ranges.append(str(start) if start == previous else f"{start}-{previous}") + return ",".join(ranges) + + +def validate_file(path: Path, require_ig: bool) -> list[str]: + try: + with path.open("r", encoding="utf-8") as stream: + data = json.load(stream) + except (OSError, json.JSONDecodeError) as exc: + return [f"unreadable JSON ({exc})"] + + hypotheses = data.get("semantic_hypotheses") + if not isinstance(hypotheses, list): + return ["semantic_hypotheses is not a list"] + + errors: list[str] = [] + types: list[str] = [] + for index, hypothesis in enumerate(hypotheses): + if not isinstance(hypothesis, dict): + errors.append(f"semantic_hypotheses[{index}] is not an object") + continue + hypothesis_type = hypothesis.get("type", hypothesis.get("hypothesis_type")) + types.append(hypothesis_type) + prompt = hypothesis.get("prompt", hypothesis.get("description")) + if not isinstance(prompt, str) or not prompt.strip(): + errors.append(f"semantic_hypotheses[{index}] has no prompt") + + if len(hypotheses) != 4: + errors.append(f"expected 4 semantic hypotheses, found {len(hypotheses)}") + if set(types) != EXPECTED_TYPES or len(types) != len(set(types)): + errors.append( + "semantic hypothesis types must be exactly " + + ",".join(sorted(EXPECTED_TYPES)) + ) + + if require_ig: + ig_hypotheses = data.get("ig_hypotheses") + if not isinstance(ig_hypotheses, list) or not ig_hypotheses: + errors.append("ig_hypotheses is missing or empty") + + return errors + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Check task0..taskN offline InfoNav hypothesis coverage" + ) + parser.add_argument( + "--env-dir", + type=Path, + default=Path("env/env_hm3dv2"), + help="directory containing task folders", + ) + parser.add_argument("--total", type=int, default=1000) + parser.add_argument( + "--require-ig", + action="store_true", + help="also require at least one information-gain hypothesis", + ) + args = parser.parse_args() + + if args.total <= 0: + parser.error("--total must be positive") + + missing: list[int] = [] + invalid: dict[int, list[str]] = {} + for task_index in range(args.total): + path = args.env_dir / f"task{task_index}" / "llm_hypothesis_analysis.json" + if not path.is_file(): + missing.append(task_index) + continue + errors = validate_file(path, args.require_ig) + if errors: + invalid[task_index] = errors + + valid = args.total - len(missing) - len(invalid) + print( + f"[hypotheses] valid={valid}/{args.total} " + f"missing={len(missing)} invalid={len(invalid)} dir={args.env_dir}" + ) + if missing: + print(f"[hypotheses] missing tasks: {compact_ranges(missing)}") + if invalid: + preview = list(sorted(invalid.items()))[:20] + for task_index, errors in preview: + print(f"[hypotheses] invalid task{task_index}: {'; '.join(errors)}") + if len(invalid) > len(preview): + print(f"[hypotheses] ... {len(invalid) - len(preview)} more invalid tasks") + + return 0 if valid == args.total else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/collect_panorama_views.py b/scripts/collect_panorama_views.py new file mode 100644 index 0000000..a515c7c --- /dev/null +++ b/scripts/collect_panorama_views.py @@ -0,0 +1,481 @@ +#!/usr/bin/env python3 +""" +ๅ…จๆ™ฏ่ง†ๅ›พ้‡‡้›†่„šๆœฌ +================ + +ๅœจ Habitat ็Žฏๅขƒไธญ่ฎฉๆ™บ่ƒฝไฝ“ๅŽŸๅœฐๆ—‹่ฝฌ้‡‡้›†ๅ››ๅผ ๅ…จๆ™ฏๅ›พ๏ผˆๆฏๅผ ้—ด้š”90ๅบฆ๏ผ‰ใ€‚ + +้‡‡้›†็š„ๅ›พ็‰‡ๅฐ†็”จไบŽ VLM ็Žฏๅขƒๅˆ†ๆž๏ผŒไธŽ vlm_environment_analyzer.py ้…ๅˆไฝฟ็”จใ€‚ + +ๅ‚ๆ•ฐ่ฏดๆ˜Ž: +--------- +- turn_angle: 30ๅบฆ (้…็ฝฎๆ–‡ไปถไธญๅฎšไน‰) +- hfov: 79ๅบฆ (็›ธๆœบๆฐดๅนณ่ง†้‡Ž) +- ๆฏ90ๅบฆ้‡‡้›†ไธ€ๅผ ๏ผŒๅ…ฑ4ๅผ ๅ›พ (0ยฐ, 90ยฐ, 180ยฐ, 270ยฐ) +- ้œ€่ฆๆ—‹่ฝฌ3ๆฌกๆ‰่ƒฝ่ฝฌ90ๅบฆ (30ร—3=90) + +่พ“ๅ‡บ: +----- +- view0.png: ๅ‰ๆ–น (0ยฐ, front) +- view1.png: ๅทฆๆ–น (90ยฐ, left) - ๅทฆ่ฝฌ90ๅบฆ +- view2.png: ๅŽๆ–น (180ยฐ, back) - ๅทฆ่ฝฌ180ๅบฆ +- view3.png: ๅณๆ–น (270ยฐ, right) - ๅทฆ่ฝฌ270ๅบฆ +- task_info.txt: ไปปๅŠกไฟกๆฏ + +ไฝฟ็”จ็คบไพ‹: +--------- +# ้‡‡้›†ๅ•ไธชไปปๅŠก็š„ๅ…จๆ™ฏๅ›พ +python scripts/collect_panorama_views.py --dataset hm3dv2 --task 0 + +# ้‡‡้›†ๅคšไธชไปปๅŠก +python scripts/collect_panorama_views.py --dataset hm3dv2 --start 0 --end 10 + +# ๆŒ‡ๅฎš่พ“ๅ‡บ็›ฎๅฝ• +python scripts/collect_panorama_views.py --dataset hm3dv2 --task 0 --output-dir test_output + +# ๅฏ่ง†ๅŒ–ๆจกๅผ๏ผˆๆ˜พ็คบ้‡‡้›†็š„ๅ›พ็‰‡๏ผ‰ +python scripts/collect_panorama_views.py --dataset hm3dv2 --task 0 --visualize + +""" + +import os +import sys +import argparse +import numpy as np +from pathlib import Path +from datetime import datetime + +# Add project root to path +sys.path.insert(0, str(Path(__file__).parent.parent)) + +try: + import habitat + from habitat.config.default import get_config + from habitat.core.env import Env + from omegaconf import DictConfig, OmegaConf + import cv2 +except ImportError as e: + print(f"Error: Missing dependency - {e}") + print("Please ensure habitat-sim and habitat-lab are installed.") + sys.exit(1) + + +# ==================== ้…็ฝฎ ==================== + +# ็›ธๆœบๅ‚ๆ•ฐ +HFOV = 79 # ๆฐดๅนณ่ง†้‡Ž่ง’ๅบฆ +TURN_ANGLE = 30 # ๆฏๆฌกๆ—‹่ฝฌ่ง’ๅบฆ +TURNS_PER_90_DEG = 3 # ่ฝฌ90ๅบฆ้œ€่ฆ็š„ๆฌกๆ•ฐ (90/30=3) + +# ่ง†ๅ›พๆ–นๅ‘ๆ˜ ๅฐ„ +VIEW_DIRECTIONS = { + 0: "front", # 0ยฐ + 1: "left", # 90ยฐ (ๅทฆ่ฝฌ) + 2: "back", # 180ยฐ + 3: "right" # 270ยฐ +} + +# ๆ•ฐๆฎ้›†้…็ฝฎ +DATASET_CONFIGS = { + "hm3dv1": "config/habitat_eval_hm3dv1.yaml", + "hm3dv2": "config/habitat_eval_hm3dv2.yaml", + "mp3d": "config/habitat_eval_mp3d.yaml" +} + + +# ==================== ๅทฅๅ…ทๅ‡ฝๆ•ฐ ==================== + +def load_config(dataset: str) -> DictConfig: + """ๅŠ ่ฝฝ Habitat ้…็ฝฎ (ไธŽ habitat_evaluation.py ไธ€่‡ด)""" + config_path = DATASET_CONFIGS.get(dataset) + if not config_path: + raise ValueError(f"Unknown dataset: {dataset}. Available: {list(DATASET_CONFIGS.keys())}") + + if not os.path.exists(config_path): + raise FileNotFoundError(f"Config file not found: {config_path}") + + # ไฝฟ็”จ Hydra ๅŠ ่ฝฝ้…็ฝฎ + from hydra import initialize_config_dir, compose + from hydra.core.global_hydra import GlobalHydra + from habitat.config.default import patch_config + + # ๆธ…็†ไน‹ๅ‰็š„ Hydra ๅฎžไพ‹ + GlobalHydra.instance().clear() + + config_dir = os.path.dirname(os.path.abspath(config_path)) + config_name = os.path.basename(config_path).replace(".yaml", "") + + with initialize_config_dir(config_dir=config_dir, version_base=None): + cfg = compose(config_name=config_name) + + # ไฝฟ็”จ patch_config ่กฅๅ…จๅฟ…่ฆ็š„้…็ฝฎๅญ—ๆฎต (ๅฆ‚ agents_order) + cfg = patch_config(cfg) + + return cfg + + +def save_image(image: np.ndarray, output_path: Path) -> None: + """ไฟๅญ˜ RGB ๅ›พๅƒไธบ PNG ๆ–‡ไปถ""" + # Habitat ่ฟ”ๅ›ž RGB๏ผŒOpenCV ้œ€่ฆ BGR + bgr_image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) + cv2.imwrite(str(output_path), bgr_image) + + +def save_task_info(output_dir: Path, episode, target_object: str, task_idx: int) -> None: + """ไฟๅญ˜ไปปๅŠกไฟกๆฏๅˆฐ task_info.txt (ไธŽ็Žฐๆœ‰ๆ ผๅผไธ€่‡ด)""" + info_path = output_dir / "task_info.txt" + + with open(info_path, 'w', encoding='utf-8') as f: + f.write(f"Task Index: {task_idx}\n") + f.write(f"Episode ID: {episode.episode_id}\n") + f.write(f"Target Object: {target_object}\n") + f.write(f"Scene ID: {episode.scene_id}\n") + + +def visualize_views(views: list, output_dir: Path) -> None: + """ๅฏ่ง†ๅŒ–้‡‡้›†็š„ๅ››ๅผ ๅ›พ็‰‡""" + import matplotlib.pyplot as plt + + fig, axes = plt.subplots(2, 2, figsize=(12, 12)) + fig.suptitle(f"Panorama Views - {output_dir.name}", fontsize=14) + + titles = ["View 0: Front (0ยฐ)", "View 1: Left (90ยฐ)", + "View 2: Back (180ยฐ)", "View 3: Right (270ยฐ)"] + + for idx, (ax, view, title) in enumerate(zip(axes.flat, views, titles)): + ax.imshow(view) + ax.set_title(title) + ax.axis('off') + + plt.tight_layout() + plt.savefig(output_dir / "panorama_overview.png", dpi=150) + plt.show() + + +# ==================== ็›ฎๆ ‡็‰ฉไฝ“ๆ˜ ๅฐ„ ==================== + +# HM3D/MP3D ็›ฎๆ ‡็‰ฉไฝ“ ID ๅˆฐๅ็งฐ็š„ๆ˜ ๅฐ„ +HM3D_CATEGORY_MAPPING = { + 0: "chair", + 1: "bed", + 2: "plant", + 3: "toilet", + 4: "tv_monitor", + 5: "sofa" +} + +MP3D_CATEGORY_MAPPING = { + # MP3D ๆœ‰ๆ›ดๅคš็ฑปๅˆซ๏ผŒ่ฟ™้‡Œๅˆ—ๅ‡บๅธธ่ง็š„ + 0: "chair", + 1: "table", + 2: "picture", + 3: "cabinet", + 4: "cushion", + 5: "sofa", + 6: "bed", + 7: "chest_of_drawers", + 8: "plant", + 9: "sink", + 10: "toilet", + 11: "stool", + 12: "towel", + 13: "tv_monitor", + 14: "shower", + 15: "bathtub", + 16: "counter", + 17: "fireplace", + 18: "gym_equipment", + 19: "seating", + 20: "clothes" +} + + +def get_target_object_name(episode, dataset: str) -> str: + """ไปŽ episode ่Žทๅ–็›ฎๆ ‡็‰ฉไฝ“ๅ็งฐ""" + # ่Žทๅ–็›ฎๆ ‡็‰ฉไฝ“็š„ category ID + if hasattr(episode, 'object_category'): + category = episode.object_category + elif hasattr(episode, 'goals') and len(episode.goals) > 0: + goal = episode.goals[0] + if hasattr(goal, 'object_category'): + category = goal.object_category + elif hasattr(goal, 'object_id'): + # ๅฐ่ฏ•ไปŽ object_id ๆๅ– + category = goal.object_id + else: + category = "unknown" + else: + category = "unknown" + + # ๆ นๆฎๆ•ฐๆฎ้›†้€‰ๆ‹ฉๆ˜ ๅฐ„ + if dataset.startswith("hm3d"): + mapping = HM3D_CATEGORY_MAPPING + else: + mapping = MP3D_CATEGORY_MAPPING + + # ๅฆ‚ๆžœๆ˜ฏๅญ—็ฌฆไธฒ๏ผŒ็›ดๆŽฅ่ฟ”ๅ›ž + if isinstance(category, str): + return category + + # ๅฆ‚ๆžœๆ˜ฏๆ•ฐๅญ—๏ผŒๆŸฅๆ‰พๆ˜ ๅฐ„ + return mapping.get(category, f"object_{category}") + + +# ==================== ไธป้‡‡้›†ๅ‡ฝๆ•ฐ ==================== + +def collect_panorama_views( + dataset: str, + task_idx: int, + output_dir: Path, + visualize: bool = False +) -> bool: + """ + ้‡‡้›†ๅ•ไธชไปปๅŠก็š„ๅ››ๅผ ๅ…จๆ™ฏๅ›พ + + Args: + dataset: ๆ•ฐๆฎ้›†ๅ็งฐ (hm3dv1, hm3dv2, mp3d) + task_idx: ไปปๅŠก็ดขๅผ• + output_dir: ่พ“ๅ‡บ็›ฎๅฝ• + visualize: ๆ˜ฏๅฆๅฏ่ง†ๅŒ– + + Returns: + bool: ๆ˜ฏๅฆๆˆๅŠŸ + """ + print(f"\n{'='*60}") + print(f"Collecting Panorama Views") + print(f"{'='*60}") + print(f"Dataset: {dataset}") + print(f"Task Index: {task_idx}") + print(f"Output Directory: {output_dir}") + print(f"Turn Angle: {TURN_ANGLE}ยฐ (3 turns = 90ยฐ)") + print(f"HFOV: {HFOV}ยฐ") + + try: + # 1. ๅŠ ่ฝฝ้…็ฝฎ + print(f"\n[1/5] Loading configuration...") + cfg = load_config(dataset) + + # 2. ๅˆ›ๅปบ็Žฏๅขƒ + print(f"[2/5] Creating Habitat environment...") + env = Env(config=cfg.habitat) + + # 3. ่ทณ่ฝฌๅˆฐๆŒ‡ๅฎšไปปๅŠก + print(f"[3/5] Navigating to task {task_idx}...") + observations = env.reset() + + # ่ทณ่ฟ‡ๅ‰้ข็š„ episodes ๅˆฐ่พพ็›ฎๆ ‡ไปปๅŠก + for _ in range(task_idx): + observations = env.reset() + + # ่Žทๅ–ๅฝ“ๅ‰ episode ไฟกๆฏ + episode = env.current_episode + target_object = get_target_object_name(episode, dataset) + + print(f" Episode ID: {episode.episode_id}") + print(f" Scene: {episode.scene_id}") + print(f" Target Object: {target_object}") + + # 4. ้‡‡้›†ๅ››ๅผ ๅ…จๆ™ฏๅ›พ + print(f"\n[4/5] Collecting 4 panorama views...") + + # ๅˆ›ๅปบ่พ“ๅ‡บ็›ฎๅฝ• + output_dir.mkdir(parents=True, exist_ok=True) + + views = [] + + for view_idx in range(4): + direction = VIEW_DIRECTIONS[view_idx] + + # ่Žทๅ–ๅฝ“ๅ‰ RGB ๅ›พๅƒ + rgb_image = observations["rgb"] + views.append(rgb_image.copy()) + + # ไฟๅญ˜ๅ›พๅƒ + view_path = output_dir / f"view{view_idx}.png" + save_image(rgb_image, view_path) + print(f" View {view_idx} ({direction}): saved to {view_path}") + + # ๅฆ‚ๆžœไธๆ˜ฏๆœ€ๅŽไธ€ๅผ ๏ผŒๅˆ™ๅทฆ่ฝฌ90ๅบฆ๏ผˆ3ๆฌกๅทฆ่ฝฌ๏ผ‰ + if view_idx < 3: + for _ in range(TURNS_PER_90_DEG): + observations = env.step({"action": "turn_left"}) + + # 5. ไฟๅญ˜ไปปๅŠกไฟกๆฏ + print(f"\n[5/5] Saving task info...") + save_task_info(output_dir, episode, target_object, task_idx) + print(f" Task info saved to {output_dir / 'task_info.txt'}") + + # ๅฏ่ง†ๅŒ– + if visualize: + print(f"\nVisualizing panorama views...") + visualize_views(views, output_dir) + + # ๅ…ณ้—ญ็Žฏๅขƒ + env.close() + + print(f"\n{'='*60}") + print(f"SUCCESS: Panorama views collected!") + print(f"Output: {output_dir}") + print(f"{'='*60}\n") + + return True + + except Exception as e: + print(f"\nERROR: {e}") + import traceback + traceback.print_exc() + return False + + +def collect_batch( + dataset: str, + start_task: int, + end_task: int, + output_base_dir: Path, + visualize: bool = False +) -> dict: + """ๆ‰น้‡้‡‡้›†ๅคšไธชไปปๅŠก็š„ๅ…จๆ™ฏๅ›พ""" + + print(f"\n{'#'*60}") + print(f"# Batch Panorama Collection") + print(f"# Dataset: {dataset}") + print(f"# Tasks: {start_task} to {end_task}") + print(f"# Output: {output_base_dir}") + print(f"{'#'*60}\n") + + stats = { + "total": end_task - start_task + 1, + "success": 0, + "failed": 0, + "failed_tasks": [] + } + + for task_idx in range(start_task, end_task + 1): + output_dir = output_base_dir / f"task{task_idx}" + + print(f"\n[Progress: {task_idx - start_task + 1}/{stats['total']}]") + + success = collect_panorama_views( + dataset=dataset, + task_idx=task_idx, + output_dir=output_dir, + visualize=visualize + ) + + if success: + stats["success"] += 1 + else: + stats["failed"] += 1 + stats["failed_tasks"].append(task_idx) + + # ๆ‰“ๅฐ็ปŸ่ฎก + print(f"\n{'#'*60}") + print(f"# Batch Collection Complete") + print(f"# Success: {stats['success']}/{stats['total']}") + print(f"# Failed: {stats['failed']}") + if stats["failed_tasks"]: + print(f"# Failed Tasks: {stats['failed_tasks']}") + print(f"{'#'*60}\n") + + return stats + + +# ==================== ๅ‘ฝไปค่กŒๆŽฅๅฃ ==================== + +def main(): + parser = argparse.ArgumentParser( + description="Collect panorama views from Habitat environment", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Single task + python scripts/collect_panorama_views.py --dataset hm3dv2 --task 0 + + # Multiple tasks + python scripts/collect_panorama_views.py --dataset hm3dv2 --start 0 --end 10 + + # With visualization + python scripts/collect_panorama_views.py --dataset hm3dv2 --task 0 --visualize + """ + ) + + parser.add_argument( + "--dataset", "-d", + type=str, + required=True, + choices=["hm3dv1", "hm3dv2", "mp3d"], + help="Dataset to use" + ) + + parser.add_argument( + "--task", "-t", + type=int, + default=None, + help="Single task index to collect" + ) + + parser.add_argument( + "--start", + type=int, + default=None, + help="Start task index for batch collection" + ) + + parser.add_argument( + "--end", + type=int, + default=None, + help="End task index for batch collection" + ) + + parser.add_argument( + "--output-dir", "-o", + type=str, + default=None, + help="Output directory (default: env/env_{dataset}/task{idx})" + ) + + parser.add_argument( + "--visualize", "-v", + action="store_true", + help="Visualize collected views" + ) + + args = parser.parse_args() + + # ็กฎๅฎš่พ“ๅ‡บ็›ฎๅฝ• + if args.output_dir: + output_base = Path(args.output_dir) + else: + output_base = Path("env") / f"env_{args.dataset}" + + # ๅ•ไปปๅŠกๆจกๅผ + if args.task is not None: + output_dir = output_base / f"task{args.task}" if args.output_dir is None else Path(args.output_dir) + success = collect_panorama_views( + dataset=args.dataset, + task_idx=args.task, + output_dir=output_dir, + visualize=args.visualize + ) + return 0 if success else 1 + + # ๆ‰น้‡ๆจกๅผ + elif args.start is not None and args.end is not None: + stats = collect_batch( + dataset=args.dataset, + start_task=args.start, + end_task=args.end, + output_base_dir=output_base, + visualize=args.visualize + ) + return 0 if stats["failed"] == 0 else 1 + + else: + parser.print_help() + print("\nError: Please specify --task or both --start and --end") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/compare_hm3dv2_value_wtrp.py b/scripts/compare_hm3dv2_value_wtrp.py new file mode 100755 index 0000000..0a98e31 --- /dev/null +++ b/scripts/compare_hm3dv2_value_wtrp.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Compare two same-ID HM3D episode_metrics.jsonl files.""" + +import argparse +import json +import math +from pathlib import Path + + +def load(path: Path): + rows = {} + with path.open("r", encoding="utf-8") as stream: + for line in stream: + if not line.strip(): + continue + row = json.loads(line) + key = (str(row["scene_id"]), str(row["episode_id"])) + if key in rows: + raise ValueError(f"duplicate episode key in {path}: {key}") + rows[key] = row + return rows + + +def mean(rows, key): + return sum(float(row[key]) for row in rows) / len(rows) if rows else math.nan + + +def exact_mcnemar_p(discordant_a, discordant_b): + """Two-sided exact McNemar p-value (binomial test, p=0.5).""" + total = discordant_a + discordant_b + if total == 0: + return 1.0 + lower = min(discordant_a, discordant_b) + tail = sum(math.comb(total, k) for k in range(lower + 1)) / (2 ** total) + return min(1.0, 2.0 * tail) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("value_greedy", type=Path) + parser.add_argument("wtrp", type=Path) + args = parser.parse_args() + + greedy = load(args.value_greedy) + wtrp = load(args.wtrp) + greedy_keys, wtrp_keys = set(greedy), set(wtrp) + if greedy_keys != wtrp_keys: + only_greedy = sorted(greedy_keys - wtrp_keys) + only_wtrp = sorted(wtrp_keys - greedy_keys) + raise SystemExit( + f"episode-ID mismatch: only_greedy={only_greedy[:5]}, only_wtrp={only_wtrp[:5]}" + ) + + keys = sorted(greedy_keys) + greedy_rows = [greedy[key] for key in keys] + wtrp_rows = [wtrp[key] for key in keys] + g_win = w_win = same = 0 + for key in keys: + g_success = int(greedy[key]["success"]) + w_success = int(wtrp[key]["success"]) + if g_success > w_success: + g_win += 1 + elif w_success > g_success: + w_win += 1 + else: + same += 1 + + print("HM3Dv2 value-greedy vs WTRP paired comparison") + print(f"paired episodes: {len(keys)}") + print( + f"value-greedy: SR={100 * mean(greedy_rows, 'success'):.2f}% " + f"SPL={100 * mean(greedy_rows, 'spl'):.2f}% " + f"SoftSPL={100 * mean(greedy_rows, 'soft_spl'):.2f}%" + ) + print( + f"WTRP: SR={100 * mean(wtrp_rows, 'success'):.2f}% " + f"SPL={100 * mean(wtrp_rows, 'spl'):.2f}% " + f"SoftSPL={100 * mean(wtrp_rows, 'soft_spl'):.2f}%" + ) + print(f"paired success outcomes: greedy-only={g_win}, WTRP-only={w_win}, same={same}") + print(f"exact McNemar p={exact_mcnemar_p(g_win, w_win):.6g}") + + +if __name__ == "__main__": + main() diff --git a/scripts/llm_hypothesis_analyzer.py b/scripts/llm_hypothesis_analyzer.py new file mode 100644 index 0000000..bf52728 --- /dev/null +++ b/scripts/llm_hypothesis_analyzer.py @@ -0,0 +1,1806 @@ +#!/usr/bin/env python3 +""" +LLM Semantic Hypothesis Analysis Tool (Unified Version) +======================================================== + +Integrates single-task analysis, batch analysis, and retry-missing-tasks functionality. +Calls LLM based on VLM analysis results to generate navigation hypotheses and directional guidance. +Output is used for multi-source Value Map construction. + +Modes: +------ +1. Single mode (single): Analyze a single task +2. Batch mode (batch): Batch analyze multiple tasks +3. Retry mode (retry): Retry missing or failed tasks + +Usage Examples: +--------------- +# Single task analysis +python llm_hypothesis_analyzer.py single --env hm3dv2 --task 0 +python llm_hypothesis_analyzer.py single --env hm3dv2 --task 0 --api-url http://localhost:11434/api/chat + +# Batch analysis +python llm_hypothesis_analyzer.py batch --env hm3dv2 +python llm_hypothesis_analyzer.py batch --env hm3dv2 --max-tasks 10 +python llm_hypothesis_analyzer.py batch --env hm3dv2 --start 5 --end 15 +python llm_hypothesis_analyzer.py batch --env hm3dv1 hm3dv2 --model qwen2:32b + +# Retry missing tasks +python llm_hypothesis_analyzer.py retry --env hm3dv2 # Auto-scan missing tasks +python llm_hypothesis_analyzer.py retry --env hm3dv2 --tasks 100 200 300 # Specify task list +python llm_hypothesis_analyzer.py retry --env hm3dv2 --start 100 --end 200 # Specify range + +Output Files: +------------- +- llm_hypothesis_analysis.json: Structured JSON result +- llm_hypothesis_analysis.txt: Human-readable text report +- batch_llm_hypothesis_summary.txt: Batch analysis summary (batch mode only) +- retry_log_YYYYMMDD_HHMMSS.json: Retry log (retry mode only) + +Dependencies: +------------- +- Requires VLM analysis (vlm_analysis.txt) to be generated first +- Requires LLM service (e.g., Ollama) running on specified port + +""" + +import json +import os +import requests +import time +import argparse +import sys +from pathlib import Path +from typing import Dict, List, Optional, Tuple +from dataclasses import dataclass, asdict +from datetime import datetime +import re + + +# ==================== Data Class Definitions ==================== + +@dataclass +class EnvironmentAnalysis: + """Environment Analysis""" + building_type: str # residential/office/museum/hospital/school/other + current_area: str + global_layout_inference: str + + +@dataclass +class TargetLocationReasoning: + """Target Location Reasoning""" + typical_locations: str + likely_path: str + key_landmarks: str + + +@dataclass +class SemanticHypothesis: + """ + Semantic Hypothesis - Corresponds to the 4-layer structure of HSVM (Hierarchical Semantic Value Map) + + hypothesis_type descriptions (corresponding to main.tex Section III-C): + - room_type: Room type layer (e.g., bedroom, kitchen) + - target_object: Target object layer (description of where target object may appear) + - co_occurrence: Co-occurrence object layer (objects that co-occur with target) + - object_part: Object part layer (parts/components of target object) + """ + id: int + hypothesis_type: str # room_type/target_object/co_occurrence/object_part + description: str # ็”จไบŽ CLIP/BLIP2 ๅŒน้…็š„ๆ–‡ๆœฌๆ่ฟฐ + reasoning: str + confidence: float # Co-occurrence Confidence C_l: P(A | B_l) (0.0-1.0) + navigation_value: float # Spatial Footprint a_l: area ratio (0.0-1.0), kept as navigation_value for backward compatibility + + @property + def weight(self) -> float: + """Hypothesis weight = C_l ร— a_l (confidence ร— spatial_footprint)""" + return self.confidence * self.navigation_value + + def to_habitat_dict(self) -> dict: + """ + Convert to habitat_evaluation.py compatible dictionary format + + Returns: + dict: Contains 'type', 'prompt', 'weight' and other compatible fields + """ + return { + 'id': self.id, + 'type': self.hypothesis_type, # habitat_evaluation expects 'type' not 'hypothesis_type' + 'hypothesis_type': self.hypothesis_type, # Keep for backward compatibility + 'prompt': self.description, # habitat_evaluation expects 'prompt' not 'description' + 'description': self.description, # Keep for backward compatibility + 'reasoning': self.reasoning, + 'confidence': self.confidence, + 'spatial_footprint': self.navigation_value, # New field name aligned with paper (a_l) + 'navigation_value': self.navigation_value, # Keep for backward compatibility + 'weight': self.weight # Pre-computed weight C_l ร— a_l for direct use by habitat_evaluation + } + + +@dataclass +class IGHypothesis: + """ + Information Gain Hypothesis - Corresponds to main.tex Section III-D / Appendix A.3 + + IG hypotheses differ from HSVM hypotheses: + - HSVM hypotheses: target-dependent (depend on target object) + - IG hypotheses: target-agnostic (independent of target, focus on connectivity structures) + + structure_type descriptions: + - corridor: Corridor/hallway + - doorway: Doorway + - passage: Passage + - archway: Archway + - opening: Opening + - other: Other + """ + id: int + prompt: str # Text description for BLIP2 matching (2-6 words) + structure_type: str # corridor/doorway/passage/archway/opening/other + reasoning: str + + def to_dict(self) -> dict: + """Convert to dictionary format""" + return { + 'id': self.id, + 'prompt': self.prompt, + 'structure_type': self.structure_type, + 'reasoning': self.reasoning + } + + +@dataclass +class IGAnalysis: + """IG Hypothesis Analysis Result""" + ig_hypotheses: List[IGHypothesis] + connectivity_summary: str + + def to_dict(self) -> dict: + """Convert to dictionary format""" + return { + 'ig_hypotheses': [h.to_dict() for h in self.ig_hypotheses], + 'connectivity_summary': self.connectivity_summary + } + + @classmethod + def from_json(cls, data: dict) -> 'IGAnalysis': + """Parse from JSON""" + hypotheses = [ + IGHypothesis( + id=h.get('id', i+1), + prompt=h.get('prompt', ''), + structure_type=h.get('structure_type', 'other'), + reasoning=h.get('reasoning', '') + ) + for i, h in enumerate(data.get('ig_hypotheses', [])) + ] + + return cls( + ig_hypotheses=hypotheses, + connectivity_summary=data.get('connectivity_summary', '') + ) + + +@dataclass +class NavigationAnalysis: + """Complete Navigation Analysis Result""" + environment_analysis: EnvironmentAnalysis + target_location_reasoning: TargetLocationReasoning + semantic_hypotheses: List[SemanticHypothesis] + exploration_strategy: Optional[str] + ig_hypotheses: Optional[List[IGHypothesis]] = None # IG hypotheses (optional) + connectivity_summary: Optional[str] = None # IG connectivity summary (optional) + + def to_dict(self) -> dict: + """ + Convert to dictionary (using habitat_evaluation compatible format) + + Output semantic_hypotheses contains the following compatible fields: + - type: alias for hypothesis_type (expected by habitat_evaluation) + - prompt: alias for description (expected by habitat_evaluation) + - weight: pre-computed confidence ร— navigation_value + """ + result = { + 'environment_analysis': asdict(self.environment_analysis), + 'target_location_reasoning': asdict(self.target_location_reasoning), + 'semantic_hypotheses': [h.to_habitat_dict() for h in self.semantic_hypotheses], + 'exploration_strategy': self.exploration_strategy + } + + # Add IG hypotheses (if present) + if self.ig_hypotheses: + result['ig_hypotheses'] = [h.to_dict() for h in self.ig_hypotheses] + if self.connectivity_summary: + result['connectivity_summary'] = self.connectivity_summary + + return result + + @classmethod + def from_json(cls, data: dict) -> 'NavigationAnalysis': + """Parse from JSON""" + # Parse environment analysis + env_data = data.get('environment_analysis', {}) + environment = EnvironmentAnalysis( + building_type=env_data.get('building_type', 'unknown'), + current_area=env_data.get('current_area', ''), + global_layout_inference=env_data.get('global_layout_inference', '') + ) + + # Parse target location reasoning + target_data = data.get('target_location_reasoning', {}) + target_reasoning = TargetLocationReasoning( + typical_locations=target_data.get('typical_locations', ''), + likely_path=target_data.get('likely_path', ''), + key_landmarks=target_data.get('key_landmarks', '') + ) + + # Parse HSVM hypothesis list + # Support both 'spatial_footprint' (new) and 'navigation_value' (legacy) field names + hypotheses = [ + SemanticHypothesis( + id=h.get('id', i+1), + hypothesis_type=h.get('hypothesis_type', 'unknown'), + description=h.get('description', ''), + reasoning=h.get('reasoning', ''), + confidence=h.get('confidence', 0.5), + navigation_value=h.get('spatial_footprint', h.get('navigation_value', 0.5)) + ) + for i, h in enumerate(data.get('semantic_hypotheses', [])) + ] + + # Parse IG hypothesis list (optional) + ig_hypotheses = None + if 'ig_hypotheses' in data: + ig_hypotheses = [ + IGHypothesis( + id=h.get('id', i+1), + prompt=h.get('prompt', ''), + structure_type=h.get('structure_type', 'other'), + reasoning=h.get('reasoning', '') + ) + for i, h in enumerate(data.get('ig_hypotheses', [])) + ] + + return cls( + environment_analysis=environment, + target_location_reasoning=target_reasoning, + semantic_hypotheses=hypotheses, + exploration_strategy=data.get('exploration_strategy'), + ig_hypotheses=ig_hypotheses, + connectivity_summary=data.get('connectivity_summary') + ) + + +# ==================== Target Object Configuration ==================== + +TARGET_CONFIGS = { + "tv": { + "part_examples": "black/dark screen, TV stand, thin rectangular display, remote control, cable box", + "cooccur_examples": "sofa, couch, coffee table, TV cabinet, entertainment center, speakers", + "room_examples": "living room, bedroom, family room" + }, + "toilet": { + "part_examples": "white porcelain bowl, toilet seat, flush handle/button, water tank", + "cooccur_examples": "sink, bathtub, shower, towel rack, toilet paper holder, bathroom tiles", + "room_examples": "bathroom, restroom" + }, + "bed": { + "part_examples": "mattress, pillows, bedframe, headboard, bedsheets, blanket, bed frame", + "cooccur_examples": "nightstand, dresser, wardrobe, bedside lamp, alarm clock, window", + "room_examples": "bedroom, guest room, master bedroom" + }, + "chair": { + "part_examples": "seat, backrest, legs, armrest, cushion, seat pad", + "cooccur_examples": "desk, table, computer, bookshelf, office", + "room_examples": "office, living room, dining room" + }, + "refrigerator": { + "part_examples": "large rectangular body, door handle, freezer compartment, stainless steel", + "cooccur_examples": "kitchen counter, stove, microwave, cabinets, sink", + "room_examples": "kitchen" + }, + "sofa": { + "part_examples": "cushions, armrests, backrest, fabric/leather surface, seat cushions", + "cooccur_examples": "coffee table, TV, rug, floor lamp, pillows", + "room_examples": "living room, family room" + }, + "couch": { + "part_examples": "cushions, armrests, backrest, fabric/leather surface, seat cushions", + "cooccur_examples": "coffee table, TV, rug, floor lamp, pillows", + "room_examples": "living room, family room" + }, + "sink": { + "part_examples": "basin, faucet, drain, countertop, white ceramic", + "cooccur_examples": "toilet, bathtub, shower, mirror, soap dispenser, towel rack", + "room_examples": "bathroom, kitchen" + }, + "table": { + "part_examples": "flat top surface, legs, wooden/metal frame", + "cooccur_examples": "chairs, place settings, decorations, lamp", + "room_examples": "dining room, living room, bedroom" + } +} + + +# ==================== LLM Prompt ==================== +# Corresponds to main.tex Section III-C: Hierarchical Semantic Value Map via Micro-Perception (HSVM) +# +# HSVM 4-layer structure: +# 1. Room Type: Current room type, largest spatial influence range +# 2. Target Object: Locations where target object may appear +# 3. Co-occurrence Objects: Objects that frequently appear with target object +# 4. Object Parts: Parts/components of target object, smallest spatial influence range +# +# Formula: V_l(p) = C_l ร— a_l ร— s_l^new(p), then V_HSVM(p) = (1/|H|) ร— ฮฃ V_l(p) +# - C_l: Co-occurrence Confidence P(A | B_l) (confidence field) +# - a_l: Spatial Footprint / Area Ratio (navigation_value field, renamed to spatial_footprint in prompt) + +# ==================== IG Hypothesis Prompt ==================== +# Corresponds to main.tex Section III-D / Appendix A.3: Information Gain Estimation +# +# IG hypotheses differ from HSVM hypotheses: +# - HSVM hypotheses: target-dependent (depend on target object) +# - IG hypotheses: target-agnostic (independent of target, focus on connectivity structures) +# +# IG hypotheses describe spatial connectivity structures (corridors, doorways, passages, openings) +# Used to estimate long-horizon information gain +# +# Important: Generated prompts must follow the style of these baseline prompts: +# - "This view shows a corridor or hallway leading to other areas" +# - "There is a doorway or opening that leads to another room" +# - "This passage connects to multiple rooms or spaces" + +IG_HYPOTHESIS_PROMPT = """Based on the following scene description, generate 3-5 COMPLETE SENTENCES describing spatial connectivity structures that may lead to unexplored areas. + +These sentences will be used for BLIP-2 Image-Text Matching to estimate exploration potential (Information Gain). + +## Scene Description (from VLM): +{vlm_perception} + +--- + +## CRITICAL: Prompt Format Requirements + +Your generated prompts MUST follow this exact sentence structure pattern (similar to our baseline prompts): + +**Baseline prompts for reference:** +1. "This view shows a corridor or hallway leading to other areas" +2. "There is a doorway or opening that leads to another room" +3. "This passage connects to multiple rooms or spaces" + +**Required sentence patterns:** +- "This view shows [connectivity structure] leading to [destination]" +- "There is [connectivity structure] that leads to [destination]" +- "This [structure] connects to [destination/other areas]" + +**Key requirements:** +1. Use COMPLETE SENTENCES (8-15 words), NOT short phrases +2. Start with "This view shows...", "There is...", or "This [structure]..." +3. Include connectivity verbs: "leading to", "leads to", "connects to", "opens to" +4. Describe WHERE the passage leads (other areas, rooms, spaces) +5. Focus on spatial transitions, NOT objects or room types + +## Output Format (MUST be valid JSON, no markdown): + +{{ + "ig_hypotheses": [ + {{ + "id": 1, + "prompt": "This view shows [structure] leading to [destination]", + "structure_type": "corridor/doorway/passage/archway/opening/other", + "reasoning": "why this suggests high exploration potential" + }}, + ... + ], + "connectivity_summary": "Brief summary of observed connectivity features" +}} + +--- + +## Good Examples (follow this style): +- "This view shows a corridor leading to private rooms" +- "There is a doorway that leads to the kitchen area" +- "This hallway connects to multiple bedrooms" +- "There is an opening that leads to the dining space" +- "This passage connects to unexplored living areas" + +## Bad Examples (DO NOT use this style): +- "corridor to kitchen" (too short, missing sentence structure) +- "bedroom" (this is a room type, not connectivity) +- "sofa area" (this is an object, not connectivity) +- "door" (too vague, missing destination) + +## Important Rules: +1. Generate 3-5 hypotheses describing CONNECTIVITY STRUCTURES only +2. Each prompt MUST be a COMPLETE SENTENCE (8-15 words) +3. Use patterns: "This view shows...", "There is...", "This [structure]..." +4. Include WHERE the passage leads to +5. Do NOT mention target objects (those belong to HSVM) +6. Output valid JSON only - NO markdown code blocks + +Generate your analysis: +""" + +NAVIGATION_HYPOTHESIS_PROMPT = """You are an expert indoor navigation planner helping a robot find **{target_object}**. + +## Current Scene Observation (from VLM): +{vlm_perception} + +--- + +## Your Task: Generate Hierarchical Semantic Hypotheses + +Based on the VLM scene description, generate **exactly 4 semantic hypotheses** at different hierarchical levels to help the robot find {target_object}. These hypotheses will be used to build a Hierarchical Semantic Value Map (HSVM) for navigation. + +**Key Knowledge about {target_object}:** +- Typical room locations: {room_examples} +- Often found near: {cooccur_examples} +- Visual features/parts: {part_examples} + +--- + +## REQUIRED Hypothesis Types (exactly 4, one for each level): + +### Level 1: Room Type - LARGEST spatial footprint +The type of room where {target_object} is typically found. +- Examples: "bedroom", "bathroom", "kitchen", "living room" +- This hypothesis has the LARGEST spatial propagation range +- Confidence: How likely is {target_object} to be in this room type? +- Spatial Footprint: Should be HIGH (0.8-1.0) as rooms cover large areas + +### Level 2: Target Object - MEDIUM-HIGH spatial footprint +The target object itself that the robot is searching for. +- This should be the {target_object} directly (e.g., "bed", "toilet", "TV") +- Confidence: MUST be 1.0 (this IS the target we are looking for) +- Spatial Footprint: MEDIUM-HIGH (0.6-0.8) + +### Level 3: Co-occurrence Objects - MEDIUM spatial footprint +Objects that frequently appear together with {target_object}. +- Examples for bed: "nightstand", "dresser", "wardrobe", "bedside lamp" +- Examples for toilet: "sink", "bathtub", "towel rack", "bathroom mirror" +- Seeing these objects suggests {target_object} is nearby +- Confidence: How often does this object appear with {target_object}? +- Spatial Footprint: MEDIUM (0.4-0.7) + +### Level 4: Object Parts - SMALLEST spatial footprint +Distinctive parts or features of {target_object} itself. +- Examples for bed: "mattress", "pillows", "headboard", "bed frame" +- Examples for toilet: "toilet seat", "flush handle", "porcelain bowl" +- Only visible when very close to the target +- Confidence: How distinctive is this part? +- Spatial Footprint: LOW (0.2-0.4) as parts indicate immediate proximity + +--- + +## Scoring Guidelines: + +### Confidence (C_l, 0.0-1.0): Co-occurrence likelihood +How likely is target A given semantic cue B_l? P(A | B_l) +- 0.8-1.0: Very strong association (e.g., beds are almost always in bedrooms) +- 0.5-0.7: Moderate association (e.g., nightstands are often near beds) +- 0.2-0.4: Weak but possible association + +### Spatial Footprint (a_l, 0.0-1.0): Area ratio / spatial influence range +Relative physical extent of this entity compared to others. Larger entities occupy more sampling space. +- 0.8-1.0: Room-level footprint (room_type covers large area) +- 0.6-0.8: Object-level footprint (target_object location) +- 0.4-0.6: Nearby object footprint (co_occurrence objects) +- 0.2-0.4: Small part footprint (object_part, immediate proximity) + +--- + +## Output Format (MUST be valid JSON, no markdown): + +{{ + "environment_analysis": {{ + "building_type": "residential/office/museum/hospital/school/other", + "current_area": "Brief description of where robot currently is", + "global_layout_inference": "Your reasoning about the overall building layout" + }}, + + "target_location_reasoning": {{ + "typical_locations": "Where {target_object} is usually found in this building type", + "likely_path": "How to navigate from current location to target", + "key_landmarks": "What to look for along the way" + }}, + + "semantic_hypotheses": [ + {{ + "id": 1, + "hypothesis_type": "room_type", + "description": "room name (2-3 words max)", + "reasoning": "why {target_object} would be in this room", + "confidence": 0.85, + "spatial_footprint": 0.9 + }}, + {{ + "id": 2, + "hypothesis_type": "target_object", + "description": "{target_object}", + "reasoning": "this is the target object we are searching for", + "confidence": 1.0, + "spatial_footprint": 0.7 + }}, + {{ + "id": 3, + "hypothesis_type": "co_occurrence", + "description": "co-occurring object name (2-4 words)", + "reasoning": "this object often appears with {target_object}", + "confidence": 0.65, + "spatial_footprint": 0.5 + }}, + {{ + "id": 4, + "hypothesis_type": "object_part", + "description": "part/feature name (2-3 words)", + "reasoning": "distinctive part of {target_object}", + "confidence": 0.6, + "spatial_footprint": 0.3 + }} + ], + + "exploration_strategy": "If target not found, describe systematic search approach" +}} + +--- + +## Important Rules: +1. Generate EXACTLY 4 hypotheses, one for each type (room_type, target_object, co_occurrence, object_part) +2. Descriptions must be SHORT (2-5 words) for effective BLIP2/CLIP matching +3. Navigation Value should DECREASE from Level 1 to Level 4 (room > target > co-occur > part) +4. Base your analysis on the VLM scene description provided +5. Output valid JSON only - NO markdown code blocks + +Generate your analysis: +""" + + +# ==================== Core Analysis Class ==================== + +class LLMHypothesisAnalyzer: + """LLM Hypothesis Analyzer - Supports Ollama and DeepSeek API""" + + # DeepSeek API Configuration + DEEPSEEK_API_KEY = os.getenv("DEEPSEEK_API_KEY", "") + DEEPSEEK_API_URL = "https://api.deepseek.com/chat/completions" + + def __init__(self, + api_url: str = "https://api.deepseek.com/chat/completions", + model: str = "deepseek-chat", + env_base_dir: str = "env", + api_key: Optional[str] = None): + """ + Args: + api_url: LLM API address + model: Model name (deepseek-chat, deepseek-reasoner, or ollama model name) + env_base_dir: Environment dataset root directory + api_key: API key (required for DeepSeek) + """ + self.api_url = api_url + self.model = model + self.env_base_dir = Path(env_base_dir) + self.api_key = api_key or self.DEEPSEEK_API_KEY + + # Auto-detect API type + self.is_deepseek = "deepseek" in api_url.lower() + + def load_vlm_analysis(self, vlm_file: Path) -> str: + """Load VLM analysis file""" + if not vlm_file.exists(): + raise FileNotFoundError(f"VLM analysis file not found: {vlm_file}") + + with open(vlm_file, 'r', encoding='utf-8') as f: + content = f.read() + + return content + + def build_prompt(self, target_object: str, vlm_perception: str) -> str: + """Build LLM prompt""" + target_lower = target_object.lower() + config = TARGET_CONFIGS.get(target_lower, { + "part_examples": f"distinctive parts of {target_object}", + "cooccur_examples": f"objects commonly found with {target_object}", + "room_examples": f"rooms where {target_object} is typically found" + }) + + return NAVIGATION_HYPOTHESIS_PROMPT.format( + target_object=target_object, + vlm_perception=vlm_perception, + **config + ) + + def call_llm(self, prompt: str, temperature: float = 0.3, timeout: int = 120) -> str: + """Call LLM to generate hypotheses - Supports DeepSeek and Ollama""" + + print(f" Calling LLM: {self.model}") + print(f" API: {self.api_url}") + print(f" Temperature: {temperature}") + + if self.is_deepseek: + return self._call_deepseek(prompt, temperature, timeout) + else: + return self._call_ollama(prompt, temperature, timeout) + + def _call_deepseek(self, prompt: str, temperature: float, timeout: int) -> str: + """Call DeepSeek API""" + headers = { + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json" + } + + payload = { + "model": self.model, + "messages": [ + { + "role": "system", + "content": "You are an expert indoor navigation planner. Always respond with valid JSON only, no markdown formatting." + }, + { + "role": "user", + "content": prompt + } + ], + "temperature": temperature, + "max_tokens": 2048, + "stream": False + } + + try: + resp = requests.post( + self.api_url, + headers=headers, + json=payload, + timeout=timeout + ) + + if resp.status_code != 200: + raise RuntimeError(f"DeepSeek API returned {resp.status_code}: {resp.text}") + + result = resp.json() + content = result.get('choices', [{}])[0].get('message', {}).get('content', '') + + if not content: + raise RuntimeError("Empty response from DeepSeek") + + return content + + except requests.Timeout: + raise TimeoutError(f"DeepSeek request timed out after {timeout}s") + except requests.RequestException as e: + raise RuntimeError(f"DeepSeek request failed: {e}") + + def _call_ollama(self, prompt: str, temperature: float, timeout: int) -> str: + """Call Ollama API""" + payload = { + "model": self.model, + "messages": [ + { + "role": "user", + "content": prompt + } + ], + "stream": False, + "options": { + "temperature": temperature, + "top_k": 40, + "top_p": 0.9 + } + } + + try: + resp = requests.post( + self.api_url, + json=payload, + timeout=timeout + ) + + if resp.status_code != 200: + raise RuntimeError(f"Ollama API returned {resp.status_code}: {resp.text}") + + result = resp.json() + content = result.get('message', {}).get('content', '') + + if not content: + raise RuntimeError("Empty response from Ollama") + + return content + + except requests.Timeout: + raise TimeoutError(f"Ollama request timed out after {timeout}s") + except requests.RequestException as e: + raise RuntimeError(f"LLM request failed: {e}") + + def parse_response(self, response: str) -> NavigationAnalysis: + """Parse LLM response into structured data""" + # Clean possible markdown code block markers + cleaned_response = response.strip() + if cleaned_response.startswith("```json"): + cleaned_response = cleaned_response[7:] + if cleaned_response.startswith("```"): + cleaned_response = cleaned_response[3:] + if cleaned_response.endswith("```"): + cleaned_response = cleaned_response[:-3] + cleaned_response = cleaned_response.strip() + + # Try to extract JSON + json_pattern = r'\{[\s\S]*\}' + matches = re.findall(json_pattern, cleaned_response) + + json_str = None + for match in matches: + try: + data = json.loads(match) + # Check required fields - new format + if 'semantic_hypotheses' in data: + json_str = match + break + except json.JSONDecodeError: + continue + + if not json_str: + # Last attempt: try full text + try: + data = json.loads(cleaned_response) + except json.JSONDecodeError: + raise ValueError(f"Failed to extract valid JSON from LLM response: {cleaned_response[:200]}...") + else: + data = json.loads(json_str) + + # Validate required fields - new format + required_fields = ['semantic_hypotheses'] + for field in required_fields: + if field not in data: + raise ValueError(f"Missing required field: {field}") + + return NavigationAnalysis.from_json(data) + + def get_target_object(self, task_dir: Path) -> Optional[str]: + """Get target object from task_info.txt""" + task_info_file = task_dir / "task_info.txt" + + if not task_info_file.exists(): + return None + + with open(task_info_file, 'r', encoding='utf-8') as f: + for line in f: + if 'target object' in line.lower(): + return line.split(':', 1)[1].strip() + + return None + + def analyze_single(self, + env_name: str, + task_num: int, + temperature: float = 0.3) -> Tuple[bool, Optional[NavigationAnalysis], Optional[str]]: + """ + Analyze a single task + + Args: + env_name: Environment name + task_num: Task number + temperature: Generation temperature (0-1) + + Returns: + (success, analysis, error_msg) + """ + task_dir = self.env_base_dir / f"env_{env_name}" / f"task{task_num}" + + print(f"\n{'='*70}") + print(f"LLM Hypothesis Analysis") + print(f"{'='*70}") + print(f"Task directory: {task_dir}") + + # Check task directory + if not task_dir.exists(): + return False, None, f"Task directory not found: {task_dir}" + + # Get target object + target_object = self.get_target_object(task_dir) + if not target_object: + return False, None, "Unable to get target object from task_info.txt" + + print(f"Target object: {target_object}") + + try: + # 1. Load VLM analysis + print(f"\n[1/4] Loading VLM analysis...") + vlm_file = task_dir / "vlm_analysis.txt" + vlm_perception = self.load_vlm_analysis(vlm_file) + print(f" Loaded VLM analysis ({len(vlm_perception)} chars)") + + # 2. Build Prompt + print(f"\n[2/4] Building analysis prompt...") + prompt = self.build_prompt(target_object, vlm_perception) + print(f" Prompt length: {len(prompt)} chars") + + # 3. Call LLM + print(f"\n[3/4] Calling LLM to generate hypotheses...") + t0 = time.time() + response = self.call_llm(prompt, temperature=temperature) + t1 = time.time() + print(f" Duration: {t1-t0:.2f}s") + print(f" Response length: {len(response)} chars") + + # 4. Parse response + print(f"\n[4/4] Parsing LLM response...") + analysis = self.parse_response(response) + print(f" Environment type: {analysis.environment_analysis.building_type}") + print(f" Generated hypotheses: {len(analysis.semantic_hypotheses)}") + + return True, analysis, None + + except Exception as e: + return False, None, str(e) + + def build_ig_prompt(self, vlm_perception: str) -> str: + """Build IG hypothesis generation prompt (target-agnostic)""" + return IG_HYPOTHESIS_PROMPT.format(vlm_perception=vlm_perception) + + def parse_ig_response(self, response: str) -> IGAnalysis: + """Parse IG hypothesis generation response""" + # Clean possible markdown code block markers + cleaned_response = response.strip() + if cleaned_response.startswith("```json"): + cleaned_response = cleaned_response[7:] + if cleaned_response.startswith("```"): + cleaned_response = cleaned_response[3:] + if cleaned_response.endswith("```"): + cleaned_response = cleaned_response[:-3] + cleaned_response = cleaned_response.strip() + + # ๅฐ่ฏ•ๆๅ–JSON + json_pattern = r'\{[\s\S]*\}' + matches = re.findall(json_pattern, cleaned_response) + + json_str = None + for match in matches: + try: + data = json.loads(match) + if 'ig_hypotheses' in data: + json_str = match + break + except json.JSONDecodeError: + continue + + if not json_str: + try: + data = json.loads(cleaned_response) + except json.JSONDecodeError: + raise ValueError(f"Failed to extract valid JSON from IG response: {cleaned_response[:200]}...") + else: + data = json.loads(json_str) + + # Validate required fields + if 'ig_hypotheses' not in data: + raise ValueError("Missing required field: ig_hypotheses") + + return IGAnalysis.from_json(data) + + def generate_ig_hypotheses(self, + vlm_perception: str, + temperature: float = 0.3) -> Tuple[bool, Optional[IGAnalysis], Optional[str]]: + """ + Generate IG hypotheses (independent of target object) + + According to paper Section III-D / Appendix A.3: + IG hypotheses are target-agnostic, focusing only on spatial connectivity structures + + Args: + vlm_perception: VLM environment analysis result + temperature: Generation temperature + + Returns: + (success, ig_analysis, error_msg) + """ + try: + print(f"\n[IG] Generating IG hypotheses...") + prompt = self.build_ig_prompt(vlm_perception) + print(f" IG Prompt length: {len(prompt)} chars") + + t0 = time.time() + response = self.call_llm(prompt, temperature=temperature) + t1 = time.time() + print(f" IG LLM duration: {t1-t0:.2f}s") + print(f" Response length: {len(response)} chars") + + ig_analysis = self.parse_ig_response(response) + print(f" Generated IG hypotheses: {len(ig_analysis.ig_hypotheses)}") + + return True, ig_analysis, None + + except Exception as e: + return False, None, str(e) + + def analyze_single_with_ig(self, + env_name: str, + task_num: int, + temperature: float = 0.3, + generate_ig: bool = True) -> Tuple[bool, Optional[NavigationAnalysis], Optional[str]]: + """ + Analyze a single task, including HSVM and IG hypothesis generation + + Args: + env_name: Environment name + task_num: Task number + temperature: Generation temperature (0-1) + generate_ig: Whether to generate IG hypotheses + + Returns: + (success, analysis, error_msg) + """ + task_dir = self.env_base_dir / f"env_{env_name}" / f"task{task_num}" + + print(f"\n{'='*70}") + print(f"LLM Hypothesis Analysis (HSVM + IG)") + print(f"{'='*70}") + print(f"Task directory: {task_dir}") + + # Check task directory + if not task_dir.exists(): + return False, None, f"Task directory not found: {task_dir}" + + # Get target object + target_object = self.get_target_object(task_dir) + if not target_object: + return False, None, "Unable to get target object from task_info.txt" + + print(f"Target object: {target_object}") + + try: + # 1. Load VLM analysis + print(f"\n[1/5] Loading VLM analysis...") + vlm_file = task_dir / "vlm_analysis.txt" + vlm_perception = self.load_vlm_analysis(vlm_file) + print(f" Loaded VLM analysis ({len(vlm_perception)} chars)") + + # 2. Build HSVM Prompt + print(f"\n[2/5] Building HSVM analysis prompt...") + prompt = self.build_prompt(target_object, vlm_perception) + print(f" Prompt length: {len(prompt)} chars") + + # 3. Call LLM to generate HSVM hypotheses + print(f"\n[3/5] Calling LLM to generate HSVM hypotheses...") + t0 = time.time() + response = self.call_llm(prompt, temperature=temperature) + t1 = time.time() + print(f" Duration: {t1-t0:.2f}s") + print(f" Response length: {len(response)} chars") + + # 4. Parse HSVM response + print(f"\n[4/5] Parsing HSVM response...") + analysis = self.parse_response(response) + print(f" Environment type: {analysis.environment_analysis.building_type}") + print(f" Generated HSVM hypotheses: {len(analysis.semantic_hypotheses)}") + + # 5. Generate IG hypotheses (optional) + if generate_ig: + print(f"\n[5/5] Generating IG hypotheses...") + ig_success, ig_analysis, ig_error = self.generate_ig_hypotheses( + vlm_perception, temperature=temperature + ) + + if ig_success and ig_analysis: + # Merge IG hypotheses into NavigationAnalysis + analysis.ig_hypotheses = ig_analysis.ig_hypotheses + analysis.connectivity_summary = ig_analysis.connectivity_summary + print(f" IG hypotheses merged: {len(ig_analysis.ig_hypotheses)}") + else: + print(f" IG hypothesis generation failed: {ig_error}") + # Does not affect HSVM hypotheses, continue returning + else: + print(f"\n[5/5] Skipping IG hypothesis generation") + + return True, analysis, None + + except Exception as e: + return False, None, str(e) + + +# ==================== Output Formatting ==================== + +class AnalysisReportWriter: + """Analysis Result Report Writer""" + + @staticmethod + def write_json(output_file: Path, analysis: NavigationAnalysis) -> None: + """Write JSON format result""" + with open(output_file, 'w', encoding='utf-8') as f: + json.dump(analysis.to_dict(), f, indent=2, ensure_ascii=False) + + @staticmethod + def write_text_report(output_file: Path, analysis: NavigationAnalysis) -> None: + """Write text format report""" + with open(output_file, 'w', encoding='utf-8') as f: + f.write("="*70 + "\n") + f.write("Global Navigation Hypothesis Analysis Report\n") + f.write("="*70 + "\n\n") + + # Environment Analysis + f.write("[Environment Analysis]\n") + env = analysis.environment_analysis + f.write(f"Building Type: {env.building_type}\n") + f.write(f"Current Location: {env.current_area}\n") + f.write(f"Global Layout Inference: {env.global_layout_inference}\n\n") + + # Target Location Reasoning + f.write("[Target Location Reasoning]\n") + target = analysis.target_location_reasoning + f.write(f"Typical Locations: {target.typical_locations}\n") + f.write(f"Likely Path: {target.likely_path}\n") + f.write(f"Key Landmarks: {target.key_landmarks}\n\n") + + # Semantic Hypotheses + f.write("[Semantic Hypotheses]\n") + f.write(f"{'ID':<3} {'Type':<18} {'Description':<25} {'C_l':<6} {'a_l':<6} {'Weight':<8}\n") + f.write("-"*70 + "\n") + + # HSVM 4-layer structure type mapping (corresponding to main.tex Section III-C) + type_map = { + # New format (HSVM 4-layer structure) + 'room_type': 'L1-Room Type', + 'target_object': 'L2-Target Object', + 'co_occurrence': 'L3-Co-occurrence', + 'object_part': 'L4-Object Part', + # Legacy format compatibility + 'building_layout': 'Building Layout', + 'target_room': 'Target Room', + 'adjacent_area': 'Adjacent Area', + 'indicator_object': 'Indicator Object', + 'target_feature': 'Target Feature', + 'spatial_navigation': 'Spatial Navigation', + 'room_context': 'Room Context', + 'part_attribute': 'Part Attribute' + } + + for h in analysis.semantic_hypotheses: + type_str = type_map.get(h.hypothesis_type, h.hypothesis_type) + desc = h.description[:25].ljust(25) + f.write(f"{h.id:<3} {type_str:<18} {desc} " + f"{h.confidence:<6.2f} {h.navigation_value:<6.2f} {h.weight:<8.4f}\n") + f.write(f" Reasoning: {h.reasoning}\n") + + f.write("\n") + + # IG Hypotheses (if present) + if analysis.ig_hypotheses: + f.write("[IG Hypotheses (Information Gain)]\n") + f.write(f"{'ID':<3} {'Type':<12} {'Prompt':<40}\n") + f.write("-"*70 + "\n") + + for h in analysis.ig_hypotheses: + prompt_str = h.prompt[:40].ljust(40) + f.write(f"{h.id:<3} {h.structure_type:<12} {prompt_str}\n") + f.write(f" Reasoning: {h.reasoning}\n") + + f.write("\n") + + if analysis.connectivity_summary: + f.write(f"Connectivity Summary: {analysis.connectivity_summary}\n\n") + + # Exploration Strategy + if analysis.exploration_strategy: + f.write("[Exploration Strategy]\n") + f.write(analysis.exploration_strategy + "\n\n") + + # Statistics + f.write("[Statistics]\n") + if analysis.semantic_hypotheses: + avg_conf = sum(h.confidence for h in analysis.semantic_hypotheses) / len(analysis.semantic_hypotheses) + avg_nav = sum(h.navigation_value for h in analysis.semantic_hypotheses) / len(analysis.semantic_hypotheses) + avg_weight = sum(h.weight for h in analysis.semantic_hypotheses) / len(analysis.semantic_hypotheses) + f.write(f"HSVM Hypothesis Count: {len(analysis.semantic_hypotheses)}\n") + f.write(f"Average Confidence (C_l): {avg_conf:.3f}\n") + f.write(f"Average Spatial Footprint (a_l): {avg_nav:.3f}\n") + f.write(f"Average Weight (C_l ร— a_l): {avg_weight:.4f}\n") + + if analysis.ig_hypotheses: + f.write(f"IG Hypothesis Count: {len(analysis.ig_hypotheses)}\n") + + f.write("\n" + "="*70 + "\n") + + +# ==================== Batch Processing Class ==================== + +class BatchAnalyzer: + """Batch Analyzer""" + + def __init__(self, analyzer: LLMHypothesisAnalyzer, delay: int = 10): + """ + Args: + analyzer: LLM hypothesis analyzer instance + delay: Delay between tasks (seconds) + """ + self.analyzer = analyzer + self.delay = delay + + def discover_tasks(self, env_name: str) -> List[int]: + """Discover all tasks with VLM analysis in the specified environment""" + env_dir = self.analyzer.env_base_dir / f"env_{env_name}" + + if not env_dir.exists(): + print(f"Warning: Environment directory not found: {env_dir}") + return [] + + tasks = [] + for task_dir in sorted(env_dir.iterdir()): + if task_dir.is_dir() and task_dir.name.startswith('task'): + try: + task_num = int(task_dir.name[4:]) + # Check if VLM analysis exists + if (task_dir / "vlm_analysis.txt").exists(): + tasks.append(task_num) + except ValueError: + continue + + return sorted(tasks) + + def analyze_batch(self, + env_name: str, + start_task: Optional[int] = None, + end_task: Optional[int] = None, + max_tasks: Optional[int] = None, + temperature: float = 0.3, + output_format: str = 'both', + generate_ig: bool = True) -> Dict: + """ + Batch analyze environment dataset + + Args: + env_name: Environment name + start_task: Start task number (inclusive) + end_task: End task number (inclusive) + max_tasks: Maximum number of tasks limit + temperature: Generation temperature + output_format: Output format ('json', 'text', 'both') + generate_ig: Whether to generate IG hypotheses (default True) + + Returns: + Statistics dictionary + """ + print(f"\n{'#'*70}") + print(f"# Batch Hypothesis Analysis: env_{env_name}") + print(f"# LLM Model: {self.analyzer.model}") + print(f"# API: {self.analyzer.api_url}") + print(f"# Generate IG Hypotheses: {generate_ig}") + print(f"# Task Delay: {self.delay} seconds") + print(f"# Start Time: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") + print(f"{'#'*70}\n") + + # Discover all tasks + all_tasks = self.discover_tasks(env_name) + + if not all_tasks: + print(f"Error: No tasks with VLM analysis found in env_{env_name}") + return None + + print(f"Found {len(all_tasks)} tasks") + + # Apply task range filtering + if start_task is not None: + all_tasks = [t for t in all_tasks if t >= start_task] + if end_task is not None: + all_tasks = [t for t in all_tasks if t <= end_task] + if max_tasks is not None: + all_tasks = all_tasks[:max_tasks] + + print(f"Will analyze {len(all_tasks)} tasks: {all_tasks[:10]}{'...' if len(all_tasks) > 10 else ''}\n") + + # Statistics + stats = { + 'total': len(all_tasks), + 'success': 0, + 'failed': 0, + 'total_time': 0, + 'results': [] + } + + writer = AnalysisReportWriter() + + # Analyze tasks one by one + for idx, task_num in enumerate(all_tasks, 1): + print(f"\nProgress: [{idx}/{len(all_tasks)}]") + + t0 = time.time() + # Use analysis method with IG + if generate_ig: + success, analysis, error_msg = self.analyzer.analyze_single_with_ig( + env_name, task_num, temperature=temperature, generate_ig=True + ) + else: + success, analysis, error_msg = self.analyzer.analyze_single( + env_name, task_num, temperature=temperature + ) + duration = time.time() - t0 + + result_file = None + if success and analysis: + task_dir = self.analyzer.env_base_dir / f"env_{env_name}" / f"task{task_num}" + + if output_format in ['json', 'both']: + json_file = task_dir / "llm_hypothesis_analysis.json" + writer.write_json(json_file, analysis) + result_file = str(json_file) + print(f" JSON result: {json_file}") + + if output_format in ['text', 'both']: + text_file = task_dir / "llm_hypothesis_analysis.txt" + writer.write_text_report(text_file, analysis) + print(f" Text report: {text_file}") + + stats['results'].append({ + 'task': task_num, + 'success': success, + 'duration': duration, + 'error': error_msg, + 'result_file': result_file + }) + + stats['total_time'] += duration + + if success: + stats['success'] += 1 + print(f" Success (duration: {duration:.2f}s)") + else: + stats['failed'] += 1 + print(f" Failed: {error_msg}") + + # Wait for specified time if not last task + if idx < len(all_tasks): + print(f"\nWaiting {self.delay} seconds before continuing...") + for remaining in range(self.delay, 0, -1): + print(f" Remaining: {remaining} seconds", end='\r') + time.sleep(1) + print() + + return stats + + def save_summary(self, env_name: str, stats: Dict) -> None: + """Save batch analysis summary""" + summary_file = self.analyzer.env_base_dir / f"env_{env_name}" / "batch_llm_hypothesis_summary.txt" + + with open(summary_file, 'w', encoding='utf-8') as f: + f.write(f"Batch LLM Hypothesis Analysis Summary: env_{env_name}\n") + f.write(f"{'='*70}\n\n") + f.write(f"Analysis Time: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n") + f.write(f"LLM Model: {self.analyzer.model}\n") + f.write(f"Total Tasks: {stats['total']}\n") + f.write(f"Success: {stats['success']}\n") + f.write(f"Failed: {stats['failed']}\n") + f.write(f"Success Rate: {stats['success']/stats['total']*100:.2f}%\n") + f.write(f"Total Time: {stats['total_time']:.2f} seconds ({stats['total_time']/60:.2f} minutes)\n") + f.write(f"Average Time: {stats['total_time']/stats['total']:.2f} seconds/task\n\n") + + f.write(f"Detailed Results:\n") + f.write(f"{'-'*70}\n") + + for result in stats['results']: + status = "OK" if result['success'] else "FAIL" + f.write(f"[{status}] task{result['task']}: {result['duration']:.2f}s") + if result['error']: + f.write(f" - {result['error']}") + f.write("\n") + + print(f"\nSummary saved to: {summary_file}") + + def print_summary(self, env_name: str, stats: Dict) -> None: + """Print analysis summary""" + print(f"\n{'#'*70}") + print(f"# Analysis Complete: env_{env_name}") + print(f"{'#'*70}\n") + + print(f"Total Tasks: {stats['total']}") + print(f"Success: {stats['success']}") + print(f"Failed: {stats['failed']}") + print(f"Success Rate: {stats['success']/stats['total']*100:.2f}%") + print(f"Total Time: {stats['total_time']:.2f} seconds ({stats['total_time']/60:.2f} minutes)") + print(f"Average Time: {stats['total_time']/stats['total']:.2f} seconds/task") + + if stats['failed'] > 0: + print(f"\nFailed Tasks:") + for result in stats['results']: + if not result['success']: + print(f" - task{result['task']}: {result['error']}") + + +# ==================== Retry Analysis Class ==================== + +class RetryAnalyzer: + """Retry Missing Tasks Analyzer""" + + def __init__(self, analyzer: LLMHypothesisAnalyzer, delay: int = 40): + """ + Args: + analyzer: LLM hypothesis analyzer instance + delay: Delay between tasks (seconds) + """ + self.analyzer = analyzer + self.delay = delay + + def find_missing_tasks(self, env_name: str, + start_task: Optional[int] = None, + end_task: Optional[int] = None) -> List[int]: + """ + Find tasks missing LLM analysis results + + Args: + env_name: Environment name + start_task: Start task number + end_task: End task number + + Returns: + List of missing task numbers + """ + env_dir = self.analyzer.env_base_dir / f"env_{env_name}" + + if not env_dir.exists(): + print(f"Warning: Environment directory not found: {env_dir}") + return [] + + missing_tasks = [] + for task_dir in sorted(env_dir.iterdir()): + if task_dir.is_dir() and task_dir.name.startswith('task'): + try: + task_num = int(task_dir.name[4:]) + + # Apply range filtering + if start_task is not None and task_num < start_task: + continue + if end_task is not None and task_num > end_task: + continue + + # Check if VLM analysis exists but LLM analysis is missing + vlm_file = task_dir / "vlm_analysis.txt" + json_file = task_dir / "llm_hypothesis_analysis.json" + + if vlm_file.exists() and not json_file.exists(): + missing_tasks.append(task_num) + + except ValueError: + continue + + return sorted(missing_tasks) + + def retry_tasks(self, + env_name: str, + tasks: List[int], + temperature: float = 0.3, + output_format: str = 'both', + skip_exists: bool = False) -> Dict: + """ + Retry specified task list + + Args: + env_name: Environment name + tasks: List of task numbers + temperature: Generation temperature + output_format: Output format + skip_exists: Whether to skip if result already exists + + Returns: + Statistics dictionary + """ + print("=" * 80) + print(f"Retry Missing LLM Analysis Tasks") + print("=" * 80) + print(f"Environment: {env_name}") + print(f"Tasks to process: {len(tasks)}") + if tasks: + print(f"Task range: task{min(tasks)} ~ task{max(tasks)}") + print(f"LLM API: {self.analyzer.api_url}") + print(f"Model: {self.analyzer.model}") + print(f"Task delay: {self.delay} seconds") + print(f"Start time: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") + print("=" * 80) + + if not tasks: + print("No tasks found to process") + return None + + # Statistics + stats = { + 'total': len(tasks), + 'success': 0, + 'failed': 0, + 'skipped': 0, + 'total_time': 0, + 'results': [], + 'failed_tasks': [] + } + + writer = AnalysisReportWriter() + + for idx, task_num in enumerate(tasks, 1): + task_dir = self.analyzer.env_base_dir / f"env_{env_name}" / f"task{task_num}" + json_file = task_dir / "llm_hypothesis_analysis.json" + + # Check if should skip + if skip_exists and json_file.exists(): + print(f"[{idx}/{len(tasks)}] SKIP task{task_num}: already exists") + stats['skipped'] += 1 + if idx < len(tasks): + time.sleep(self.delay) + continue + + print(f"[{idx}/{len(tasks)}] Processing task{task_num}...", end=' ', flush=True) + + t0 = time.time() + success, analysis, error_msg = self.analyzer.analyze_single( + env_name, task_num, temperature=temperature + ) + duration = time.time() - t0 + + stats['total_time'] += duration + + if success and analysis: + if output_format in ['json', 'both']: + writer.write_json(json_file, analysis) + + if output_format in ['text', 'both']: + text_file = task_dir / "llm_hypothesis_analysis.txt" + writer.write_text_report(text_file, analysis) + + stats['success'] += 1 + print(f"OK ({duration:.2f}s)") + else: + stats['failed'] += 1 + stats['failed_tasks'].append((task_num, error_msg)) + print(f"FAIL ({duration:.2f}s) - {error_msg[:60] if error_msg else 'Unknown'}") + + stats['results'].append({ + 'task': task_num, + 'success': success, + 'duration': duration, + 'error': error_msg + }) + + # Task delay + if idx < len(tasks): + print(f" Waiting {self.delay} seconds...", end=' ', flush=True) + time.sleep(self.delay) + print("done") + + return stats + + def save_retry_log(self, env_name: str, stats: Dict) -> Path: + """Save retry log""" + log_file = self.analyzer.env_base_dir / f"env_{env_name}" / f"retry_log_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json" + + log_data = { + 'timestamp': datetime.now().isoformat(), + 'env': env_name, + 'model': self.analyzer.model, + 'api_url': self.analyzer.api_url, + 'total_tasks': stats['total'], + 'successful': stats['success'], + 'failed': stats['failed'], + 'skipped': stats['skipped'], + 'total_time': stats['total_time'], + 'failed_tasks': [{'task_id': tid, 'error': err} for tid, err in stats['failed_tasks']], + 'results': stats['results'] + } + + with open(log_file, 'w', encoding='utf-8') as f: + json.dump(log_data, f, ensure_ascii=False, indent=2) + + print(f"\nLog saved to: {log_file}") + return log_file + + def print_retry_summary(self, stats: Dict) -> None: + """Print retry summary""" + print("\n" + "=" * 80) + print("Processing Complete") + print("=" * 80) + print(f"Total Tasks: {stats['total']}") + print(f"Success: {stats['success']} ({stats['success']/stats['total']*100:.1f}%)" if stats['total'] > 0 else "Success: 0") + print(f"Failed: {stats['failed']} ({stats['failed']/stats['total']*100:.1f}%)" if stats['total'] > 0 else "Failed: 0") + print(f"Skipped: {stats['skipped']}") + print(f"Total Time: {stats['total_time']:.1f} seconds") + if stats['total'] > 0: + print(f"Average Time: {stats['total_time']/stats['total']:.1f} seconds/task") + + if stats['failed_tasks']: + print(f"\nFailed Tasks ({len(stats['failed_tasks'])}):") + for task_id, error in stats['failed_tasks'][:10]: + print(f" task{task_id}: {error[:60] if error else 'Unknown error'}") + if len(stats['failed_tasks']) > 10: + print(f" ... and {len(stats['failed_tasks']) - 10} more failed tasks") + + print(f"\nEnd Time: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") + print("=" * 80) + + +# ==================== Command Line Interface ==================== + +def cmd_single(args): + """Single task analysis command""" + analyzer = LLMHypothesisAnalyzer( + api_url=args.api_url, + model=args.model, + env_base_dir=args.env_base_dir + ) + + # Select analysis method based on whether IG hypotheses are needed + generate_ig = getattr(args, 'generate_ig', True) + if generate_ig: + success, analysis, error_msg = analyzer.analyze_single_with_ig( + args.env, args.task, temperature=args.temperature, generate_ig=True + ) + else: + success, analysis, error_msg = analyzer.analyze_single( + args.env, args.task, temperature=args.temperature + ) + + if not success: + print(f"\nError: {error_msg}") + return 1 + + # Save results + task_dir = analyzer.env_base_dir / f"env_{args.env}" / f"task{args.task}" + writer = AnalysisReportWriter() + + print(f"\n{'='*70}") + print("Saving analysis results...") + print(f"{'='*70}") + + if args.output_format in ['json', 'both']: + json_file = task_dir / "llm_hypothesis_analysis.json" + writer.write_json(json_file, analysis) + print(f"JSON result: {json_file}") + + if args.output_format in ['text', 'both']: + text_file = task_dir / "llm_hypothesis_analysis.txt" + writer.write_text_report(text_file, analysis) + print(f"Text report: {text_file}") + + print(f"\n{'='*70}") + print("Analysis complete!") + print(f"{'='*70}\n") + + return 0 + + +def cmd_batch(args): + """Batch analysis command""" + analyzer = LLMHypothesisAnalyzer( + api_url=args.api_url, + model=args.model, + env_base_dir=args.env_base_dir + ) + + batch = BatchAnalyzer(analyzer, delay=args.delay) + + all_stats = {} + + # Get whether to generate IG hypotheses option + generate_ig = getattr(args, 'generate_ig', True) + + for env_name in args.env: + stats = batch.analyze_batch( + env_name, + start_task=args.start, + end_task=args.end, + max_tasks=args.max_tasks, + temperature=args.temperature, + output_format=args.output_format, + generate_ig=generate_ig + ) + + if stats: + all_stats[env_name] = stats + batch.print_summary(env_name, stats) + batch.save_summary(env_name, stats) + + # Wait if not the last environment + if env_name != args.env[-1]: + print(f"\n{'='*70}") + print(f"Preparing to analyze next environment...") + print(f"Waiting {args.delay} seconds...") + print(f"{'='*70}\n") + time.sleep(args.delay) + + # Print overall summary + if len(all_stats) > 1: + print(f"\n{'#'*70}") + print(f"# Overall Summary") + print(f"{'#'*70}\n") + + total_tasks = sum(s['total'] for s in all_stats.values()) + total_success = sum(s['success'] for s in all_stats.values()) + total_failed = sum(s['failed'] for s in all_stats.values()) + total_time = sum(s['total_time'] for s in all_stats.values()) + + print(f"Environments Analyzed: {len(all_stats)}") + print(f"Total Tasks: {total_tasks}") + print(f"Total Success: {total_success}") + print(f"Total Failed: {total_failed}") + print(f"Overall Success Rate: {total_success/total_tasks*100:.2f}%") + print(f"Total Time: {total_time:.2f} seconds ({total_time/60:.2f} minutes)") + print() + + for env_name, stats in all_stats.items(): + print(f" {env_name}: {stats['success']}/{stats['total']} " + f"({stats['success']/stats['total']*100:.2f}%)") + + print(f"\n{'='*70}") + print("All analysis tasks complete!") + print(f"{'='*70}\n") + + return 0 + + +def cmd_retry(args): + """Retry missing tasks command""" + analyzer = LLMHypothesisAnalyzer( + api_url=args.api_url, + model=args.model, + env_base_dir=args.env_base_dir + ) + + retry = RetryAnalyzer(analyzer, delay=args.delay) + + # Determine tasks to process + if args.tasks: + # Use specified task list + tasks_to_process = sorted(args.tasks) + else: + # Auto-scan for missing tasks + print(f"Scanning env_{args.env} for tasks missing LLM analysis...") + tasks_to_process = retry.find_missing_tasks( + args.env, + start_task=args.start, + end_task=args.end + ) + + if not tasks_to_process: + print("No tasks missing LLM analysis found, all tasks are complete!") + return 0 + + print(f"Found {len(tasks_to_process)} missing tasks: {tasks_to_process[:20]}{'...' if len(tasks_to_process) > 20 else ''}") + + # Execute retry + stats = retry.retry_tasks( + args.env, + tasks_to_process, + temperature=args.temperature, + output_format=args.output_format, + skip_exists=args.skip_exists + ) + + if stats: + retry.print_retry_summary(stats) + retry.save_retry_log(args.env, stats) + + return 0 + + +def main(): + parser = argparse.ArgumentParser( + description="LLM Semantic Hypothesis Analysis Tool - Supports single task, batch analysis, and retry missing tasks", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Usage Examples: +=============== + +Single task analysis: + python llm_hypothesis_analyzer.py single --env hm3dv2 --task 0 + python llm_hypothesis_analyzer.py single --env hm3dv2 --task 0 --api-url http://localhost:11434/api/chat + +Batch analysis: + python llm_hypothesis_analyzer.py batch --env hm3dv2 + python llm_hypothesis_analyzer.py batch --env hm3dv2 --max-tasks 10 + python llm_hypothesis_analyzer.py batch --env hm3dv2 --start 5 --end 15 + python llm_hypothesis_analyzer.py batch --env hm3dv1 hm3dv2 --model qwen2:32b + +Retry missing tasks: + python llm_hypothesis_analyzer.py retry --env hm3dv2 # Auto-scan missing tasks + python llm_hypothesis_analyzer.py retry --env hm3dv2 --tasks 100 200 300 # Specify task list + python llm_hypothesis_analyzer.py retry --env hm3dv2 --start 100 --end 200 # Specify range + python llm_hypothesis_analyzer.py retry --env hm3dv2 --skip-exists # Skip existing results + """ + ) + + subparsers = parser.add_subparsers(dest='command', help='Execution mode') + + # Single task mode + single_parser = subparsers.add_parser('single', help='Single task analysis mode') + single_parser.add_argument('--env', type=str, required=True, + choices=['hm3dv1', 'hm3dv2', 'mp3d'], + help='Environment category') + single_parser.add_argument('--task', type=int, required=True, + help='Task number') + single_parser.add_argument('--api-url', type=str, default='https://api.deepseek.com/chat/completions', + help='LLM API address (default: DeepSeek API)') + single_parser.add_argument('--model', type=str, default='deepseek-chat', + help='LLM model name (default: deepseek-chat)') + single_parser.add_argument('--temperature', type=float, default=0.3, + help='Generation temperature 0.0-1.0 (default: 0.3)') + single_parser.add_argument('--env-base-dir', type=str, default='env', + help='Environment dataset root directory') + single_parser.add_argument('--output-format', type=str, default='both', + choices=['json', 'text', 'both'], + help='Output format (default: both)') + single_parser.add_argument('--generate-ig', action='store_true', default=True, + help='Generate IG hypotheses (default: True)') + single_parser.add_argument('--no-ig', action='store_false', dest='generate_ig', + help='Do not generate IG hypotheses') + + # Batch mode + batch_parser = subparsers.add_parser('batch', help='Batch analysis mode') + batch_parser.add_argument('--env', type=str, nargs='+', required=True, + choices=['hm3dv1', 'hm3dv2', 'mp3d'], + help='Environments to analyze (can specify multiple)') + batch_parser.add_argument('--start', type=int, default=None, + help='Start task number (inclusive)') + batch_parser.add_argument('--end', type=int, default=None, + help='End task number (inclusive)') + batch_parser.add_argument('--max-tasks', type=int, default=None, + help='Maximum number of tasks limit') + batch_parser.add_argument('--delay', type=int, default=5, + help='Delay between tasks (seconds), default 5s') + batch_parser.add_argument('--api-url', type=str, default='https://api.deepseek.com/chat/completions', + help='LLM API address (default: DeepSeek API)') + batch_parser.add_argument('--model', type=str, default='deepseek-chat', + help='LLM model name (default: deepseek-chat)') + batch_parser.add_argument('--temperature', type=float, default=0.2, + help='Generation temperature 0.0-1.0 (default: 0.2)') + batch_parser.add_argument('--env-base-dir', type=str, default='env', + help='Environment dataset root directory') + batch_parser.add_argument('--output-format', type=str, default='both', + choices=['json', 'text', 'both'], + help='Output format (default: both)') + batch_parser.add_argument('--generate-ig', action='store_true', default=True, + help='Generate IG hypotheses (default: True)') + batch_parser.add_argument('--no-ig', action='store_false', dest='generate_ig', + help='Do not generate IG hypotheses') + + # Retry mode + retry_parser = subparsers.add_parser('retry', help='Retry missing tasks mode') + retry_parser.add_argument('--env', type=str, required=True, + choices=['hm3dv1', 'hm3dv2', 'mp3d'], + help='Environment category') + retry_parser.add_argument('--tasks', type=int, nargs='+', default=None, + help='Task numbers to retry (optional, auto-scans if not specified)') + retry_parser.add_argument('--start', type=int, default=None, + help='Start task number (inclusive)') + retry_parser.add_argument('--end', type=int, default=None, + help='End task number (inclusive)') + retry_parser.add_argument('--delay', type=int, default=5, + help='Delay between tasks (seconds), default 5s') + retry_parser.add_argument('--api-url', type=str, default='https://api.deepseek.com/chat/completions', + help='LLM API address (default: DeepSeek API)') + retry_parser.add_argument('--model', type=str, default='deepseek-chat', + help='LLM model name (default: deepseek-chat)') + retry_parser.add_argument('--temperature', type=float, default=0.3, + help='Generation temperature 0.0-1.0 (default: 0.3)') + retry_parser.add_argument('--env-base-dir', type=str, default='env', + help='Environment dataset root directory') + retry_parser.add_argument('--output-format', type=str, default='both', + choices=['json', 'text', 'both'], + help='Output format (default: both)') + retry_parser.add_argument('--skip-exists', action='store_true', + help='Skip if result already exists (default: overwrite)') + + args = parser.parse_args() + + if args.command is None: + parser.print_help() + return 1 + + if args.command == 'single': + return cmd_single(args) + elif args.command == 'batch': + return cmd_batch(args) + elif args.command == 'retry': + return cmd_retry(args) + + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except KeyboardInterrupt: + print("\n\nAnalysis interrupted by user") + sys.exit(1) diff --git a/scripts/probe_cloud_models.py b/scripts/probe_cloud_models.py new file mode 100644 index 0000000..3768027 --- /dev/null +++ b/scripts/probe_cloud_models.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Make minimal authenticated calls to the cloud models used by InfoNav.""" + +from __future__ import annotations + +import argparse +import os +import time + +import requests +from openai import OpenAI + + +def probe_qwen(model: str) -> None: + api_key = os.environ.get("DASHSCOPE_API_KEY", "") + if not api_key: + raise RuntimeError("DASHSCOPE_API_KEY is unset") + base_url = os.environ.get( + "DASHSCOPE_BASE_URL", + "https://dashscope.aliyuncs.com/compatible-mode/v1", + ).rstrip("/") + client = OpenAI(api_key=api_key, base_url=base_url, timeout=60.0) + started = time.monotonic() + response = client.chat.completions.create( + model=model, + messages=[ + { + "role": "user", + "content": "Reply with only: OK", + } + ], + temperature=0, + max_tokens=8, + ) + content = response.choices[0].message.content + if not content: + raise RuntimeError("empty response") + print(f"[probe] qwen model={model} status=available latency={time.monotonic() - started:.2f}s") + + +def probe_deepseek(model: str) -> None: + api_key = os.environ.get("DEEPSEEK_API_KEY", "") + if not api_key: + raise RuntimeError("DEEPSEEK_API_KEY is unset") + api_url = os.environ.get( + "DEEPSEEK_BASE_URL", "https://api.deepseek.com/chat/completions" + ) + if not api_url.rstrip("/").endswith("chat/completions"): + api_url = f"{api_url.rstrip('/')}/chat/completions" + started = time.monotonic() + response = requests.post( + api_url, + headers={ + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + }, + json={ + "model": model, + "messages": [{"role": "user", "content": "Reply with only: OK"}], + "temperature": 0, + "max_tokens": 8, + "stream": False, + }, + timeout=60, + ) + if response.status_code != 200: + raise RuntimeError(f"HTTP {response.status_code}: {response.text[:300]}") + content = response.json().get("choices", [{}])[0].get("message", {}).get("content") + if not content: + raise RuntimeError("empty response") + print(f"[probe] deepseek model={model} status=available latency={time.monotonic() - started:.2f}s") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--qwen-model", default="qwen3-vl-flash") + parser.add_argument("--deepseek-model", default="deepseek-chat") + args = parser.parse_args() + + failures = 0 + for name, probe in ( + ("qwen", lambda: probe_qwen(args.qwen_model)), + ("deepseek", lambda: probe_deepseek(args.deepseek_model)), + ): + try: + probe() + except Exception as exc: + failures += 1 + print(f"[probe] {name} status=unavailable error={type(exc).__name__}: {exc}") + return 1 if failures else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/report_hm3dv2_greedy1000.py b/scripts/report_hm3dv2_greedy1000.py new file mode 100755 index 0000000..a8222cb --- /dev/null +++ b/scripts/report_hm3dv2_greedy1000.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 +"""Print or send a concise status report for the HM3Dv2 greedy-1000 run.""" + +from __future__ import annotations + +import argparse +import datetime as dt +import os +from pathlib import Path +import re +import socket +import subprocess +import sys + + +ROOT = Path(__file__).resolve().parents[1] +STATE_FILE = ROOT / ".run_state" / "hm3dv2_greedy1000.env" +CC_DATA_DIR = os.environ.get("CC_CONNECT_DATA_DIR", "") +CC_CONNECT = os.environ.get("CC_CONNECT_BIN", "cc-connect") +FEISHU_SESSION_KEY = os.environ.get("CC_CONNECT_SESSION", "") + + +def read_state() -> dict[str, str]: + state: dict[str, str] = {} + if not STATE_FILE.exists(): + return state + for raw_line in STATE_FILE.read_text(errors="replace").splitlines(): + line = raw_line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + state[key.strip()] = value.strip() + return state + + +def first_match(text: str, pattern: str, default: str = "N/A") -> str: + match = re.search(pattern, text, flags=re.MULTILINE) + return match.group(1).strip() if match else default + + +def parse_continue(output_dir: Path) -> dict[str, str]: + path = output_dir / "continue.txt" + if not path.exists(): + return { + "completed": "0", + "success": "0", + "spl": "N/A", + "soft_spl": "N/A", + "episode_id": "N/A", + "target": "N/A", + "result": "N/A", + "age": "N/A", + } + + # write_record prepends the newest result, so the first block is sufficient. + text = path.read_text(errors="replace")[:12000] + record_path = output_dir / "record.txt" + record_text = ( + record_path.read_text(errors="replace")[:12000] + if record_path.exists() + else "" + ) + completed = first_match(text, r"No\.(\d+)\s+task is finished", "0") + success = first_match(text, r"\|\s*Total Success\s*\|\s*([0-9.]+)\s*\|", "0") + # record.txt stores the already-computed average with two decimal places; + # continue.txt stores a rounded sum, which loses precision after division. + spl = first_match( + record_text, r"\|\s*Average SPL\s*\|\s*([0-9.]+%)\s*\|" + ) + soft_spl = first_match( + record_text, r"\|\s*Average Soft SPL\s*\|\s*([0-9.]+%)\s*\|" + ) + + modified = dt.datetime.fromtimestamp(path.stat().st_mtime).astimezone() + age_minutes = max( + 0, int((dt.datetime.now().astimezone() - modified).total_seconds() // 60) + ) + return { + "completed": completed, + "success": success, + "spl": spl, + "soft_spl": soft_spl, + "episode_id": first_match(text, r"Episode ID:\s*(\S+)"), + "target": first_match(text, r"target to find is\s+(.+)"), + "result": first_match(text, r"success or not:\s*(.+)"), + "age": f"{age_minutes} ๅˆ†้’Ÿๅ‰", + } + + +def process_running(pattern: str) -> bool: + result = subprocess.run( + ["pgrep", "-f", pattern], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + return result.returncode == 0 + + +def port_open(port: int) -> bool: + try: + with socket.create_connection(("127.0.0.1", port), timeout=0.4): + return True + except OSError: + return False + + +def gpu_status(power_limit_max: float = 220.0) -> tuple[bool, str]: + command = [ + "nvidia-smi", + "--query-gpu=temperature.gpu,power.draw,power.limit,memory.used," + "memory.total,utilization.gpu", + "--format=csv,noheader,nounits", + ] + try: + result = subprocess.run( + command, capture_output=True, text=True, timeout=5, check=False + ) + except (OSError, subprocess.TimeoutExpired): + return False, "ไธๅฏ่ฎฟ้—ฎ" + if result.returncode != 0 or not result.stdout.strip(): + return False, "ไธๅฏ่ฎฟ้—ฎ๏ผˆ้ฉฑๅŠจ/PCIe ๆ•…้šœ๏ผ‰" + values = [item.strip() for item in result.stdout.strip().split(",")] + if len(values) != 6: + return False, "่ฟ”ๅ›žๆ ผๅผๅผ‚ๅธธ" + temp, power, power_limit, used, total, util = values + summary = ( + f"{temp}ยฐC, {power}W/{power_limit}W, " + f"ๆ˜พๅญ˜ {used}/{total}MiB, ๅˆฉ็”จ็Ž‡ {util}%" + ) + try: + if float(power_limit) > power_limit_max: + return False, f"ๅŠŸ่€—ไธŠ้™่ฟ‡้ซ˜๏ผš{summary}๏ผˆ่ฆๆฑ‚ โ‰ค{power_limit_max:g}W๏ผ‰" + except ValueError: + return False, f"ๅŠŸ่€—ไธŠ้™ๆ ผๅผๅผ‚ๅธธ๏ผš{summary}" + return True, summary + + +def build_report() -> str: + state = read_state() + output_value = state.get("OUTPUT_DIR", "") + output_dir = Path(output_value) if output_value else ROOT / "videos" / "ๆœชๅˆ›ๅปบ" + metrics = parse_continue(output_dir) + total = int(state.get("TOTAL", "1000")) + completed = int(metrics["completed"]) + success = int(float(metrics["success"])) + failed = max(0, completed - success) + sr = f"{success / completed * 100:.2f}%" if completed else "N/A" + progress = f"{completed / total * 100:.1f}%" if total else "N/A" + + power_limit_max = float(state.get("GPU_POWER_LIMIT_MAX", "220")) + gpu_ok, gpu = gpu_status(power_limit_max) + eval_running = process_running( + r"/python([0-9.]*)? .*habitat_evaluation\.py" + ) + planner_running = process_running(r"/exploration_node([ ]|$)") + verifier_running = process_running( + r"/python([0-9.]*)? .*vlm_approach_verifier\.py" + ) + services = { + "BLIP2": port_open(12182), + "MobileSAM": port_open(12183), + "D-FINE": port_open(12185), + } + services_text = ", ".join( + f"{name}={'ๆญฃๅธธ' if healthy else 'ๆœช่ฟ่กŒ'}" + for name, healthy in services.items() + ) + + run_status = state.get("STATUS", "not_prepared") + if completed >= total: + human_status = "ๅทฒๅฎŒๆˆ" + elif not gpu_ok: + human_status = "ๆš‚ๅœ๏ผšGPU ไธๅฏ่ฎฟ้—ฎ" + elif eval_running and planner_running: + human_status = "่ฟ่กŒไธญ" + elif run_status == "starting": + human_status = "ๆœๅŠกๅฏๅŠจไธญ" + else: + human_status = f"ๆœช่ฟ่กŒ๏ผˆ{run_status}๏ผ‰" + + now = dt.datetime.now().astimezone().strftime("%Y-%m-%d %H:%M:%S %Z") + return "\n".join( + [ + "[InfoNav HM3Dv2 Greedy-1000 ๅฎšๆ—ถๆ’ญๆŠฅ]", + f"ๆ—ถ้—ด๏ผš{now}", + "็ญ–็•ฅ๏ผšpolicy=1๏ผŒvalue-greedy๏ผˆไปทๅ€ผๅœฐๅ›พ่ดชๅฟƒ๏ผ‰", + ( + f"่ดŸ่ฝฝ๏ผš{state.get('CONTROL_RATE_HZ', '4')}Hz ๆŽงๅˆถ๏ผ›" + "ๆœฌๅœฐไป… BLIP2 + MobileSAM + D-FINE๏ผ›" + f"Qwen={state.get('QWEN_VL_MODEL', 'qwen3-vl-plus')}" + ), + f"็Šถๆ€๏ผš{human_status}", + f"่ฟ›ๅบฆ๏ผš{completed}/{total}๏ผˆ{progress}๏ผ‰", + f"็ป“ๆžœ๏ผšๆˆๅŠŸ {success}๏ผŒๅคฑ่ดฅ {failed}๏ผŒSR {sr}", + f"SPL๏ผš{metrics['spl']}๏ผ›Soft SPL๏ผš{metrics['soft_spl']}", + ( + f"ๆœ€่ฟ‘๏ผšEpisode {metrics['episode_id']}๏ผŒ็›ฎๆ ‡ {metrics['target']}๏ผŒ" + f"็ป“ๆžœ {metrics['result']}" + ), + f"ๆœ€ๅŽๅ†™ๅ…ฅ๏ผš{metrics['age']}", + ( + "่ฟ›็จ‹๏ผš" + f"eval={'่ฟ่กŒ' if eval_running else 'ๆœช่ฟ่กŒ'}๏ผŒ" + f"planner={'่ฟ่กŒ' if planner_running else 'ๆœช่ฟ่กŒ'}๏ผŒ" + f"verifier={'่ฟ่กŒ' if verifier_running else 'ๆœช่ฟ่กŒ'}" + ), + f"VLM๏ผš{services_text}", + f"GPU๏ผš{gpu}", + f"็›ฎๅฝ•๏ผš{output_dir}", + ] + ) + + +def send_report(report: str) -> int: + if not CC_DATA_DIR or not FEISHU_SESSION_KEY: + print( + "Set CC_CONNECT_DATA_DIR and CC_CONNECT_SESSION before using --send.", + file=sys.stderr, + ) + return 2 + command = [ + CC_CONNECT, + "send", + "--data-dir", + CC_DATA_DIR, + "--project", + "claude_bridge", + "--session", + FEISHU_SESSION_KEY, + "--stdin", + ] + result = subprocess.run(command, input=report, text=True, check=False) + return result.returncode + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "--send", action="store_true", help="send through the active cc-connect session" + ) + args = parser.parse_args() + report = build_report() + if args.send: + return send_report(report) + print(report) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/report_hm3dv2_value_wtrp_pair100.py b/scripts/report_hm3dv2_value_wtrp_pair100.py new file mode 100755 index 0000000..c7e10b1 --- /dev/null +++ b/scripts/report_hm3dv2_value_wtrp_pair100.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python3 +"""Report the active HM3Dv2 value-greedy/WTRP paired experiment.""" + +from __future__ import annotations + +import datetime as dt +import json +import os +from pathlib import Path +import re +import socket +import subprocess + + +ROOT = Path(__file__).resolve().parents[1] +VIDEOS = ROOT / "videos" +STATE_DIR = ROOT / ".run_state" +CRON_ID = os.environ.get("CC_CONNECT_CRON_ID", "") +CC_DATA_DIR = os.environ.get("CC_CONNECT_DATA_DIR", "") + + +def read_env(path: Path) -> dict[str, str]: + values: dict[str, str] = {} + if not path.exists(): + return values + for raw_line in path.read_text(errors="replace").splitlines(): + if "=" not in raw_line or raw_line.lstrip().startswith("#"): + continue + key, value = raw_line.split("=", 1) + values[key.strip()] = value.strip() + return values + + +def latest_pair_root() -> Path | None: + candidates = [] + for path in VIDEOS.glob("test_hm3dv2_value_wtrp_pair100_*_val"): + protocol = read_env(path / "value_greedy" / "run_protocol.env") + if protocol.get("TOTAL_EPISODES") == "100": + candidates.append(path) + return max(candidates, key=lambda path: path.stat().st_mtime) if candidates else None + + +def load_metrics(path: Path) -> list[dict]: + rows: dict[tuple[str, str], dict] = {} + if not path.exists(): + return [] + for line in path.read_text(errors="replace").splitlines(): + if not line.strip(): + continue + try: + row = json.loads(line) + key = (str(row["scene_id"]), str(row["episode_id"])) + except (json.JSONDecodeError, KeyError, TypeError): + continue + rows[key] = row + return sorted(rows.values(), key=lambda row: int(row.get("episode_num", 0))) + + +def arm_summary(root: Path, name: str, state: dict[str, str]) -> dict[str, object]: + metrics_path = root / name / "episode_metrics.jsonl" + rows = load_metrics(metrics_path) + completed = len(rows) + success = sum(int(float(row.get("success", 0))) for row in rows) + spl = sum(float(row.get("spl", 0)) for row in rows) / completed if completed else None + soft_spl = ( + sum(float(row.get("soft_spl", 0)) for row in rows) / completed + if completed + else None + ) + age = "N/A" + if metrics_path.exists(): + modified = dt.datetime.fromtimestamp(metrics_path.stat().st_mtime).astimezone() + minutes = max( + 0, + int((dt.datetime.now().astimezone() - modified).total_seconds() // 60), + ) + age = f"{minutes} ๅˆ†้’Ÿๅ‰" + return { + "completed": completed, + "success": success, + "failed": completed - success, + "sr": 100.0 * success / completed if completed else None, + "spl": 100.0 * spl if spl is not None else None, + "soft_spl": 100.0 * soft_spl if soft_spl is not None else None, + "latest": rows[-1] if rows else {}, + "age": age, + "status": state.get("STATUS", "not_started"), + } + + +def percent(value: object) -> str: + return "N/A" if value is None else f"{float(value):.2f}%" + + +def process_running(pattern: str) -> bool: + return subprocess.run( + ["pgrep", "-f", pattern], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ).returncode == 0 + + +def tmux_running(session: str) -> bool: + return subprocess.run( + ["tmux", "has-session", "-t", session], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ).returncode == 0 + + +def port_open(port: int) -> bool: + try: + with socket.create_connection(("127.0.0.1", port), timeout=0.4): + return True + except OSError: + return False + + +def gpu_status() -> str: + command = [ + "nvidia-smi", + "--query-gpu=temperature.gpu,power.draw,power.limit,memory.used," + "memory.total,utilization.gpu", + "--format=csv,noheader,nounits", + ] + try: + result = subprocess.run( + command, capture_output=True, text=True, timeout=5, check=False + ) + except (OSError, subprocess.TimeoutExpired): + return "ไธๅฏ่ฎฟ้—ฎ" + values = [item.strip() for item in result.stdout.strip().split(",")] + if result.returncode != 0 or len(values) != 6: + return "ไธๅฏ่ฎฟ้—ฎ๏ผˆ้ฉฑๅŠจๆˆ– PCIe ๅผ‚ๅธธ๏ผ‰" + temp, power, limit, used, total, util = values + return f"{temp}ยฐC, {power}/{limit}W, ๆ˜พๅญ˜ {used}/{total}MiB, ๅˆฉ็”จ็Ž‡ {util}%" + + +def latest_text(summary: dict[str, object]) -> str: + latest = summary["latest"] + if not latest: + return "N/A" + return ( + f"Episode {latest.get('episode_id', 'N/A')}๏ผŒ" + f"็›ฎๆ ‡ {latest.get('target', 'N/A')}๏ผŒ" + f"็ป“ๆžœ {latest.get('result', 'N/A')}๏ผŒ" + f"{int(latest.get('steps', 0))} ๆญฅ" + ) + + +def disable_after_completion() -> None: + if not CRON_ID or not CC_DATA_DIR: + return + subprocess.run( + [ + "cc-connect", + "cron", + "edit", + CRON_ID, + "enabled", + "false", + "--data-dir", + CC_DATA_DIR, + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + + +def main() -> int: + pair_root = latest_pair_root() + now = dt.datetime.now().astimezone().strftime("%Y-%m-%d %H:%M:%S %Z") + if pair_root is None: + print(f"[InfoNav HM3Dv2 Value-Greedy vs WTRP ๅฎšๆ—ถๆ’ญๆŠฅ]\nๆ—ถ้—ด๏ผš{now}\n็Šถๆ€๏ผšๆœชๆ‰พๅˆฐ้…ๅฏนๅฎž้ชŒ็›ฎๅฝ•") + return 1 + + match = re.search(r"pair100_(\d{8}_\d{6})_val$", pair_root.name) + stamp = match.group(1) if match else "unknown" + greedy_state = read_env(STATE_DIR / f"hm3dv2_value_greedy100_{stamp}.env") + wtrp_state = read_env(STATE_DIR / f"hm3dv2_wtrp100_{stamp}.env") + greedy = arm_summary(pair_root, "value_greedy", greedy_state) + wtrp = arm_summary(pair_root, "wtrp", wtrp_state) + + greedy_n = int(greedy["completed"]) + wtrp_n = int(wtrp["completed"]) + overall = greedy_n + wtrp_n + if wtrp_n >= 100: + active = "ๅ…จ้ƒจๅฎŒๆˆ" + elif wtrp_state.get("STATUS") in {"starting", "running"}: + active = "B๏ผšWTRP๏ผˆpolicy=4๏ผ‰" + elif greedy_state.get("STATUS") in {"starting", "running"}: + active = "A๏ผšไปทๅ€ผ่ดชๅฟƒ๏ผˆpolicy=1๏ผ‰" + elif greedy_n >= 100: + active = "A ๅทฒๅฎŒๆˆ๏ผŒๆญฃๅœจๅˆ‡ๆขๅˆฐ B" + else: + active = "ๆœช่ฟ่กŒๆˆ–ๅผ‚ๅธธๅœๆญข" + + services = { + "BLIP2": port_open(12182), + "GroundingDINO": port_open(12181), + "MobileSAM": port_open(12183), + "D-FINE": port_open(12185), + } + services_text = "๏ผŒ".join( + f"{name}={'ๆญฃๅธธ' if healthy else 'ๅผ‚ๅธธ'}" for name, healthy in services.items() + ) + eval_running = process_running(r"habitat_evaluation\.py") + planner_running = process_running(r"/exploration_node([ ]|$)") + manager_running = tmux_running("infonav_value_wtrp_pair100") + + lines = [ + "[InfoNav HM3Dv2 Value-Greedy vs WTRP ๅฎšๆ—ถๆ’ญๆŠฅ]", + f"ๆ—ถ้—ด๏ผš{now}", + "ๅ่ฎฎ๏ผšๅ‰ 100 ไธช็›ธๅŒ episode๏ผŒseed=100๏ผŒsuccess distance=1m๏ผ›ๅฝ’ไธ€ๅŒ–ๅคšๆบ HSVM+IG", + f"ๅฝ“ๅ‰ๅฎž้ชŒ่‡‚๏ผš{active}", + f"ๆ€ป่ฟ›ๅบฆ๏ผš{overall}/200๏ผˆ{overall / 2:.1f}%๏ผ‰", + ( + f"A ไปทๅ€ผ่ดชๅฟƒ๏ผš{greedy_n}/100๏ผŒๆˆๅŠŸ {greedy['success']}๏ผŒ" + f"ๅคฑ่ดฅ {greedy['failed']}๏ผŒSR {percent(greedy['sr'])}๏ผŒ" + f"SPL {percent(greedy['spl'])}๏ผŒSoft SPL {percent(greedy['soft_spl'])}" + ), + ( + f"B WTRP๏ผš{wtrp_n}/100๏ผŒๆˆๅŠŸ {wtrp['success']}๏ผŒ" + f"ๅคฑ่ดฅ {wtrp['failed']}๏ผŒSR {percent(wtrp['sr'])}๏ผŒ" + f"SPL {percent(wtrp['spl'])}๏ผŒSoft SPL {percent(wtrp['soft_spl'])}" + ), + f"ๆœ€่ฟ‘๏ผš{latest_text(wtrp if wtrp_n else greedy)}", + f"ๆœ€ๅŽๅ†™ๅ…ฅ๏ผš{(wtrp if wtrp_n else greedy)['age']}", + ( + f"่ฟ›็จ‹๏ผševal={'่ฟ่กŒ' if eval_running else 'ๆœช่ฟ่กŒ'}๏ผŒ" + f"planner={'่ฟ่กŒ' if planner_running else 'ๆœช่ฟ่กŒ'}๏ผŒ" + f"pair-manager={'่ฟ่กŒ' if manager_running else 'ๆœช่ฟ่กŒ'}" + ), + f"VLM๏ผš{services_text}", + f"GPU๏ผš{gpu_status()}", + f"็›ฎๅฝ•๏ผš{pair_root}", + ] + print("\n".join(lines)) + if greedy_n >= 100 and wtrp_n >= 100: + disable_after_completion() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_evaluation_with_restart.sh b/scripts/run_evaluation_with_restart.sh index 92a40f6..fed0e58 100755 --- a/scripts/run_evaluation_with_restart.sh +++ b/scripts/run_evaluation_with_restart.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Auto-restart evaluation script for ApexNav +# Auto-restart evaluation script for InfoNav # Runs habitat_evaluation in batches with exploration_node restart # Only restarts exploration.launch, keeps roscore and rviz running @@ -12,7 +12,7 @@ START_EPISODE=${2:-0} # Starting episode number TOTAL_EPISODES=${3:-1000} # Total episodes to run # Conda environment -CONDA_ENV="sg_apexnav" +CONDA_ENV="${INFONAV_CONDA_ENV:-infonav}" # Colors for output RED='\033[0;31m' diff --git a/scripts/run_hm3dv2_greedy1000.sh b/scripts/run_hm3dv2_greedy1000.sh new file mode 100755 index 0000000..69264ec --- /dev/null +++ b/scripts/run_hm3dv2_greedy1000.sh @@ -0,0 +1,845 @@ +#!/usr/bin/env bash +# Resumable and GPU-safe HM3Dv2 evaluation runner. + +set -Eeuo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd -- "${SCRIPT_DIR}/.." && pwd)" +PYTHON="${INFONAV_PYTHON:-$(command -v python || command -v python3)}" +CC_CONNECT_DATA_DIR="${CC_CONNECT_DATA_DIR:-}" +STATE_DIR="${ROOT}/.run_state" +RUN_TAG="${RUN_TAG:-hm3dv2_greedy1000}" +STATE_FILE="${STATE_DIR}/${RUN_TAG}.env" +CRON_ID_FILE="${STATE_DIR}/${RUN_TAG}_cron_id" +SESSION="${SESSION:-infonav_greedy1000}" +TOTAL="${TOTAL:-1000}" +PROTOCOL_ID="${PROTOCOL_ID:-hm3dv2_distance_greedy_hsvm_ig_gdino_success1m_v1}" +SUCCESS_DISTANCE="${SUCCESS_DISTANCE:-1.0}" +EXPLORATION_POLICY="${EXPLORATION_POLICY:-0}" +USE_MULTI_SOURCE="${USE_MULTI_SOURCE:-true}" +FUSION_ALPHA="${FUSION_ALPHA:-0.8}" +USE_IG_FUSION="${USE_IG_FUSION:-true}" +USE_ADAPTIVE_ALPHA="${USE_ADAPTIVE_ALPHA:-false}" +ALPHA_BASE="${ALPHA_BASE:-0.8}" +TAU_CONF="${TAU_CONF:-0.8}" +TOP_K_VALUE="${TOP_K_VALUE:-8}" +WTRP_TEMPERATURE="${WTRP_TEMPERATURE:-0.3}" +WTRP_HYSTERESIS_RATIO="${WTRP_HYSTERESIS_RATIO:-0.8}" +WTRP_MAX_BRUTE_FORCE="${WTRP_MAX_BRUTE_FORCE:-8}" +ENABLE_FEISHU_REPORT="${ENABLE_FEISHU_REPORT:-false}" +HABITAT_SEED="${HABITAT_SEED:-100}" +VLM_VALIDATION_ENABLED="${VLM_VALIDATION_ENABLED:-false}" +EXPERIMENT_GIT_COMMIT="${EXPERIMENT_GIT_COMMIT:-$(git -C "${ROOT}" rev-parse HEAD)}" +if [[ -z "${EXPERIMENT_GIT_DIRTY_TRACKED+x}" ]]; then + if [[ -n "$(git -C "${ROOT}" status --porcelain --untracked-files=no)" ]]; then + EXPERIMENT_GIT_DIRTY_TRACKED=true + else + EXPERIMENT_GIT_DIRTY_TRACKED=false + fi +fi +BATCH_SIZE="${BATCH_SIZE:-25}" +STALL_TIMEOUT="${STALL_TIMEOUT:-2700}" +GPU_MEMORY_LIMIT="${GPU_MEMORY_LIMIT:-11000}" +GPU_TEMPERATURE_LIMIT="${GPU_TEMPERATURE_LIMIT:-82}" +GPU_POWER_LIMIT_MAX="${GPU_POWER_LIMIT_MAX:-220}" +GPU_IDLE_MEMORY_LIMIT="${GPU_IDLE_MEMORY_LIMIT:-1500}" +GPU_IDLE_STABLE_SECONDS="${GPU_IDLE_STABLE_SECONDS:-60}" +GPU_IDLE_TIMEOUT="${GPU_IDLE_TIMEOUT:-300}" +CONTROL_RATE_HZ="${CONTROL_RATE_HZ:-4}" +MODEL_LOAD_COOLDOWN="${MODEL_LOAD_COOLDOWN:-30}" +VLM_REQUEST_TIMEOUT="${VLM_REQUEST_TIMEOUT:-120}" +VLM_REQUEST_ATTEMPTS="${VLM_REQUEST_ATTEMPTS:-3}" +QWEN_VL_MODEL="${QWEN_VL_MODEL:-qwen3-vl-plus}" +DASHSCOPE_REGION="${DASHSCOPE_REGION:-beijing}" +DEEPSEEK_MODEL="${DEEPSEEK_MODEL:-deepseek-v4-flash}" +QWEN_HYPOTHESIS_MODEL="${QWEN_HYPOTHESIS_MODEL:-qwen3-vl-flash}" +DEEPSEEK_HYPOTHESIS_MODEL="${DEEPSEEK_HYPOTHESIS_MODEL:-deepseek-chat}" +FORCE_REGENERATE_HYPOTHESES="${FORCE_REGENERATE_HYPOTHESES:-0}" + +# ~/.bashrc keeps API exports below its non-interactive early return. Import +# only the named runtime variables through an interactive shell, without +# printing values or embedding them in process command lines. +load_interactive_export() { + local name="$1" value + if [[ -n "${!name:-}" ]]; then + return 0 + fi + value="$(bash -ic "printf '%s' \"\${${name}:-}\"" 2>/dev/null)" + if [[ -n "${value}" ]]; then + printf -v "${name}" '%s' "${value}" + export "${name}" + fi +} +for runtime_name in DASHSCOPE_API_KEY DASHSCOPE_BASE_URL DASHSCOPE_WORKSPACE_ID \ + DEEPSEEK_API_KEY DEEPSEEK_BASE_URL; do + load_interactive_export "${runtime_name}" +done + +PREFLIGHT_ONLY=0 +START_VLM_ONLY=0 +if [[ "${1:-}" == "--preflight-only" ]]; then + PREFLIGHT_ONLY=1 + shift +elif [[ "${1:-}" == "--start-vlm-only" ]]; then + START_VLM_ONLY=1 + shift +fi +if (( START_VLM_ONLY == 1 )); then + OUTPUT_DIR="${1:-${STATE_DIR}/vlm_start}" +else + OUTPUT_DIR="${1:-${ROOT}/videos/${RUN_TAG}_$(date +%Y%m%d_%H%M%S)_val}" +fi +LOG_DIR="${OUTPUT_DIR}/run_logs" +RUN_PROTOCOL_FILE="${OUTPUT_DIR}/run_protocol.env" +VLM_SESSION="${VLM_SESSION:-infonav_vlm_persistent}" +VLM_LOG_DIR="${STATE_DIR}/vlm_logs" +GPU_TRACE_LOG="${LOG_DIR}/gpu.csv" +KERNEL_GPU_LOG="${LOG_DIR}/kernel_gpu.log" +HYPOTHESIS_DIR="${ROOT}/env/env_hm3dv2" + +# One BLIP2 instance scores every HSVM and IG channel in a combined request. +# GroundingDINO remains mandatory for non-COCO contextual hypotheses. +export QWEN_VL_MODEL DASHSCOPE_REGION DEEPSEEK_MODEL +export QWEN_HYPOTHESIS_MODEL DEEPSEEK_HYPOTHESIS_MODEL +export USE_PARALLEL_BLIP2=false +export VLM_REQUEST_TIMEOUT VLM_REQUEST_ATTEMPTS + +declare -A PIDS=() + +protocol_value() { + local key="$1" + sed -n "s/^${key}=//p" "${RUN_PROTOCOL_FILE}" | head -n 1 +} + +protocol_matches() { + [[ "$(protocol_value PROTOCOL_ID)" == "${PROTOCOL_ID}" ]] && + [[ "$(protocol_value TOTAL_EPISODES)" == "${TOTAL}" ]] && + [[ "$(protocol_value SUCCESS_DISTANCE_M)" == "${SUCCESS_DISTANCE}" ]] && + [[ "$(protocol_value EXPLORATION_POLICY)" == "${EXPLORATION_POLICY}" ]] && + [[ "$(protocol_value USE_MULTI_SOURCE)" == "${USE_MULTI_SOURCE}" ]] && + [[ "$(protocol_value FUSION_ALPHA)" == "${FUSION_ALPHA}" ]] && + [[ "$(protocol_value USE_IG_FUSION)" == "${USE_IG_FUSION}" ]] && + [[ "$(protocol_value USE_ADAPTIVE_ALPHA)" == "${USE_ADAPTIVE_ALPHA}" ]] && + [[ "$(protocol_value ALPHA_BASE)" == "${ALPHA_BASE}" ]] && + [[ "$(protocol_value TAU_CONF)" == "${TAU_CONF}" ]] && + [[ "$(protocol_value TOP_K_VALUE)" == "${TOP_K_VALUE}" ]] && + [[ "$(protocol_value WTRP_TEMPERATURE)" == "${WTRP_TEMPERATURE}" ]] && + [[ "$(protocol_value WTRP_HYSTERESIS_RATIO)" == "${WTRP_HYSTERESIS_RATIO}" ]] && + [[ "$(protocol_value WTRP_MAX_BRUTE_FORCE)" == "${WTRP_MAX_BRUTE_FORCE}" ]] && + [[ "$(protocol_value HABITAT_SEED)" == "${HABITAT_SEED}" ]] && + [[ "$(protocol_value VLM_VALIDATION_ENABLED)" == "${VLM_VALIDATION_ENABLED}" ]] +} + +write_run_protocol() { + local tmp_file="${RUN_PROTOCOL_FILE}.tmp.$$" + { + printf 'PROTOCOL_ID=%s\n' "${PROTOCOL_ID}" + printf 'DATASET=hm3dv2\n' + printf 'SPLIT=val\n' + printf 'TOTAL_EPISODES=%s\n' "${TOTAL}" + printf 'HABITAT_SEED=%s\n' "${HABITAT_SEED}" + printf 'SUCCESS_DISTANCE_M=%s\n' "${SUCCESS_DISTANCE}" + printf 'EXPLORATION_POLICY=%s\n' "${EXPLORATION_POLICY}" + printf 'USE_MULTI_SOURCE=%s\n' "${USE_MULTI_SOURCE}" + printf 'FUSION_ALPHA=%s\n' "${FUSION_ALPHA}" + printf 'USE_IG_FUSION=%s\n' "${USE_IG_FUSION}" + printf 'USE_ADAPTIVE_ALPHA=%s\n' "${USE_ADAPTIVE_ALPHA}" + printf 'ALPHA_BASE=%s\n' "${ALPHA_BASE}" + printf 'TAU_CONF=%s\n' "${TAU_CONF}" + printf 'TOP_K_VALUE=%s\n' "${TOP_K_VALUE}" + printf 'WTRP_TEMPERATURE=%s\n' "${WTRP_TEMPERATURE}" + printf 'WTRP_HYSTERESIS_RATIO=%s\n' "${WTRP_HYSTERESIS_RATIO}" + printf 'WTRP_MAX_BRUTE_FORCE=%s\n' "${WTRP_MAX_BRUTE_FORCE}" + printf 'VLM_VALIDATION_ENABLED=%s\n' "${VLM_VALIDATION_ENABLED}" + printf 'VALUE_DEFINITION=normalized_multi_source_HSVM_plus_IG\n' + printf 'GIT_COMMIT=%s\n' "${EXPERIMENT_GIT_COMMIT}" + printf 'GIT_DIRTY_TRACKED=%s\n' "${EXPERIMENT_GIT_DIRTY_TRACKED}" + printf 'DETECTOR_MODE=2\n' + printf 'LOCAL_MODELS=blip2_itm,grounding_dino,mobile_sam,dfine\n' + printf 'BLIP2_REPLICAS=1\n' + printf 'CREATED_AT=%s\n' "$(date --iso-8601=seconds)" + } >"${tmp_file}" + mv "${tmp_file}" "${RUN_PROTOCOL_FILE}" +} + +write_state() { + local status="$1" + local reason="${2:-none}" + mkdir -p "${STATE_DIR}" + local tmp_file="${STATE_FILE}.tmp.$$" + { + printf 'STATUS=%s\n' "${status}" + printf 'REASON=%s\n' "${reason// /_}" + printf 'OUTPUT_DIR=%s\n' "${OUTPUT_DIR}" + printf 'TOTAL=%s\n' "${TOTAL}" + printf 'HABITAT_SEED=%s\n' "${HABITAT_SEED}" + printf 'PROTOCOL_ID=%s\n' "${PROTOCOL_ID}" + printf 'SUCCESS_DISTANCE_M=%s\n' "${SUCCESS_DISTANCE}" + printf 'POLICY=%s\n' "${EXPLORATION_POLICY}" + printf 'USE_MULTI_SOURCE=%s\n' "${USE_MULTI_SOURCE}" + printf 'FUSION_ALPHA=%s\n' "${FUSION_ALPHA}" + printf 'USE_IG_FUSION=%s\n' "${USE_IG_FUSION}" + printf 'USE_ADAPTIVE_ALPHA=%s\n' "${USE_ADAPTIVE_ALPHA}" + printf 'ALPHA_BASE=%s\n' "${ALPHA_BASE}" + printf 'TAU_CONF=%s\n' "${TAU_CONF}" + printf 'TOP_K_VALUE=%s\n' "${TOP_K_VALUE}" + printf 'WTRP_TEMPERATURE=%s\n' "${WTRP_TEMPERATURE}" + printf 'WTRP_HYSTERESIS_RATIO=%s\n' "${WTRP_HYSTERESIS_RATIO}" + printf 'WTRP_MAX_BRUTE_FORCE=%s\n' "${WTRP_MAX_BRUTE_FORCE}" + printf 'VLM_VALIDATION_ENABLED=%s\n' "${VLM_VALIDATION_ENABLED}" + printf 'BATCH_SIZE=%s\n' "${BATCH_SIZE}" + printf 'CONTROL_RATE_HZ=%s\n' "${CONTROL_RATE_HZ}" + printf 'GPU_POWER_LIMIT_MAX=%s\n' "${GPU_POWER_LIMIT_MAX}" + printf 'VLM_SESSION=%s\n' "${VLM_SESSION}" + printf 'VLM_REQUEST_TIMEOUT=%s\n' "${VLM_REQUEST_TIMEOUT}" + printf 'LOCAL_MODELS=blip2_itm,grounding_dino,mobile_sam,dfine\n' + printf 'BLIP2_REPLICAS=1\n' + printf 'BLIP2_ENABLED=true\n' + printf 'QWEN_VL_MODEL=%s\n' "${QWEN_VL_MODEL}" + printf 'DASHSCOPE_REGION=%s\n' "${DASHSCOPE_REGION}" + printf 'DEEPSEEK_MODEL=%s\n' "${DEEPSEEK_MODEL}" + printf 'QWEN_HYPOTHESIS_MODEL=%s\n' "${QWEN_HYPOTHESIS_MODEL}" + printf 'DEEPSEEK_HYPOTHESIS_MODEL=%s\n' "${DEEPSEEK_HYPOTHESIS_MODEL}" + printf 'FORCE_REGENERATE_HYPOTHESES=%s\n' "${FORCE_REGENERATE_HYPOTHESES}" + printf 'UPDATED_AT=%s\n' "$(date --iso-8601=seconds)" + } >"${tmp_file}" + mv "${tmp_file}" "${STATE_FILE}" +} + +gpu_query() { + timeout 5 nvidia-smi --id=0 \ + --query-gpu=memory.used,temperature.gpu,power.draw,power.limit,utilization.gpu,pstate,pcie.link.gen.current,pcie.link.width.current \ + --format=csv,noheader,nounits 2>/dev/null +} + +kernel_gpu_fault() { + [[ -s "${KERNEL_GPU_LOG}" ]] && + grep -E -q -i 'NVRM: Xid|fallen off the bus|PCIe Bus Error|Physical Layer.*RxErr' \ + "${KERNEL_GPU_LOG}" +} + +gpu_health_reason() { + local sample used temperature power_draw power_limit utilization _rest + sample="$(gpu_query)" || { + printf 'gpu_nvidia_smi_unavailable\n' + return + } + IFS=',' read -r used temperature power_draw power_limit utilization _rest <<<"${sample}" + used="${used//[[:space:]]/}" + temperature="${temperature//[[:space:]]/}" + power_limit="${power_limit//[[:space:]]/}" + if kernel_gpu_fault; then + printf 'gpu_pcie_or_xid_kernel_fault\n' + elif ! [[ "${used}" =~ ^[0-9]+$ && "${temperature}" =~ ^[0-9]+$ && + "${power_limit}" =~ ^[0-9]+([.][0-9]+)?$ ]]; then + printf 'gpu_metrics_invalid\n' + elif awk -v actual="${power_limit}" -v maximum="${GPU_POWER_LIMIT_MAX}" \ + 'BEGIN { exit !(actual > maximum) }'; then + printf 'gpu_power_limit_%sW_above_%sW\n' \ + "${power_limit}" "${GPU_POWER_LIMIT_MAX}" + elif (( used >= GPU_MEMORY_LIMIT )); then + printf 'gpu_memory_limit_%sMiB\n' "${used}" + elif (( temperature >= GPU_TEMPERATURE_LIMIT )); then + printf 'gpu_temperature_limit_%sC\n' "${temperature}" + else + printf 'healthy\n' + fi +} + +gpu_healthy() { + [[ "$(gpu_health_reason)" == "healthy" ]] +} + +gpu_compute_idle() { + local apps + apps="$(timeout 5 nvidia-smi --id=0 --query-compute-apps=pid \ + --format=csv,noheader,nounits 2>/dev/null)" || return 1 + [[ -z "${apps//[[:space:]]/}" ]] +} + +wait_for_gpu_idle() { + local waited=0 stable=0 sample used + printf '[cooldown] Waiting for an idle GPU before the persistent VLM cold start\n' | + tee -a "${LOG_DIR}/controller.log" + while (( waited < GPU_IDLE_TIMEOUT )); do + if ! gpu_healthy; then + printf '[error] GPU is unhealthy during idle wait: %s\n' \ + "$(gpu_health_reason)" >&2 + return 1 + fi + sample="$(gpu_query)" + IFS=',' read -r used _ <<<"${sample}" + used="${used//[[:space:]]/}" + if gpu_compute_idle && [[ "${used}" =~ ^[0-9]+$ ]] && + (( used <= GPU_IDLE_MEMORY_LIMIT )); then + stable=$((stable + 5)) + if (( stable >= GPU_IDLE_STABLE_SECONDS )); then + printf '[cooldown] GPU idle for %ss; starting persistent VLM services\n' \ + "${stable}" | tee -a "${LOG_DIR}/controller.log" + return 0 + fi + else + stable=0 + fi + sleep 5 + waited=$((waited + 5)) + done + printf '[error] GPU did not remain idle for %ss within %ss\n' \ + "${GPU_IDLE_STABLE_SECONDS}" "${GPU_IDLE_TIMEOUT}" >&2 + return 1 +} + +port_open() { + timeout 1 bash -c "exec 3<>/dev/tcp/127.0.0.1/$1" >/dev/null 2>&1 +} + +wait_for_port() { + local port="$1" + local name="$2" + local max_wait="${3:-300}" + local waited=0 + while (( waited < max_wait )); do + if port_open "${port}"; then + printf '[ready] %s on port %s\n' "${name}" "${port}" + return 0 + fi + if ! gpu_healthy; then + printf '[error] GPU became unhealthy while loading %s\n' "${name}" >&2 + return 1 + fi + sleep 2 + waited=$((waited + 2)) + done + printf '[error] %s did not open port %s in %ss\n' "${name}" "${port}" "${max_wait}" >&2 + return 1 +} + +vlm_stack_ready() { + tmux has-session -t "${VLM_SESSION}" 2>/dev/null && + port_open 12181 && port_open 12182 && port_open 12183 && port_open 12185 && + pgrep -f ' -m vlm\.detector\.grounding_dino( |$)' >/dev/null && + pgrep -f ' -m vlm\.itm\.blip2itm( |$)' >/dev/null && + pgrep -f ' -m vlm\.segmentor\.sam( |$)' >/dev/null && + pgrep -f ' -m vlm\.detector\.dfine( |$)' >/dev/null +} + +vlm_stack_present() { + tmux has-session -t "${VLM_SESSION}" 2>/dev/null || + port_open 12181 || port_open 12182 || port_open 12183 || port_open 12185 || + pgrep -f ' -m vlm\.(itm\.blip2itm|segmentor\.sam|detector\.(dfine|grounding_dino))( |$)' \ + >/dev/null +} + +wait_for_persistent_service() { + local name="$1" port="$2" process_pattern="$3" max_wait="$4" waited=0 + while (( waited < max_wait )); do + if port_open "${port}" && pgrep -f "${process_pattern}" >/dev/null; then + printf '[ready] persistent %s on port %s after %ss\n' \ + "${name}" "${port}" "${waited}" | tee -a "${LOG_DIR}/controller.log" + return 0 + fi + if ! tmux has-session -t "${VLM_SESSION}" 2>/dev/null || + ! gpu_healthy; then + printf '[error] persistent %s failed during cold start: %s\n' \ + "${name}" "$(gpu_health_reason)" >&2 + return 1 + fi + sleep 2 + waited=$((waited + 2)) + done + printf '[error] persistent %s did not open port %s in %ss\n' \ + "${name}" "${port}" "${max_wait}" >&2 + return 1 +} + +start_vlm_window() { + local mode="$1" name="$2" module="$3" port="$4" command + printf -v command \ + "cd %q; exec env -u ALL_PROXY -u all_proxy -u HTTPS_PROXY -u https_proxy -u HTTP_PROXY -u http_proxy VLM_SERVER_THREADS=1 %q -m %q --port %q >>%q 2>&1" \ + "${ROOT}" "${PYTHON}" "${module}" "${port}" "${VLM_LOG_DIR}/${name}.log" + if [[ "${mode}" == "session" ]]; then + tmux new-session -d -s "${VLM_SESSION}" -n "${name}" "${command}" + else + tmux new-window -d -t "${VLM_SESSION}" -n "${name}" "${command}" + fi +} + +ensure_vlm_stack() { + if vlm_stack_ready; then + printf '[reuse] Persistent single BLIP2, GroundingDINO, MobileSAM, and D-FINE are healthy\n' | + tee -a "${LOG_DIR}/controller.log" + return 0 + fi + if vlm_stack_present; then + printf '[error] Partial or foreign VLM stack detected; refusing an automatic restart\n' >&2 + printf '[hint] Inspect with: tmux attach -t %s\n' "${VLM_SESSION}" >&2 + printf '[hint] Stop explicitly with: tmux kill-session -t %s\n' \ + "${VLM_SESSION}" >&2 + return 1 + fi + + wait_for_gpu_idle || return 1 + mkdir -p "${VLM_LOG_DIR}" + for name in blip2 sam dfine gdino; do + printf '\n=== persistent VLM cold start %s at %s ===\n' \ + "${name}" "$(date --iso-8601=seconds)" >>"${VLM_LOG_DIR}/${name}.log" + done + + start_vlm_window session blip2 vlm.itm.blip2itm 12182 + if ! wait_for_persistent_service blip2 12182 \ + ' -m vlm\.itm\.blip2itm( |$)' 600; then + tmux kill-session -t "${VLM_SESSION}" 2>/dev/null || true + return 1 + fi + printf '[cooldown] BLIP2 loaded; waiting %ss before MobileSAM\n' \ + "${MODEL_LOAD_COOLDOWN}" | tee -a "${LOG_DIR}/controller.log" + sleep "${MODEL_LOAD_COOLDOWN}" + + start_vlm_window window sam vlm.segmentor.sam 12183 + if ! wait_for_persistent_service sam 12183 \ + ' -m vlm\.segmentor\.sam( |$)' 240; then + tmux kill-session -t "${VLM_SESSION}" 2>/dev/null || true + return 1 + fi + printf '[cooldown] MobileSAM loaded; waiting %ss before D-FINE\n' \ + "${MODEL_LOAD_COOLDOWN}" | tee -a "${LOG_DIR}/controller.log" + sleep "${MODEL_LOAD_COOLDOWN}" + + start_vlm_window window dfine vlm.detector.dfine 12185 + if ! wait_for_persistent_service dfine 12185 \ + ' -m vlm\.detector\.dfine( |$)' 300; then + tmux kill-session -t "${VLM_SESSION}" 2>/dev/null || true + return 1 + fi + printf '[cooldown] D-FINE loaded; waiting %ss before GroundingDINO\n' \ + "${MODEL_LOAD_COOLDOWN}" | tee -a "${LOG_DIR}/controller.log" + sleep "${MODEL_LOAD_COOLDOWN}" + + start_vlm_window window gdino vlm.detector.grounding_dino 12181 + if ! wait_for_persistent_service gdino 12181 \ + ' -m vlm\.detector\.grounding_dino( |$)' 300; then + tmux kill-session -t "${VLM_SESSION}" 2>/dev/null || true + return 1 + fi + printf '[cooldown] Persistent VLM stack ready; waiting %ss before Habitat\n' \ + "${MODEL_LOAD_COOLDOWN}" | tee -a "${LOG_DIR}/controller.log" + sleep "${MODEL_LOAD_COOLDOWN}" +} + +start_group() { + local name="$1" + local log_file="$2" + shift 2 + setsid "$@" >>"${log_file}" 2>&1 & + PIDS["${name}"]=$! + printf '[start] %s pid=%s\n' "${name}" "${PIDS[${name}]}" +} + +stop_group() { + local name="$1" + local signal="${2:-TERM}" + local pid="${PIDS[${name}]:-}" + if [[ -n "${pid}" ]] && kill -0 "${pid}" 2>/dev/null; then + kill "-${signal}" -- "-${pid}" 2>/dev/null || true + for _ in {1..20}; do + kill -0 "${pid}" 2>/dev/null || break + sleep 1 + done + if kill -0 "${pid}" 2>/dev/null; then + kill -KILL -- "-${pid}" 2>/dev/null || true + fi + fi + unset 'PIDS['"${name}"']' || true +} + +completed_count() { + local continue_file="${OUTPUT_DIR}/continue.txt" + if [[ ! -f "${continue_file}" ]]; then + printf '0\n' + return + fi + sed -n 's/^No\.\([0-9][0-9]*\) task is finished$/\1/p' "${continue_file}" | + head -n 1 +} + +send_status() { + [[ "${ENABLE_FEISHU_REPORT}" == "true" ]] || return 0 + "${PYTHON}" "${ROOT}/scripts/report_hm3dv2_greedy1000.py" --send \ + >>"${LOG_DIR}/feishu.log" 2>&1 || true +} + +disable_reporter() { + [[ "${ENABLE_FEISHU_REPORT}" == "true" ]] || return 0 + [[ -n "${CC_CONNECT_DATA_DIR}" ]] || return 0 + if [[ -s "${CRON_ID_FILE}" ]]; then + local cron_id + cron_id="$(tr -d '[:space:]' <"${CRON_ID_FILE}")" + if [[ -n "${cron_id}" ]]; then + cc-connect cron edit "${cron_id}" enabled false \ + --data-dir "${CC_CONNECT_DATA_DIR}" \ + >>"${LOG_DIR}/feishu.log" 2>&1 || true + fi + fi +} + +cleanup_stack() { + stop_group eval INT + stop_group verifier TERM + stop_group planner TERM +} + +cleanup_runtime() { + cleanup_stack + stop_group roscore TERM + stop_group kernel_trace TERM + stop_group gpu_trace TERM +} + +start_gpu_monitors() { + start_group gpu_trace "${GPU_TRACE_LOG}" nvidia-smi --id=0 \ + --query-gpu=timestamp,pstate,temperature.gpu,power.draw,power.limit,memory.used,memory.total,utilization.gpu,utilization.memory,pcie.link.gen.current,pcie.link.width.current \ + --format=csv -l 1 + start_group kernel_trace "${KERNEL_GPU_LOG}" journalctl -kf --since now -o short-iso +} + +fatal() { + local reason="$1" + printf '[fatal] %s\n' "${reason}" | tee -a "${LOG_DIR}/controller.log" + write_state "stopped" "${reason}" + send_status + cleanup_runtime + exit 1 +} + +trap 'write_state "stopped" "signal"; cleanup_runtime' INT TERM +trap 'cleanup_runtime' EXIT + +if (( START_VLM_ONLY == 1 )); then + : +elif [[ -e "${OUTPUT_DIR}/continue.txt" ]]; then + if [[ ! -f "${OUTPUT_DIR}/evaluation_protocol.json" || + ! -f "${RUN_PROTOCOL_FILE}" ]]; then + printf '[error] Refusing legacy checkpoint without protocol manifests: %s\n' \ + "${OUTPUT_DIR}" >&2 + printf '[hint] Start the 1 m protocol in a new output directory\n' >&2 + exit 2 + fi + if ! protocol_matches; then + printf '[error] Refusing checkpoint whose run protocol differs from this launch: %s\n' \ + "${OUTPUT_DIR}" >&2 + exit 2 + fi + printf '[resume] Using existing checkpoint: %s\n' "${OUTPUT_DIR}/continue.txt" +elif [[ -d "${OUTPUT_DIR}" ]] && + find "${OUTPUT_DIR}" -mindepth 1 -maxdepth 1 -print -quit 2>/dev/null | + grep -q .; then + printf '[error] Refusing a non-empty output directory without a checkpoint: %s\n' \ + "${OUTPUT_DIR}" >&2 + exit 2 +fi + +if pgrep -f '/python([0-9.]*)? .*habitat_evaluation\.py' >/dev/null || + pgrep -f '/exploration_node([ ]|$)' >/dev/null; then + printf '[error] Another evaluation/planner process is already running\n' >&2 + exit 2 +fi +if ! [[ -x "${PYTHON}" ]]; then + printf '[error] Python environment not found: %s\n' "${PYTHON}" >&2 + exit 2 +fi +if (( START_VLM_ONLY == 0 )); then + if ! "${PYTHON}" "${ROOT}/scripts/check_hypothesis_coverage.py" \ + --env-dir "${HYPOTHESIS_DIR}" --total "${TOTAL}"; then + printf '[cloud] Missing hypotheses will be generated at episode start\n' + fi +fi +if ! command -v tmux >/dev/null; then + printf '[error] tmux is required for the resumable evaluation\n' >&2 + exit 2 +fi +if ! [[ "${CONTROL_RATE_HZ}" =~ ^[0-9]+([.][0-9]+)?$ ]] || + ! awk -v rate="${CONTROL_RATE_HZ}" 'BEGIN { exit !(rate > 0) }' || + ! [[ "${TOTAL}" =~ ^[1-9][0-9]*$ ]] || + ! [[ "${HABITAT_SEED}" =~ ^[0-9]+$ ]] || + ! [[ "${BATCH_SIZE}" =~ ^[1-9][0-9]*$ ]] || + ! [[ "${VLM_REQUEST_TIMEOUT}" =~ ^[0-9]+([.][0-9]+)?$ ]] || + ! awk -v timeout="${VLM_REQUEST_TIMEOUT}" 'BEGIN { exit !(timeout > 0) }' || + ! [[ "${VLM_REQUEST_ATTEMPTS}" =~ ^[1-9][0-9]*$ ]] || + ! [[ "${MODEL_LOAD_COOLDOWN}" =~ ^[0-9]+$ ]] || + ! awk -v cooldown="${MODEL_LOAD_COOLDOWN}" 'BEGIN { exit !(cooldown >= 30) }' || + ! [[ "${GPU_POWER_LIMIT_MAX}" =~ ^[1-9][0-9]*([.][0-9]+)?$ ]] || + ! awk -v maximum="${GPU_POWER_LIMIT_MAX}" 'BEGIN { exit !(maximum <= 220) }' || + ! [[ "${GPU_IDLE_MEMORY_LIMIT}" =~ ^[1-9][0-9]*$ ]] || + ! [[ "${GPU_IDLE_STABLE_SECONDS}" =~ ^[1-9][0-9]*$ ]] || + ! awk -v stable="${GPU_IDLE_STABLE_SECONDS}" 'BEGIN { exit !(stable >= 60) }' || + ! [[ "${GPU_IDLE_TIMEOUT}" =~ ^[1-9][0-9]*$ ]] || + ! [[ "${FORCE_REGENERATE_HYPOTHESES}" =~ ^[01]$ ]] || + ! [[ "${SUCCESS_DISTANCE}" =~ ^[0-9]+([.][0-9]+)?$ ]] || + ! awk -v distance="${SUCCESS_DISTANCE}" 'BEGIN { exit !(distance > 0) }' || + ! [[ "${EXPLORATION_POLICY}" =~ ^(0|1|4)$ ]] || + ! [[ "${USE_MULTI_SOURCE}" =~ ^(true|false)$ ]] || + ! [[ "${USE_IG_FUSION}" =~ ^(true|false)$ ]] || + ! [[ "${USE_ADAPTIVE_ALPHA}" =~ ^(true|false)$ ]] || + ! [[ "${FUSION_ALPHA}" =~ ^(0(\.[0-9]+)?|1(\.0+)?)$ ]] || + ! [[ "${ALPHA_BASE}" =~ ^(0(\.[0-9]+)?|1(\.0+)?)$ ]] || + ! [[ "${TAU_CONF}" =~ ^[0-9]+([.][0-9]+)?$ ]] || + ! awk -v threshold="${TAU_CONF}" 'BEGIN { exit !(threshold > 0) }' || + ! [[ "${TOP_K_VALUE}" =~ ^[1-9][0-9]*$ ]] || + ! [[ "${WTRP_MAX_BRUTE_FORCE}" =~ ^[1-9][0-9]*$ ]] || + ! awk -v top_k="${TOP_K_VALUE}" -v brute="${WTRP_MAX_BRUTE_FORCE}" \ + 'BEGIN { exit !(top_k <= 8 && brute <= 8 && brute <= top_k) }' || + ! [[ "${WTRP_TEMPERATURE}" =~ ^[0-9]+([.][0-9]+)?$ ]] || + ! awk -v temperature="${WTRP_TEMPERATURE}" 'BEGIN { exit !(temperature > 0) }' || + ! [[ "${WTRP_HYSTERESIS_RATIO}" =~ ^(0(\.[0-9]+)?|1(\.0+)?)$ ]] || + ! awk -v hysteresis="${WTRP_HYSTERESIS_RATIO}" 'BEGIN { exit !(hysteresis > 0) }' || + ! [[ "${ENABLE_FEISHU_REPORT}" =~ ^(true|false)$ ]] || + ! [[ "${VLM_VALIDATION_ENABLED}" =~ ^(true|false)$ ]] || + ! [[ "${EXPERIMENT_GIT_COMMIT}" =~ ^[0-9a-f]{40}$ ]] || + ! [[ "${EXPERIMENT_GIT_DIRTY_TRACKED}" =~ ^(true|false)$ ]] || + ! awk -v stable="${GPU_IDLE_STABLE_SECONDS}" -v timeout="${GPU_IDLE_TIMEOUT}" \ + 'BEGIN { exit !(timeout >= stable) }'; then + printf '[error] Evaluation, cooldown, or GPU guard configuration is invalid\n' >&2 + exit 2 +fi +if (( START_VLM_ONLY == 0 )) && [[ -z "${DASHSCOPE_API_KEY:-}" ]]; then + printf '[error] DASHSCOPE_API_KEY is required for the HM3D approach verifier\n' >&2 + exit 2 +fi +if (( START_VLM_ONLY == 0 )) && [[ -z "${DEEPSEEK_API_KEY:-}" ]]; then + printf '[error] DEEPSEEK_API_KEY is required for cloud hypothesis generation\n' >&2 + exit 2 +fi +if ! [[ "${DASHSCOPE_REGION}" =~ ^(beijing|singapore|tokyo|virginia)$ ]]; then + printf '[error] DASHSCOPE_REGION must be beijing, singapore, tokyo, or virginia\n' >&2 + exit 2 +fi +if (( START_VLM_ONLY == 0 )) && [[ "${DASHSCOPE_REGION}" == "tokyo" ]] && + [[ -z "${DASHSCOPE_WORKSPACE_ID:-}" ]] && + [[ -z "${DASHSCOPE_BASE_URL:-}" ]]; then + printf '[error] Tokyo DashScope requires DASHSCOPE_WORKSPACE_ID or DASHSCOPE_BASE_URL\n' >&2 + exit 2 +fi + +# All six HM3D targets should be cached. DeepSeek V4 Flash is the online +# fallback only when a cache entry is missing, so local Ollama is unnecessary. +missing_cache=() +for label in couch tv chair toilet bed 'potted plant'; do + if ! grep -F -q -- "${label}:" "${ROOT}/llm/answers/llm_answer_hm3d.txt"; then + missing_cache+=("${label}") + fi +done +if (( START_VLM_ONLY == 0 )) && (( ${#missing_cache[@]} > 0 )) && + [[ -z "${DEEPSEEK_API_KEY:-}" ]]; then + printf '[error] Missing HM3D LLM cache entries (%s) and DEEPSEEK_API_KEY is unset\n' \ + "${missing_cache[*]}" >&2 + exit 2 +fi + +gpu_reason="$(gpu_health_reason)" +if [[ "${gpu_reason}" != "healthy" ]]; then + if (( PREFLIGHT_ONLY == 0 )); then + write_state "waiting_for_gpu" "${gpu_reason}" + fi + printf '[blocked] GPU is not healthy: %s\n' "${gpu_reason}" >&2 + if [[ "${gpu_reason}" == gpu_power_limit_* ]]; then + printf '[hint] Apply the validated Prism guard before retrying: sudo nvidia-smi -pl %s\n' \ + "${GPU_POWER_LIMIT_MAX}" >&2 + fi + exit 3 +fi + +# The navigation stack is per-run. The four expected local VLM services are +# handled separately and intentionally persist across runner invocations. +guarded_ports=(12184 12192) +if (( START_VLM_ONLY == 0 )); then + guarded_ports+=(11311) +fi +for port in "${guarded_ports[@]}"; do + if port_open "${port}"; then + printf '[error] Unexpected ROS/VLM service on port %s\n' \ + "${port}" >&2 + exit 2 + fi +done + +if (( PREFLIGHT_ONLY == 1 )); then + if vlm_stack_present && ! vlm_stack_ready; then + printf '[preflight] Partial or foreign VLM stack detected\n' >&2 + exit 2 + fi + printf '[preflight] Passed without starting tmux, VLM, ROS, or evaluation services\n' + exit 0 +fi + +if (( START_VLM_ONLY == 1 )); then + mkdir -p "${LOG_DIR}" + printf '\n[vlm-only] start requested at %s\n' "$(date --iso-8601=seconds)" | + tee -a "${LOG_DIR}/controller.log" + start_gpu_monitors + if ! ensure_vlm_stack; then + printf '[vlm-only] Persistent VLM startup failed\n' | + tee -a "${LOG_DIR}/controller.log" >&2 + exit 1 + fi + printf '[vlm-only] Persistent VLM stack is ready in tmux session %s\n' \ + "${VLM_SESSION}" | tee -a "${LOG_DIR}/controller.log" + exit 0 +fi + +# Re-enter inside tmux so the long run survives terminal/bridge disconnects. +if [[ "${INFONAV_GREEDY1000_INNER:-0}" != "1" ]]; then + if tmux has-session -t "${SESSION}" 2>/dev/null; then + printf '[error] tmux session %s already exists\n' "${SESSION}" >&2 + exit 2 + fi + # Create a placeholder pane, copy the selected runtime environment into the + # session without exposing API keys in the process command line, then respawn. + tmux new-session -d -s "${SESSION}" 'sleep 86400' + for key in DASHSCOPE_API_KEY DASHSCOPE_BASE_URL DASHSCOPE_WORKSPACE_ID \ + DEEPSEEK_API_KEY DEEPSEEK_BASE_URL; do + if [[ -n "${!key:-}" ]]; then + tmux set-environment -t "${SESSION}" "${key}" "${!key}" + fi + done + tmux set-environment -t "${SESSION}" QWEN_VL_MODEL "${QWEN_VL_MODEL}" + tmux set-environment -t "${SESSION}" DEEPSEEK_MODEL "${DEEPSEEK_MODEL}" + tmux set-environment -t "${SESSION}" QWEN_HYPOTHESIS_MODEL "${QWEN_HYPOTHESIS_MODEL}" + tmux set-environment -t "${SESSION}" DEEPSEEK_HYPOTHESIS_MODEL "${DEEPSEEK_HYPOTHESIS_MODEL}" + tmux respawn-pane -k -t "${SESSION}:0.0" \ + "env INFONAV_GREEDY1000_INNER=1 RUN_TAG='${RUN_TAG}' TOTAL='${TOTAL}' PROTOCOL_ID='${PROTOCOL_ID}' HABITAT_SEED='${HABITAT_SEED}' VLM_VALIDATION_ENABLED='${VLM_VALIDATION_ENABLED}' EXPERIMENT_GIT_COMMIT='${EXPERIMENT_GIT_COMMIT}' EXPERIMENT_GIT_DIRTY_TRACKED='${EXPERIMENT_GIT_DIRTY_TRACKED}' SUCCESS_DISTANCE='${SUCCESS_DISTANCE}' EXPLORATION_POLICY='${EXPLORATION_POLICY}' USE_MULTI_SOURCE='${USE_MULTI_SOURCE}' FUSION_ALPHA='${FUSION_ALPHA}' USE_IG_FUSION='${USE_IG_FUSION}' USE_ADAPTIVE_ALPHA='${USE_ADAPTIVE_ALPHA}' ALPHA_BASE='${ALPHA_BASE}' TAU_CONF='${TAU_CONF}' TOP_K_VALUE='${TOP_K_VALUE}' WTRP_TEMPERATURE='${WTRP_TEMPERATURE}' WTRP_HYSTERESIS_RATIO='${WTRP_HYSTERESIS_RATIO}' WTRP_MAX_BRUTE_FORCE='${WTRP_MAX_BRUTE_FORCE}' ENABLE_FEISHU_REPORT='${ENABLE_FEISHU_REPORT}' BATCH_SIZE='${BATCH_SIZE}' STALL_TIMEOUT='${STALL_TIMEOUT}' GPU_MEMORY_LIMIT='${GPU_MEMORY_LIMIT}' GPU_TEMPERATURE_LIMIT='${GPU_TEMPERATURE_LIMIT}' GPU_POWER_LIMIT_MAX='${GPU_POWER_LIMIT_MAX}' GPU_IDLE_MEMORY_LIMIT='${GPU_IDLE_MEMORY_LIMIT}' GPU_IDLE_STABLE_SECONDS='${GPU_IDLE_STABLE_SECONDS}' GPU_IDLE_TIMEOUT='${GPU_IDLE_TIMEOUT}' VLM_SESSION='${VLM_SESSION}' CONTROL_RATE_HZ='${CONTROL_RATE_HZ}' MODEL_LOAD_COOLDOWN='${MODEL_LOAD_COOLDOWN}' VLM_REQUEST_TIMEOUT='${VLM_REQUEST_TIMEOUT}' VLM_REQUEST_ATTEMPTS='${VLM_REQUEST_ATTEMPTS}' DASHSCOPE_REGION='${DASHSCOPE_REGION}' QWEN_HYPOTHESIS_MODEL='${QWEN_HYPOTHESIS_MODEL}' DEEPSEEK_HYPOTHESIS_MODEL='${DEEPSEEK_HYPOTHESIS_MODEL}' FORCE_REGENERATE_HYPOTHESES='${FORCE_REGENERATE_HYPOTHESES}' '${BASH_SOURCE[0]}' '${OUTPUT_DIR}'" + printf '[started] tmux=%s output=%s\n' "${SESSION}" "${OUTPUT_DIR}" + exit 0 +fi + +mkdir -p "${LOG_DIR}" +if [[ ! -f "${RUN_PROTOCOL_FILE}" ]]; then + write_run_protocol +fi +write_state "starting" +printf '[config] protocol=%s output=%s total=%s batch=%s policy=%s success_distance=%sm HSVM_multi=%s fusion=(adaptive=%s,base=%s,IG=%s) WTRP=(top_k=%s,temp=%s,hysteresis=%s,brute=%s) rate=%sHz local_models=BLIP2x1,GroundingDINO,MobileSAM,D-FINE terminal_verifier=%s(%s) hypothesis_models=%s+%s\n' \ + "${PROTOCOL_ID}" "${OUTPUT_DIR}" "${TOTAL}" "${BATCH_SIZE}" \ + "${EXPLORATION_POLICY}" "${SUCCESS_DISTANCE}" "${USE_MULTI_SOURCE}" \ + "${USE_ADAPTIVE_ALPHA}" "${ALPHA_BASE}" "${USE_IG_FUSION}" \ + "${TOP_K_VALUE}" "${WTRP_TEMPERATURE}" "${WTRP_HYSTERESIS_RATIO}" \ + "${WTRP_MAX_BRUTE_FORCE}" "${CONTROL_RATE_HZ}" \ + "${VLM_VALIDATION_ENABLED}" "${QWEN_VL_MODEL}" \ + "${QWEN_HYPOTHESIS_MODEL}" "${DEEPSEEK_HYPOTHESIS_MODEL}" | tee -a "${LOG_DIR}/controller.log" + +start_gpu_monitors +ensure_vlm_stack || fatal "persistent_vlm_start_failed" + +start_group roscore "${LOG_DIR}/roscore.log" bash -lc \ + "source /opt/ros/noetic/setup.bash; exec roscore" +wait_for_port 11311 roscore 60 || fatal "roscore_start_failed" + +start_stack() { + local force_regenerate_arg="" + if [[ "${FORCE_REGENERATE_HYPOTHESES}" == "1" ]]; then + force_regenerate_arg="--force-regenerate" + fi + start_group planner "${LOG_DIR}/planner.log" bash -lc \ + "cd '${ROOT}'; source /opt/ros/noetic/setup.bash; source devel/setup.bash; exec roslaunch exploration_manager exploration.launch dataset:=hm3dv2 vlm_validation_enabled:='${VLM_VALIDATION_ENABLED}' exploration_policy:='${EXPLORATION_POLICY}' use_multi_source:='${USE_MULTI_SOURCE}' fusion_alpha:='${FUSION_ALPHA}' use_ig_fusion:='${USE_IG_FUSION}' use_adaptive_alpha:='${USE_ADAPTIVE_ALPHA}' alpha_base:='${ALPHA_BASE}' tau_conf:='${TAU_CONF}' top_k_value:='${TOP_K_VALUE}' wtrp_temperature:='${WTRP_TEMPERATURE}' wtrp_hysteresis_ratio:='${WTRP_HYSTERESIS_RATIO}' wtrp_max_brute_force:='${WTRP_MAX_BRUTE_FORCE}'" + sleep 8 + if ! kill -0 "${PIDS[planner]}" 2>/dev/null; then + fatal "planner_start_failed" + fi + + if [[ "${VLM_VALIDATION_ENABLED}" == "true" ]]; then + start_group verifier "${LOG_DIR}/verifier.log" bash -ic \ + "cd '${ROOT}'; source /opt/ros/noetic/setup.bash; source devel/setup.bash; exec env -u ALL_PROXY -u all_proxy -u HTTPS_PROXY -u https_proxy -u HTTP_PROXY -u http_proxy '${PYTHON}' src/planner/plan_env/scripts/vlm_approach_verifier.py _model:='${QWEN_VL_MODEL}' _api_region:='${DASHSCOPE_REGION}' _debug_save_images:=false _max_completion_tokens:=384" + sleep 3 + if ! kill -0 "${PIDS[verifier]}" 2>/dev/null; then + fatal "verifier_start_failed" + fi + else + printf '[config] terminal VLM validation disabled for this planning-policy comparison\n' \ + | tee -a "${LOG_DIR}/controller.log" + fi + + start_group eval "${LOG_DIR}/eval.log" bash -ic \ + "cd '${ROOT}'; source /opt/ros/noetic/setup.bash; source devel/setup.bash; exec env -u ALL_PROXY -u all_proxy -u HTTPS_PROXY -u https_proxy -u HTTP_PROXY -u http_proxy APEXNAV_LOG_LEVEL=MINIMAL PYTHONUNBUFFERED=1 USE_PARALLEL_BLIP2=false '${PYTHON}' habitat_evaluation.py --dataset hm3dv2 --detector-mode 2 --control-rate-hz '${CONTROL_RATE_HZ}' --max-episodes-per-process '${BATCH_SIZE}' --realtime-hypothesis --vlm-model '${QWEN_HYPOTHESIS_MODEL}' --llm-model '${DEEPSEEK_HYPOTHESIS_MODEL}' ${force_regenerate_arg} habitat.seed='${HABITAT_SEED}' habitat.task.measurements.success.success_distance='${SUCCESS_DISTANCE}' llm.llm_client.llm_client=deepseek need_video=false video_output_path='${OUTPUT_DIR}'" + sleep 5 + if ! kill -0 "${PIDS[eval]}" 2>/dev/null; then + fatal "evaluation_start_failed" + fi +} + +start_stack +write_state "running" +send_status + +last_count="$(completed_count)" +last_progress_time="$(date +%s)" +batch_start="${last_count}" +last_gpu_log_time=0 +printf '[monitor] resume count=%s\n' "${last_count}" | tee -a "${LOG_DIR}/controller.log" + +while (( last_count < TOTAL )); do + sleep 10 + + gpu_reason="$(gpu_health_reason)" + if [[ "${gpu_reason}" != "healthy" ]]; then + fatal "${gpu_reason}" + fi + vlm_stack_ready || fatal "persistent_vlm_unhealthy" + + current_count="$(completed_count)" + if ! [[ "${current_count}" =~ ^[0-9]+$ ]]; then + fatal "checkpoint_parse_failed" + fi + if (( current_count > last_count )); then + last_count="${current_count}" + last_progress_time="$(date +%s)" + printf '[progress] %s/%s\n' "${last_count}" "${TOTAL}" | + tee -a "${LOG_DIR}/controller.log" + fi + if (( current_count >= TOTAL )); then + break + fi + + now="$(date +%s)" + if (( now - last_gpu_log_time >= 60 )); then + printf '[gpu] %s\n' "$(gpu_query)" | tee -a "${LOG_DIR}/controller.log" + last_gpu_log_time="${now}" + fi + if (( now - last_progress_time > STALL_TIMEOUT )); then + printf '[recover] no completed episode for %ss; restarting navigation stack\n' \ + "$((now - last_progress_time))" | tee -a "${LOG_DIR}/controller.log" + cleanup_stack + sleep 5 + start_stack + last_progress_time="$(date +%s)" + batch_start="${current_count}" + continue + fi + + verifier_exited=false + if [[ "${VLM_VALIDATION_ENABLED}" == "true" ]] && + ! kill -0 "${PIDS[verifier]:-}" 2>/dev/null; then + verifier_exited=true + fi + if ! kill -0 "${PIDS[eval]}" 2>/dev/null || + ! kill -0 "${PIDS[planner]}" 2>/dev/null || + [[ "${verifier_exited}" == "true" ]]; then + if (( current_count - batch_start >= BATCH_SIZE )); then + printf '[batch] evaluator completed %s episodes and exited cleanly; restarting navigation stack\n' \ + "$((current_count - batch_start))" | tee -a "${LOG_DIR}/controller.log" + else + printf '[recover] navigation process exited; restarting from checkpoint %s\n' \ + "${current_count}" | tee -a "${LOG_DIR}/controller.log" + fi + cleanup_stack + sleep 5 + start_stack + last_progress_time="$(date +%s)" + batch_start="${current_count}" + continue + fi + +done + +write_state "completed" +send_status +disable_reporter +printf '[done] completed %s/%s episodes\n' "${last_count}" "${TOTAL}" | + tee -a "${LOG_DIR}/controller.log" diff --git a/scripts/run_hm3dv2_value_wtrp_pair100.sh b/scripts/run_hm3dv2_value_wtrp_pair100.sh new file mode 100755 index 0000000..9ca541b --- /dev/null +++ b/scripts/run_hm3dv2_value_wtrp_pair100.sh @@ -0,0 +1,148 @@ +#!/usr/bin/env bash +# Run a strict same-episode HM3Dv2 comparison: +# arm A: policy=1, greedy maximum of the fused frontier value +# arm B: policy=4, WTRP over the same fused frontier values + +set -Eeuo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd -- "${SCRIPT_DIR}/.." && pwd)" +RUNNER="${ROOT}/scripts/run_hm3dv2_greedy1000.sh" +PAIR_SESSION="${PAIR_SESSION:-infonav_value_wtrp_pair100}" +PAIR_STAMP="${PAIR_STAMP:-$(date +%Y%m%d_%H%M%S)}" +PAIR_ROOT="${1:-${ROOT}/videos/test_hm3dv2_value_wtrp_pair100_${PAIR_STAMP}_val}" +VALUE_PROTOCOL="${PAIR_ROOT}/value_greedy/run_protocol.env" +if [[ -f "${VALUE_PROTOCOL}" ]]; then + EXPERIMENT_GIT_COMMIT="$(sed -n 's/^GIT_COMMIT=//p' "${VALUE_PROTOCOL}" | head -n 1)" + EXPERIMENT_GIT_DIRTY_TRACKED="$(sed -n 's/^GIT_DIRTY_TRACKED=//p' "${VALUE_PROTOCOL}" | head -n 1)" +else + EXPERIMENT_GIT_COMMIT="$(git -C "${ROOT}" rev-parse HEAD)" + if [[ -n "$(git -C "${ROOT}" status --porcelain --untracked-files=no)" ]]; then + EXPERIMENT_GIT_DIRTY_TRACKED=true + else + EXPERIMENT_GIT_DIRTY_TRACKED=false + fi +fi + +common_env=( + TOTAL=100 + HABITAT_SEED=100 + SUCCESS_DISTANCE=1.0 + USE_MULTI_SOURCE=true + FUSION_ALPHA=0.8 + USE_IG_FUSION=true + USE_ADAPTIVE_ALPHA=true + ALPHA_BASE=0.8 + TAU_CONF=0.8 + TOP_K_VALUE=8 + WTRP_TEMPERATURE=0.3 + WTRP_HYSTERESIS_RATIO=0.8 + WTRP_MAX_BRUTE_FORCE=8 + ENABLE_FEISHU_REPORT=false + VLM_VALIDATION_ENABLED=false + EXPERIMENT_GIT_COMMIT="${EXPERIMENT_GIT_COMMIT}" + EXPERIMENT_GIT_DIRTY_TRACKED="${EXPERIMENT_GIT_DIRTY_TRACKED}" + FORCE_REGENERATE_HYPOTHESES=0 +) + +state_value() { + local file="$1" key="$2" + if [[ ! -f "${file}" ]]; then + return 0 + fi + sed -n "s/^${key}=//p" "${file}" | head -n 1 +} + +metrics_count() { + local metrics_file="$1" + if [[ ! -f "${metrics_file}" ]]; then + printf '0\n' + return + fi + wc -l <"${metrics_file}" | tr -d '[:space:]' +} + +wait_for_arm() { + local state_file="$1" session="$2" metrics_file="$3" status completed + while true; do + completed="$(metrics_count "${metrics_file}")" + if [[ "${completed}" =~ ^[0-9]+$ ]] && (( completed >= 100 )); then + printf '[pair] %s completed according to episode metrics (%s/100)\n' \ + "${session}" "${completed}" + return 0 + fi + status="$(state_value "${state_file}" STATUS)" + case "${status}" in + completed) + printf '[pair] %s completed\n' "${session}" + return 0 + ;; + stopped) + printf '[pair:error] %s stopped: %s\n' \ + "${session}" "$(state_value "${state_file}" REASON)" >&2 + return 1 + ;; + esac + if [[ -n "${status}" ]] && ! tmux has-session -t "${session}" 2>/dev/null; then + printf '[pair:error] %s disappeared with status=%s\n' "${session}" "${status}" >&2 + return 1 + fi + sleep 15 + done +} + +run_arm() { + local name="$1" policy="$2" + local run_tag="hm3dv2_${name}100_${PAIR_STAMP}" + local arm_session="infonav_${name}100_${PAIR_STAMP}" + local protocol_id="hm3dv2_${name}_hsvm_ig_gdino_success1m_pair100_v1" + local output_dir="${PAIR_ROOT}/${name}" + local state_file="${ROOT}/.run_state/${run_tag}.env" + local metrics_file="${output_dir}/episode_metrics.jsonl" + + local status completed + status="$(state_value "${state_file}" STATUS)" + completed="$(metrics_count "${metrics_file}")" + if [[ "${status}" == "completed" ]] || + { [[ "${completed}" =~ ^[0-9]+$ ]] && (( completed >= 100 )); }; then + printf '[pair] %s was already completed; continuing\n' "${name}" + return 0 + fi + if tmux has-session -t "${arm_session}" 2>/dev/null; then + printf '[pair] reusing active %s arm session %s\n' "${name}" "${arm_session}" + wait_for_arm "${state_file}" "${arm_session}" "${metrics_file}" + return + fi + if [[ "${status}" == "running" || "${status}" == "starting" ]]; then + printf '[pair:error] %s state is %s but its tmux session is absent\n' \ + "${name}" "${status}" >&2 + return 1 + fi + printf '[pair] starting %s (policy=%s) -> %s\n' "${name}" "${policy}" "${output_dir}" + env "${common_env[@]}" \ + RUN_TAG="${run_tag}" SESSION="${arm_session}" \ + PROTOCOL_ID="${protocol_id}" EXPLORATION_POLICY="${policy}" \ + "${RUNNER}" "${output_dir}" + wait_for_arm "${state_file}" "${arm_session}" "${metrics_file}" +} + +if [[ "${PAIR_INNER:-0}" != "1" ]]; then + if tmux has-session -t "${PAIR_SESSION}" 2>/dev/null; then + printf '[pair:error] manager session already exists: %s\n' "${PAIR_SESSION}" >&2 + exit 2 + fi + mkdir -p "${PAIR_ROOT}" + tmux new-session -d -s "${PAIR_SESSION}" \ + "env PAIR_INNER=1 PAIR_SESSION='${PAIR_SESSION}' PAIR_STAMP='${PAIR_STAMP}' '${ROOT}/scripts/run_hm3dv2_value_wtrp_pair100.sh' '${PAIR_ROOT}' >'${PAIR_ROOT}/pair_controller.log' 2>&1" + printf '[pair:ready] tmux=%s root=%s\n' "${PAIR_SESSION}" "${PAIR_ROOT}" + printf '[pair:note] arms run sequentially and reuse the existing persistent VLM services\n' + exit 0 +fi + +run_arm value_greedy 1 +run_arm wtrp 4 +"${ROOT}/scripts/compare_hm3dv2_value_wtrp.py" \ + "${PAIR_ROOT}/value_greedy/episode_metrics.jsonl" \ + "${PAIR_ROOT}/wtrp/episode_metrics.jsonl" \ + >"${PAIR_ROOT}/paired_comparison.txt" +printf '[pair:done] comparison=%s\n' "${PAIR_ROOT}/paired_comparison.txt" diff --git a/scripts/run_retry_missing.sh b/scripts/run_retry_missing.sh index 1d28189..18b5f6c 100644 --- a/scripts/run_retry_missing.sh +++ b/scripts/run_retry_missing.sh @@ -5,11 +5,12 @@ echo "======================================================================" echo "๐Ÿ”„ ๅผ€ๅง‹้‡ๆ–ฐๆต‹่ฏ•็ผบๅคฑ็š„ LLM ๅˆ†ๆžไปปๅŠก" echo "======================================================================" -cd /home/yangz/Nav/ApexNav +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +cd "${SCRIPT_DIR}/.." -# ๆฃ€ๆŸฅ llm_hypothesis_analysis.py ๆ˜ฏๅฆๅญ˜ๅœจ -if [ ! -f "llm_hypothesis_analysis.py" ]; then - echo "โŒ ้”™่ฏฏ: llm_hypothesis_analysis.py ไธๅญ˜ๅœจ" +# ๆฃ€ๆŸฅ็ปŸไธ€ๅˆ†ๆž่„šๆœฌๆ˜ฏๅฆๅญ˜ๅœจ +if [ ! -f "scripts/llm_hypothesis_analyzer.py" ]; then + echo "โŒ ้”™่ฏฏ: scripts/llm_hypothesis_analyzer.py ไธๅญ˜ๅœจ" exit 1 fi @@ -31,16 +32,16 @@ EOF echo "" echo "๐Ÿ“Œ ไฝฟ็”จๆ–นๆณ•:" echo " 1. ๅค„็†ๆ‰€ๆœ‰็ผบๅคฑ็š„ไปปๅŠก (ๅฎŒๆ•ด):" -echo " python3 retry_missing_tasks.py --env hm3dv2" +echo " python3 scripts/llm_hypothesis_analyzer.py retry --env hm3dv2" echo "" echo " 2. ๅค„็†ๅ‰ 10 ไธชไปปๅŠก (ๆต‹่ฏ•):" -echo " python3 retry_missing_tasks.py --env hm3dv2 --start 127 --end 140" +echo " python3 scripts/llm_hypothesis_analyzer.py retry --env hm3dv2 --start 127 --end 140" echo "" echo " 3. ๅค„็†็‰นๅฎš่Œƒๅ›ด (ๅฆ‚ task300-400):" -echo " python3 retry_missing_tasks.py --env hm3dv2 --start 300 --end 400" +echo " python3 scripts/llm_hypothesis_analyzer.py retry --env hm3dv2 --start 300 --end 400" echo "" echo " 4. ๅขžๅŠ ่ถ…ๆ—ถๆ—ถ้—ด (ๅฆ‚ๆžœๅ‡บ็Žฐ่ถ…ๆ—ถ้”™่ฏฏ):" -echo " python3 retry_missing_tasks.py --env hm3dv2 --timeout 180" +echo " python3 scripts/llm_hypothesis_analyzer.py retry --env hm3dv2 --delay 10" echo "" echo "======================================================================" echo "" diff --git a/scripts/run_with_periodic_restart.sh b/scripts/run_with_periodic_restart.sh index 978dda6..7db06fb 100755 --- a/scripts/run_with_periodic_restart.sh +++ b/scripts/run_with_periodic_restart.sh @@ -60,7 +60,7 @@ BLUE='\033[0;34m' NC='\033[0m' echo -e "${BLUE}โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•—${NC}" -echo -e "${BLUE}โ•‘ ApexNav Periodic Restart Evaluation โ•‘${NC}" +echo -e "${BLUE}โ•‘ InfoNav Periodic Restart Evaluation โ•‘${NC}" echo -e "${BLUE}โ• โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•ฃ${NC}" echo -e "${BLUE}โ•‘ Dataset: ${DATASET}${NC}" echo -e "${BLUE}โ•‘ Episodes per batch: ${EPISODES_PER_BATCH}${NC}" diff --git a/scripts/set_logging_env.sh b/scripts/set_logging_env.sh index 8cf6758..e7463cf 100755 --- a/scripts/set_logging_env.sh +++ b/scripts/set_logging_env.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Set environment variables for ApexNav logging control +# Set legacy-compatible environment variables for InfoNav logging control # # Usage: # source scripts/set_logging_env.sh MINIMAL @@ -9,7 +9,7 @@ LOG_LEVEL=${1:-NORMAL} -echo "Setting ApexNav logging level to: $LOG_LEVEL" +echo "Setting InfoNav logging level to: $LOG_LEVEL" # Set Python logging level export APEXNAV_LOG_LEVEL=$LOG_LEVEL diff --git a/scripts/start_vlm_servers.sh b/scripts/start_vlm_servers.sh index d1dc26f..d19b2a9 100755 --- a/scripts/start_vlm_servers.sh +++ b/scripts/start_vlm_servers.sh @@ -1,8 +1,7 @@ #!/bin/bash -# ApexNav VLM Servers Manager using tmux # Usage: ./start_vlm_servers.sh [start|stop|restart|status] [env_name] -# Example: ./start_vlm_servers.sh start infonav +# Example: ./start_vlm_servers.sh restart infonav #่ฟ่กŒๆŸฅ็œ‹่„šๆœฌ็š„็”จๆˆทๅฟ…้กป็กฎไฟๅทฒๅฎ‰่ฃ…tmuxๅนถไธ”ๅทฒๅˆ›ๅปบๆ‰€้œ€็š„conda็Žฏๅขƒใ€‚ #่ฟ่กŒ diff --git a/scripts/system_monitor.sh b/scripts/system_monitor.sh index 21455c3..16a89b3 100755 --- a/scripts/system_monitor.sh +++ b/scripts/system_monitor.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Comprehensive System Monitor for ApexNav +# Comprehensive System Monitor for InfoNav # Shows: VLM servers, GPU, habitat_evaluation process, disk usage RED='\033[0;31m' @@ -14,7 +14,7 @@ NC='\033[0m' while true; do clear echo -e "${CYAN}โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•—${NC}" - echo -e "${CYAN}โ•‘${NC} ${GREEN}ApexNav System Monitor${NC} - $(date '+%Y-%m-%d %H:%M:%S') ${CYAN}โ•‘${NC}" + echo -e "${CYAN}โ•‘${NC} ${GREEN}InfoNav System Monitor${NC} - $(date '+%Y-%m-%d %H:%M:%S') ${CYAN}โ•‘${NC}" echo -e "${CYAN}โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•${NC}" echo "" diff --git a/scripts/test_api_services.py b/scripts/test_api_services.py new file mode 100644 index 0000000..751649a --- /dev/null +++ b/scripts/test_api_services.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +""" +ๆต‹่ฏ• VLM ๅ’Œ LLM API ๆœๅŠกๆ˜ฏๅฆๆญฃๅธธๅทฅไฝœ +""" + +import os +import sys +import time + +def test_dashscope_vlm(): + """ๆต‹่ฏ• DashScope VLM API""" + print("\n" + "="*60) + print("ๆต‹่ฏ• DashScope VLM API") + print("="*60) + + api_key = os.getenv("DASHSCOPE_API_KEY") + if not api_key: + print("[ERROR] DASHSCOPE_API_KEY ็Žฏๅขƒๅ˜้‡ๆœช่ฎพ็ฝฎ") + print("่ฏท่ฟ่กŒ: export DASHSCOPE_API_KEY='your-api-key'") + return False + + print(f"[OK] DASHSCOPE_API_KEY ๅทฒ่ฎพ็ฝฎ (้•ฟๅบฆ: {len(api_key)})") + + try: + from openai import OpenAI + + client = OpenAI( + api_key=api_key, + base_url="https://dashscope.aliyuncs.com/compatible-mode/v1", + ) + + print("[INFO] ๅ‘้€ๆต‹่ฏ•่ฏทๆฑ‚ๅˆฐ DashScope VLM...") + t0 = time.time() + + # ็ฎ€ๅ•็š„ๆ–‡ๆœฌๆต‹่ฏ•๏ผˆไธ้œ€่ฆๅ›พ็‰‡๏ผ‰ + completion = client.chat.completions.create( + model="qwen-vl-max", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "่ฏทๅ›žๅค 'VLMๆœๅŠกๆญฃๅธธ' ่ฟ™ๅ››ไธชๅญ—"} + ] + } + ], + temperature=0.1, + max_tokens=50 + ) + + t1 = time.time() + response = completion.choices[0].message.content + + print(f"[OK] VLM ๅ“ๅบ”ๆˆๅŠŸ!") + print(f" ่€—ๆ—ถ: {t1-t0:.2f}s") + print(f" ๅ“ๅบ”: {response[:100]}") + return True + + except Exception as e: + print(f"[ERROR] VLM ๆต‹่ฏ•ๅคฑ่ดฅ: {e}") + return False + + +def test_deepseek_llm(): + """ๆต‹่ฏ• DeepSeek LLM API""" + print("\n" + "="*60) + print("ๆต‹่ฏ• DeepSeek LLM API") + print("="*60) + + api_key = os.getenv("DEEPSEEK_API_KEY") + if not api_key: + print("[ERROR] DEEPSEEK_API_KEY ็Žฏๅขƒๅ˜้‡ๆœช่ฎพ็ฝฎ") + print("่ฏท่ฟ่กŒ: export DEEPSEEK_API_KEY='your-api-key'") + return False + + print(f"[OK] DEEPSEEK_API_KEY ๅทฒ่ฎพ็ฝฎ (้•ฟๅบฆ: {len(api_key)})") + + try: + import requests + + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json" + } + + payload = { + "model": "deepseek-chat", + "messages": [ + {"role": "user", "content": "่ฏทๅ›žๅค 'LLMๆœๅŠกๆญฃๅธธ' ่ฟ™ๅ››ไธชๅญ—"} + ], + "temperature": 0.1, + "max_tokens": 50 + } + + print("[INFO] ๅ‘้€ๆต‹่ฏ•่ฏทๆฑ‚ๅˆฐ DeepSeek LLM...") + t0 = time.time() + + resp = requests.post( + "https://api.deepseek.com/chat/completions", + headers=headers, + json=payload, + timeout=30 + ) + + t1 = time.time() + + if resp.status_code != 200: + print(f"[ERROR] HTTP {resp.status_code}: {resp.text}") + return False + + result = resp.json() + response = result.get('choices', [{}])[0].get('message', {}).get('content', '') + + print(f"[OK] LLM ๅ“ๅบ”ๆˆๅŠŸ!") + print(f" ่€—ๆ—ถ: {t1-t0:.2f}s") + print(f" ๅ“ๅบ”: {response[:100]}") + return True + + except Exception as e: + print(f"[ERROR] LLM ๆต‹่ฏ•ๅคฑ่ดฅ: {e}") + return False + + +def test_ollama_vlm(api_url="http://localhost:20004/api/chat"): + """ๆต‹่ฏ• Ollama VLM API (ๅฏ้€‰)""" + print("\n" + "="*60) + print(f"ๆต‹่ฏ• Ollama VLM API @ {api_url}") + print("="*60) + + try: + import requests + + payload = { + "model": "qwen3-vl:32b", + "messages": [ + {"role": "user", "content": "Hello"} + ], + "stream": False + } + + print("[INFO] ๅ‘้€ๆต‹่ฏ•่ฏทๆฑ‚ๅˆฐ Ollama...") + t0 = time.time() + + resp = requests.post(api_url, json=payload, timeout=10) + + t1 = time.time() + + if resp.status_code != 200: + print(f"[ERROR] HTTP {resp.status_code}") + return False + + print(f"[OK] Ollama ๅ“ๅบ”ๆˆๅŠŸ! ่€—ๆ—ถ: {t1-t0:.2f}s") + return True + + except requests.exceptions.ConnectionError: + print("[WARNING] Ollama ๆœๅŠกๆœช่ฟ่กŒ (่ฟ™ๆ˜ฏๅฏ้€‰็š„)") + return False + except Exception as e: + print(f"[WARNING] Ollama ๆต‹่ฏ•ๅคฑ่ดฅ: {e}") + return False + + +def main(): + print("\n" + "#"*60) + print("# API ๆœๅŠก่ฟž้€šๆ€งๆต‹่ฏ•") + print("#"*60) + + results = {} + + # ๆต‹่ฏ• DashScope VLM + results['dashscope_vlm'] = test_dashscope_vlm() + + # ๆต‹่ฏ• DeepSeek LLM + results['deepseek_llm'] = test_deepseek_llm() + + # ๆต‹่ฏ• Ollama (ๅฏ้€‰) + results['ollama_vlm'] = test_ollama_vlm() + + # ๆ€ป็ป“ + print("\n" + "="*60) + print("ๆต‹่ฏ•็ป“ๆžœๆ€ป็ป“") + print("="*60) + + for service, passed in results.items(): + status = "โœ“ PASS" if passed else "โœ— FAIL" + print(f" {service}: {status}") + + # ๆฃ€ๆŸฅๅฟ…้œ€็š„ๆœๅŠก + required_pass = results['dashscope_vlm'] and results['deepseek_llm'] + + if required_pass: + print("\n[SUCCESS] ๅฟ…้œ€็š„ API ๆœๅŠก (DashScope VLM + DeepSeek LLM) ๅ‡ๆญฃๅธธ!") + print("ๅฏไปฅไฝฟ็”จ --realtime-hypothesis ๆจกๅผ่ฟ่กŒ habitat_evaluation.py") + else: + print("\n[FAILED] ้ƒจๅˆ†ๅฟ…้œ€ๆœๅŠกไธๅฏ็”จ๏ผŒ่ฏทๆฃ€ๆŸฅ API Key ้…็ฝฎ:") + if not results['dashscope_vlm']: + print(" - ่ฎพ็ฝฎ DASHSCOPE_API_KEY ็Žฏๅขƒๅ˜้‡") + if not results['deepseek_llm']: + print(" - ่ฎพ็ฝฎ DEEPSEEK_API_KEY ็Žฏๅขƒๅ˜้‡") + + return 0 if required_pass else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/verify_panorama_views.py b/scripts/verify_panorama_views.py new file mode 100644 index 0000000..5c4fae0 --- /dev/null +++ b/scripts/verify_panorama_views.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +""" +ๅ…จๆ™ฏ่ง†ๅ›พ้ชŒ่ฏ่„šๆœฌ +================ + +้ชŒ่ฏ้‡‡้›†็š„ๅ››ๅผ ๅ…จๆ™ฏๅ›พๆ˜ฏๅฆๆญฃ็กฎ๏ผˆๆ ผๅผใ€ๆ–นๅ‘ใ€่ฆ†็›–่Œƒๅ›ด๏ผ‰ใ€‚ + +ไฝฟ็”จ็คบไพ‹: +--------- +python scripts/verify_panorama_views.py --task-dir env/env_hm3dv2/task0 +python scripts/verify_panorama_views.py --task-dir env/env_hm3dv2/task0 --show + +""" + +import argparse +import sys +from pathlib import Path + +try: + import cv2 + import numpy as np +except ImportError: + print("Please install opencv-python: pip install opencv-python") + sys.exit(1) + + +def verify_views(task_dir: Path, show: bool = False) -> bool: + """ + ้ชŒ่ฏไปปๅŠก็›ฎๅฝ•ไธญ็š„ๅ››ๅผ ๅ…จๆ™ฏๅ›พ + + ๆฃ€ๆŸฅ้กน็›ฎ: + 1. ๆ–‡ไปถๅญ˜ๅœจๆ€ง (view0.png ~ view3.png) + 2. ๅ›พๅƒๅฐบๅฏธไธ€่‡ดๆ€ง + 3. ๅ›พๅƒๆ ผๅผ (RGB/BGR, 8-bit) + 4. task_info.txt ๅญ˜ๅœจ + """ + print(f"\n{'='*60}") + print(f"Verifying Panorama Views") + print(f"{'='*60}") + print(f"Task Directory: {task_dir}") + + errors = [] + warnings = [] + + # 1. ๆฃ€ๆŸฅ task_info.txt + task_info_path = task_dir / "task_info.txt" + if task_info_path.exists(): + print(f"\n[1/4] task_info.txt: OK") + with open(task_info_path, 'r') as f: + content = f.read() + print(f" Content:\n{content}") + else: + errors.append("task_info.txt not found") + print(f"\n[1/4] task_info.txt: MISSING") + + # 2. ๆฃ€ๆŸฅๅ››ๅผ ๅ›พ็‰‡ + print(f"\n[2/4] Checking view images...") + views = [] + view_info = [] + + directions = ["front (0ยฐ)", "left (90ยฐ)", "back (180ยฐ)", "right (270ยฐ)"] + + for i in range(4): + view_path = task_dir / f"view{i}.png" + + if not view_path.exists(): + errors.append(f"view{i}.png not found") + print(f" view{i}.png ({directions[i]}): MISSING") + continue + + # ่ฏปๅ–ๅ›พๅƒ + img = cv2.imread(str(view_path)) + if img is None: + errors.append(f"view{i}.png cannot be read") + print(f" view{i}.png ({directions[i]}): CANNOT READ") + continue + + views.append(img) + height, width, channels = img.shape + view_info.append({ + "index": i, + "path": view_path, + "width": width, + "height": height, + "channels": channels, + "dtype": str(img.dtype) + }) + + print(f" view{i}.png ({directions[i]}): {width}x{height}, {channels}ch, {img.dtype}") + + # 3. ๆฃ€ๆŸฅๅฐบๅฏธไธ€่‡ดๆ€ง + print(f"\n[3/4] Checking size consistency...") + if len(view_info) >= 2: + sizes = set((v["width"], v["height"]) for v in view_info) + if len(sizes) == 1: + print(f" All images have consistent size: {view_info[0]['width']}x{view_info[0]['height']}") + else: + warnings.append("Images have inconsistent sizes") + print(f" WARNING: Inconsistent sizes detected: {sizes}") + else: + print(f" Cannot check (need at least 2 images)") + + # 4. ่ฎก็ฎ—่ง†ๅ›พ้—ด็š„ๅทฎๅผ‚๏ผˆๆฃ€ๆŸฅๆ˜ฏๅฆ็œŸ็š„ๆ˜ฏไธๅŒๆ–นๅ‘๏ผ‰ + print(f"\n[4/4] Checking view diversity...") + if len(views) == 4: + # ่ฎก็ฎ—็›ธ้‚ป่ง†ๅ›พ็š„ๅทฎๅผ‚ + differences = [] + for i in range(4): + j = (i + 1) % 4 + diff = np.mean(np.abs(views[i].astype(float) - views[j].astype(float))) + differences.append(diff) + print(f" Diff(view{i}, view{j}): {diff:.2f}") + + avg_diff = np.mean(differences) + if avg_diff < 10: + warnings.append(f"Views may be too similar (avg diff: {avg_diff:.2f})") + print(f" WARNING: Views might not cover different directions (avg diff: {avg_diff:.2f})") + else: + print(f" Views appear to cover different directions (avg diff: {avg_diff:.2f})") + else: + print(f" Cannot check (need all 4 images)") + + # ๆ˜พ็คบๅ›พๅƒ + if show and len(views) == 4: + print(f"\nDisplaying panorama overview...") + try: + import matplotlib.pyplot as plt + + fig, axes = plt.subplots(2, 2, figsize=(12, 9)) + fig.suptitle(f"Panorama Views - {task_dir.name}", fontsize=14) + + titles = ["View 0: Front (0ยฐ)", "View 1: Left (90ยฐ)", + "View 2: Back (180ยฐ)", "View 3: Right (270ยฐ)"] + + for idx, (ax, view, title) in enumerate(zip(axes.flat, views, titles)): + # OpenCV ่ฏปๅ–็š„ๆ˜ฏ BGR๏ผŒ่ฝฌๆขไธบ RGB ๆ˜พ็คบ + rgb = cv2.cvtColor(view, cv2.COLOR_BGR2RGB) + ax.imshow(rgb) + ax.set_title(title) + ax.axis('off') + + plt.tight_layout() + + # ไฟๅญ˜ๆฆ‚่งˆๅ›พ + overview_path = task_dir / "panorama_overview.png" + plt.savefig(overview_path, dpi=150) + print(f" Overview saved to: {overview_path}") + + plt.show() + + except ImportError: + print(" matplotlib not available, skipping visualization") + + # ๆ€ป็ป“ + print(f"\n{'='*60}") + print(f"Verification Summary") + print(f"{'='*60}") + + if errors: + print(f"ERRORS ({len(errors)}):") + for e in errors: + print(f" - {e}") + else: + print(f"No errors found.") + + if warnings: + print(f"\nWARNINGS ({len(warnings)}):") + for w in warnings: + print(f" - {w}") + + success = len(errors) == 0 + print(f"\nResult: {'PASS' if success else 'FAIL'}") + print(f"{'='*60}\n") + + return success + + +def main(): + parser = argparse.ArgumentParser( + description="Verify panorama views collected from Habitat" + ) + + parser.add_argument( + "--task-dir", "-t", + type=str, + required=True, + help="Task directory containing view*.png files" + ) + + parser.add_argument( + "--show", "-s", + action="store_true", + help="Show the panorama overview" + ) + + args = parser.parse_args() + + task_dir = Path(args.task_dir) + if not task_dir.exists(): + print(f"Error: Directory not found: {task_dir}") + return 1 + + success = verify_views(task_dir, show=args.show) + return 0 if success else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/vlm_environment_analyzer.py b/scripts/vlm_environment_analyzer.py new file mode 100644 index 0000000..64ff2a8 --- /dev/null +++ b/scripts/vlm_environment_analyzer.py @@ -0,0 +1,748 @@ +#!/usr/bin/env python3 +""" +ไฝฟ็”จ่ฏดๆ˜Ž๏ผš2026-2-1 +#python vlm_environment_analyzer.py single --env hm3dv2 --task 0 --api-type dashscope --model qwen3-vl-flash --env-base-dir /path/to/InfoNav/env + +VLM ็Žฏๅขƒๆ„Ÿ็Ÿฅๅˆ†ๆžๅทฅๅ…ท (Unified Version) +===================================== + +ๆ•ดๅˆไบ†ๅ•ไปปๅŠกVLMๅˆ†ๆžๅ’Œๆ‰น้‡ๅˆ†ๆžๅŠŸ่ƒฝ๏ผŒไฝฟ็”จVLM๏ผˆๅฆ‚Qwen3-VL๏ผ‰ๅˆ†ๆž็Žฏๅขƒๅ›พ็‰‡๏ผŒ +็”Ÿๆˆ็Žฏๅขƒๆ„Ÿ็Ÿฅไฟกๆฏไพ›ๅŽ็ปญLLMๅ‡่ฏดๅˆ†ๆžไฝฟ็”จใ€‚ + +ๅŠŸ่ƒฝๆจกๅผ: +--------- +1. ๅ•ไปปๅŠกๆจกๅผ (single): ๅˆ†ๆžๅ•ไธชไปปๅŠก็š„4่ง†่ง’ๅ›พ็‰‡ +2. ๆ‰น้‡ๆจกๅผ (batch): ๆ‰น้‡ๅˆ†ๆžๅคšไธชไปปๅŠก + +ไฝฟ็”จ็คบไพ‹: +--------- +# ๅ•ไปปๅŠกๅˆ†ๆž (Ollama ๆœฌๅœฐๆจกๅผ) +python vlm_environment_analyzer.py single --env hm3dv2 --task 0 +python vlm_environment_analyzer.py single --env hm3dv2 --task 0 --api-url http://localhost:20004/api/chat + +# ๅ•ไปปๅŠกๅˆ†ๆž (DashScope ้˜ฟ้‡Œไบ‘ๆจกๅผ) +export DASHSCOPE_API_KEY="your-api-key" +python vlm_environment_analyzer.py single --env hm3dv2 --task 0 --api-type dashscope --model qwen3-vl-flash + +# ๆ‰น้‡ๅˆ†ๆž (Ollama) +python vlm_environment_analyzer.py batch --env hm3dv2 +python vlm_environment_analyzer.py batch --env hm3dv2 --max-tasks 10 +python vlm_environment_analyzer.py batch --env hm3dv2 --start 5 --end 15 +python vlm_environment_analyzer.py batch --env hm3dv1 hm3dv2 --model qwen3-vl:32b + +# ๆ‰น้‡ๅˆ†ๆž (DashScope) +python vlm_environment_analyzer.py batch --env hm3dv2 --api-type dashscope --model qwen3-vl-flash + +่พ“ๅ…ฅๆ–‡ไปถ: +--------- +- task_info.txt: ไปปๅŠกไฟกๆฏ๏ผˆ็›ฎๆ ‡็‰ฉไฝ“็ญ‰๏ผ‰ +- view0.png ~ view3.png: 4่ง†่ง’็Žฏๅขƒๅ›พ็‰‡ + - view0.png: ๅ‰ๆ–น (front) + - view1.png: ๅทฆๆ–น (left) + - view2.png: ๅŽๆ–น (back) + - view3.png: ๅณๆ–น (right) + +่พ“ๅ‡บๆ–‡ไปถ: +--------- +- vlm_analysis.txt: VLM็Žฏๅขƒๅˆ†ๆž็ป“ๆžœ +- batch_eval_summary.txt: ๆ‰น้‡ๅˆ†ๆžๆ‘˜่ฆ๏ผˆไป…ๆ‰น้‡ๆจกๅผ๏ผ‰ + +ไพ่ต–: +----- +- Ollamaๆจกๅผ: ้œ€่ฆVLMๆœๅŠก๏ผˆๅฆ‚Ollama + qwen3-vl๏ผ‰่ฟ่กŒๅœจๆŒ‡ๅฎš็ซฏๅฃ +- DashScopeๆจกๅผ: ้œ€่ฆ่ฎพ็ฝฎ DASHSCOPE_API_KEY ็Žฏๅขƒๅ˜้‡๏ผŒๅฎ‰่ฃ… openai ๅบ“ (pip install openai) +- ้œ€่ฆไปปๅŠก็›ฎๅฝ•ๅŒ…ๅซ task_info.txt ๅ’Œ 4ๅผ ่ง†่ง’ๅ›พ็‰‡ + +DashScope ๆ”ฏๆŒ็š„ๆจกๅž‹: +------------------- +- qwen-vl-plus: ้€š็”จ่ง†่ง‰่ฏญ่จ€ๆจกๅž‹ +- qwen-vl-max: ้ซ˜ๆ€ง่ƒฝ่ง†่ง‰่ฏญ่จ€ๆจกๅž‹ +- qwen3-vl-flash: ๅฟซ้€ŸๆŽจ็†ๆจกๅž‹ (ๆŽจ่) + +""" + +import base64 +import requests +import time +import argparse +import sys +import os +from pathlib import Path +from typing import Dict, List, Optional, Tuple +from datetime import datetime +from openai import OpenAI + + +# ==================== ๅทฅๅ…ทๅ‡ฝๆ•ฐ ==================== + +def encode_image(path: Path) -> str: + """ๅฐ†ๅ›พ็‰‡็ผ–็ ไธบ base64 ๅญ—็ฌฆไธฒ""" + with open(path, "rb") as f: + return base64.b64encode(f.read()).decode("utf-8") + + +def load_task_info(task_dir: Path) -> Dict[str, str]: + """ไปŽ task_info.txt ่ฏปๅ–ไปปๅŠกไฟกๆฏ""" + info_file = task_dir / "task_info.txt" + task_info = {} + + if not info_file.exists(): + return task_info + + with open(info_file, 'r', encoding='utf-8') as f: + for line in f: + line = line.strip() + if ':' in line: + key, value = line.split(':', 1) + key = key.strip().lower().replace(' ', '_') + value = value.strip() + task_info[key] = value + + return task_info + + +def build_prompt(target_object: str) -> str: + """ๆž„ๅปบ็Žฏๅขƒๆ„Ÿ็Ÿฅ prompt - ไธบLLMๅ…จๅฑ€ๆŽจ็†ๆไพ›ๅœบๆ™ฏไธŠไธ‹ๆ–‡""" + prompt = f"""You are helping a robot find **{target_object}** in an indoor environment. +Describe the scene to help plan a building-wide search: +1. **Scene**: What building type (house/apartment/office/etc)? What room is this? +2. **Layout**: Describe visible doors, hallways, or passages. Where might they lead? +3. **Objects**: Key furniture/items visible. Any clues about nearby rooms? +4. **Global hint**: In this building, where would {target_object} typically be? +""" + return prompt + + +# ==================== ๆ ธๅฟƒๅˆ†ๆž็ฑป ==================== + +class VLMEnvironmentAnalyzer: + """VLM็Žฏๅขƒๆ„Ÿ็Ÿฅๅˆ†ๆžๅ™จ""" + + # ๆ”ฏๆŒ็š„ API ๅŽ็ซฏ็ฑปๅž‹ + API_OLLAMA = "ollama" + API_DASHSCOPE = "dashscope" + + def __init__(self, + api_url: str = "http://localhost:20004/api/chat", + model: str = "qwen3-vl:32b", + env_base_dir: str = "env", + timeout: int = 300, + api_type: str = "ollama"): + """ + Args: + api_url: VLM APIๅœฐๅ€ (Ollamaๆจกๅผ) ๆˆ– base_url (DashScopeๆจกๅผ) + model: ๆจกๅž‹ๅ็งฐ + env_base_dir: ็Žฏๅขƒๆ•ฐๆฎ้›†ๆ น็›ฎๅฝ• + timeout: ่ฏทๆฑ‚่ถ…ๆ—ถๆ—ถ้—ด๏ผˆ็ง’๏ผ‰ + api_type: API็ฑปๅž‹ ("ollama" ๆˆ– "dashscope") + """ + self.api_url = api_url + self.model = model + self.env_base_dir = Path(env_base_dir) + self.timeout = timeout + self.api_type = api_type + + # ๅˆๅง‹ๅŒ– DashScope ๅฎขๆˆท็ซฏ + if self.api_type == self.API_DASHSCOPE: + api_key = os.getenv("DASHSCOPE_API_KEY") + if not api_key: + raise ValueError("ไฝฟ็”จ DashScope API ้œ€่ฆ่ฎพ็ฝฎ DASHSCOPE_API_KEY ็Žฏๅขƒๅ˜้‡") + self.openai_client = OpenAI( + api_key=api_key, + base_url=api_url or "https://dashscope.aliyuncs.com/compatible-mode/v1", + ) + + def load_images(self, task_dir: Path) -> Tuple[List[Path], List[str]]: + """ + ๅŠ ่ฝฝ4ๅผ ่ง†่ง’ๅ›พ็‰‡ + + Args: + task_dir: ไปปๅŠก็›ฎๅฝ• + + Returns: + (image_paths, missing_images) + """ + image_paths = [] + missing = [] + + for i in range(4): + img_path = task_dir / f"view{i}.png" + if img_path.exists(): + image_paths.append(img_path) + else: + missing.append(f"view{i}.png") + + return image_paths, missing + + def call_vlm(self, prompt: str, image_paths: List[Path], temperature: float = 0.2) -> Tuple[str, float]: + """ + ่ฐƒ็”จVLM API่ฟ›่กŒๅˆ†ๆž + + Args: + prompt: ๅˆ†ๆžprompt + image_paths: ๅ›พ็‰‡่ทฏๅพ„ๅˆ—่กจ + temperature: ็”Ÿๆˆๆธฉๅบฆ + + Returns: + (response_content, duration) + """ + if self.api_type == self.API_DASHSCOPE: + return self._call_dashscope(prompt, image_paths, temperature) + else: + return self._call_ollama(prompt, image_paths, temperature) + + def _call_ollama(self, prompt: str, image_paths: List[Path], temperature: float = 0.2) -> Tuple[str, float]: + """่ฐƒ็”จ Ollama ๆœฌๅœฐ API""" + payload = { + "model": self.model, + "messages": [ + { + "role": "user", + "content": prompt, + "images": [encode_image(p) for p in image_paths] + } + ], + "stream": False, + "options": { + "temperature": temperature + } + } + + t0 = time.time() + resp = requests.post(self.api_url, json=payload, timeout=self.timeout) + duration = time.time() - t0 + + if resp.status_code != 200: + raise RuntimeError(f"VLM API่ฟ”ๅ›ž้”™่ฏฏ: {resp.status_code} - {resp.text}") + + result = resp.json() + content = result.get("message", {}).get("content", "") + + if not content: + raise RuntimeError("VLM่ฟ”ๅ›ž็ฉบๅ“ๅบ”") + + return content, duration + + def _call_dashscope(self, prompt: str, image_paths: List[Path], temperature: float = 0.2) -> Tuple[str, float]: + """่ฐƒ็”จ DashScope API (้˜ฟ้‡Œไบ‘้€šไน‰ๅƒ้—ฎ)""" + # ๆž„ๅปบๅคšๆจกๆ€ๆถˆๆฏๅ†…ๅฎน + content = [] + + # ๆทปๅŠ ๅ›พ็‰‡ (ไฝฟ็”จ base64 data URL ๆ ผๅผ) + for img_path in image_paths: + img_base64 = encode_image(img_path) + # ๆ นๆฎๆ–‡ไปถๆ‰ฉๅฑ•ๅ็กฎๅฎš MIME ็ฑปๅž‹ + suffix = img_path.suffix.lower() + mime_type = { + '.png': 'image/png', + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.gif': 'image/gif', + '.webp': 'image/webp' + }.get(suffix, 'image/png') + + content.append({ + "type": "image_url", + "image_url": {"url": f"data:{mime_type};base64,{img_base64}"} + }) + + # ๆทปๅŠ ๆ–‡ๆœฌ prompt + content.append({ + "type": "text", + "text": prompt + }) + + t0 = time.time() + completion = self.openai_client.chat.completions.create( + model=self.model, + messages=[{"role": "user", "content": content}], + temperature=temperature, + ) + duration = time.time() - t0 + + response_content = completion.choices[0].message.content + + if not response_content: + raise RuntimeError("VLM่ฟ”ๅ›ž็ฉบๅ“ๅบ”") + + return response_content, duration + + def analyze_single(self, + env_name: str, + task_num: int, + temperature: float = 0.2, + output_file: Optional[Path] = None) -> Tuple[bool, Optional[str], float, Optional[str]]: + """ + ๅˆ†ๆžๅ•ไธชไปปๅŠก + + Args: + env_name: ็Žฏๅขƒๅ็งฐ + task_num: ไปปๅŠก็ผ–ๅท + temperature: ็”Ÿๆˆๆธฉๅบฆ + output_file: ่พ“ๅ‡บๆ–‡ไปถ่ทฏๅพ„๏ผˆๅฏ้€‰๏ผ‰ + + Returns: + (success, content, duration, error_msg) + """ + task_dir = self.env_base_dir / f"env_{env_name}" / f"task{task_num}" + + print("=" * 70) + print(f"VLM ็Žฏๅขƒๆ„Ÿ็Ÿฅๅˆ†ๆž") + print(f"็Žฏๅขƒ: {env_name}") + print(f"ไปปๅŠก: {task_num}") + print(f"็›ฎๅฝ•: {task_dir}") + print("=" * 70) + + # ๆฃ€ๆŸฅไปปๅŠก็›ฎๅฝ• + if not task_dir.exists(): + return False, None, 0, f"ไปปๅŠก็›ฎๅฝ•ไธๅญ˜ๅœจ: {task_dir}" + + # 1. ่ฏปๅ–ไปปๅŠกไฟกๆฏ + print("\n[1/4] ่ฏปๅ–ไปปๅŠกไฟกๆฏ...") + task_info = load_task_info(task_dir) + target_object = task_info.get('target_object', 'unknown') + + print(f" Task Index: {task_info.get('task_index', 'N/A')}") + print(f" Episode ID: {task_info.get('episode_id', 'N/A')}") + print(f" Target Object: {target_object}") + print(f" Scene ID: {task_info.get('scene_id', 'N/A')}") + + # 2. ๅŠ ่ฝฝ4ๅผ ่ง†่ง’ๅ›พ็‰‡ + print("\n[2/4] ๅŠ ่ฝฝ็Žฏๅขƒๅ›พ็‰‡...") + image_paths, missing = self.load_images(task_dir) + + for path in image_paths: + print(f" OK: {path.name}") + for name in missing: + print(f" MISSING: {name}") + + if len(image_paths) != 4: + return False, None, 0, f"้œ€่ฆ4ๅผ ๅ›พ็‰‡๏ผŒไฝ†ๅชๆ‰พๅˆฐ {len(image_paths)} ๅผ " + + # 3. ๆž„ๅปบ prompt + print("\n[3/4] ๆž„ๅปบๅˆ†ๆž prompt...") + prompt = build_prompt(target_object) + print(f" Prompt ้•ฟๅบฆ: {len(prompt)} ๅญ—็ฌฆ") + + # 4. ่ฐƒ็”จ VLM API + print(f"\n[4/4] ่ฐƒ็”จ VLM API ่ฟ›่กŒๅˆ†ๆž...") + print(f" ๆจกๅž‹: {self.model}") + print(f" API: {self.api_url}") + print(f" ็ญ‰ๅพ…ๅ“ๅบ”ไธญ... (่ถ…ๆ—ถ่ฎพ็ฝฎ: {self.timeout}็ง’)") + + try: + content, duration = self.call_vlm(prompt, image_paths, temperature) + + print(f" HTTP็Šถๆ€: 200") + print(f" ่€—ๆ—ถ: {duration:.2f} ็ง’") + print(f" ่พ“ๅ‡บ้•ฟๅบฆ: {len(content)} ๅญ—็ฌฆ") + + except requests.Timeout: + return False, None, 0, f"่ฏทๆฑ‚่ถ…ๆ—ถ (่ถ…่ฟ‡{self.timeout}็ง’)" + except Exception as e: + return False, None, 0, str(e) + + # 5. ๆ˜พ็คบๅ’Œไฟๅญ˜็ป“ๆžœ + print("\n" + "=" * 70) + print("VLM ๅˆ†ๆž็ป“ๆžœ") + print("=" * 70) + print(content) + print("=" * 70) + + # ็กฎๅฎš่พ“ๅ‡บๆ–‡ไปถ่ทฏๅพ„ + if output_file is None: + output_file = task_dir / "vlm_analysis.txt" + + # ไฟๅญ˜่พ“ๅ‡บ + with open(output_file, "w", encoding="utf-8") as f: + f.write(f"็Žฏๅขƒ: {env_name}\n") + f.write(f"ไปปๅŠก: {task_num}\n") + f.write(f"็›ฎๆ ‡็‰ฉไฝ“: {target_object}\n") + f.write(f"่€—ๆ—ถ: {duration:.2f} ็ง’\n") + f.write("=" * 70 + "\n") + f.write(content) + + print(f"\n็ป“ๆžœๅทฒไฟๅญ˜ๅˆฐ: {output_file}") + + return True, content, duration, None + + +# ==================== ๆ‰น้‡ๅค„็†็ฑป ==================== + +class BatchEvaluator: + """ๆ‰น้‡่ฏ„ไผฐๅ™จ""" + + def __init__(self, analyzer: VLMEnvironmentAnalyzer, delay: int = 30): + """ + Args: + analyzer: VLMๅˆ†ๆžๅ™จๅฎžไพ‹ + delay: ไปปๅŠก้—ด้š”ๆ—ถ้—ด๏ผˆ็ง’๏ผ‰ + """ + self.analyzer = analyzer + self.delay = delay + + def discover_tasks(self, env_name: str) -> List[int]: + """ๅ‘็ŽฐๆŒ‡ๅฎš็Žฏๅขƒไธ‹็š„ๆ‰€ๆœ‰ๆœ‰ๆ•ˆไปปๅŠก""" + env_dir = self.analyzer.env_base_dir / f"env_{env_name}" + + if not env_dir.exists(): + print(f"่ญฆๅ‘Š: ็Žฏๅขƒ็›ฎๅฝ•ไธๅญ˜ๅœจ: {env_dir}") + return [] + + tasks = [] + for task_dir in sorted(env_dir.iterdir()): + if task_dir.is_dir() and task_dir.name.startswith('task'): + try: + task_num = int(task_dir.name[4:]) + # ๆฃ€ๆŸฅๆ˜ฏๅฆๅŒ…ๅซๅฟ…่ฆๆ–‡ไปถ + if (task_dir / "task_info.txt").exists(): + tasks.append(task_num) + except ValueError: + continue + + return sorted(tasks) + + def evaluate_batch(self, + env_name: str, + start_task: Optional[int] = None, + end_task: Optional[int] = None, + max_tasks: Optional[int] = None, + temperature: float = 0.2) -> Dict: + """ + ๆ‰น้‡่ฏ„ไผฐ็Žฏๅขƒๆ•ฐๆฎ้›† + + Args: + env_name: ็Žฏๅขƒๅ็งฐ + start_task: ่ตทๅง‹ไปปๅŠก็ผ–ๅท๏ผˆๅŒ…ๅซ๏ผ‰ + end_task: ็ป“ๆŸไปปๅŠก็ผ–ๅท๏ผˆๅŒ…ๅซ๏ผ‰ + max_tasks: ๆœ€ๅคงไปปๅŠกๆ•ฐ้‡้™ๅˆถ + temperature: ็”Ÿๆˆๆธฉๅบฆ + + Returns: + ็ปŸ่ฎกไฟกๆฏๅญ—ๅ…ธ + """ + print(f"\n{'#'*70}") + print(f"# ๆ‰น้‡VLM่ฏ„ไผฐ: env_{env_name}") + print(f"# VLMๆจกๅž‹: {self.analyzer.model}") + print(f"# API: {self.analyzer.api_url}") + print(f"# ไปปๅŠก้—ด้š”: {self.delay} ็ง’") + print(f"# ๅผ€ๅง‹ๆ—ถ้—ด: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") + print(f"{'#'*70}\n") + + # ๅ‘็Žฐๆ‰€ๆœ‰ไปปๅŠก + all_tasks = self.discover_tasks(env_name) + + if not all_tasks: + print(f"้”™่ฏฏ: ๅœจ env_{env_name} ไธญๆœชๆ‰พๅˆฐไปปไฝ•ๆœ‰ๆ•ˆไปปๅŠก") + return None + + print(f"ๅ‘็Žฐ {len(all_tasks)} ไธชไปปๅŠก: {all_tasks[:20]}{'...' if len(all_tasks) > 20 else ''}") + + # ๅบ”็”จไปปๅŠก่Œƒๅ›ด่ฟ‡ๆปค + if start_task is not None: + all_tasks = [t for t in all_tasks if t >= start_task] + if end_task is not None: + all_tasks = [t for t in all_tasks if t <= end_task] + if max_tasks is not None: + all_tasks = all_tasks[:max_tasks] + + print(f"ๅฐ†่ฏ„ไผฐ {len(all_tasks)} ไธชไปปๅŠก\n") + + # ็ปŸ่ฎกไฟกๆฏ + stats = { + 'total': len(all_tasks), + 'success': 0, + 'failed': 0, + 'total_time': 0, + 'results': [] + } + + # ้€ไธช่ฏ„ไผฐไปปๅŠก + for idx, task_num in enumerate(all_tasks, 1): + print(f"\n่ฟ›ๅบฆ: [{idx}/{len(all_tasks)}]") + + success, content, duration, error_msg = self.analyzer.analyze_single( + env_name, task_num, temperature=temperature + ) + + stats['results'].append({ + 'task': task_num, + 'success': success, + 'duration': duration, + 'error': error_msg + }) + + stats['total_time'] += duration + + if success: + stats['success'] += 1 + else: + stats['failed'] += 1 + print(f" ๅคฑ่ดฅ: {error_msg}") + + # ๅฆ‚ๆžœไธๆ˜ฏๆœ€ๅŽไธ€ไธชไปปๅŠก๏ผŒ็ญ‰ๅพ…ๆŒ‡ๅฎšๆ—ถ้—ด + if idx < len(all_tasks): + print(f"\n็ญ‰ๅพ… {self.delay} ็ง’ๅŽ็ปง็ปญ...") + for remaining in range(self.delay, 0, -1): + print(f" ๅ‰ฉไฝ™: {remaining} ็ง’", end='\r') + time.sleep(1) + print() + + return stats + + def save_summary(self, env_name: str, stats: Dict) -> None: + """ไฟๅญ˜่ฏ„ไผฐๆ‘˜่ฆ""" + summary_file = self.analyzer.env_base_dir / f"env_{env_name}" / "batch_eval_summary.txt" + + with open(summary_file, 'w', encoding='utf-8') as f: + f.write(f"ๆ‰น้‡VLM่ฏ„ไผฐๆ‘˜่ฆ: env_{env_name}\n") + f.write(f"{'='*70}\n\n") + f.write(f"่ฏ„ไผฐๆ—ถ้—ด: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n") + f.write(f"VLMๆจกๅž‹: {self.analyzer.model}\n") + f.write(f"ๆ€ปไปปๅŠกๆ•ฐ: {stats['total']}\n") + f.write(f"ๆˆๅŠŸ: {stats['success']}\n") + f.write(f"ๅคฑ่ดฅ: {stats['failed']}\n") + f.write(f"ๆˆๅŠŸ็Ž‡: {stats['success']/stats['total']*100:.2f}%\n") + f.write(f"ๆ€ป่€—ๆ—ถ: {stats['total_time']:.2f} ็ง’ ({stats['total_time']/60:.2f} ๅˆ†้’Ÿ)\n") + f.write(f"ๅนณๅ‡่€—ๆ—ถ: {stats['total_time']/stats['total']:.2f} ็ง’/ไปปๅŠก\n\n") + + f.write(f"่ฏฆ็ป†็ป“ๆžœ:\n") + f.write(f"{'-'*70}\n") + + for result in stats['results']: + status = "OK" if result['success'] else "FAIL" + f.write(f"[{status}] task{result['task']}: {result['duration']:.2f}s") + if result['error']: + f.write(f" - {result['error']}") + f.write("\n") + + print(f"\nๆ‘˜่ฆๅทฒไฟๅญ˜ๅˆฐ: {summary_file}") + + def print_summary(self, env_name: str, stats: Dict) -> None: + """ๆ‰“ๅฐ่ฏ„ไผฐๆ‘˜่ฆ""" + print(f"\n{'#'*70}") + print(f"# ่ฏ„ไผฐๅฎŒๆˆ: env_{env_name}") + print(f"{'#'*70}\n") + + print(f"ๆ€ปไปปๅŠกๆ•ฐ: {stats['total']}") + print(f"ๆˆๅŠŸ: {stats['success']}") + print(f"ๅคฑ่ดฅ: {stats['failed']}") + print(f"ๆˆๅŠŸ็Ž‡: {stats['success']/stats['total']*100:.2f}%") + print(f"ๆ€ป่€—ๆ—ถ: {stats['total_time']:.2f} ็ง’ ({stats['total_time']/60:.2f} ๅˆ†้’Ÿ)") + print(f"ๅนณๅ‡่€—ๆ—ถ: {stats['total_time']/stats['total']:.2f} ็ง’/ไปปๅŠก") + + if stats['failed'] > 0: + print(f"\nๅคฑ่ดฅ็š„ไปปๅŠก:") + for result in stats['results']: + if not result['success']: + print(f" - task{result['task']}: {result['error']}") + + +# ==================== ๅ‘ฝไปค่กŒๆŽฅๅฃ ==================== + +def cmd_single(args): + """ๅ•ไปปๅŠกๅˆ†ๆžๅ‘ฝไปค""" + analyzer = VLMEnvironmentAnalyzer( + api_url=args.api_url, + model=args.model, + env_base_dir=args.env_base_dir, + timeout=args.timeout, + api_type=args.api_type + ) + + output_file = Path(args.output) if args.output else None + + success, content, duration, error_msg = analyzer.analyze_single( + args.env, args.task, temperature=args.temperature, output_file=output_file + ) + + if not success: + print(f"\n้”™่ฏฏ: {error_msg}") + return 1 + + return 0 + + +def cmd_batch(args): + """ๆ‰น้‡ๅˆ†ๆžๅ‘ฝไปค""" + analyzer = VLMEnvironmentAnalyzer( + api_url=args.api_url, + model=args.model, + env_base_dir=args.env_base_dir, + timeout=args.timeout, + api_type=args.api_type + ) + + batch = BatchEvaluator(analyzer, delay=args.delay) + + all_stats = {} + + for env_name in args.env: + stats = batch.evaluate_batch( + env_name, + start_task=args.start, + end_task=args.end, + max_tasks=args.max_tasks, + temperature=args.temperature + ) + + if stats: + all_stats[env_name] = stats + batch.print_summary(env_name, stats) + batch.save_summary(env_name, stats) + + # ๅฆ‚ๆžœไธๆ˜ฏๆœ€ๅŽไธ€ไธช็Žฏๅขƒ๏ผŒ็ญ‰ๅพ… + if env_name != args.env[-1]: + print(f"\n{'='*70}") + print(f"ๅ‡†ๅค‡่ฏ„ไผฐไธ‹ไธ€ไธช็Žฏๅขƒ...") + print(f"็ญ‰ๅพ… {args.delay} ็ง’...") + print(f"{'='*70}\n") + time.sleep(args.delay) + + # ๆ‰“ๅฐๆ€ปไฝ“ๆ‘˜่ฆ + if len(all_stats) > 1: + print(f"\n{'#'*70}") + print(f"# ๆ€ปไฝ“ๆ‘˜่ฆ") + print(f"{'#'*70}\n") + + total_tasks = sum(s['total'] for s in all_stats.values()) + total_success = sum(s['success'] for s in all_stats.values()) + total_failed = sum(s['failed'] for s in all_stats.values()) + total_time = sum(s['total_time'] for s in all_stats.values()) + + print(f"่ฏ„ไผฐ็Žฏๅขƒๆ•ฐ: {len(all_stats)}") + print(f"ๆ€ปไปปๅŠกๆ•ฐ: {total_tasks}") + print(f"ๆ€ปๆˆๅŠŸ: {total_success}") + print(f"ๆ€ปๅคฑ่ดฅ: {total_failed}") + print(f"ๆ€ปๆˆๅŠŸ็Ž‡: {total_success/total_tasks*100:.2f}%") + print(f"ๆ€ป่€—ๆ—ถ: {total_time:.2f} ็ง’ ({total_time/60:.2f} ๅˆ†้’Ÿ)") + print() + + for env_name, stats in all_stats.items(): + print(f" {env_name}: {stats['success']}/{stats['total']} " + f"({stats['success']/stats['total']*100:.2f}%)") + + print(f"\n{'='*70}") + print("ๆ‰€ๆœ‰่ฏ„ไผฐไปปๅŠกๅฎŒๆˆ!") + print(f"{'='*70}\n") + + return 0 + + +def main(): + parser = argparse.ArgumentParser( + description="VLM ็Žฏๅขƒๆ„Ÿ็Ÿฅๅˆ†ๆžๅทฅๅ…ท - ๆ”ฏๆŒๅ•ไปปๅŠกๅ’Œๆ‰น้‡ๅˆ†ๆž", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +ไฝฟ็”จ็คบไพ‹: +========= + +ๅ•ไปปๅŠกๅˆ†ๆž: + python vlm_environment_analyzer.py single --env hm3dv2 --task 0 + python vlm_environment_analyzer.py single --env hm3dv2 --task 0 --api-url http://localhost:20004/api/chat + python vlm_environment_analyzer.py single --env hm3dv2 --task 0 --output result.txt + +ๆ‰น้‡ๅˆ†ๆž: + python vlm_environment_analyzer.py batch --env hm3dv2 + python vlm_environment_analyzer.py batch --env hm3dv2 --max-tasks 10 + python vlm_environment_analyzer.py batch --env hm3dv2 --start 5 --end 15 + python vlm_environment_analyzer.py batch --env hm3dv1 hm3dv2 --delay 60 + +ๅทฅไฝœๆต็จ‹: +========= +1. ๅ…ˆ่ฟ่กŒๆœฌๅทฅๅ…ท็”Ÿๆˆ vlm_analysis.txt +2. ๅ†่ฟ่กŒ llm_hypothesis_analyzer.py ็”Ÿๆˆๅ‡่ฏดๅˆ†ๆž + +ๅฎŒๆ•ด็คบไพ‹: + # Step 1: VLM็Žฏๅขƒๅˆ†ๆž + python vlm_environment_analyzer.py batch --env hm3dv2 --max-tasks 10 + + # Step 2: LLMๅ‡่ฏดๅˆ†ๆž + python llm_hypothesis_analyzer.py batch --env hm3dv2 --max-tasks 10 + """ + ) + + subparsers = parser.add_subparsers(dest='command', help='่ฟ่กŒๆจกๅผ') + + # ๅ•ไปปๅŠกๆจกๅผ + single_parser = subparsers.add_parser('single', help='ๅ•ไปปๅŠกๅˆ†ๆžๆจกๅผ') + single_parser.add_argument('--env', type=str, required=True, + choices=['hm3dv1', 'hm3dv2', 'mp3d'], + help='็Žฏๅขƒ็ฑปๅˆซ') + single_parser.add_argument('--task', type=int, required=True, + help='ไปปๅŠก็ผ–ๅท') + single_parser.add_argument('--api-type', type=str, default='ollama', + choices=['ollama', 'dashscope'], + help='API็ฑปๅž‹: ollama(ๆœฌๅœฐ) ๆˆ– dashscope(้˜ฟ้‡Œไบ‘)') + single_parser.add_argument('--api-url', type=str, default=None, + help='VLM API ๅœฐๅ€ (ollama้ป˜่ฎค: localhost:20004, dashscope้ป˜่ฎค: ้˜ฟ้‡Œไบ‘)') + single_parser.add_argument('--model', type=str, default=None, + help='VLM ๆจกๅž‹ๅ็งฐ (ollama้ป˜่ฎค: qwen3-vl:32b, dashscope้ป˜่ฎค: qwen-vl-max)') + single_parser.add_argument('--temperature', type=float, default=0.2, + help='็”Ÿๆˆๆธฉๅบฆ 0.0-1.0 (้ป˜่ฎค: 0.2)') + single_parser.add_argument('--timeout', type=int, default=600, + help='่ฏทๆฑ‚่ถ…ๆ—ถๆ—ถ้—ด๏ผˆ็ง’๏ผ‰๏ผŒ้ป˜่ฎค600็ง’') + single_parser.add_argument('--env-base-dir', type=str, default='env', + help='็Žฏๅขƒๆ•ฐๆฎ้›†ๆ น็›ฎๅฝ•') + single_parser.add_argument('--output', type=str, default=None, + help='่พ“ๅ‡บๆ–‡ไปถ่ทฏๅพ„๏ผˆ้ป˜่ฎค่‡ชๅŠจไฟๅญ˜ๅˆฐไปปๅŠก็›ฎๅฝ•๏ผ‰') + + # ๆ‰น้‡ๆจกๅผ + batch_parser = subparsers.add_parser('batch', help='ๆ‰น้‡ๅˆ†ๆžๆจกๅผ') + batch_parser.add_argument('--env', type=str, nargs='+', required=True, + choices=['hm3dv1', 'hm3dv2', 'mp3d'], + help='่ฆ่ฏ„ไผฐ็š„็Žฏๅขƒ๏ผˆๅฏๆŒ‡ๅฎšๅคšไธช๏ผ‰') + batch_parser.add_argument('--start', type=int, default=None, + help='่ตทๅง‹ไปปๅŠก็ผ–ๅท๏ผˆๅŒ…ๅซ๏ผ‰') + batch_parser.add_argument('--end', type=int, default=None, + help='็ป“ๆŸไปปๅŠก็ผ–ๅท๏ผˆๅŒ…ๅซ๏ผ‰') + batch_parser.add_argument('--max-tasks', type=int, default=None, + help='ๆœ€ๅคงไปปๅŠกๆ•ฐ้‡้™ๅˆถ') + batch_parser.add_argument('--delay', type=int, default=20, + help='ไปปๅŠก้—ด้š”ๆ—ถ้—ด๏ผˆ็ง’๏ผ‰๏ผŒ้ป˜่ฎค20็ง’') + batch_parser.add_argument('--api-type', type=str, default='ollama', + choices=['ollama', 'dashscope'], + help='API็ฑปๅž‹: ollama(ๆœฌๅœฐ) ๆˆ– dashscope(้˜ฟ้‡Œไบ‘)') + batch_parser.add_argument('--api-url', type=str, default=None, + help='VLM API ๅœฐๅ€ (ollama้ป˜่ฎค: localhost:20004, dashscope้ป˜่ฎค: ้˜ฟ้‡Œไบ‘)') + batch_parser.add_argument('--model', type=str, default=None, + help='VLM ๆจกๅž‹ๅ็งฐ (ollama้ป˜่ฎค: qwen3-vl:32b, dashscope้ป˜่ฎค: qwen-vl-max)') + batch_parser.add_argument('--temperature', type=float, default=0.2, + help='็”Ÿๆˆๆธฉๅบฆ 0.0-1.0 (้ป˜่ฎค: 0.2)') + batch_parser.add_argument('--timeout', type=int, default=600, + help='่ฏทๆฑ‚่ถ…ๆ—ถๆ—ถ้—ด๏ผˆ็ง’๏ผ‰๏ผŒ้ป˜่ฎค600็ง’') + batch_parser.add_argument('--env-base-dir', type=str, default='env', + help='็Žฏๅขƒๆ•ฐๆฎ้›†ๆ น็›ฎๅฝ•') + + args = parser.parse_args() + + if args.command is None: + parser.print_help() + return 1 + + # ๆ นๆฎ api_type ่ฎพ็ฝฎ้ป˜่ฎคๅ€ผ + if args.api_type == 'dashscope': + if args.api_url is None: + args.api_url = "https://dashscope.aliyuncs.com/compatible-mode/v1" + if args.model is None: + args.model = "qwen-vl-max" # ๅฏ้€‰: qwen-vl-plus, qwen-vl-max, qwen3-vl-flash + else: # ollama + if args.api_url is None: + args.api_url = "http://localhost:20004/api/chat" + if args.model is None: + args.model = "qwen3-vl:32b" + + if args.command == 'single': + return cmd_single(args) + elif args.command == 'batch': + return cmd_batch(args) + + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except KeyboardInterrupt: + print("\n\n็”จๆˆทไธญๆ–ญ่ฏ„ไผฐ") + sys.exit(1) diff --git a/src/planner/exploration_manager/include/exploration_manager/exploration_manager.h b/src/planner/exploration_manager/include/exploration_manager/exploration_manager.h index ed99387..36fd8e8 100644 --- a/src/planner/exploration_manager/include/exploration_manager/exploration_manager.h +++ b/src/planner/exploration_manager/include/exploration_manager/exploration_manager.h @@ -105,6 +105,14 @@ class ExplorationManager { typedef shared_ptr Ptr; private: + /// Return the active semantic value field. HM3D value-based policies use + /// the fused multi-source HSVM; the legacy scalar map is only a fallback. + double getHSVMValue(const Eigen::Vector2i& idx); + double getLocalHSVMValue(const Eigen::Vector2i& idx); + double getLocalIGValue(const Eigen::Vector2i& idx); + void computeNormalizedFrontierComponents(const vector& frontiers, + vector& semantic_values, vector& ig_values, bool use_ig); + // Exploration Policy void chooseExplorationPolicy(Vector2d cur_pos, vector frontiers, Vector2d& next_best_pos, vector& next_best_path); @@ -282,4 +290,4 @@ inline vector ExplorationManager::allNeighbors( } // namespace apexnav_planner -#endif \ No newline at end of file +#endif diff --git a/src/planner/exploration_manager/launch/algorithm.xml b/src/planner/exploration_manager/launch/algorithm.xml index 34dbfb1..1e9b55c 100644 --- a/src/planner/exploration_manager/launch/algorithm.xml +++ b/src/planner/exploration_manager/launch/algorithm.xml @@ -13,6 +13,18 @@ + + + + + + + + + + + + @@ -56,22 +68,23 @@ - - + + + - - + + - - - + + + - + - - + + @@ -141,4 +154,4 @@ - \ No newline at end of file + diff --git a/src/planner/exploration_manager/launch/exploration.launch b/src/planner/exploration_manager/launch/exploration.launch index a11629a..f86c2f3 100644 --- a/src/planner/exploration_manager/launch/exploration.launch +++ b/src/planner/exploration_manager/launch/exploration.launch @@ -10,6 +10,17 @@ + + + + + + + + + + + @@ -43,6 +54,17 @@ + + + + + + + + + + + diff --git a/src/planner/exploration_manager/package.xml b/src/planner/exploration_manager/package.xml index b0d7714..fa702bd 100755 --- a/src/planner/exploration_manager/package.xml +++ b/src/planner/exploration_manager/package.xml @@ -2,12 +2,12 @@ exploration_manager 0.0.0 - The apexnav_planner package + InfoNav frontier exploration and scheduling - zagerzhang + Zhi Yang diff --git a/src/planner/exploration_manager/src/exploration_fsm.cpp b/src/planner/exploration_manager/src/exploration_fsm.cpp index 5344f14..cc19543 100644 --- a/src/planner/exploration_manager/src/exploration_fsm.cpp +++ b/src/planner/exploration_manager/src/exploration_fsm.cpp @@ -410,7 +410,8 @@ int ExplorationFSM::callActionPlanner() } else { // Use VLM Approach Verifier for soft arrival validation int current_obj_id = expl_manager_->object_map2d_->getCurrentTargetObjectId(); - if (current_obj_id >= 0 && !fd_->vlm_disabled_this_episode_) { + // Only use VLM verification if globally enabled AND not disabled this episode + if (current_obj_id >= 0 && vlm_validation_enabled_ && !fd_->vlm_disabled_this_episode_) { // Check if VLM already verified this object bool vlm_verified = expl_manager_->sdf_map_->object_map2d_->isObjectVLMVerified(current_obj_id); if (vlm_verified) { @@ -450,9 +451,12 @@ int ExplorationFSM::callActionPlanner() // Wait for VLM result return fd_->final_result_; } - } else if (fd_->vlm_disabled_this_episode_) { - // VLM disabled this episode - trust detector - ROS_WARN("[Soft Arrival] VLM disabled, trusting detector at dist=%.2fm", dist_to_target); + } else if (!vlm_validation_enabled_ || fd_->vlm_disabled_this_episode_) { + // VLM globally disabled OR disabled this episode - trust detector + ROS_WARN("[Soft Arrival] VLM disabled (global=%s, episode=%s), trusting detector at dist=%.2fm", + vlm_validation_enabled_ ? "false" : "true", + fd_->vlm_disabled_this_episode_ ? "true" : "false", + dist_to_target); if (expl_manager_->isSuspiciousTargetLocked()) { expl_manager_->setSuspiciousTargetLock(false); } @@ -517,7 +521,7 @@ int ExplorationFSM::callActionPlanner() bool vlm_passed = false; // Default to false - require VLM verification bool vlm_uncertain = false; - if (current_obj_id >= 0 && !fd_->vlm_disabled_this_episode_) { + if (current_obj_id >= 0 && vlm_validation_enabled_ && !fd_->vlm_disabled_this_episode_) { vlm_verified = expl_manager_->sdf_map_->object_map2d_->isObjectVLMVerified(current_obj_id); if (vlm_verified) { vlm_passed = expl_manager_->sdf_map_->object_map2d_->getObjectVLMResult(current_obj_id); @@ -535,10 +539,12 @@ int ExplorationFSM::callActionPlanner() return fd_->final_result_; } } - } else if (fd_->vlm_disabled_this_episode_) { - // VLM disabled - trust detector as fallback + } else if (!vlm_validation_enabled_ || fd_->vlm_disabled_this_episode_) { + // VLM globally disabled OR disabled this episode - trust detector as fallback vlm_passed = true; - ROS_WARN("[Final Approach] VLM disabled, trusting detector"); + ROS_WARN("[Final Approach] VLM disabled (global=%s, episode=%s), trusting detector", + vlm_validation_enabled_ ? "true" : "false", + fd_->vlm_disabled_this_episode_ ? "true" : "false"); } ROS_INFO("[Final Approach] VLM validation: obj_id=%d, verified=%s, passed=%s, uncertain=%s", @@ -622,7 +628,7 @@ int ExplorationFSM::callActionPlanner() (current_pos - expl_manager_->ed_->next_pos_).norm() < soft_reach_distance) { // Stuck near target - require VLM verification before confirming success int current_obj_id = expl_manager_->object_map2d_->getCurrentTargetObjectId(); - if (current_obj_id >= 0 && !fd_->vlm_disabled_this_episode_) { + if (current_obj_id >= 0 && vlm_validation_enabled_ && !fd_->vlm_disabled_this_episode_) { bool vlm_verified = expl_manager_->sdf_map_->object_map2d_->isObjectVLMVerified(current_obj_id); if (vlm_verified) { bool vlm_passed = expl_manager_->sdf_map_->object_map2d_->getObjectVLMResult(current_obj_id); @@ -646,9 +652,11 @@ int ExplorationFSM::callActionPlanner() } // Don't return REACH_OBJECT, wait for VLM result } - } else if (fd_->vlm_disabled_this_episode_) { - // VLM disabled - trust detector - ROS_WARN("[Stuck Near Target] VLM disabled, trusting detector"); + } else if (!vlm_validation_enabled_ || fd_->vlm_disabled_this_episode_) { + // VLM globally disabled OR disabled this episode - trust detector + ROS_WARN("[Stuck Near Target] VLM disabled (global=%s, episode=%s), trusting detector", + vlm_validation_enabled_ ? "true" : "false", + fd_->vlm_disabled_this_episode_ ? "true" : "false"); resetApproachDetectionCheck(); resetSoftArrivalTracking(); final_res = FINAL_RESULT::REACH_OBJECT; @@ -2678,6 +2686,12 @@ void ExplorationFSM::vlmVerificationResultCallback(const plan_env::VLMVerificati void ExplorationFSM::triggerVLMVerification(int object_id, int trigger_type, double distance_to_target) { + // Check if VLM validation is globally disabled + if (!vlm_validation_enabled_) { + ROS_DEBUG("[VLM] VLM validation globally disabled, skipping verification"); + return; + } + // Check if VLM is disabled for this episode if (fd_->vlm_disabled_this_episode_) { ROS_WARN("[VLM] VLM disabled for this episode (previous timeout), skipping verification"); @@ -2755,8 +2769,8 @@ void ExplorationFSM::publishObjectUpdateInfo(int object_id, double confidence, bool ExplorationFSM::checkVLMTriggerConditions(double dist_to_target, int object_id) { - // Skip if VLM disabled or already waiting - if (fd_->vlm_disabled_this_episode_ || fd_->vlm_waiting_) { + // Skip if VLM globally disabled, disabled this episode, or already waiting + if (!vlm_validation_enabled_ || fd_->vlm_disabled_this_episode_ || fd_->vlm_waiting_) { return false; } diff --git a/src/planner/exploration_manager/src/exploration_manager.cpp b/src/planner/exploration_manager/src/exploration_manager.cpp index a9bb0ea..73b1a41 100644 --- a/src/planner/exploration_manager/src/exploration_manager.cpp +++ b/src/planner/exploration_manager/src/exploration_manager.cpp @@ -1,7 +1,6 @@ /** * @file exploration_manager.cpp * @brief Implementation of exploration manager for autonomous semantic navigation - * @author Zager-Zhang * * This file implements the ExplorationManager class that handles various * exploration strategies including distance-based, semantic-based, hybrid, @@ -471,28 +470,17 @@ void ExplorationManager::findHighestSemanticsFrontierPolicy(Vector2d cur_pos, bool use_ig = ep_->use_ig_fusion_ && sdf_map_->use_ig_value_map_ && sdf_map_->ig_value_map_; + vector semantic_values, ig_values; + computeNormalizedFrontierComponents(frontiers, semantic_values, ig_values, use_ig); // Compute total value for each frontier - for (auto frontier : frontiers) { - Vector2i idx; - sdf_map_->posToIndex(frontier, idx); - auto nbrs = allNeighbors(idx, 2); // 5x5 neighborhood - - // Find maximum semantic value in local neighborhood - double sem_value = sdf_map_->value_map_->getValue(idx); - for (auto nbr : nbrs) sem_value = max(sem_value, sdf_map_->value_map_->getValue(nbr)); - - // Find maximum IG value in local neighborhood (if enabled) - double ig_value = 0.0; - if (use_ig) { - ig_value = sdf_map_->ig_value_map_->getIGValue(idx); - for (auto nbr : nbrs) ig_value = max(ig_value, sdf_map_->ig_value_map_->getIGValue(nbr)); - } - - // Compute total value using dual-value fusion (HSVM/4 to reduce intensity) - double total_value = use_ig ? (alpha * (sem_value / 4.0) + (1.0 - alpha) * ig_value) : (sem_value / 4.0); - - frontier_values.emplace_back(frontier, total_value); + for (size_t i = 0; i < frontiers.size(); ++i) { + // Both arms use exactly the same normalized HSVM/IG value. Policy 1 + // greedily takes its maximum; policy 4 plans a weighted arrival-time tour. + const double total_value = use_ig + ? alpha * semantic_values[i] + (1.0 - alpha) * ig_values[i] + : semantic_values[i]; + frontier_values.emplace_back(frontiers[i], total_value); } // Sort by semantic value (descending), then by distance (ascending) @@ -923,6 +911,77 @@ bool ExplorationManager::searchObjectPath(const Vector3d& start, return false; } +double ExplorationManager::getHSVMValue(const Vector2i& idx) +{ + if (!sdf_map_->isInMap(idx)) + return 0.0; + if (sdf_map_->use_multi_source_value_map_ && sdf_map_->multi_source_value_map_) + return sdf_map_->multi_source_value_map_->getFusedSemanticValue(idx); + return sdf_map_->value_map_->getValue(idx); +} + +double ExplorationManager::getLocalHSVMValue(const Vector2i& idx) +{ + double value = getHSVMValue(idx); + for (const auto& nbr : allNeighbors(idx, 2)) { + if (!sdf_map_->isInMap(nbr) || sdf_map_->getInflateOccupancy(nbr) == 1 || + sdf_map_->getOccupancy(nbr) == SDFMap2D::OCCUPIED) + continue; + value = std::max(value, getHSVMValue(nbr)); + } + return value; +} + +double ExplorationManager::getLocalIGValue(const Vector2i& idx) +{ + if (!sdf_map_->use_ig_value_map_ || !sdf_map_->ig_value_map_ || !sdf_map_->isInMap(idx)) + return 0.0; + double value = sdf_map_->ig_value_map_->getIGValue(idx); + for (const auto& nbr : allNeighbors(idx, 2)) { + if (!sdf_map_->isInMap(nbr) || sdf_map_->getInflateOccupancy(nbr) == 1 || + sdf_map_->getOccupancy(nbr) == SDFMap2D::OCCUPIED) + continue; + value = std::max(value, sdf_map_->ig_value_map_->getIGValue(nbr)); + } + return value; +} + +void ExplorationManager::computeNormalizedFrontierComponents( + const vector& frontiers, vector& semantic_values, + vector& ig_values, bool use_ig) +{ + semantic_values.clear(); + ig_values.clear(); + semantic_values.reserve(frontiers.size()); + ig_values.reserve(frontiers.size()); + for (const auto& frontier : frontiers) { + Vector2i idx; + sdf_map_->posToIndex(frontier, idx); + semantic_values.push_back(getLocalHSVMValue(idx)); + ig_values.push_back(use_ig ? getLocalIGValue(idx) : 0.0); + } + + auto min_max_normalize = [](vector& values) { + if (values.empty()) + return; + const auto bounds = std::minmax_element(values.begin(), values.end()); + const double minimum = *bounds.first; + const double range = *bounds.second - minimum; + if (range <= 1e-9) { + std::fill(values.begin(), values.end(), 0.0); + return; + } + for (double& value : values) + value = (value - minimum) / range; + }; + + // HSVM and IG have different native scales. Frontier-wise min-max + // normalization makes alpha meaningful and identical in both policies. + min_max_normalize(semantic_values); + if (use_ig) + min_max_normalize(ig_values); +} + double ExplorationManager::computeAdaptiveAlpha(const vector& frontiers) { // Adaptive Alpha (Paper Equation): @@ -941,24 +1000,13 @@ double ExplorationManager::computeAdaptiveAlpha(const vector& frontier Vector2i idx; sdf_map_->posToIndex(frontier, idx); - // Get semantic value at frontier position - double sem_value = sdf_map_->value_map_->getValue(idx); - - // Also check 5x5 neighborhood for maximum - auto nbrs = allNeighbors(idx, 2); - for (const auto& nbr : nbrs) { - if (sdf_map_->getInflateOccupancy(nbr) == 1 || - sdf_map_->getOccupancy(nbr) == SDFMap2D::OCCUPIED) - continue; - sem_value = std::max(sem_value, sdf_map_->value_map_->getValue(nbr)); - } - - max_hsvm = std::max(max_hsvm, sem_value); + max_hsvm = std::max(max_hsvm, getLocalHSVMValue(idx)); } // Compute adaptive alpha with clamping to [alpha_base, 1.0] - double alpha = ep_->alpha_base_ + (1.0 - ep_->alpha_base_) * (max_hsvm / ep_->tau_conf_); - alpha = std::min(alpha, 1.0); // Clamp to maximum of 1.0 + const double tau_conf = std::max(ep_->tau_conf_, 1e-9); + double alpha = ep_->alpha_base_ + (1.0 - ep_->alpha_base_) * (max_hsvm / tau_conf); + alpha = std::max(ep_->alpha_base_, std::min(alpha, 1.0)); ROS_DEBUG("[AdaptiveAlpha] max_hsvm=%.3f, alpha=%.3f (base=%.2f, tau=%.2f)", max_hsvm, alpha, ep_->alpha_base_, ep_->tau_conf_); @@ -978,45 +1026,20 @@ void ExplorationManager::getSortedSemanticFrontiers(const Vector2d& cur_pos, bool use_ig = ep_->use_ig_fusion_ && sdf_map_->use_ig_value_map_ && sdf_map_->ig_value_map_; + vector semantic_values, ig_values; + computeNormalizedFrontierComponents(frontiers, semantic_values, ig_values, use_ig); - for (auto& frontier : frontiers) { + for (size_t i = 0; i < frontiers.size(); ++i) { SemanticFrontier sem_frontier; - sem_frontier.position = frontier; - - // Compute semantic value from local neighborhood - Vector2i idx; - sdf_map_->posToIndex(frontier, idx); - auto nbrs = allNeighbors(idx, 2); // 5x5 grid neighborhood - double sem_value = sdf_map_->value_map_->getValue(idx); - - // Find maximum semantic value in neighborhood (ignoring occupied cells) - for (auto& nbr : nbrs) { - if (sdf_map_->getInflateOccupancy(idx) == 1 || - sdf_map_->getOccupancy(idx) == SDFMap2D::OCCUPIED) - continue; - sem_value = std::max(sem_value, sdf_map_->value_map_->getValue(nbr)); - } - - // Compute IG value from local neighborhood (if enabled) - double ig_value = 0.0; - if (use_ig) { - ig_value = sdf_map_->ig_value_map_->getIGValue(idx); - for (auto& nbr : nbrs) { - if (sdf_map_->getInflateOccupancy(idx) == 1 || - sdf_map_->getOccupancy(idx) == SDFMap2D::OCCUPIED) - continue; - ig_value = std::max(ig_value, sdf_map_->ig_value_map_->getIGValue(nbr)); - } - } - - // Compute total value using dual-value fusion (HSVM/4 to reduce intensity) - double total_value = use_ig ? (alpha * (sem_value / 4.0) + (1.0 - alpha) * ig_value) : (sem_value / 4.0); - sem_frontier.semantic_value = total_value; // Store total value + sem_frontier.position = frontiers[i]; + sem_frontier.semantic_value = use_ig + ? alpha * semantic_values[i] + (1.0 - alpha) * ig_values[i] + : semantic_values[i]; // Validate reachability and compute path cost Vector2d tmp_pos; vector tmp_path; - if (!searchFrontierPath(cur_pos, frontier, tmp_pos, tmp_path)) { + if (!searchFrontierPath(cur_pos, frontiers[i], tmp_pos, tmp_path)) { // Assign high cost penalty for unreachable frontiers sem_frontier.path_length = 1000000; sem_frontier.path.clear(); diff --git a/src/planner/path_searching/package.xml b/src/planner/path_searching/package.xml index 810a30a..c22e7b5 100755 --- a/src/planner/path_searching/package.xml +++ b/src/planner/path_searching/package.xml @@ -7,7 +7,7 @@ - zagerzhang + Zhi Yang diff --git a/src/planner/plan_env/CMakeLists.txt b/src/planner/plan_env/CMakeLists.txt index a7f7623..5fa684e 100755 --- a/src/planner/plan_env/CMakeLists.txt +++ b/src/planner/plan_env/CMakeLists.txt @@ -77,6 +77,10 @@ add_library(plan_env src/multi_source_value_map.cpp src/ig_value_map.cpp ) +add_dependencies(plan_env + ${${PROJECT_NAME}_EXPORTED_TARGETS} + ${catkin_EXPORTED_TARGETS} +) target_link_libraries(plan_env ${catkin_LIBRARIES} ${PCL_LIBRARIES} diff --git a/src/planner/plan_env/include/plan_env/ig_value_map.h b/src/planner/plan_env/include/plan_env/ig_value_map.h index a1a3dd5..03d4af9 100644 --- a/src/planner/plan_env/include/plan_env/ig_value_map.h +++ b/src/planner/plan_env/include/plan_env/ig_value_map.h @@ -18,8 +18,6 @@ * 1. "This view shows a corridor or hallway leading to other areas" * 2. "There is a doorway or opening that leads to another room" * 3. "This passage connects to multiple rooms or spaces" - * - * @author Zhi Yang */ #include @@ -120,11 +118,6 @@ class IGValueMap { */ double normalizeAngle(double angle); - /** - * @brief Callback for IGScore messages from Habitat - */ - void igScoreCallback(const plan_env::IGScore::ConstPtr& msg); - // ==================== Data Members ==================== // Data buffers @@ -137,10 +130,7 @@ class IGValueMap { // Parameters double fov_angle_; ///< Field of view angle (radians) - // ROS communication - ros::Subscriber ig_score_sub_; ///< Subscriber for IGScore messages - - // Cached sensor state for ROS callback updates + // Cached sensor state retained for backwards-compatible direct callers Vector2d current_sensor_pos_; double current_sensor_yaw_; vector current_free_grids_; diff --git a/src/planner/plan_env/include/plan_env/map_ros.h b/src/planner/plan_env/include/plan_env/map_ros.h index ccae685..b08630a 100644 --- a/src/planner/plan_env/include/plan_env/map_ros.h +++ b/src/planner/plan_env/include/plan_env/map_ros.h @@ -79,6 +79,7 @@ class MapROS { void detectedObjectCloudCallback(const plan_env::MultipleMasksWithConfidenceConstPtr& msg); void semanticPromptsCallback(const plan_env::SemanticPromptsConstPtr& msg); void semanticScoresCallback(const plan_env::SemanticScoresConstPtr& msg); + void igScoreCallback(const plan_env::IGScoreConstPtr& msg); void episodeResetCallback(const std_msgs::Empty::ConstPtr& msg); ///< Reset callback for new episodes void visCallback(const ros::TimerEvent& /*event*/); @@ -134,7 +135,7 @@ class MapROS { // ROS subscribers for sensor data ros::Subscriber detected_object_cloud_sub_; - ros::Subscriber semantic_prompts_sub_, semantic_scores_sub_; + ros::Subscriber semantic_prompts_sub_, semantic_scores_sub_, ig_score_sub_; ros::Subscriber episode_reset_sub_; ///< Subscriber for episode reset signals // ROS timers for periodic updates @@ -173,6 +174,8 @@ class MapROS { vector current_semantic_scores_; ///< Current ITM scores for each semantic prompt string current_target_object_; ///< Current navigation target object bool use_multi_source_semantic_; ///< Flag to enable multi-source semantic mode + double current_ig_score_; ///< Latest IG score to project with depth/pose + bool has_current_ig_score_; ///< Whether an IG score arrived this episode // Dual-Value Fusion parameter (shared with exploration_manager) double fusion_alpha_; ///< Weight for semantic value in V_total = alpha * V_sem + (1-alpha) * V_ig @@ -730,4 +733,4 @@ inline void MapROS::publishUpdateRange() } // namespace apexnav_planner -#endif \ No newline at end of file +#endif diff --git a/src/planner/plan_env/include/plan_env/multi_source_value_map.h b/src/planner/plan_env/include/plan_env/multi_source_value_map.h index 6dae1a1..4fc0715 100644 --- a/src/planner/plan_env/include/plan_env/multi_source_value_map.h +++ b/src/planner/plan_env/include/plan_env/multi_source_value_map.h @@ -48,7 +48,7 @@ struct SemanticSource { * * 1. Multi-source semantic value maps: Each semantic hypothesis gets its own value map * 2. Rotation sampling: VLM evaluates environment at multiple yaw angles before LLM reasoning - * 3. Weighted fusion: Combines multiple semantic sources with confidence weighting + * 3. Paper HSVM aggregation: averages C_l a_l s_l over hypotheses * 4. Geometry value integration: Supports fusion with frontier-based geometric exploration * * Reference: Value_Map_Construction_and_Fusion.md @@ -63,7 +63,7 @@ class MultiSourceValueMap { /** * @brief Add a new semantic source (semantic hypothesis) * @param prompt Semantic text prompt for VLM evaluation - * @param weight Fusion weight (default 1.0, will be normalized) + * @param weight Paper weight C_l*a_l (default 1.0; not sum-normalized) */ void addSemanticSource(const string& prompt, double weight = 1.0); @@ -122,7 +122,7 @@ class MultiSourceValueMap { /** * @brief Get fused semantic value at a position * - * Implements: V_semantic(p) = ฮฃ w_k * V^(k)_sem(p) + * Implements: V_semantic(p) = (1/|H|) ฮฃ w_k * V^(k)_sem(p) * * @param pos Query position in world coordinates * @return Fused semantic value @@ -186,11 +186,6 @@ class MultiSourceValueMap { */ double normalizeAngle(double angle); - /** - * @brief Normalize semantic source weights to sum to 1.0 - */ - void normalizeWeights(); - /** * @brief Callback for SemanticScores messages from Habitat * @param msg SemanticScores message containing multi-source semantic prompts and scores diff --git a/src/planner/plan_env/package.xml b/src/planner/plan_env/package.xml index e114f31..1dce58d 100755 --- a/src/planner/plan_env/package.xml +++ b/src/planner/plan_env/package.xml @@ -7,7 +7,7 @@ - zagerzhang + Zhi Yang diff --git a/src/planner/plan_env/scripts/multi_source_itm_node.py b/src/planner/plan_env/scripts/multi_source_itm_node.py index 28419aa..61d2a85 100755 --- a/src/planner/plan_env/scripts/multi_source_itm_node.py +++ b/src/planner/plan_env/scripts/multi_source_itm_node.py @@ -10,9 +10,6 @@ 2. Subscribes to /map_ros/depth (or RGB topic) to get current images 3. Queries BLIP2 ITM service for each prompt 4. Publishes ITM scores to /semantic_scores - -Author: Claude (Anthropic) -Date: 2025-12-17 """ import rospy diff --git a/src/planner/plan_env/scripts/qwen_vlm_validation_node.py b/src/planner/plan_env/scripts/qwen_vlm_validation_node.py index 82fdcf2..93ce1a0 100755 --- a/src/planner/plan_env/scripts/qwen_vlm_validation_node.py +++ b/src/planner/plan_env/scripts/qwen_vlm_validation_node.py @@ -25,9 +25,6 @@ API Documentation: https://help.aliyun.com/zh/model-studio/ - -Author: InfoNav Team -Date: 2026-01-23 """ import os diff --git a/src/planner/plan_env/scripts/vlm_approach_verifier.py b/src/planner/plan_env/scripts/vlm_approach_verifier.py index 5ac7c56..4365894 100644 --- a/src/planner/plan_env/scripts/vlm_approach_verifier.py +++ b/src/planner/plan_env/scripts/vlm_approach_verifier.py @@ -17,9 +17,6 @@ - When VLM verification is triggered, FSM stops outputting actions - VLM result determines whether to boost or penalize object confidence - 30 second timeout disables VLM for the rest of the episode - -Author: InfoNav Team -Date: 2026-02 """ import os diff --git a/src/planner/plan_env/src/frontier_map2d.cpp b/src/planner/plan_env/src/frontier_map2d.cpp index 9432ae9..06073a7 100644 --- a/src/planner/plan_env/src/frontier_map2d.cpp +++ b/src/planner/plan_env/src/frontier_map2d.cpp @@ -7,8 +7,6 @@ * known free space and unknown regions, performs intelligent clustering using * PCA-based subdivision, and provides visibility-based validation for efficient * exploration planning. - * - * @author Zager-Zhang */ #include #include diff --git a/src/planner/plan_env/src/ig_value_map.cpp b/src/planner/plan_env/src/ig_value_map.cpp index c28c7fd..7c08f5a 100644 --- a/src/planner/plan_env/src/ig_value_map.cpp +++ b/src/planner/plan_env/src/ig_value_map.cpp @@ -6,8 +6,6 @@ * * This file implements the IGValueMap class which provides exploration * guidance by tracking visual-semantic connectivity scores from VLM. - * - * @author Zhi Yang */ #include @@ -19,8 +17,9 @@ namespace apexnav_planner { IGValueMap::IGValueMap(SDFMap2D* sdf_map, ros::NodeHandle& nh) : sdf_map_(sdf_map), - has_sensor_state_(false), - current_sensor_yaw_(0.0) + current_sensor_pos_(Vector2d::Zero()), + current_sensor_yaw_(0.0), + has_sensor_state_(false) { // Initialize buffers int voxel_num = sdf_map_->getVoxelNum(); @@ -31,16 +30,8 @@ IGValueMap::IGValueMap(SDFMap2D* sdf_map, ros::NodeHandle& nh) nh.param("ig_value_map/fov_angle_deg", fov_angle_, 79.0); fov_angle_ = fov_angle_ * M_PI / 180.0; // Convert to radians - // Subscribe to IGScore messages from Habitat - ig_score_sub_ = nh.subscribe( - "/habitat/ig_score", - 10, - &IGValueMap::igScoreCallback, - this - ); - ROS_INFO("[IGValueMap] Initialized with FOV angle %.1f deg", fov_angle_ * 180.0 / M_PI); - ROS_INFO("[IGValueMap] Subscribed to /habitat/ig_score"); + ROS_INFO("[IGValueMap] Updates are synchronized and routed by MapROS"); } void IGValueMap::reset() @@ -115,33 +106,6 @@ double IGValueMap::getFovConfidence(const Vector2d& sensor_pos, const double& se return value * value; // Square for stronger center weighting } -void IGValueMap::igScoreCallback(const plan_env::IGScore::ConstPtr& msg) -{ - if (!has_sensor_state_) { - ROS_WARN_THROTTLE(5.0, "[IGValueMap] No sensor state available for IG update. " - "Make sure to call updateSensorState() first."); - return; - } - - if (current_free_grids_.empty()) { - ROS_WARN_THROTTLE(5.0, "[IGValueMap] No free grids available for update."); - return; - } - - // Update IG map with received score - updateIGMap( - current_sensor_pos_, - current_sensor_yaw_, - current_free_grids_, - msg->ig_score - ); - - ROS_DEBUG_THROTTLE(2.0, "[IGValueMap] Updated IG map with score %.3f " - "(corridor: %.3f, doorway: %.3f, passage: %.3f)", - msg->ig_score, msg->corridor_score, - msg->doorway_score, msg->passage_score); -} - double IGValueMap::getAverageIGScore() const { double sum = 0.0; diff --git a/src/planner/plan_env/src/map_ros.cpp b/src/planner/plan_env/src/map_ros.cpp index 466e3b0..0fbdb5c 100644 --- a/src/planner/plan_env/src/map_ros.cpp +++ b/src/planner/plan_env/src/map_ros.cpp @@ -5,8 +5,6 @@ * This file implements the MapROS class which provides the ROS interface for * the 2D signed distance field mapping system. It handles sensor data processing, * object detection integration, and real-time map visualization. - * - * @author Zager-Zhang */ #include @@ -114,7 +112,9 @@ void MapROS::init() semantic_prompts_sub_ = node_.subscribe( "/semantic_prompts", 1, &MapROS::semanticPromptsCallback, this); semantic_scores_sub_ = node_.subscribe( - "/semantic_scores", 10, &MapROS::semanticScoresCallback, this); + "/habitat/semantic_scores", 10, &MapROS::semanticScoresCallback, this); + ig_score_sub_ = node_.subscribe( + "/habitat/ig_score", 10, &MapROS::igScoreCallback, this); // Setup subscriber for episode reset signals episode_reset_sub_ = node_.subscribe( @@ -125,6 +125,8 @@ void MapROS::init() if (use_multi_source_semantic_) { ROS_INFO("[MapROS] Multi-source semantic value map enabled"); } + current_ig_score_ = 0.0; + has_current_ig_score_ = false; // Load Dual-Value Fusion parameter (same as exploration_manager) node_.param("exploration/fusion_alpha", fusion_alpha_, 0.5); @@ -204,20 +206,50 @@ void MapROS::semanticScoresCallback(const plan_env::SemanticScoresConstPtr& msg) return; } - // Validate that scores match the number of current prompts - if (msg->scores.size() != current_semantic_prompts_.size()) { - ROS_WARN("[MapROS] Received %zu scores but have %zu prompts, ignoring", - msg->scores.size(), current_semantic_prompts_.size()); + if (msg->prompts.empty() || msg->scores.size() != msg->prompts.size() || + (!msg->weights.empty() && msg->weights.size() != msg->prompts.size())) { + ROS_WARN_THROTTLE(5.0, + "[MapROS] Invalid SemanticScores shape: prompts=%zu scores=%zu weights=%zu", + msg->prompts.size(), msg->scores.size(), msg->weights.size()); return; } - // Update semantic scores + // The evaluator publishes the complete prompt/weight/score tuple. Build the + // per-hypothesis maps directly instead of waiting for /semantic_prompts, + // which is not emitted by the main evaluation pipeline. + const bool sources_changed = + msg->target_object != current_target_object_ || + msg->prompts != current_semantic_prompts_ || + map_->multi_source_value_map_->getNumSemanticSources() != + static_cast(msg->prompts.size()); + if (sources_changed) { + current_target_object_ = msg->target_object; + current_semantic_prompts_ = msg->prompts; + map_->multi_source_value_map_->clearSemanticSources(); + for (size_t i = 0; i < msg->prompts.size(); ++i) { + const double weight = msg->weights.empty() ? 1.0 : msg->weights[i]; + map_->multi_source_value_map_->addSemanticSource(msg->prompts[i], weight); + } + ROS_INFO("[MapROS] Initialized %zu HSVM sources for target '%s' from SemanticScores", + msg->prompts.size(), msg->target_object.c_str()); + } + + // depthPoseCallback projects this vector onto the current free-space grid. current_semantic_scores_ = msg->scores; ROS_DEBUG_THROTTLE(2.0, "[MapROS] Updated semantic scores for %zu sources", msg->scores.size()); } +void MapROS::igScoreCallback(const plan_env::IGScoreConstPtr& msg) +{ + current_ig_score_ = msg->ig_score; + has_current_ig_score_ = true; + ROS_DEBUG_THROTTLE(2.0, + "[MapROS] Buffered IG score %.3f (corridor=%.3f doorway=%.3f passage=%.3f)", + msg->ig_score, msg->corridor_score, msg->doorway_score, msg->passage_score); +} + void MapROS::detectedObjectCloudCallback(const plan_env::MultipleMasksWithConfidenceConstPtr& msg) { // Validate message structure consistency @@ -450,9 +482,10 @@ void MapROS::depthPoseCallback( } } - // Update IG value map sensor state (actual update happens via ROS callback) - if (map_->use_ig_value_map_ && map_->ig_value_map_) { - map_->ig_value_map_->updateSensorState(camera_pos, camera_yaw, free_grids); + // Project IG with the same pose/free-space grid used for HSVM. + if (map_->use_ig_value_map_ && map_->ig_value_map_ && has_current_ig_score_) { + map_->ig_value_map_->updateIGMap( + camera_pos, camera_yaw, free_grids, current_ig_score_); } double value_map_time = (ros::Time::now() - t1).toSec(); @@ -757,6 +790,8 @@ void MapROS::resetMapForNewEpisode() current_semantic_prompts_.clear(); current_semantic_scores_.clear(); current_target_object_.clear(); + current_ig_score_ = 0.0; + has_current_ig_score_ = false; current_nav_target_object_id_ = -1; // Reset VLM target tracking local_updated_ = false; esdf_need_update_ = false; @@ -773,4 +808,4 @@ void MapROS::episodeResetCallback(const std_msgs::Empty::ConstPtr& msg) resetMapForNewEpisode(); } -} // namespace apexnav_planner \ No newline at end of file +} // namespace apexnav_planner diff --git a/src/planner/plan_env/src/multi_source_value_map.cpp b/src/planner/plan_env/src/multi_source_value_map.cpp index 51f4988..bd017e1 100644 --- a/src/planner/plan_env/src/multi_source_value_map.cpp +++ b/src/planner/plan_env/src/multi_source_value_map.cpp @@ -9,8 +9,6 @@ * 4. Dynamic semantic-geometric fusion with reliability estimation * * Reference: "Value Map Construction and Fusion" design document - * - * @author Zager-Zhang */ #include @@ -37,33 +35,18 @@ MultiSourceValueMap::MultiSourceValueMap(SDFMap2D* sdf_map, ros::NodeHandle& nh) nh.param("value_map/min_lambda", min_lambda_, 0.2); nh.param("value_map/max_lambda", max_lambda_, 0.8); - // Subscribe to SemanticScores messages from Habitat - semantic_scores_sub_ = nh.subscribe( - "/habitat/semantic_scores", - 10, - &MultiSourceValueMap::semanticScoresCallback, - this - ); - - // Subscribe to episode reset signals - episode_reset_sub_ = nh.subscribe( - "/habitat/episode_reset", - 10, - &MultiSourceValueMap::episodeResetCallback, - this - ); + // MapROS owns score/reset subscriptions so a frame is projected exactly + // once with synchronized pose and free-space geometry. ROS_INFO("[MultiSourceValueMap] Initialized with rotation sampling %s, %d samples", enable_rotation_sampling_ ? "enabled" : "disabled", num_rotation_samples_); - ROS_INFO("[MultiSourceValueMap] Subscribed to /habitat/semantic_scores"); - ROS_INFO("[MultiSourceValueMap] Subscribed to /habitat/episode_reset"); + ROS_INFO("[MultiSourceValueMap] Updates are synchronized and routed by MapROS"); } void MultiSourceValueMap::addSemanticSource(const string& prompt, double weight) { int buffer_size = sdf_map_->getVoxelNum(); semantic_sources_.emplace_back(prompt, weight, buffer_size); - normalizeWeights(); ROS_INFO("[MultiSourceValueMap] Added semantic source: '%s' (weight: %.3f)", prompt.c_str(), weight); @@ -92,23 +75,6 @@ void MultiSourceValueMap::resetForNewEpisode() ROS_INFO("[MultiSourceValueMap] Reset for new episode"); } -void MultiSourceValueMap::normalizeWeights() -{ - if (semantic_sources_.empty()) - return; - - double sum = 0.0; - for (const auto& source : semantic_sources_) { - sum += source.weight; - } - - if (sum > 1e-6) { - for (auto& source : semantic_sources_) { - source.weight /= sum; - } - } -} - void MultiSourceValueMap::updateSensorState(const Vector2d& sensor_pos, const double& sensor_yaw, const vector& free_grids) { @@ -121,7 +87,8 @@ void MultiSourceValueMap::updateSemanticSource(int source_idx, const Vector2d& sensor_pos, const double& sensor_yaw, const vector& free_grids, const double& itm_score) { - if (source_idx < 0 || source_idx >= semantic_sources_.size()) { + if (source_idx < 0 || + static_cast(source_idx) >= semantic_sources_.size()) { ROS_ERROR("[MultiSourceValueMap] Invalid source index: %d", source_idx); return; } @@ -198,12 +165,11 @@ double MultiSourceValueMap::getFusedSemanticValue(const Vector2i& idx) int adr = sdf_map_->toAddress(idx); double fused_value = 0.0; - // Weighted sum: V_semantic(p) = ฮฃ w_k * V^(k)_sem(p) + // Paper HSVM: V_HSVM(p) = (1/|H|) * ฮฃ_l C_l a_l s_l(p). for (const auto& source : semantic_sources_) { fused_value += source.weight * source.value_buffer_[adr]; } - - return fused_value; + return fused_value / static_cast(semantic_sources_.size()); } double MultiSourceValueMap::getFusedConfidence(const Vector2i& idx) @@ -218,8 +184,7 @@ double MultiSourceValueMap::getFusedConfidence(const Vector2i& idx) for (const auto& source : semantic_sources_) { fused_confidence += source.weight * source.confidence_buffer_[adr]; } - - return fused_confidence; + return fused_confidence / static_cast(semantic_sources_.size()); } double MultiSourceValueMap::getTotalValue(const Vector2i& idx, @@ -279,8 +244,6 @@ double MultiSourceValueMap::estimateSemanticReliability() } variance /= all_semantic_values.size(); - double std_dev = std::sqrt(variance); - // Find max value double max_value = *std::max_element(all_semantic_values.begin(), all_semantic_values.end()); diff --git a/src/planner/plan_env/src/object_map2d.cpp b/src/planner/plan_env/src/object_map2d.cpp index 11d7c7f..a120934 100644 --- a/src/planner/plan_env/src/object_map2d.cpp +++ b/src/planner/plan_env/src/object_map2d.cpp @@ -6,8 +6,6 @@ * providing semantic object detection, multi-view fusion, confidence scoring, * and 3D point cloud processing capabilities. The system integrates vision-based * object detection with occupancy mapping to create robust semantic representations. - * - * @author Zhiyang */ #include diff --git a/src/planner/plan_env/src/sdf_map2d.cpp b/src/planner/plan_env/src/sdf_map2d.cpp index b29ea9f..1a72722 100644 --- a/src/planner/plan_env/src/sdf_map2d.cpp +++ b/src/planner/plan_env/src/sdf_map2d.cpp @@ -7,8 +7,6 @@ * raycasting-based sensor fusion, and obstacle inflation capabilities. The system integrates * depth sensor data with probabilistic occupancy models to create robust spatial representations * for path planning and collision avoidance. - * - * @author Zager-Zhang */ #include diff --git a/src/planner/plan_env/src/value_map2d.cpp b/src/planner/plan_env/src/value_map2d.cpp index 52f7d2e..459839b 100644 --- a/src/planner/plan_env/src/value_map2d.cpp +++ b/src/planner/plan_env/src/value_map2d.cpp @@ -7,8 +7,6 @@ * of ITM (Image-Text Matching) scores using field-of-view based confidence modeling. * * Reference paper "VLFM: Vision-Language Frontier Maps for Zero-Shot Semantic Navigation" - * - * @author Zager-Zhang */ #include diff --git a/src/planner/utils/lkh_mtsp_solver/package.xml b/src/planner/utils/lkh_mtsp_solver/package.xml index e8817f2..b17545d 100644 --- a/src/planner/utils/lkh_mtsp_solver/package.xml +++ b/src/planner/utils/lkh_mtsp_solver/package.xml @@ -4,9 +4,9 @@ 0.0.0 The mtsp solver package - zagerzhang + Zhi Yang - GPLv3 + Research-only (LKH-3 upstream terms) catkin roscpp diff --git a/src/planner/utils/vis_utils/package.xml b/src/planner/utils/vis_utils/package.xml index 99d7e7a..4053db8 100755 --- a/src/planner/utils/vis_utils/package.xml +++ b/src/planner/utils/vis_utils/package.xml @@ -7,7 +7,7 @@ - zagerzhang + Zhi Yang diff --git a/vlm/detector/D-FINE.backup/.github/ISSUE_TEMPLATE/bug_report.md b/vlm/detector/D-FINE.backup/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index dd84ea7..0000000 --- a/vlm/detector/D-FINE.backup/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -name: Bug report -about: Create a report to help us improve -title: '' -labels: '' -assignees: '' - ---- - -**Describe the bug** -A clear and concise description of what the bug is. - -**To Reproduce** -Steps to reproduce the behavior: -1. Go to '...' -2. Click on '....' -3. Scroll down to '....' -4. See error - -**Expected behavior** -A clear and concise description of what you expected to happen. - -**Screenshots** -If applicable, add screenshots to help explain your problem. - -**Desktop (please complete the following information):** - - OS: [e.g. iOS] - - Browser [e.g. chrome, safari] - - Version [e.g. 22] - -**Smartphone (please complete the following information):** - - Device: [e.g. iPhone6] - - OS: [e.g. iOS8.1] - - Browser [e.g. stock browser, safari] - - Version [e.g. 22] - -**Additional context** -Add any other context about the problem here. diff --git a/vlm/detector/D-FINE.backup/.github/ISSUE_TEMPLATE/feature_request.md b/vlm/detector/D-FINE.backup/.github/ISSUE_TEMPLATE/feature_request.md deleted file mode 100644 index bbcbbe7..0000000 --- a/vlm/detector/D-FINE.backup/.github/ISSUE_TEMPLATE/feature_request.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -name: Feature request -about: Suggest an idea for this project -title: '' -labels: '' -assignees: '' - ---- - -**Is your feature request related to a problem? Please describe.** -A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] - -**Describe the solution you'd like** -A clear and concise description of what you want to happen. - -**Describe alternatives you've considered** -A clear and concise description of any alternative solutions or features you've considered. - -**Additional context** -Add any other context or screenshots about the feature request here. diff --git a/vlm/detector/D-FINE.backup/.github/workflows/is.yml b/vlm/detector/D-FINE.backup/.github/workflows/is.yml deleted file mode 100644 index 4d986ee..0000000 --- a/vlm/detector/D-FINE.backup/.github/workflows/is.yml +++ /dev/null @@ -1,63 +0,0 @@ -name: Issue Screening - -on: - issues: - types: [opened, edited] - -jobs: - screen-issues: - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Get details and check for keywords - id: issue_check - uses: actions/github-script@v5 - with: - script: | - const issue = context.payload.issue; - const issueNumber = issue.number; - const title = (issue.title || "").toLowerCase(); - const body = (issue.body || "").toLowerCase(); - core.setOutput('number', issueNumber); - - const keywords = ["spam", "badword", "inappropriate", "suspicious", "unusual", "star", "stars", "buy", "buying"]; - let containsKeyword = false; - - console.log(`Checking issue #${issueNumber} for whole word keywords...`); - for (const keyword of keywords) { - const regex = new RegExp(`\\b${keyword}\\b`); - if (regex.test(title) || regex.test(body)) { - containsKeyword = true; - console.log(`Whole word keyword '${keyword}' found in issue #${issueNumber}.`); - break; - } - } - - console.log(`Keyword check for issue #${issueNumber} completed. contains_keyword=${containsKeyword}`); - core.setOutput('contains_keyword', containsKeyword); - - - name: Close and Modify Issue if it contains keywords - if: steps.issue_check.outputs.contains_keyword == 'true' - uses: actions/github-script@v5 - with: - github-token: ${{ secrets.ISSUE }} - script: | - const issueNumber = ${{ steps.issue_check.outputs.number }}; - try { - console.log(`Attempting to close, clear body, and rename title of issue #${issueNumber} due to keyword.`); - await github.rest.issues.update({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: issueNumber, - state: 'closed', - title: "Cleared suspicious issues", - body: "" - }); - console.log(`Successfully closed, cleared body, and renamed title of issue #${issueNumber}.`); - } catch (error) { - console.error(`Failed to update issue #${issueNumber}:`, error); - throw error; - } diff --git a/vlm/detector/D-FINE.backup/.github/workflows/pre-commit.yml b/vlm/detector/D-FINE.backup/.github/workflows/pre-commit.yml deleted file mode 100644 index 81eb8e7..0000000 --- a/vlm/detector/D-FINE.backup/.github/workflows/pre-commit.yml +++ /dev/null @@ -1,15 +0,0 @@ -name: pre-commit - -on: - pull_request: - branches: [master] - push: - branches: [master] - -jobs: - pre-commit: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v3 - - uses: pre-commit/action@v3.0.1 diff --git a/vlm/detector/D-FINE.backup/.gitignore b/vlm/detector/D-FINE.backup/.gitignore deleted file mode 100644 index 16062a9..0000000 --- a/vlm/detector/D-FINE.backup/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -output/ -weight/ -*.pyc -wandb/ -*.pth -*.onnx diff --git a/vlm/detector/D-FINE.backup/.pre-commit-config.yaml b/vlm/detector/D-FINE.backup/.pre-commit-config.yaml deleted file mode 100644 index c7d884a..0000000 --- a/vlm/detector/D-FINE.backup/.pre-commit-config.yaml +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright The Lightning team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -default_language_version: - python: python3 - -ci: - autofix_prs: true - autoupdate_commit_msg: "[pre-commit.ci] pre-commit suggestions" - autoupdate_schedule: quarterly - # submodules: true - -repos: - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 - hooks: - - id: end-of-file-fixer - - id: trailing-whitespace - # - id: check-json # skip for incompatibility with .devcontainer/devcontainer.json - - id: check-yaml - - id: check-toml - - id: check-docstring-first - - id: check-executables-have-shebangs - - id: check-case-conflict - # - id: check-added-large-files - # args: ["--maxkb=100", "--enforce-all"] - - id: detect-private-key - - # - repo: https://github.com/PyCQA/docformatter - # rev: v1.7.5 - # hooks: - # - id: docformatter - # additional_dependencies: [tomli] - # args: ["--in-place"] - - - repo: https://github.com/executablebooks/mdformat - rev: 0.7.17 - hooks: - - id: mdformat - exclude: '^.*\.md$' - args: ["--number"] - additional_dependencies: - - mdformat-gfm - - mdformat-black - - mdformat_frontmatter - - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.9 - hooks: - # try to fix what is possible - - id: ruff - args: ["--fix", "--ignore", "E501,F401,F403,F841,E741"] - # # perform formatting updates - # - id: ruff-format - # validate if all is fine with preview mode - - id: ruff - args: ["--ignore", "E501,F401,F403,F841,E741"] diff --git a/vlm/detector/D-FINE.backup/Dockerfile b/vlm/detector/D-FINE.backup/Dockerfile deleted file mode 100644 index 9d8af8b..0000000 --- a/vlm/detector/D-FINE.backup/Dockerfile +++ /dev/null @@ -1,48 +0,0 @@ -FROM registry.cn-hangzhou.aliyuncs.com/peterande/dfine:v1 - -# FULL BUILDING INFO: - -# docker login --username=xxx registry.cn-hangzhou.aliyuncs.com -# cd [PATH_2_Dockerfile] -# docker build -t xxx:v1 . -# docker tag xxx:v1 registry.cn-hangzhou.aliyuncs.com/xxx/xxx:v1 -# docker push registry.cn-hangzhou.aliyuncs.com/xxx/xxx:v1 - -# FROM dockerpull.com/nvidia/cuda:12.0.1-cudnn8-devel-ubuntu18.04 -# ARG DEBIAN_FRONTEND=noninteractive -# ENV PATH="/root/miniconda3/bin:${PATH}" -# ARG PATH="/root/miniconda3/bin:${PATH}" - -# RUN sed -i "s/archive.ubuntu./mirrors.aliyun./g" /etc/apt/sources.list -# RUN sed -i "s/deb.debian.org/mirrors.aliyun.com/g" /etc/apt/sources.list -# RUN sed -i "s/security.debian.org/mirrors.aliyun.com\/debian-security/g" /etc/apt/sources.list -# RUN sed -i 's/archive.ubuntu.com/mirrors.ustc.edu.cn/g' /etc/apt/sources.list - -# RUN apt-get update && apt-get install -y --no-install-recommends apt-utils && \ -# apt-get upgrade -y && \ -# apt-get install -y vim git libgl1-mesa-glx libglib2.0-0 libsm6 && \ -# apt-get install -y libxrender1 libxext6 tmux wget htop && \ -# apt-get install -y build-essential gcc g++ gdb binutils pciutils net-tools iputils-ping iproute2 git vim wget curl make openssh-server openssh-client tmux tree man unzip unrar - -# ENV PYTHONIOENCODING=UTF-8 - -# RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ -# mkdir /root/.conda && \ -# bash Miniconda3-latest-Linux-x86_64.sh -b && \ -# rm -f Miniconda3-latest-Linux-x86_64.sh && \ -# conda init bash - -# RUN conda config --set show_channel_urls yes \ -# && echo "channels:" > ~/.condarc \ -# && echo " - https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/free/" >> ~/.condarc \ -# && echo " - https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/main/" >> ~/.condarc \ -# && echo "show_channel_urls: true" \ -# && cat ~/.condarc \ -# && pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple \ -# && cat ~/.config/pip/pip.conf - -# RUN python3 -m pip install --upgrade pip && \ -# python3 -m pip install --upgrade setuptools - -# RUN python3 -m pip install jupyterlab pycocotools PyYAML tensorboard scipy -# RUN python3 -m pip --default-timeout=10000 install torch torchvision diff --git a/vlm/detector/D-FINE.backup/LICENSE b/vlm/detector/D-FINE.backup/LICENSE deleted file mode 100644 index 261eeb9..0000000 --- a/vlm/detector/D-FINE.backup/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/vlm/detector/D-FINE.backup/README.md b/vlm/detector/D-FINE.backup/README.md deleted file mode 100644 index 4b0cc42..0000000 --- a/vlm/detector/D-FINE.backup/README.md +++ /dev/null @@ -1,700 +0,0 @@ - - -English | [็ฎ€ไฝ“ไธญๆ–‡](README_cn.md) | [ๆ—ฅๆœฌ่ชž](README_ja.md) | [English Blog](src/zoo/dfine/blog.md) | [ไธญๆ–‡ๅšๅฎข](src/zoo/dfine/blog_cn.md) - -

- D-FINE: Redefine Regression Task of DETRs as Fine‑grained Distribution Refinement -

- - - -

- - hf - - - license - - - prs - - - issues - - - arXiv - - - - pre-commit.ci status - - - stars - -

- - - -

- ๐Ÿ“„ This is the official implementation of the paper: -
- D-FINE: Redefine Regression Task of DETRs as Fine-grained Distribution Refinement -

- - - -

-Yansong Peng, Hebei Li, Peixi Wu, Yueyi Zhang, Xiaoyan Sun, and Feng Wu -

- -

-University of Science and Technology of China -

- -

- - sota - -

- - - -

-If you like D-FINE, please give us a โญ! Your support motivates us to keep improving! -

- -

- -

- -D-FINE is a powerful real-time object detector that redefines the bounding box regression task in DETRs as Fine-grained Distribution Refinement (FDR) and introduces Global Optimal Localization Self-Distillation (GO-LSD), achieving outstanding performance without introducing additional inference and training costs. - -
- Video - -We conduct object detection using D-FINE and YOLO11 on a complex street scene video from [YouTube](https://www.youtube.com/watch?v=CfhEWj9sd9A). Despite challenging conditions such as backlighting, motion blur, and dense crowds, D-FINE-X successfully detects nearly all targets, including subtle small objects like backpacks, bicycles, and traffic lights. Its confidence scores and the localization precision for blurred edges are significantly higher than those of YOLO11. - - - -https://github.com/user-attachments/assets/e5933d8e-3c8a-400e-870b-4e452f5321d9 - -
- -## ๐Ÿš€ Updates -- [x] **\[2024.10.18\]** Release D-FINE series. -- [x] **\[2024.10.25\]** Add custom dataset finetuning configs ([#7](https://github.com/Peterande/D-FINE/issues/7)). -- [x] **\[2024.10.30\]** Update D-FINE-L (E25) pretrained model, with performance improved by 2.0%. -- [x] **\[2024.11.07\]** Release **D-FINE-N**, achiving 42.8% APval on COCO @ 472 FPST4! - -## Model Zoo - -### COCO -| Model | Dataset | APval | #Params | Latency | GFLOPs | config | checkpoint | logs | -| :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | -**D‑FINE‑N** | COCO | **42.8** | 4M | 2.12ms | 7 | [yml](./configs/dfine/dfine_hgnetv2_n_coco.yml) | [42.8](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_n_coco.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/coco/dfine_n_coco_log.txt) -**D‑FINE‑S** | COCO | **48.5** | 10M | 3.49ms | 25 | [yml](./configs/dfine/dfine_hgnetv2_s_coco.yml) | [48.5](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_s_coco.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/coco/dfine_s_coco_log.txt) -**D‑FINE‑M** | COCO | **52.3** | 19M | 5.62ms | 57 | [yml](./configs/dfine/dfine_hgnetv2_m_coco.yml) | [52.3](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_m_coco.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/coco/dfine_m_coco_log.txt) -**D‑FINE‑L** | COCO | **54.0** | 31M | 8.07ms | 91 | [yml](./configs/dfine/dfine_hgnetv2_l_coco.yml) | [54.0](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_l_coco.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/coco/dfine_l_coco_log.txt) -**D‑FINE‑X** | COCO | **55.8** | 62M | 12.89ms | 202 | [yml](./configs/dfine/dfine_hgnetv2_x_coco.yml) | [55.8](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_x_coco.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/coco/dfine_x_coco_log.txt) - - -### Objects365+COCO -| Model | Dataset | APval | #Params | Latency | GFLOPs | config | checkpoint | logs | -| :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | -**D‑FINE‑S** | Objects365+COCO | **50.7** | 10M | 3.49ms | 25 | [yml](./configs/dfine/objects365/dfine_hgnetv2_s_obj2coco.yml) | [50.7](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_s_obj2coco.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/obj2coco/dfine_s_obj2coco_log.txt) -**D‑FINE‑M** | Objects365+COCO | **55.1** | 19M | 5.62ms | 57 | [yml](./configs/dfine/objects365/dfine_hgnetv2_m_obj2coco.yml) | [55.1](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_m_obj2coco.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/obj2coco/dfine_m_obj2coco_log.txt) -**D‑FINE‑L** | Objects365+COCO | **57.3** | 31M | 8.07ms | 91 | [yml](./configs/dfine/objects365/dfine_hgnetv2_l_obj2coco.yml) | [57.3](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_l_obj2coco_e25.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/obj2coco/dfine_l_obj2coco_log_e25.txt) -**D‑FINE‑X** | Objects365+COCO | **59.3** | 62M | 12.89ms | 202 | [yml](./configs/dfine/objects365/dfine_hgnetv2_x_obj2coco.yml) | [59.3](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_x_obj2coco.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/obj2coco/dfine_x_obj2coco_log.txt) - -**We highly recommend that you use the Objects365 pre-trained model for fine-tuning:** - -โš ๏ธ **Important**: Please note that this is generally beneficial for complex scene understanding. If your categories are very simple, it might lead to overfitting and suboptimal performance. -
- ๐Ÿ”ฅ Pretrained Models on Objects365 (Best generalization) - -| Model | Dataset | APval | AP5000 | #Params | Latency | GFLOPs | config | checkpoint | logs | -| :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | -**D‑FINE‑S** | Objects365 | **31.0** | **30.5** | 10M | 3.49ms | 25 | [yml](./configs/dfine/objects365/dfine_hgnetv2_s_obj365.yml) | [30.5](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_s_obj365.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/obj365/dfine_s_obj365_log.txt) -**D‑FINE‑M** | Objects365 | **38.6** | **37.4** | 19M | 5.62ms | 57 | [yml](./configs/dfine/objects365/dfine_hgnetv2_m_obj365.yml) | [37.4](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_m_obj365.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/obj365/dfine_m_obj365_log.txt) -**D‑FINE‑L** | Objects365 | - | **40.6** | 31M | 8.07ms | 91 | [yml](./configs/dfine/objects365/dfine_hgnetv2_l_obj365.yml) | [40.6](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_l_obj365.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/obj365/dfine_l_obj365_log.txt) -**D‑FINE‑L (E25)** | Objects365 | **44.7** | **42.6** | 31M | 8.07ms | 91 | [yml](./configs/dfine/objects365/dfine_hgnetv2_l_obj365.yml) | [42.6](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_l_obj365_e25.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/obj365/dfine_l_obj365_log_e25.txt) -**D‑FINE‑X** | Objects365 | **49.5** | **46.5** | 62M | 12.89ms | 202 | [yml](./configs/dfine/objects365/dfine_hgnetv2_x_obj365.yml) | [46.5](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_x_obj365.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/obj365/dfine_x_obj365_log.txt) -- **E25**: Re-trained and extended the pretraining to 25 epochs. -- **APval** is evaluated on *Objects365* full validation set. -- **AP5000** is evaluated on the first 5000 samples of the *Objects365* validation set. -
- -**Notes:** -- **APval** is evaluated on *MSCOCO val2017* dataset. -- **Latency** is evaluated on a single T4 GPU with $batch\\_size = 1$, $fp16$, and $TensorRT==10.4.0$. -- **Objects365+COCO** means finetuned model on *COCO* using pretrained weights trained on *Objects365*. - - - -## Quick start - -### Setup - -```shell -conda create -n dfine python=3.11.9 -conda activate dfine -pip install -r requirements.txt -``` - - -### Data Preparation - -
- COCO2017 Dataset - -1. Download COCO2017 from [OpenDataLab](https://opendatalab.com/OpenDataLab/COCO_2017) or [COCO](https://cocodataset.org/#download). -1. Modify paths in [coco_detection.yml](./configs/dataset/coco_detection.yml) - - ```yaml - train_dataloader: - img_folder: /data/COCO2017/train2017/ - ann_file: /data/COCO2017/annotations/instances_train2017.json - val_dataloader: - img_folder: /data/COCO2017/val2017/ - ann_file: /data/COCO2017/annotations/instances_val2017.json - ``` - -
- -
- Objects365 Dataset - -1. Download Objects365 from [OpenDataLab](https://opendatalab.com/OpenDataLab/Objects365). - -2. Set the Base Directory: -```shell -export BASE_DIR=/data/Objects365/data -``` - -3. Extract and organize the downloaded files, resulting directory structure: - -```shell -${BASE_DIR}/train -โ”œโ”€โ”€ images -โ”‚ โ”œโ”€โ”€ v1 -โ”‚ โ”‚ โ”œโ”€โ”€ patch0 -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ 000000000.jpg -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ 000000001.jpg -โ”‚ โ”‚ โ”‚ โ””โ”€โ”€ ... (more images) -โ”‚ โ”œโ”€โ”€ v2 -โ”‚ โ”‚ โ”œโ”€โ”€ patchx -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ 000000000.jpg -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ 000000001.jpg -โ”‚ โ”‚ โ”‚ โ””โ”€โ”€ ... (more images) -โ”œโ”€โ”€ zhiyuan_objv2_train.json -``` - -```shell -${BASE_DIR}/val -โ”œโ”€โ”€ images -โ”‚ โ”œโ”€โ”€ v1 -โ”‚ โ”‚ โ”œโ”€โ”€ patch0 -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ 000000000.jpg -โ”‚ โ”‚ โ”‚ โ””โ”€โ”€ ... (more images) -โ”‚ โ”œโ”€โ”€ v2 -โ”‚ โ”‚ โ”œโ”€โ”€ patchx -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ 000000000.jpg -โ”‚ โ”‚ โ”‚ โ””โ”€โ”€ ... (more images) -โ”œโ”€โ”€ zhiyuan_objv2_val.json -``` - -4. Create a New Directory to Store Images from the Validation Set: -```shell -mkdir -p ${BASE_DIR}/train/images_from_val -``` - -5. Copy the v1 and v2 folders from the val directory into the train/images_from_val directory -```shell -cp -r ${BASE_DIR}/val/images/v1 ${BASE_DIR}/train/images_from_val/ -cp -r ${BASE_DIR}/val/images/v2 ${BASE_DIR}/train/images_from_val/ -``` - -6. Run remap_obj365.py to merge a subset of the validation set into the training set. Specifically, this script moves samples with indices between 5000 and 800000 from the validation set to the training set. -```shell -python tools/remap_obj365.py --base_dir ${BASE_DIR} -``` - - -7. Run the resize_obj365.py script to resize any images in the dataset where the maximum edge length exceeds 640 pixels. Use the updated JSON file generated in Step 5 to process the sample data. Ensure that you resize images in both the train and val datasets to maintain consistency. -```shell -python tools/resize_obj365.py --base_dir ${BASE_DIR} -``` - -8. Modify paths in [obj365_detection.yml](./configs/dataset/obj365_detection.yml) - - ```yaml - train_dataloader: - img_folder: /data/Objects365/data/train - ann_file: /data/Objects365/data/train/new_zhiyuan_objv2_train_resized.json - val_dataloader: - img_folder: /data/Objects365/data/val/ - ann_file: /data/Objects365/data/val/new_zhiyuan_objv2_val_resized.json - ``` - - -
- -
-CrowdHuman - -Download COCO format dataset here: [url](https://aistudio.baidu.com/datasetdetail/231455) - -
- -
-Custom Dataset - -To train on your custom dataset, you need to organize it in the COCO format. Follow the steps below to prepare your dataset: - -1. **Set `remap_mscoco_category` to `False`:** - - This prevents the automatic remapping of category IDs to match the MSCOCO categories. - - ```yaml - remap_mscoco_category: False - ``` - -2. **Organize Images:** - - Structure your dataset directories as follows: - - ```shell - dataset/ - โ”œโ”€โ”€ images/ - โ”‚ โ”œโ”€โ”€ train/ - โ”‚ โ”‚ โ”œโ”€โ”€ image1.jpg - โ”‚ โ”‚ โ”œโ”€โ”€ image2.jpg - โ”‚ โ”‚ โ””โ”€โ”€ ... - โ”‚ โ”œโ”€โ”€ val/ - โ”‚ โ”‚ โ”œโ”€โ”€ image1.jpg - โ”‚ โ”‚ โ”œโ”€โ”€ image2.jpg - โ”‚ โ”‚ โ””โ”€โ”€ ... - โ””โ”€โ”€ annotations/ - โ”œโ”€โ”€ instances_train.json - โ”œโ”€โ”€ instances_val.json - โ””โ”€โ”€ ... - ``` - - - **`images/train/`**: Contains all training images. - - **`images/val/`**: Contains all validation images. - - **`annotations/`**: Contains COCO-formatted annotation files. - -3. **Convert Annotations to COCO Format:** - - If your annotations are not already in COCO format, you'll need to convert them. You can use the following Python script as a reference or utilize existing tools: - - ```python - import json - - def convert_to_coco(input_annotations, output_annotations): - # Implement conversion logic here - pass - - if __name__ == "__main__": - convert_to_coco('path/to/your_annotations.json', 'dataset/annotations/instances_train.json') - ``` - -4. **Update Configuration Files:** - - Modify your [custom_detection.yml](./configs/dataset/custom_detection.yml). - - ```yaml - task: detection - - evaluator: - type: CocoEvaluator - iou_types: ['bbox', ] - - num_classes: 777 # your dataset classes - remap_mscoco_category: False - - train_dataloader: - type: DataLoader - dataset: - type: CocoDetection - img_folder: /data/yourdataset/train - ann_file: /data/yourdataset/train/train.json - return_masks: False - transforms: - type: Compose - ops: ~ - shuffle: True - num_workers: 4 - drop_last: True - collate_fn: - type: BatchImageCollateFunction - - val_dataloader: - type: DataLoader - dataset: - type: CocoDetection - img_folder: /data/yourdataset/val - ann_file: /data/yourdataset/val/ann.json - return_masks: False - transforms: - type: Compose - ops: ~ - shuffle: False - num_workers: 4 - drop_last: False - collate_fn: - type: BatchImageCollateFunction - ``` - -
- - -## Usage -
- COCO2017 - - -1. Set Model -```shell -export model=l # n s m l x -``` - -2. Training -```shell -CUDA_VISIBLE_DEVICES=0,1,2,3 torchrun --master_port=7777 --nproc_per_node=4 train.py -c configs/dfine/dfine_hgnetv2_${model}_coco.yml --use-amp --seed=0 -``` - - -3. Testing -```shell -CUDA_VISIBLE_DEVICES=0,1,2,3 torchrun --master_port=7777 --nproc_per_node=4 train.py -c configs/dfine/dfine_hgnetv2_${model}_coco.yml --test-only -r model.pth -``` - - -4. Tuning -```shell -CUDA_VISIBLE_DEVICES=0,1,2,3 torchrun --master_port=7777 --nproc_per_node=4 train.py -c configs/dfine/dfine_hgnetv2_${model}_coco.yml --use-amp --seed=0 -t model.pth -``` -
- - -
- Objects365 to COCO2017 - -1. Set Model -```shell -export model=l # n s m l x -``` - -2. Training on Objects365 -```shell -CUDA_VISIBLE_DEVICES=0,1,2,3 torchrun --master_port=7777 --nproc_per_node=4 train.py -c configs/dfine/objects365/dfine_hgnetv2_${model}_obj365.yml --use-amp --seed=0 -``` - -3. Tuning on COCO2017 -```shell -CUDA_VISIBLE_DEVICES=0,1,2,3 torchrun --master_port=7777 --nproc_per_node=4 train.py -c configs/dfine/objects365/dfine_hgnetv2_${model}_obj2coco.yml --use-amp --seed=0 -t model.pth -``` - - -4. Testing -```shell -CUDA_VISIBLE_DEVICES=0,1,2,3 torchrun --master_port=7777 --nproc_per_node=4 train.py -c configs/dfine/dfine_hgnetv2_${model}_coco.yml --test-only -r model.pth -``` -
- - -
- Custom Dataset - -1. Set Model -```shell -export model=l # n s m l x -``` - -2. Training on Custom Dataset -```shell -CUDA_VISIBLE_DEVICES=0,1,2,3 torchrun --master_port=7777 --nproc_per_node=4 train.py -c configs/dfine/custom/dfine_hgnetv2_${model}_custom.yml --use-amp --seed=0 -``` - -3. Testing -```shell -CUDA_VISIBLE_DEVICES=0,1,2,3 torchrun --master_port=7777 --nproc_per_node=4 train.py -c configs/dfine/custom/dfine_hgnetv2_${model}_custom.yml --test-only -r model.pth -``` - -4. Tuning on Custom Dataset -```shell -CUDA_VISIBLE_DEVICES=0,1,2,3 torchrun --master_port=7777 --nproc_per_node=4 train.py -c configs/dfine/custom/objects365/dfine_hgnetv2_${model}_obj2custom.yml --use-amp --seed=0 -t model.pth -``` - -5. **[Optional]** Modify Class Mappings: - -When using the Objects365 pre-trained weights to train on your custom dataset, the example assumes that your dataset only contains the classes `'Person'` and `'Car'`. For faster convergence, you can modify `self.obj365_ids` in `src/solver/_solver.py` as follows: - - -```python -self.obj365_ids = [0, 5] # Person, Cars -``` -You can replace these with any corresponding classes from your dataset. The list of Objects365 classes with their corresponding IDs: -https://github.com/Peterande/D-FINE/blob/352a94ece291e26e1957df81277bef00fe88a8e3/src/solver/_solver.py#L330 - -New training command: - -```shell -CUDA_VISIBLE_DEVICES=0,1,2,3 torchrun --master_port=7777 --nproc_per_node=4 train.py -c configs/dfine/custom/dfine_hgnetv2_${model}_custom.yml --use-amp --seed=0 -t model.pth -``` - -However, if you don't wish to modify the class mappings, the pre-trained Objects365 weights will still work without any changes. Modifying the class mappings is optional and can potentially accelerate convergence for specific tasks. - - - -
- -
- Customizing Batch Size - -For example, if you want to double the total batch size when training D-FINE-L on COCO2017, here are the steps you should follow: - -1. **Modify your [dataloader.yml](./configs/dfine/include/dataloader.yml)** to increase the `total_batch_size`: - - ```yaml - train_dataloader: - total_batch_size: 64 # Previously it was 32, now doubled - ``` - -2. **Modify your [dfine_hgnetv2_l_coco.yml](./configs/dfine/dfine_hgnetv2_l_coco.yml)**. Hereโ€™s how the key parameters should be adjusted: - - ```yaml - optimizer: - type: AdamW - params: - - - params: '^(?=.*backbone)(?!.*norm|bn).*$' - lr: 0.000025 # doubled, linear scaling law - - - params: '^(?=.*(?:encoder|decoder))(?=.*(?:norm|bn)).*$' - weight_decay: 0. - - lr: 0.0005 # doubled, linear scaling law - betas: [0.9, 0.999] - weight_decay: 0.0001 # need a grid search - - ema: # added EMA settings - decay: 0.9998 # adjusted by 1 - (1 - decay) * 2 - warmups: 500 # halved - - lr_warmup_scheduler: - warmup_duration: 250 # halved - ``` - -
- - -
- Customizing Input Size - -If you'd like to train **D-FINE-L** on COCO2017 with an input size of 320x320, follow these steps: - -1. **Modify your [dataloader.yml](./configs/dfine/include/dataloader.yml)**: - - ```yaml - - train_dataloader: - dataset: - transforms: - ops: - - {type: Resize, size: [320, 320], } - collate_fn: - base_size: 320 - dataset: - transforms: - ops: - - {type: Resize, size: [320, 320], } - ``` - -2. **Modify your [dfine_hgnetv2.yml](./configs/dfine/include/dfine_hgnetv2.yml)**: - - ```yaml - eval_spatial_size: [320, 320] - ``` - -
- -## Tools -
- Deployment - - -1. Setup -```shell -pip install onnx onnxsim -export model=l # n s m l x -``` - -2. Export onnx -```shell -python tools/deployment/export_onnx.py --check -c configs/dfine/dfine_hgnetv2_${model}_coco.yml -r model.pth -``` - -3. Export [tensorrt](https://docs.nvidia.com/deeplearning/tensorrt/install-guide/index.html) -```shell -trtexec --onnx="model.onnx" --saveEngine="model.engine" --fp16 -``` - -
- -
- Inference (Visualization) - - -1. Setup -```shell -pip install -r tools/inference/requirements.txt -export model=l # n s m l x -``` - - - -2. Inference (onnxruntime / tensorrt / torch) - -Inference on images and videos is now supported. -```shell -python tools/inference/onnx_inf.py --onnx model.onnx --input image.jpg # video.mp4 -python tools/inference/trt_inf.py --trt model.engine --input image.jpg -python tools/inference/torch_inf.py -c configs/dfine/dfine_hgnetv2_${model}_coco.yml -r model.pth --input image.jpg --device cuda:0 -``` -
- -
- Benchmark - -1. Setup -```shell -pip install -r tools/benchmark/requirements.txt -export model=l # n s m l x -``` - - -2. Model FLOPs, MACs, and Params -```shell -python tools/benchmark/get_info.py -c configs/dfine/dfine_hgnetv2_${model}_coco.yml -``` - -2. TensorRT Latency -```shell -python tools/benchmark/trt_benchmark.py --COCO_dir path/to/COCO2017 --engine_dir model.engine -``` -
- -
- Fiftyone Visualization - -1. Setup -```shell -pip install fiftyone -export model=l # n s m l x -``` -4. Voxel51 Fiftyone Visualization ([fiftyone](https://github.com/voxel51/fiftyone)) -```shell -python tools/visualization/fiftyone_vis.py -c configs/dfine/dfine_hgnetv2_${model}_coco.yml -r model.pth -``` -
- -
- Others - -1. Auto Resume Training -```shell -bash reference/safe_training.sh -``` - -2. Converting Model Weights -```shell -python reference/convert_weight.py model.pth -``` -
- -## Figures and Visualizations - -
- FDR and GO-LSD - -1. Overview of D-FINE with FDR. The probability distributions that act as a more fine- -grained intermediate representation are iteratively refined by the decoder layers in a residual manner. -Non-uniform weighting functions are applied to allow for finer localization. - -

- Fine-grained Distribution Refinement Process -

- -2. Overview of GO-LSD process. Localization knowledge from the final layerโ€™s refined -distributions is distilled into earlier layers through DDF loss with decoupled weighting strategies. - -

- GO-LSD Process -

- -
- -
- Distributions - -Visualizations of FDR across detection scenarios with initial and refined bounding boxes, along with unweighted and weighted distributions. - -

- -

- -
- -
- Hard Cases - -The following visualization demonstrates D-FINE's predictions in various complex detection scenarios. These include cases with occlusion, low-light conditions, motion blur, depth of field effects, and densely populated scenes. Despite these challenges, D-FINE consistently produces accurate localization results. - -

- D-FINE Predictions in Challenging Scenarios -

- -
- - - - - - - -## Citation -If you use `D-FINE` or its methods in your work, please cite the following BibTeX entries: -
- bibtex - -```latex -@misc{peng2024dfine, - title={D-FINE: Redefine Regression Task in DETRs as Fine-grained Distribution Refinement}, - author={Yansong Peng and Hebei Li and Peixi Wu and Yueyi Zhang and Xiaoyan Sun and Feng Wu}, - year={2024}, - eprint={2410.13842}, - archivePrefix={arXiv}, - primaryClass={cs.CV} -} -``` -
- -## Acknowledgement -Our work is built upon [RT-DETR](https://github.com/lyuwenyu/RT-DETR). -Thanks to the inspirations from [RT-DETR](https://github.com/lyuwenyu/RT-DETR), [GFocal](https://github.com/implus/GFocal), [LD](https://github.com/HikariTJU/LD), and [YOLOv9](https://github.com/WongKinYiu/yolov9). - -โœจ Feel free to contribute and reach out if you have any questions! โœจ diff --git a/vlm/detector/D-FINE.backup/README_cn.md b/vlm/detector/D-FINE.backup/README_cn.md deleted file mode 100644 index c9beb8e..0000000 --- a/vlm/detector/D-FINE.backup/README_cn.md +++ /dev/null @@ -1,673 +0,0 @@ - - -[English](README.md) | ็ฎ€ไฝ“ไธญๆ–‡ | [ๆ—ฅๆœฌ่ชž](README_ja.md) | [English Blog](src/zoo/dfine/blog.md) | [ไธญๆ–‡ๅšๅฎข](src/zoo/dfine/blog_cn.md) - -

- D-FINE: Redefine Regression Task of DETRs as Fine‑grained Distribution Refinement -

- -

- - - license - - - prs - - - issues - - - arXiv - - - - pre-commit.ci status - - - stars - -

- -

- ๐Ÿ“„ ่ฟ™ๆ˜ฏ่ฏฅๆ–‡็ซ ็š„ๅฎ˜ๆ–นๅฎž็Žฐ: -
- D-FINE: Redefine Regression Task of DETRs as Fine-grained Distribution Refinement -

- - -

-ๅฝญๅฒฉๆพ๏ผŒๆŽๅ’Œๅ€๏ผŒๅดๆฒ›็†น๏ผŒๅผ ่ถŠไธ€๏ผŒๅญ™ๆ™“่‰ณ๏ผŒๅดๆžซ -

- -

-ไธญๅ›ฝ็ง‘ๅญฆๆŠ€ๆœฏๅคงๅญฆ -

- -

- - sota - -

- - -

ๅฆ‚ๆžœไฝ ๅ–œๆฌข D-FINE๏ผŒ่ฏท็ป™ๆˆ‘ไปฌไธ€ไธช โญ๏ผไฝ ็š„ๆ”ฏๆŒๆฟ€ๅŠฑๆˆ‘ไปฌไธๆ–ญๅ‰่ฟ›๏ผ

- -

- -

- -D-FINE ๆ˜ฏไธ€ไธชๅผบๅคง็š„ๅฎžๆ—ถ็›ฎๆ ‡ๆฃ€ๆต‹ๅ™จ๏ผŒๅฐ† DETR ไธญ็š„่พน็•Œๆก†ๅ›žๅฝ’ไปปๅŠก้‡ๆ–ฐๅฎšไน‰ไธบไบ†็ป†็ฒ’ๅบฆ็š„ๅˆ†ๅธƒไผ˜ๅŒ–๏ผˆFDR๏ผ‰๏ผŒๅนถๅผ•ๅ…ฅๅ…จๅฑ€ๆœ€ไผ˜็š„ๅฎšไฝ่‡ช่’ธ้ฆ๏ผˆGO-LSD๏ผ‰๏ผŒๅœจไธๅขžๅŠ ้ขๅค–ๆŽจ็†ๅ’Œ่ฎญ็ปƒๆˆๆœฌ็š„ๆƒ…ๅ†ตไธ‹๏ผŒๅฎž็Žฐไบ†ๅ“่ถŠ็š„ๆ€ง่ƒฝใ€‚ - -
- ่ง†้ข‘ - -ๆˆ‘ไปฌๅˆ†ๅˆซไฝฟ็”จ D-FINE ๅ’Œ YOLO11 ๅฏน [YouTube](https://www.youtube.com/watch?v=CfhEWj9sd9A) ไธŠ็š„ไธ€ๆฎตๅคๆ‚่ก—ๆ™ฏ่ง†้ข‘่ฟ›่กŒไบ†็›ฎๆ ‡ๆฃ€ๆต‹ใ€‚ๅฐฝ็ฎกๅญ˜ๅœจ้€†ๅ…‰ใ€่™šๅŒ–ๆจก็ณŠๅ’Œๅฏ†้›†้ฎๆŒก็ญ‰ไธๅˆฉๅ› ็ด ๏ผŒD-FINE-X ไพ็„ถๆˆๅŠŸๆฃ€ๆต‹ๅ‡บๅ‡ ไนŽๆ‰€ๆœ‰็›ฎๆ ‡๏ผŒๅŒ…ๆ‹ฌ่ƒŒๅŒ…ใ€่‡ช่กŒ่ฝฆๅ’Œไฟกๅท็ฏ็ญ‰้šพไปฅๅฏŸ่ง‰็š„ๅฐ็›ฎๆ ‡๏ผŒๅ…ถ็ฝฎไฟกๅบฆใ€ไปฅๅŠๆจก็ณŠ่พน็ผ˜็š„ๅฎšไฝๅ‡†็กฎๅบฆๆ˜Žๆ˜พ้ซ˜ไบŽ YOLO11xใ€‚ - -https://github.com/user-attachments/assets/e5933d8e-3c8a-400e-870b-4e452f5321d9 - -
- -## ๐Ÿš€ Updates -- [x] **\[2024.10.18\]** ๅ‘ๅธƒ D-FINE ็ณปๅˆ—ใ€‚ -- [x] **\[2024.10.25\]** ๆทปๅŠ ไบ†่‡ชๅฎšไน‰ๆ•ฐๆฎ้›†ๅพฎ่ฐƒ้…็ฝฎๆ–‡ไปถ ([#7](https://github.com/Peterande/D-FINE/issues/7))ใ€‚ -- [x] **\[2024.10.30\]** ๆ›ดๆ–ฐ D-FINE-L (E25) ้ข„่ฎญ็ปƒๆจกๅž‹๏ผŒๆ€ง่ƒฝๆๅ‡ไบ† 2.0%ใ€‚ -- [x] **\[2024.11.07\]** ๅ‘ๅธƒ **D-FINE-N**, ๅœจ COCO ไธŠ่พพๅˆฐ 42.8% APval @ 472 FPST4! - -## ๆจกๅž‹ๅบ“ - -### COCO -| ๆจกๅž‹ | ๆ•ฐๆฎ้›† | APval | ๅ‚ๆ•ฐ้‡ | ๆ—ถๅปถ (ms) | GFLOPs | ้…็ฝฎ | ๆƒ้‡ | ๆ—ฅๅฟ— | -| :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | -**D‑FINE‑N** | COCO | **42.8** | 4M | 2.12ms | 7 | [yml](./configs/dfine/dfine_hgnetv2_n_coco.yml) | [42.8](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_n_coco.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/coco/dfine_n_coco_log.txt) -**D‑FINE‑S** | COCO | **48.5** | 10M | 3.49ms | 25 | [yml](./configs/dfine/dfine_hgnetv2_s_coco.yml) | [48.5](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_s_coco.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/coco/dfine_s_coco_log.txt) -**D‑FINE‑M** | COCO | **52.3** | 19M | 5.62ms | 57 | [yml](./configs/dfine/dfine_hgnetv2_m_coco.yml) | [52.3](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_m_coco.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/coco/dfine_m_coco_log.txt) -**D‑FINE‑L** | COCO | **54.0** | 31M | 8.07ms | 91 | [yml](./configs/dfine/dfine_hgnetv2_l_coco.yml) | [54.0](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_l_coco.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/coco/dfine_l_coco_log.txt) -**D‑FINE‑X** | COCO | **55.8** | 62M | 12.89ms | 202 | [yml](./configs/dfine/dfine_hgnetv2_x_coco.yml) | [55.8](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_x_coco.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/coco/dfine_x_coco_log.txt) - -### Objects365+COCO -| ๆจกๅž‹ | ๆ•ฐๆฎ้›† | APval | ๅ‚ๆ•ฐ้‡ | ๆ—ถๅปถ (ms) | GFLOPs | ้…็ฝฎ | ๆƒ้‡ | ๆ—ฅๅฟ— | -| :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | -**D‑FINE‑S** | Objects365+COCO | **50.7** | 10M | 3.49ms | 25 | [yml](./configs/dfine/objects365/dfine_hgnetv2_s_obj2coco.yml) | [50.7](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_s_obj2coco.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/obj2coco/dfine_s_obj2coco_log.txt) -**D‑FINE‑M** | Objects365+COCO | **55.1** | 19M | 5.62ms | 57 | [yml](./configs/dfine/objects365/dfine_hgnetv2_m_obj2coco.yml) | [55.1](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_m_obj2coco.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/obj2coco/dfine_m_obj2coco_log.txt) -**D‑FINE‑L** | Objects365+COCO | **57.3** | 31M | 8.07ms | 91 | [yml](./configs/dfine/objects365/dfine_hgnetv2_l_obj2coco.yml) | [57.3](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_l_obj2coco_e25.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/obj2coco/dfine_l_obj2coco_log_e25.txt) -**D‑FINE‑X** | Objects365+COCO | **59.3** | 62M | 12.89ms | 202 | [yml](./configs/dfine/objects365/dfine_hgnetv2_x_obj2coco.yml) | [59.3](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_x_obj2coco.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/obj2coco/dfine_x_obj2coco_log.txt) - -**ๆˆ‘ไปฌๅผบ็ƒˆๆŽจ่ๆ‚จไฝฟ็”จ Objects365 ้ข„่ฎญ็ปƒๆจกๅž‹่ฟ›่กŒๅพฎ่ฐƒ๏ผš** - -โš ๏ธ ้‡่ฆๆ้†’๏ผš้€šๅธธ่ฟ™็ง้ข„่ฎญ็ปƒๆจกๅž‹ๅฏนๅคๆ‚ๅœบๆ™ฏ็š„็†่งฃ้žๅธธๆœ‰็”จใ€‚ๅฆ‚ๆžœๆ‚จ็š„็ฑปๅˆซ้žๅธธ็ฎ€ๅ•๏ผŒ่ฏทๆณจๆ„๏ผŒ่ฟ™ๅฏ่ƒฝไผšๅฏผ่‡ด่ฟ‡ๆ‹Ÿๅˆๅ’Œๆฌกไผ˜ๆ€ง่ƒฝใ€‚ - -
๐Ÿ”ฅ Objects365 ้ข„่ฎญ็ปƒๆจกๅž‹๏ผˆๆณ›ๅŒ–ๆ€งๆœ€ๅฅฝ๏ผ‰ - -| ๆจกๅž‹ | ๆ•ฐๆฎ้›† | APval | AP5000 | ๅ‚ๆ•ฐ้‡ | ๆ—ถๅปถ (ms) | GFLOPs | ้…็ฝฎ | ๆƒ้‡ | ๆ—ฅๅฟ— | -| :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | -**D‑FINE‑S** | Objects365 | **31.0** | **30.5** | 10M | 3.49ms | 25 | [yml](./configs/dfine/objects365/dfine_hgnetv2_s_obj365.yml) | [30.5](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_s_obj365.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/obj365/dfine_s_obj365_log.txt) -**D‑FINE‑M** | Objects365 | **38.6** | **37.4** | 19M | 5.62ms | 57 | [yml](./configs/dfine/objects365/dfine_hgnetv2_m_obj365.yml) | [37.4](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_m_obj365.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/obj365/dfine_m_obj365_log.txt) -**D‑FINE‑L** | Objects365 | - | **40.6** | 31M | 8.07ms | 91 | [yml](./configs/dfine/objects365/dfine_hgnetv2_l_obj365.yml) | [40.6](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_l_obj365.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/obj365/dfine_l_obj365_log.txt) -**D‑FINE‑L (E25)** | Objects365 | **44.7** | **42.6** | 31M | 8.07ms | 91 | [yml](./configs/dfine/objects365/dfine_hgnetv2_l_obj365.yml) | [42.6](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_l_obj365_e25.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/obj365/dfine_l_obj365_log_e25.txt) -**D‑FINE‑X** | Objects365 | **49.5** | **46.5** | 62M | 12.89ms | 202 | [yml](./configs/dfine/objects365/dfine_hgnetv2_x_obj365.yml) | [46.5](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_x_obj365.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/obj365/dfine_x_obj365_log.txt) -- **E25**: ้‡ๆ–ฐ่ฎญ็ปƒ๏ผŒๅนถๅฐ†่ฎญ็ปƒๅปถ้•ฟ่‡ณ 25 ไธช epochใ€‚ -- **APval** ๆ˜ฏๅœจ *Objects365* ๅฎŒๆ•ด็š„้ชŒ่ฏ้›†ไธŠ่ฟ›่กŒ่ฏ„ไผฐ็š„ใ€‚ -- **AP5000** ๆ˜ฏๅœจ *Objects365* ้ชŒ่ฏ้›†็š„ๅ‰5000ไธชๆ ทๆœฌไธŠ่ฏ„ไผฐ็š„ใ€‚ -
- -**ๆณจๆ„๏ผš** -- **APval** ๆ˜ฏๅœจ *MSCOCO val2017* ๆ•ฐๆฎ้›†ไธŠ่ฏ„ไผฐ็š„ใ€‚ -- **ๆ—ถๅปถ** ๆ˜ฏๅœจๅ•ๅผ  T4 GPU ไธŠไปฅ $batch\\_size = 1$, $fp16$, ๅ’Œ $TensorRT==10.4.0$ ่ฏ„ไผฐ็š„ใ€‚ -- **Objects365+COCO** ่กจ็คบไฝฟ็”จๅœจ *Objects365* ไธŠ้ข„่ฎญ็ปƒ็š„ๆƒ้‡ๅœจ *COCO* ไธŠๅพฎ่ฐƒ็š„ๆจกๅž‹ใ€‚ - - - -## ๅฟซ้€Ÿๅผ€ๅง‹ - -### ่ฎพ็ฝฎ - -```shell -conda create -n dfine python=3.11.9 -conda activate dfine -pip install -r requirements.txt -``` - - - - - -### ๆ•ฐๆฎ้›†ๅ‡†ๅค‡ - - -
- - COCO2017 ๆ•ฐๆฎ้›† - -1. ไปŽ [OpenDataLab](https://opendatalab.com/OpenDataLab/COCO_2017) ๆˆ–่€… [COCO](https://cocodataset.org/#download) ไธ‹่ฝฝ COCO2017ใ€‚ -1.ไฟฎๆ”น [coco_detection.yml](./configs/dataset/coco_detection.yml) ไธญ็š„่ทฏๅพ„ใ€‚ - - ```yaml - train_dataloader: - img_folder: /data/COCO2017/train2017/ - ann_file: /data/COCO2017/annotations/instances_train2017.json - val_dataloader: - img_folder: /data/COCO2017/val2017/ - ann_file: /data/COCO2017/annotations/instances_val2017.json - ``` - -
- -
- Objects365 ๆ•ฐๆฎ้›† - -1. ไปŽ [OpenDataLab](https://opendatalab.com/OpenDataLab/Objects365) ไธ‹่ฝฝ Objects365ใ€‚ - -2. ่ฎพ็ฝฎๆ•ฐๆฎ้›†็š„ๅŸบ็ก€็›ฎๅฝ•๏ผš -```shell -export BASE_DIR=/data/Objects365/data -``` - -3. ่งฃๅŽ‹ๅนถๆ•ด็†็›ฎๅฝ•็ป“ๆž„ๅฆ‚ไธ‹๏ผš - -```shell -${BASE_DIR}/train -โ”œโ”€โ”€ images -โ”‚ โ”œโ”€โ”€ v1 -โ”‚ โ”‚ โ”œโ”€โ”€ patch0 -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ 000000000.jpg -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ 000000001.jpg -โ”‚ โ”‚ โ”‚ โ””โ”€โ”€ ... (more images) -โ”‚ โ”œโ”€โ”€ v2 -โ”‚ โ”‚ โ”œโ”€โ”€ patchx -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ 000000000.jpg -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ 000000001.jpg -โ”‚ โ”‚ โ”‚ โ””โ”€โ”€ ... (more images) -โ”œโ”€โ”€ zhiyuan_objv2_train.json -``` - -```shell -${BASE_DIR}/val -โ”œโ”€โ”€ images -โ”‚ โ”œโ”€โ”€ v1 -โ”‚ โ”‚ โ”œโ”€โ”€ patch0 -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ 000000000.jpg -โ”‚ โ”‚ โ”‚ โ””โ”€โ”€ ... (more images) -โ”‚ โ”œโ”€โ”€ v2 -โ”‚ โ”‚ โ”œโ”€โ”€ patchx -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ 000000000.jpg -โ”‚ โ”‚ โ”‚ โ””โ”€โ”€ ... (more images) -โ”œโ”€โ”€ zhiyuan_objv2_val.json -``` - - -4. ๅˆ›ๅปบไธ€ไธชๆ–ฐ็›ฎๅฝ•ๆฅๅญ˜ๅ‚จ้ชŒ่ฏ้›†ไธญ็š„ๅ›พๅƒ๏ผš -```shell -mkdir -p ${BASE_DIR}/train/images_from_val -``` - -5. ๅฐ† val ็›ฎๅฝ•ไธญ็š„ v1 ๅ’Œ v2 ๆ–‡ไปถๅคนๅคๅˆถๅˆฐ train/images_from_val ็›ฎๅฝ•ไธญ -```shell -cp -r ${BASE_DIR}/val/images/v1 ${BASE_DIR}/train/images_from_val/ -cp -r ${BASE_DIR}/val/images/v2 ${BASE_DIR}/train/images_from_val/ -``` - - -6. ่ฟ่กŒ remap_obj365.py ๅฐ†้ชŒ่ฏ้›†ไธญ็š„้ƒจๅˆ†ๆ ทๆœฌๅˆๅนถๅˆฐ่ฎญ็ปƒ้›†ไธญใ€‚ๅ…ทไฝ“ๆฅ่ฏด๏ผŒ่ฏฅ่„šๆœฌๅฐ†็ดขๅผ•ๅœจ 5000 ๅˆฐ 800000 ไน‹้—ด็š„ๆ ทๆœฌไปŽ้ชŒ่ฏ้›†็งปๅŠจๅˆฐ่ฎญ็ปƒ้›†ใ€‚ -```shell -python tools/remap_obj365.py --base_dir ${BASE_DIR} -``` - - -7. ่ฟ่กŒ resize_obj365.py ่„šๆœฌ๏ผŒๅฐ†ๆ•ฐๆฎ้›†ไธญไปปไฝ•ๆœ€ๅคง่พน้•ฟ่ถ…่ฟ‡ 640 ๅƒ็ด ็š„ๅ›พๅƒ่ฟ›่กŒๅคงๅฐ่ฐƒๆ•ดใ€‚ไฝฟ็”จๆญฅ้ชค 5 ไธญ็”Ÿๆˆ็š„ๆ›ดๆ–ฐๅŽ็š„ JSON ๆ–‡ไปถๅค„็†ๆ ทๆœฌๆ•ฐๆฎใ€‚ -```shell -python tools/resize_obj365.py --base_dir ${BASE_DIR} -``` - -8. ไฟฎๆ”น [obj365_detection.yml](./configs/dataset/obj365_detection.yml) ไธญ็š„่ทฏๅพ„ใ€‚ - - ```yaml - train_dataloader: - img_folder: /data/Objects365/data/train - ann_file: /data/Objects365/data/train/new_zhiyuan_objv2_train_resized.json - val_dataloader: - img_folder: /data/Objects365/data/val/ - ann_file: /data/Objects365/data/val/new_zhiyuan_objv2_val_resized.json - ``` - - -
- -
-CrowdHuman - -ๅœจๆญคไธ‹่ฝฝ COCO ๆ ผๅผ็š„ๆ•ฐๆฎ้›†๏ผš[้“พๆŽฅ](https://aistudio.baidu.com/datasetdetail/231455) - -
- -
-่‡ชๅฎšไน‰ๆ•ฐๆฎ้›† - -่ฆๅœจไฝ ็š„่‡ชๅฎšไน‰ๆ•ฐๆฎ้›†ไธŠ่ฎญ็ปƒ๏ผŒไฝ ้œ€่ฆๅฐ†ๅ…ถ็ป„็ป‡ไธบ COCO ๆ ผๅผใ€‚่ฏทๆŒ‰็…งไปฅไธ‹ๆญฅ้ชคๅ‡†ๅค‡ไฝ ็š„ๆ•ฐๆฎ้›†๏ผš - -1. **ๅฐ† `remap_mscoco_category` ่ฎพ็ฝฎไธบ `False`:** - - ่ฟ™ๅฏไปฅ้˜ฒๆญข็ฑปๅˆซ ID ่‡ชๅŠจๆ˜ ๅฐ„ไปฅๅŒน้… MSCOCO ็ฑปๅˆซใ€‚ - - ```yaml - remap_mscoco_category: False - ``` - -2. **็ป„็ป‡ๅ›พๅƒ๏ผš** - - ๆŒ‰ไปฅไธ‹็ป“ๆž„็ป„็ป‡ไฝ ็š„ๆ•ฐๆฎ้›†็›ฎๅฝ•๏ผš - - ```shell - dataset/ - โ”œโ”€โ”€ images/ - โ”‚ โ”œโ”€โ”€ train/ - โ”‚ โ”‚ โ”œโ”€โ”€ image1.jpg - โ”‚ โ”‚ โ”œโ”€โ”€ image2.jpg - โ”‚ โ”‚ โ””โ”€โ”€ ... - โ”‚ โ”œโ”€โ”€ val/ - โ”‚ โ”‚ โ”œโ”€โ”€ image1.jpg - โ”‚ โ”‚ โ”œโ”€โ”€ image2.jpg - โ”‚ โ”‚ โ””โ”€โ”€ ... - โ””โ”€โ”€ annotations/ - โ”œโ”€โ”€ instances_train.json - โ”œโ”€โ”€ instances_val.json - โ””โ”€โ”€ ... - ``` - - - **`images/train/`**: ๅŒ…ๅซๆ‰€ๆœ‰่ฎญ็ปƒๅ›พๅƒใ€‚ - - **`images/val/`**: ๅŒ…ๅซๆ‰€ๆœ‰้ชŒ่ฏๅ›พๅƒใ€‚ - - **`annotations/`**: ๅŒ…ๅซ COCO ๆ ผๅผ็š„ๆณจ้‡Šๆ–‡ไปถใ€‚ - -3. **ๅฐ†ๆณจ้‡Š่ฝฌๆขไธบ COCO ๆ ผๅผ๏ผš** - - ๅฆ‚ๆžœไฝ ็š„ๆณจ้‡Šๅฐšๆœชไธบ COCO ๆ ผๅผ๏ผŒไฝ ้œ€่ฆ่ฟ›่กŒ่ฝฌๆขใ€‚ไฝ ๅฏไปฅๅ‚่€ƒไปฅไธ‹ Python ่„šๆœฌๆˆ–ไฝฟ็”จ็Žฐๆœ‰ๅทฅๅ…ท๏ผš - - ```python - import json - - def convert_to_coco(input_annotations, output_annotations): - # Implement conversion logic here - pass - - if __name__ == "__main__": - convert_to_coco('path/to/your_annotations.json', 'dataset/annotations/instances_train.json') - ``` - -4. **ๆ›ดๆ–ฐ้…็ฝฎๆ–‡ไปถ๏ผš** - - ไฟฎๆ”นไฝ ็š„ [custom_detection.yml](./configs/dataset/custom_detection.yml)ใ€‚ - - ```yaml - task: detection - - evaluator: - type: CocoEvaluator - iou_types: ['bbox', ] - - num_classes: 777 # your dataset classes - remap_mscoco_category: False - - train_dataloader: - type: DataLoader - dataset: - type: CocoDetection - img_folder: /data/yourdataset/train - ann_file: /data/yourdataset/train/train.json - return_masks: False - transforms: - type: Compose - ops: ~ - shuffle: True - num_workers: 4 - drop_last: True - collate_fn: - type: BatchImageCollateFunction - - val_dataloader: - type: DataLoader - dataset: - type: CocoDetection - img_folder: /data/yourdataset/val - ann_file: /data/yourdataset/val/ann.json - return_masks: False - transforms: - type: Compose - ops: ~ - shuffle: False - num_workers: 4 - drop_last: False - collate_fn: - type: BatchImageCollateFunction - ``` -
- - -## ไฝฟ็”จๆ–นๆณ• -
- COCO2017 - - -1. ่ฎพ็ฝฎๆจกๅž‹ -```shell -export model=l # n s m l x -``` - -2. ่ฎญ็ปƒ -```shell -CUDA_VISIBLE_DEVICES=0,1,2,3 torchrun --master_port=7777 --nproc_per_node=4 train.py -c configs/dfine/dfine_hgnetv2_${model}_coco.yml --use-amp --seed=0 -``` - - -3. ๆต‹่ฏ• -```shell -CUDA_VISIBLE_DEVICES=0,1,2,3 torchrun --master_port=7777 --nproc_per_node=4 train.py -c configs/dfine/dfine_hgnetv2_${model}_coco.yml --test-only -r model.pth -``` - - -4. ๅพฎ่ฐƒ -```shell -CUDA_VISIBLE_DEVICES=0,1,2,3 torchrun --master_port=7777 --nproc_per_node=4 train.py -c configs/dfine/dfine_hgnetv2_${model}_coco.yml --use-amp --seed=0 -t model.pth -``` -
- - -
- ๅœจ Objects365 ไธŠ่ฎญ็ปƒ๏ผŒๅœจCOCO2017ไธŠๅพฎ่ฐƒ - -1. ่ฎพ็ฝฎๆจกๅž‹ -```shell -export model=l # n s m l x -``` - -2. ๅœจ Objects365 ไธŠ่ฎญ็ปƒ -```shell -CUDA_VISIBLE_DEVICES=0,1,2,3 torchrun --master_port=7777 --nproc_per_node=4 train.py -c configs/dfine/objects365/dfine_hgnetv2_${model}_obj365.yml --use-amp --seed=0 -``` - -3. ๅœจ COCO2017 ไธŠๅพฎ่ฐƒ -```shell -CUDA_VISIBLE_DEVICES=0,1,2,3 torchrun --master_port=7777 --nproc_per_node=4 train.py -c configs/dfine/objects365/dfine_hgnetv2_${model}_obj2coco.yml --use-amp --seed=0 -t model.pth -``` - - -4. ๆต‹่ฏ• -```shell -CUDA_VISIBLE_DEVICES=0,1,2,3 torchrun --master_port=7777 --nproc_per_node=4 train.py -c configs/dfine/dfine_hgnetv2_${model}_coco.yml --test-only -r model.pth -``` -
- - -
- ่‡ชๅฎšไน‰ๆ•ฐๆฎ้›† - -1. ่ฎพ็ฝฎๆจกๅž‹ -```shell -export model=l # n s m l x -``` - -2. ๅœจ่‡ชๅฎšไน‰ๆ•ฐๆฎ้›†ไธŠ่ฎญ็ปƒ -```shell -CUDA_VISIBLE_DEVICES=0,1,2,3 torchrun --master_port=7777 --nproc_per_node=4 train.py -c configs/dfine/custom/dfine_hgnetv2_${model}_custom.yml --use-amp --seed=0 -``` - -3. ๆต‹่ฏ• -```shell -CUDA_VISIBLE_DEVICES=0,1,2,3 torchrun --master_port=7777 --nproc_per_node=4 train.py -c configs/dfine/custom/dfine_hgnetv2_${model}_custom.yml --test-only -r model.pth -``` - -4. ๅœจ่‡ชๅฎšไน‰ๆ•ฐๆฎ้›†ไธŠๅพฎ่ฐƒ -```shell -CUDA_VISIBLE_DEVICES=0,1,2,3 torchrun --master_port=7777 --nproc_per_node=4 train.py -c configs/dfine/custom/objects365/dfine_hgnetv2_${model}_obj2custom.yml --use-amp --seed=0 -t model.pth -``` - -5. **[ๅฏ้€‰้กน]** ไฟฎๆ”น็ฑปๆ˜ ๅฐ„: - -ๅœจไฝฟ็”จ Objects365 ้ข„่ฎญ็ปƒๆƒ้‡่ฎญ็ปƒ่‡ชๅฎšไน‰ๆ•ฐๆฎ้›†ๆ—ถ๏ผŒ็คบไพ‹ไธญๅ‡่ฎพ่‡ชๅฎšไน‰ๆ•ฐๆฎ้›†ไป…ๆœ‰ `'Person'` ๅ’Œ `'Car'` ็ฑป๏ผŒๆ‚จๅฏไปฅๅฐ†ๅ…ถๆ›ฟๆขไธบๆ•ฐๆฎ้›†ไธญๅฏนๅบ”็š„ไปปไฝ•็ฑปๅˆซใ€‚ไธบไบ†ๅŠ ๅฟซๆ”ถๆ•›๏ผŒๅฏไปฅๅœจ `src/solver/_solver.py` ไธญไฟฎๆ”น `self.obj365_ids`๏ผŒๅฆ‚ไธ‹ๆ‰€็คบ๏ผš - -```python -self.obj365_ids = [0, 5] # Person, Cars -``` -Objects365 ็ฑปๅŠๅ…ถๅฏนๅบ” ID ็š„ๅฎŒๆ•ดๅˆ—่กจ: -https://github.com/Peterande/D-FINE/blob/352a94ece291e26e1957df81277bef00fe88a8e3/src/solver/_solver.py#L330 - -ๆ–ฐ็š„่ฎญ็ปƒๅฏๅŠจๅ‘ฝไปค๏ผš - -```shell -CUDA_VISIBLE_DEVICES=0,1,2,3 torchrun --master_port=7777 --nproc_per_node=4 train.py -c configs/dfine/custom/dfine_hgnetv2_${model}_custom.yml --use-amp --seed=0 -t model.pth -``` - -ๅฆ‚ๆžœๆ‚จไธๆƒณไฟฎๆ”น็ฑปๆ˜ ๅฐ„๏ผŒ้ข„่ฎญ็ปƒ็š„ Objects365 ๆƒ้‡ไพ็„ถๅฏไปฅไธๅšไปปไฝ•ๆ›ดๆ”น็›ดๆŽฅไฝฟ็”จใ€‚ไฟฎๆ”น็ฑปๆ˜ ๅฐ„ๆ˜ฏๅฏ้€‰็š„๏ผŒไฝ†้’ˆๅฏน็‰นๅฎšไปปๅŠกๅฏ่ƒฝไผšๅŠ ๅฟซๆ”ถๆ•›้€Ÿๅบฆใ€‚ - -
- -
- ่‡ชๅฎšไน‰ๆ‰นๆฌกๅคงๅฐ - -ไพ‹ๅฆ‚๏ผŒๅฆ‚ๆžœไฝ ๆƒณๅœจ่ฎญ็ปƒ D-FINE-L ๆ—ถๅฐ† COCO2017 ็š„ๆ€ปๆ‰นๆฌกๅคงๅฐๅขžๅŠ ไธ€ๅ€๏ผŒ่ฏทๆŒ‰็…งไปฅไธ‹ๆญฅ้ชคๆ“ไฝœ๏ผš - -1. **ไฟฎๆ”นไฝ ็š„ [dataloader.yml](./configs/dfine/include/dataloader.yml)**๏ผŒๅขžๅŠ  `total_batch_size`๏ผš - - ```yaml - train_dataloader: - total_batch_size: 64 # ๅŽŸๆฅๆ˜ฏ 32๏ผŒ็ŽฐๅœจๅขžๅŠ ไบ†ไธ€ๅ€ - ``` - -2. **ไฟฎๆ”นไฝ ็š„ [dfine_hgnetv2_l_coco.yml](./configs/dfine/dfine_hgnetv2_l_coco.yml)**ใ€‚ - - ```yaml - optimizer: - type: AdamW - params: - - - params: '^(?=.*backbone)(?!.*norm|bn).*$' - lr: 0.000025 # ็ฟปๅ€๏ผŒ็บฟๆ€ง็ผฉๆ”พๅŽŸๅˆ™ - - - params: '^(?=.*(?:encoder|decoder))(?=.*(?:norm|bn)).*$' - weight_decay: 0. - - lr: 0.0005 # ็ฟปๅ€๏ผŒ็บฟๆ€ง็ผฉๆ”พๅŽŸๅˆ™ - betas: [0.9, 0.999] - weight_decay: 0.0001 # ้œ€่ฆ็ฝ‘ๆ ผๆœ็ดขๆ‰พๅˆฐๆœ€ไผ˜ๅ€ผ - - ema: # ๆทปๅŠ  EMA ่ฎพ็ฝฎ - decay: 0.9998 # ๆ นๆฎ 1 - (1 - decay) * 2 ่ฐƒๆ•ด - warmups: 500 # ๅ‡ๅŠ - - lr_warmup_scheduler: - warmup_duration: 250 # ๅ‡ๅŠ - ``` - -
- -
- ่‡ชๅฎšไน‰่พ“ๅ…ฅๅฐบๅฏธ - -ๅฆ‚ๆžœไฝ ๆƒณๅœจ COCO2017 ไธŠไฝฟ็”จ **D-FINE-L** ่ฟ›่กŒ 320x320 ๅฐบๅฏธ็š„ๅ›พ็‰‡่ฎญ็ปƒ๏ผŒๆŒ‰็…งไปฅไธ‹ๆญฅ้ชคๆ“ไฝœ๏ผš - -1. **ไฟฎๆ”นไฝ ็š„ [dataloader.yml](./configs/dfine/include/dataloader.yml)**๏ผš - - ```yaml - - train_dataloader: - dataset: - transforms: - ops: - - {type: Resize, size: [320, 320], } - collate_fn: - base_size: 320 - dataset: - transforms: - ops: - - {type: Resize, size: [320, 320], } - ``` - -2. **ไฟฎๆ”นไฝ ็š„ [dfine_hgnetv2.yml](./configs/dfine/include/dfine_hgnetv2.yml)**๏ผš - - ```yaml - eval_spatial_size: [320, 320] - ``` - -
- - -## ๅทฅๅ…ท - -
- ้ƒจ็ฝฒ - - -1. ่ฎพ็ฝฎ -```shell -pip install onnx onnxsim onnxruntime -export model=l # n s m l x -``` - -2. ๅฏผๅ‡บ onnx -```shell -python tools/export_onnx.py --check -c configs/dfine/dfine_hgnetv2_${model}_coco.yml -r model.pth -``` - -3. ๅฏผๅ‡บ [tensorrt](https://docs.nvidia.com/deeplearning/tensorrt/install-guide/index.html) -```shell -trtexec --onnx="model.onnx" --saveEngine="model.engine" --fp16 -``` - -
- -
- ๆŽจ็†๏ผˆๅฏ่ง†ๅŒ–๏ผ‰ - - -1. ่ฎพ็ฝฎ -```shell -pip install -r tools/inference/requirements.txt -export model=l # n s m l x -``` - - - -2. ๆŽจ็† (onnxruntime / tensorrt / torch) - -็›ฎๅ‰ๆ”ฏๆŒๅฏนๅ›พๅƒๅ’Œ่ง†้ข‘็š„ๆŽจ็†ใ€‚ -```shell -python tools/inference/onnx_inf.py --onnx model.onnx --input image.jpg # video.mp4 -python tools/inference/trt_inf.py --trt model.engine --input image.jpg -python tools/inference/torch_inf.py -c configs/dfine/dfine_hgnetv2_${model}_coco.yml -r model.pth --input image.jpg --device cuda:0 -``` -
- -
- ๅŸบๅ‡†ๆต‹่ฏ• - -1. ่ฎพ็ฝฎ -```shell -pip install -r tools/benchmark/requirements.txt -export model=l # n s m l x -``` - - -2. ๆจกๅž‹ FLOPsใ€MACsใ€ๅ‚ๆ•ฐ้‡ -```shell -python tools/benchmark/get_info.py -c configs/dfine/dfine_hgnetv2_${model}_coco.yml -``` - -2. TensorRT ๅปถ่ฟŸ -```shell -python tools/benchmark/trt_benchmark.py --COCO_dir path/to/COCO2017 --engine_dir model.engine -``` -
- -
- Voxel51 Fiftyone ๅฏ่ง†ๅŒ– - -1. ่ฎพ็ฝฎ -```shell -pip install fiftyone -export model=l # n s m l x -``` -4. Voxel51 Fiftyone ๅฏ่ง†ๅŒ– ([fiftyone](https://github.com/voxel51/fiftyone)) -```shell -python tools/visualization/fiftyone_vis.py -c configs/dfine/dfine_hgnetv2_${model}_coco.yml -r model.pth -``` -
- -
- ๅ…ถไป– - -1. ่‡ชๅŠจๆขๅค๏ผˆAuto Resume๏ผ‰่ฎญ็ปƒ -```shell -bash reference/safe_training.sh -``` - -2. ๆจกๅž‹ๆƒ้‡่ฝฌๆข -```shell -python reference/convert_weight.py model.pth -``` -
- -## ๅ›พ่กจไธŽๅฏ่ง†ๅŒ– - -
- FDR ๅ’Œ GO-LSD - -D-FINEไธŽFDRๆฆ‚่งˆใ€‚ๆฆ‚็Ž‡ๅˆ†ๅธƒไฝœไธบๆ›ด็ป†็ฒ’ๅบฆ็š„ไธญ้—ด่กจๅพ๏ผŒ้€š่ฟ‡่งฃ็ ๅ™จๅฑ‚ไปฅๆฎ‹ๅทฎๆ–นๅผ่ฟ›่กŒ่ฟญไปฃไผ˜ๅŒ–ใ€‚ๅบ”็”จ้žๅ‡ๅŒ€ๅŠ ๆƒๅ‡ฝๆ•ฐไปฅๅฎž็Žฐๆ›ด็ฒพ็ป†็š„ๅฎšไฝใ€‚ -

-็ป†็ฒ’ๅบฆๅˆ†ๅธƒไผ˜ๅŒ–่ฟ‡็จ‹

-GO-LSDๆต็จ‹ๆฆ‚่งˆใ€‚้€š่ฟ‡DDFๆŸๅคฑๅ‡ฝๆ•ฐๅ’Œ่งฃ่€ฆๅŠ ๆƒ็ญ–็•ฅๅฐ†ๆœ€็ปˆๅฑ‚ๅˆ†ๅธƒไธญ็š„ๅฎšไฝ็Ÿฅ่ฏ†่’ธ้ฆๅˆฐๅ‰้ข็š„ๅฑ‚ไธญใ€‚ -

GO-LSDๆต็จ‹

- -
- -
- ๅˆ†ๅธƒๅฏ่ง†ๅŒ– - -FDRๅœจๆฃ€ๆต‹ๅœบๆ™ฏไธญ็š„ๅฏ่ง†ๅŒ–๏ผŒๅŒ…ๆ‹ฌๅˆๅง‹ๅ’Œไผ˜ๅŒ–ๅŽ็š„่พน็•Œๆก†๏ผŒไปฅๅŠๆœชๅŠ ๆƒๅ’ŒๅŠ ๆƒ็š„ๅˆ†ๅธƒๅ›พใ€‚ - -

- -

- -
- -
- ๅ›ฐ้šพๅœบๆ™ฏ - -ไปฅไธ‹ๅฏ่ง†ๅŒ–ๅฑ•็คบไบ†D-FINEๅœจๅ„็งๅคๆ‚ๆฃ€ๆต‹ๅœบๆ™ฏไธญ็š„้ข„ๆต‹็ป“ๆžœใ€‚่ฟ™ไบ›ๅœบๆ™ฏๅŒ…ๆ‹ฌ้ฎๆŒกใ€ไฝŽๅ…‰ๆกไปถใ€่ฟๅŠจๆจก็ณŠใ€ๆ™ฏๆทฑๆ•ˆๆžœๅ’Œๅฏ†้›†ๅœบๆ™ฏใ€‚ๅฐฝ็ฎก้ขไธด่ฟ™ไบ›ๆŒ‘ๆˆ˜๏ผŒD-FINEไพ็„ถ่ƒฝๅคŸ็”Ÿๆˆๅ‡†็กฎ็š„ๅฎšไฝ็ป“ๆžœใ€‚ - -

-D-FINEๅœจๆŒ‘ๆˆ˜ๆ€งๅœบๆ™ฏไธญ็š„้ข„ๆต‹ -

- -
- - - - -## ๅผ•็”จ -ๅฆ‚ๆžœไฝ ๅœจๅทฅไฝœไธญไฝฟ็”จไบ† `D-FINE` ๆˆ–ๅ…ถๆ–นๆณ•๏ผŒ่ฏทๅผ•็”จไปฅไธ‹ BibTeX ๆก็›ฎ๏ผš -
- bibtex - -```latex -@misc{peng2024dfine, - title={D-FINE: Redefine Regression Task in DETRs as Fine-grained Distribution Refinement}, - author={Yansong Peng and Hebei Li and Peixi Wu and Yueyi Zhang and Xiaoyan Sun and Feng Wu}, - year={2024}, - eprint={2410.13842}, - archivePrefix={arXiv}, - primaryClass={cs.CV} -} -``` -
- -## ่‡ด่ฐข -ๆˆ‘ไปฌ็š„ๅทฅไฝœๅŸบไบŽ [RT-DETR](https://github.com/lyuwenyu/RT-DETR)ใ€‚ -ๆ„Ÿ่ฐข [RT-DETR](https://github.com/lyuwenyu/RT-DETR), [GFocal](https://github.com/implus/GFocal), [LD](https://github.com/HikariTJU/LD), ๅ’Œ [YOLOv9](https://github.com/WongKinYiu/yolov9) ็š„ๅฏๅ‘ใ€‚ - -โœจ ๆฌข่ฟŽ่ดก็Œฎๅนถๅœจๆœ‰ไปปไฝ•้—ฎ้ข˜ๆ—ถ่”็ณปๆˆ‘๏ผ โœจ diff --git a/vlm/detector/D-FINE.backup/README_ja.md b/vlm/detector/D-FINE.backup/README_ja.md deleted file mode 100644 index e67c239..0000000 --- a/vlm/detector/D-FINE.backup/README_ja.md +++ /dev/null @@ -1,698 +0,0 @@ - - -[English](README.md) | [็ฎ€ไฝ“ไธญๆ–‡](README_cn.md) | ๆ—ฅๆœฌ่ชž | [English Blog](src/zoo/dfine/blog.md) | [ไธญๆ–‡ๅšๅฎข](src/zoo/dfine/blog_cn.md) - -

- D-FINE: Redefine Regression Task of DETRs as Fine‑grained Distribution Refinement -

- - - -

- - license - - - prs - - - issues - - - arXiv - - - - pre-commit.ci status - - - stars - -

- - - -

- ๐Ÿ“„ ใ“ใ‚Œใฏ่ซ–ๆ–‡ใฎๅ…ฌๅผๅฎŸ่ฃ…ใงใ™: -
- D-FINE: Redefine Regression Task of DETRs as Fine-grained Distribution Refinement -

-

- D-FINE: DETRใฎๅ›žๅธฐใ‚ฟใ‚นใ‚ฏใ‚’็ดฐ็ฒ’ๅบฆๅˆ†ๅธƒๆœ€้ฉๅŒ–ใจใ—ใฆๅ†ๅฎš็พฉ -

- - - -

-Yansong Peng, Hebei Li, Peixi Wu, Yueyi Zhang, Xiaoyan Sun, and Feng Wu -

- -

-ไธญๅ›ฝ็ง‘ๅญฆๆŠ€่ก“ๅคงๅญฆ -

- -

- - sota - -

- - - -

-ใ‚‚ใ—D-FINEใŒๆฐ—ใซๅ…ฅใฃใŸใ‚‰ใ€ใœใฒโญใ‚’ใใ ใ•ใ„๏ผใ‚ใชใŸใฎใ‚ตใƒใƒผใƒˆใŒ็งใŸใกใฎใƒขใƒใƒ™ใƒผใ‚ทใƒงใƒณใซใชใ‚Šใพใ™๏ผ -

- -

- -

- -D-FINEใฏใ€DETRใฎๅขƒ็•Œใƒœใƒƒใ‚ฏใ‚นๅ›žๅธฐใ‚ฟใ‚นใ‚ฏใ‚’็ดฐ็ฒ’ๅบฆๅˆ†ๅธƒๆœ€้ฉๅŒ–๏ผˆFDR๏ผ‰ใจใ—ใฆๅ†ๅฎš็พฉใ—ใ€ใ‚ฐใƒญใƒผใƒใƒซๆœ€้ฉใชไฝ็ฝฎ็‰นๅฎš่‡ชๅทฑ่’ธ็•™๏ผˆGO-LSD๏ผ‰ใ‚’ๅฐŽๅ…ฅใ™ใ‚‹ใ“ใจใงใ€่ฟฝๅŠ ใฎๆŽจ่ซ–ใŠใ‚ˆใณใƒˆใƒฌใƒผใƒ‹ใƒณใ‚ฐใ‚ณใ‚นใƒˆใ‚’ๅข—ใ‚„ใ™ใ“ใจใชใใ€ๅ„ชใ‚ŒใŸใƒ‘ใƒ•ใ‚ฉใƒผใƒžใƒณใ‚นใ‚’ๅฎŸ็พใ™ใ‚‹ๅผทๅŠ›ใชใƒชใ‚ขใƒซใ‚ฟใ‚คใƒ ใ‚ชใƒ–ใ‚ธใ‚งใ‚ฏใƒˆๆคœๅ‡บๅ™จใงใ™ใ€‚ - -
- ใƒ“ใƒ‡ใ‚ช - -D-FINEใจYOLO11ใ‚’ไฝฟ็”จใ—ใฆใ€[YouTube](https://www.youtube.com/watch?v=CfhEWj9sd9A)ใฎ่ค‡้›‘ใช่ก—ไธฆใฟใฎใƒ“ใƒ‡ใ‚ชใงใ‚ชใƒ–ใ‚ธใ‚งใ‚ฏใƒˆๆคœๅ‡บใ‚’่กŒใ„ใพใ—ใŸใ€‚้€†ๅ…‰ใ€ใƒขใƒผใ‚ทใƒงใƒณใƒ–ใƒฉใƒผใ€ๅฏ†้›†ใ—ใŸ็พค่ก†ใชใฉใฎๅŽณใ—ใ„ๆกไปถใซใ‚‚ใ‹ใ‹ใ‚ใ‚‰ใšใ€D-FINE-Xใฏใปใผใ™ในใฆใฎใ‚ฟใƒผใ‚ฒใƒƒใƒˆใ‚’ๆคœๅ‡บใ—ใ€ใƒใƒƒใ‚ฏใƒ‘ใƒƒใ‚ฏใ€่‡ช่ปข่ปŠใ€ไฟกๅทๆฉŸใชใฉใฎๅพฎๅฆ™ใชๅฐใ•ใชใ‚ชใƒ–ใ‚ธใ‚งใ‚ฏใƒˆใ‚‚ๅซใพใ‚Œใพใ™ใ€‚ใใฎไฟก้ ผใ‚นใ‚ณใ‚ขใจใผใ‚„ใ‘ใŸใ‚จใƒƒใ‚ธใฎไฝ็ฝฎ็‰นๅฎš็ฒพๅบฆใฏYOLO11ใ‚ˆใ‚Šใ‚‚ใฏใ‚‹ใ‹ใซ้ซ˜ใ„ใงใ™ใ€‚ - - - -https://github.com/user-attachments/assets/e5933d8e-3c8a-400e-870b-4e452f5321d9 - -
- -## ๐Ÿš€ ๆ›ดๆ–ฐๆƒ…ๅ ฑ -- [x] **\[2024.10.18\]** D-FINEใ‚ทใƒชใƒผใ‚บใ‚’ใƒชใƒชใƒผใ‚นใ€‚ -- [x] **\[2024.10.25\]** ใ‚ซใ‚นใ‚ฟใƒ ใƒ‡ใƒผใ‚ฟใ‚ปใƒƒใƒˆใฎๅพฎ่ชฟๆ•ด่จญๅฎšใ‚’่ฟฝๅŠ  ([#7](https://github.com/Peterande/D-FINE/issues/7))ใ€‚ -- [x] **\[2024.10.30\]** D-FINE-L (E25) ไบ‹ๅ‰ใƒˆใƒฌใƒผใƒ‹ใƒณใ‚ฐใƒขใƒ‡ใƒซใ‚’ๆ›ดๆ–ฐใ—ใ€ใƒ‘ใƒ•ใ‚ฉใƒผใƒžใƒณใ‚นใŒ2.0%ๅ‘ไธŠใ€‚ -- [x] **\[2024.11.07\]** **D-FINE-N** ใ‚’ใƒชใƒชใƒผใ‚น, COCO ใง 42.8% ใฎ APval ใ‚’้”ๆˆ @ 472 FPST4! - -## ใƒขใƒ‡ใƒซใ‚บใƒผ - -### COCO -| ใƒขใƒ‡ใƒซ | ใƒ‡ใƒผใ‚ฟใ‚ปใƒƒใƒˆ | APval | ใƒ‘ใƒฉใƒกใƒผใ‚ฟๆ•ฐ | ใƒฌใ‚คใƒ†ใƒณใ‚ท | GFLOPs | config | checkpoint | logs | -| :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | -**D‑FINE‑N** | COCO | **42.8** | 4M | 2.12ms | 7 | [yml](./configs/dfine/dfine_hgnetv2_n_coco.yml) | [42.8](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_n_coco.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/coco/dfine_n_coco_log.txt) -**D‑FINE‑S** | COCO | **48.5** | 10M | 3.49ms | 25 | [yml](./configs/dfine/dfine_hgnetv2_s_coco.yml) | [48.5](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_s_coco.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/coco/dfine_s_coco_log.txt) -**D‑FINE‑M** | COCO | **52.3** | 19M | 5.62ms | 57 | [yml](./configs/dfine/dfine_hgnetv2_m_coco.yml) | [52.3](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_m_coco.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/coco/dfine_m_coco_log.txt) -**D‑FINE‑L** | COCO | **54.0** | 31M | 8.07ms | 91 | [yml](./configs/dfine/dfine_hgnetv2_l_coco.yml) | [54.0](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_l_coco.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/coco/dfine_l_coco_log.txt) -**D‑FINE‑X** | COCO | **55.8** | 62M | 12.89ms | 202 | [yml](./configs/dfine/dfine_hgnetv2_x_coco.yml) | [55.8](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_x_coco.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/coco/dfine_x_coco_log.txt) - - -### Objects365+COCO -| ใƒขใƒ‡ใƒซ | ใƒ‡ใƒผใ‚ฟใ‚ปใƒƒใƒˆ | APval | ใƒ‘ใƒฉใƒกใƒผใ‚ฟๆ•ฐ | ใƒฌใ‚คใƒ†ใƒณใ‚ท | GFLOPs | config | checkpoint | logs | -| :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | -**D‑FINE‑S** | Objects365+COCO | **50.7** | 10M | 3.49ms | 25 | [yml](./configs/dfine/objects365/dfine_hgnetv2_s_obj2coco.yml) | [50.7](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_s_obj2coco.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/obj2coco/dfine_s_obj2coco_log.txt) -**D‑FINE‑M** | Objects365+COCO | **55.1** | 19M | 5.62ms | 57 | [yml](./configs/dfine/objects365/dfine_hgnetv2_m_obj2coco.yml) | [55.1](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_m_obj2coco.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/obj2coco/dfine_m_obj2coco_log.txt) -**D‑FINE‑L** | Objects365+COCO | **57.3** | 31M | 8.07ms | 91 | [yml](./configs/dfine/objects365/dfine_hgnetv2_l_obj2coco.yml) | [57.3](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_l_obj2coco_e25.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/obj2coco/dfine_l_obj2coco_log_e25.txt) -**D‑FINE‑X** | Objects365+COCO | **59.3** | 62M | 12.89ms | 202 | [yml](./configs/dfine/objects365/dfine_hgnetv2_x_obj2coco.yml) | [59.3](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_x_obj2coco.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/obj2coco/dfine_x_obj2coco_log.txt) - -**ๅพฎ่ชฟๆ•ดใฎใŸใ‚ใซ Objects365 ใฎไบ‹ๅ‰ๅญฆ็ฟ’ใƒขใƒ‡ใƒซใ‚’ไฝฟ็”จใ™ใ‚‹ใ“ใจใ‚’ๅผทใใŠๅ‹งใ‚ใ—ใพใ™๏ผš** - -โš ๏ธ ้‡่ฆใชใŠ็Ÿฅใ‚‰ใ›๏ผšใ“ใฎใƒ—ใƒชใƒˆใƒฌใ‚คใƒณใƒขใƒ‡ใƒซใฏ่ค‡้›‘ใชใ‚ทใƒผใƒณใฎ็†่งฃใซๆœ‰็›Šใงใ™ใŒใ€ใ‚ซใƒ†ใ‚ดใƒชใŒ้žๅธธใซๅ˜็ด”ใชๅ ดๅˆใ€้Žๅญฆ็ฟ’ใ‚„ๆœ€้ฉใงใฏใชใ„ๆ€ง่ƒฝใซใคใชใŒใ‚‹ๅฏ่ƒฝๆ€งใŒใ‚ใ‚Šใพใ™ใฎใงใ€ใ”ๆณจๆ„ใใ ใ•ใ„ใ€‚ - -
๐Ÿ”ฅ Objects365ใงไบ‹ๅ‰ใƒˆใƒฌใƒผใƒ‹ใƒณใ‚ฐใ•ใ‚ŒใŸใƒขใƒ‡ใƒซ๏ผˆๆœ€่‰ฏใฎๆฑŽๅŒ–ๆ€ง่ƒฝ๏ผ‰ - - -| ใƒขใƒ‡ใƒซ | ใƒ‡ใƒผใ‚ฟใ‚ปใƒƒใƒˆ | APval | AP5000 | ใƒ‘ใƒฉใƒกใƒผใ‚ฟๆ•ฐ | ใƒฌใ‚คใƒ†ใƒณใ‚ท | GFLOPs | config | checkpoint | logs | -| :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | -**D‑FINE‑S** | Objects365 | **31.0** | **30.5** | 10M | 3.49ms | 25 | [yml](./configs/dfine/objects365/dfine_hgnetv2_s_obj365.yml) | [30.5](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_s_obj365.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/obj365/dfine_s_obj365_log.txt) -**D‑FINE‑M** | Objects365 | **38.6** | **37.4** | 19M | 5.62ms | 57 | [yml](./configs/dfine/objects365/dfine_hgnetv2_m_obj365.yml) | [37.4](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_m_obj365.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/obj365/dfine_m_obj365_log.txt) -**D‑FINE‑L** | Objects365 | - | **40.6** | 31M | 8.07ms | 91 | [yml](./configs/dfine/objects365/dfine_hgnetv2_l_obj365.yml) | [40.6](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_l_obj365.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/obj365/dfine_l_obj365_log.txt) -**D‑FINE‑L (E25)** | Objects365 | **44.7** | **42.6** | 31M | 8.07ms | 91 | [yml](./configs/dfine/objects365/dfine_hgnetv2_l_obj365.yml) | [42.6](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_l_obj365_e25.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/obj365/dfine_l_obj365_log_e25.txt) -**D‑FINE‑X** | Objects365 | **49.5** | **46.5** | 62M | 12.89ms | 202 | [yml](./configs/dfine/objects365/dfine_hgnetv2_x_obj365.yml) | [46.5](https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_x_obj365.pth) | [url](https://raw.githubusercontent.com/Peterande/storage/refs/heads/master/logs/obj365/dfine_x_obj365_log.txt) -- **E25**: ๅ†ใƒˆใƒฌใƒผใƒ‹ใƒณใ‚ฐใ—ใ€ไบ‹ๅ‰ใƒˆใƒฌใƒผใƒ‹ใƒณใ‚ฐใ‚’25ใ‚จใƒใƒƒใ‚ฏใซๅปถ้•ทใ€‚ -- **APval** ใฏ *Objects365* ใฎใƒ•ใƒซใƒใƒชใƒ‡ใƒผใ‚ทใƒงใƒณใ‚ปใƒƒใƒˆใง่ฉ•ไพกใ•ใ‚Œใพใ™ใ€‚ -- **AP5000** ใฏ *Objects365* ๆคœ่จผใ‚ปใƒƒใƒˆใฎๆœ€ๅˆใฎ5000ใ‚ตใƒณใƒ—ใƒซใง่ฉ•ไพกใ•ใ‚Œใพใ™ใ€‚ -
- -**ๆณจๆ„ไบ‹้ …:** -- **APval** ใฏ *MSCOCO val2017* ใƒ‡ใƒผใ‚ฟใ‚ปใƒƒใƒˆใง่ฉ•ไพกใ•ใ‚Œใพใ™ใ€‚ -- **ใƒฌใ‚คใƒ†ใƒณใ‚ท** ใฏๅ˜ไธ€ใฎT4 GPUใง $batch\\_size = 1$, $fp16$, ใŠใ‚ˆใณ $TensorRT==10.4.0$ ใง่ฉ•ไพกใ•ใ‚Œใพใ™ใ€‚ -- **Objects365+COCO** ใฏ *Objects365* ใงไบ‹ๅ‰ใƒˆใƒฌใƒผใƒ‹ใƒณใ‚ฐใ•ใ‚ŒใŸ้‡ใฟใ‚’ไฝฟ็”จใ—ใฆ *COCO* ใงๅพฎ่ชฟๆ•ดใ•ใ‚ŒใŸใƒขใƒ‡ใƒซใ‚’ๆ„ๅ‘ณใ—ใพใ™ใ€‚ - - - -## ใ‚ฏใ‚คใƒƒใ‚ฏใ‚นใ‚ฟใƒผใƒˆ - -### ใ‚ปใƒƒใƒˆใ‚ขใƒƒใƒ— - -```shell -conda create -n dfine python=3.11.9 -conda activate dfine -pip install -r requirements.txt -``` - - -### ใƒ‡ใƒผใ‚ฟๆบ–ๅ‚™ - -
- COCO2017 ใƒ‡ใƒผใ‚ฟใ‚ปใƒƒใƒˆ - -1. [OpenDataLab](https://opendatalab.com/OpenDataLab/COCO_2017) ใพใŸใฏ [COCO](https://cocodataset.org/#download) ใ‹ใ‚‰COCO2017ใ‚’ใƒ€ใ‚ฆใƒณใƒญใƒผใƒ‰ใ—ใพใ™ใ€‚ -1. [coco_detection.yml](./configs/dataset/coco_detection.yml) ใฎใƒ‘ใ‚นใ‚’ไฟฎๆญฃใ—ใพใ™ใ€‚ - - ```yaml - train_dataloader: - img_folder: /data/COCO2017/train2017/ - ann_file: /data/COCO2017/annotations/instances_train2017.json - val_dataloader: - img_folder: /data/COCO2017/val2017/ - ann_file: /data/COCO2017/annotations/instances_val2017.json - ``` - -
- -
- Objects365 ใƒ‡ใƒผใ‚ฟใ‚ปใƒƒใƒˆ - -1. [OpenDataLab](https://opendatalab.com/OpenDataLab/Objects365) ใ‹ใ‚‰Objects365ใ‚’ใƒ€ใ‚ฆใƒณใƒญใƒผใƒ‰ใ—ใพใ™ใ€‚ - -2. ใƒ™ใƒผใ‚นใƒ‡ใ‚ฃใƒฌใ‚ฏใƒˆใƒชใ‚’่จญๅฎšใ—ใพใ™๏ผš -```shell -export BASE_DIR=/data/Objects365/data -``` - -3. ใƒ€ใ‚ฆใƒณใƒญใƒผใƒ‰ใ—ใŸใƒ•ใ‚กใ‚คใƒซใ‚’่งฃๅ‡ใ—ใ€ไปฅไธ‹ใฎใƒ‡ใ‚ฃใƒฌใ‚ฏใƒˆใƒชๆง‹้€ ใซๆ•ด็†ใ—ใพใ™๏ผš - -```shell -${BASE_DIR}/train -โ”œโ”€โ”€ images -โ”‚ โ”œโ”€โ”€ v1 -โ”‚ โ”‚ โ”œโ”€โ”€ patch0 -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ 000000000.jpg -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ 000000001.jpg -โ”‚ โ”‚ โ”‚ โ””โ”€โ”€ ... (more images) -โ”‚ โ”œโ”€โ”€ v2 -โ”‚ โ”‚ โ”œโ”€โ”€ patchx -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ 000000000.jpg -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ 000000001.jpg -โ”‚ โ”‚ โ”‚ โ””โ”€โ”€ ... (more images) -โ”œโ”€โ”€ zhiyuan_objv2_train.json -``` - -```shell -${BASE_DIR}/val -โ”œโ”€โ”€ images -โ”‚ โ”œโ”€โ”€ v1 -โ”‚ โ”‚ โ”œโ”€โ”€ patch0 -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ 000000000.jpg -โ”‚ โ”‚ โ”‚ โ””โ”€โ”€ ... (more images) -โ”‚ โ”œโ”€โ”€ v2 -โ”‚ โ”‚ โ”œโ”€โ”€ patchx -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ 000000000.jpg -โ”‚ โ”‚ โ”‚ โ””โ”€โ”€ ... (more images) -โ”œโ”€โ”€ zhiyuan_objv2_val.json -``` - -4. ๆคœ่จผใ‚ปใƒƒใƒˆใฎ็”ปๅƒใ‚’ไฟๅญ˜ใ™ใ‚‹ๆ–ฐใ—ใ„ใƒ‡ใ‚ฃใƒฌใ‚ฏใƒˆใƒชใ‚’ไฝœๆˆใ—ใพใ™๏ผš -```shell -mkdir -p ${BASE_DIR}/train/images_from_val -``` - -5. valใƒ‡ใ‚ฃใƒฌใ‚ฏใƒˆใƒชใฎv1ใŠใ‚ˆใณv2ใƒ•ใ‚ฉใƒซใƒ€ใ‚’train/images_from_valใƒ‡ใ‚ฃใƒฌใ‚ฏใƒˆใƒชใซใ‚ณใƒ”ใƒผใ—ใพใ™ -```shell -cp -r ${BASE_DIR}/val/images/v1 ${BASE_DIR}/train/images_from_val/ -cp -r ${BASE_DIR}/val/images/v2 ${BASE_DIR}/train/images_from_val/ -``` - -6. remap_obj365.pyใ‚’ๅฎŸ่กŒใ—ใฆใ€ๆคœ่จผใ‚ปใƒƒใƒˆใฎไธ€้ƒจใ‚’ใƒˆใƒฌใƒผใƒ‹ใƒณใ‚ฐใ‚ปใƒƒใƒˆใซใƒžใƒผใ‚ธใ—ใพใ™ใ€‚ๅ…ทไฝ“็š„ใซใฏใ€ใ“ใฎใ‚นใ‚ฏใƒชใƒ—ใƒˆใฏใ‚คใƒณใƒ‡ใƒƒใ‚ฏใ‚นใŒ5000ใ‹ใ‚‰800000ใฎใ‚ตใƒณใƒ—ใƒซใ‚’ๆคœ่จผใ‚ปใƒƒใƒˆใ‹ใ‚‰ใƒˆใƒฌใƒผใƒ‹ใƒณใ‚ฐใ‚ปใƒƒใƒˆใซ็งปๅ‹•ใ—ใพใ™ใ€‚ -```shell -python tools/remap_obj365.py --base_dir ${BASE_DIR} -``` - - -7. resize_obj365.pyใ‚นใ‚ฏใƒชใƒ—ใƒˆใ‚’ๅฎŸ่กŒใ—ใฆใ€ใƒ‡ใƒผใ‚ฟใ‚ปใƒƒใƒˆๅ†…ใฎๆœ€ๅคงใ‚จใƒƒใ‚ธ้•ทใŒ640ใƒ”ใ‚ฏใ‚ปใƒซใ‚’่ถ…ใˆใ‚‹็”ปๅƒใ‚’ใƒชใ‚ตใ‚คใ‚บใ—ใพใ™ใ€‚ใ‚นใƒ†ใƒƒใƒ—5ใง็”Ÿๆˆใ•ใ‚ŒใŸๆ›ดๆ–ฐใ•ใ‚ŒใŸJSONใƒ•ใ‚กใ‚คใƒซใ‚’ไฝฟ็”จใ—ใฆใ‚ตใƒณใƒ—ใƒซใƒ‡ใƒผใ‚ฟใ‚’ๅ‡ฆ็†ใ—ใพใ™ใ€‚ใƒˆใƒฌใƒผใƒ‹ใƒณใ‚ฐใ‚ปใƒƒใƒˆใจๆคœ่จผใ‚ปใƒƒใƒˆใฎไธกๆ–นใฎ็”ปๅƒใ‚’ใƒชใ‚ตใ‚คใ‚บใ—ใฆใ€ไธ€่ฒซๆ€งใ‚’ไฟใกใพใ™ใ€‚ -```shell -python tools/resize_obj365.py --base_dir ${BASE_DIR} -``` - -8. [obj365_detection.yml](./configs/dataset/obj365_detection.yml) ใฎใƒ‘ใ‚นใ‚’ไฟฎๆญฃใ—ใพใ™ใ€‚ - - ```yaml - train_dataloader: - img_folder: /data/Objects365/data/train - ann_file: /data/Objects365/data/train/new_zhiyuan_objv2_train_resized.json - val_dataloader: - img_folder: /data/Objects365/data/val/ - ann_file: /data/Objects365/data/val/new_zhiyuan_objv2_val_resized.json - ``` - - -
- -
-CrowdHuman - -ใ“ใกใ‚‰ใ‹ใ‚‰COCOใƒ•ใ‚ฉใƒผใƒžใƒƒใƒˆใฎใƒ‡ใƒผใ‚ฟใ‚ปใƒƒใƒˆใ‚’ใƒ€ใ‚ฆใƒณใƒญใƒผใƒ‰ใ—ใฆใใ ใ•ใ„๏ผš[ใƒชใƒณใ‚ฏ](https://aistudio.baidu.com/datasetdetail/231455) - -
- -
-ใ‚ซใ‚นใ‚ฟใƒ ใƒ‡ใƒผใ‚ฟใ‚ปใƒƒใƒˆ - -ใ‚ซใ‚นใ‚ฟใƒ ใƒ‡ใƒผใ‚ฟใ‚ปใƒƒใƒˆใงใƒˆใƒฌใƒผใƒ‹ใƒณใ‚ฐใ™ใ‚‹ใซใฏใ€COCOๅฝขๅผใงๆ•ด็†ใ™ใ‚‹ๅฟ…่ฆใŒใ‚ใ‚Šใพใ™ใ€‚ไปฅไธ‹ใฎๆ‰‹้ †ใซๅพ“ใฃใฆใƒ‡ใƒผใ‚ฟใ‚ปใƒƒใƒˆใ‚’ๆบ–ๅ‚™ใ—ใฆใใ ใ•ใ„๏ผš - -1. **`remap_mscoco_category` ใ‚’ `False` ใซ่จญๅฎšใ—ใพใ™**๏ผš - - ใ“ใ‚Œใซใ‚ˆใ‚Šใ€ใ‚ซใƒ†ใ‚ดใƒชIDใŒMSCOCOใ‚ซใƒ†ใ‚ดใƒชใซ่‡ชๅ‹•็š„ใซใƒžใƒƒใƒ”ใƒณใ‚ฐใ•ใ‚Œใ‚‹ใฎใ‚’้˜ฒใŽใพใ™ใ€‚ - - ```yaml - remap_mscoco_category: False - ``` - -2. **็”ปๅƒใ‚’ๆ•ด็†ใ—ใพใ™**๏ผš - - ใƒ‡ใƒผใ‚ฟใ‚ปใƒƒใƒˆใƒ‡ใ‚ฃใƒฌใ‚ฏใƒˆใƒชใ‚’ไปฅไธ‹ใฎใ‚ˆใ†ใซๆง‹้€ ๅŒ–ใ—ใพใ™๏ผš - - ```shell - dataset/ - โ”œโ”€โ”€ images/ - โ”‚ โ”œโ”€โ”€ train/ - โ”‚ โ”‚ โ”œโ”€โ”€ image1.jpg - โ”‚ โ”‚ โ”œโ”€โ”€ image2.jpg - โ”‚ โ”‚ โ””โ”€โ”€ ... - โ”‚ โ”œโ”€โ”€ val/ - โ”‚ โ”‚ โ”œโ”€โ”€ image1.jpg - โ”‚ โ”‚ โ”œโ”€โ”€ image2.jpg - โ”‚ โ”‚ โ””โ”€โ”€ ... - โ””โ”€โ”€ annotations/ - โ”œโ”€โ”€ instances_train.json - โ”œโ”€โ”€ instances_val.json - โ””โ”€โ”€ ... - ``` - - - **`images/train/`**: ใ™ในใฆใฎใƒˆใƒฌใƒผใƒ‹ใƒณใ‚ฐ็”ปๅƒใ‚’ๅซใฟใพใ™ใ€‚ - - **`images/val/`**: ใ™ในใฆใฎๆคœ่จผ็”ปๅƒใ‚’ๅซใฟใพใ™ใ€‚ - - **`annotations/`**: COCOๅฝขๅผใฎๆณจ้‡ˆใƒ•ใ‚กใ‚คใƒซใ‚’ๅซใฟใพใ™ใ€‚ - -3. **ๆณจ้‡ˆใ‚’COCOๅฝขๅผใซๅค‰ๆ›ใ—ใพใ™**๏ผš - - ๆณจ้‡ˆใŒใพใ COCOๅฝขๅผใงใชใ„ๅ ดๅˆใฏใ€ๅค‰ๆ›ใ™ใ‚‹ๅฟ…่ฆใŒใ‚ใ‚Šใพใ™ใ€‚ไปฅไธ‹ใฎPythonใ‚นใ‚ฏใƒชใƒ—ใƒˆใ‚’ๅ‚่€ƒใซใ™ใ‚‹ใ‹ใ€ๆ—ขๅญ˜ใฎใƒ„ใƒผใƒซใ‚’ๅˆฉ็”จใ—ใฆใใ ใ•ใ„๏ผš - - ```python - import json - - def convert_to_coco(input_annotations, output_annotations): - # ๅค‰ๆ›ใƒญใ‚ธใƒƒใ‚ฏใ‚’ใ“ใ“ใซๅฎŸ่ฃ…ใ—ใพใ™ - pass - - if __name__ == "__main__": - convert_to_coco('path/to/your_annotations.json', 'dataset/annotations/instances_train.json') - ``` - -4. **่จญๅฎšใƒ•ใ‚กใ‚คใƒซใ‚’ๆ›ดๆ–ฐใ—ใพใ™**๏ผš - - [custom_detection.yml](./configs/dataset/custom_detection.yml) ใ‚’ไฟฎๆญฃใ—ใพใ™ใ€‚ - - ```yaml - task: detection - - evaluator: - type: CocoEvaluator - iou_types: ['bbox', ] - - num_classes: 777 # ใƒ‡ใƒผใ‚ฟใ‚ปใƒƒใƒˆใฎใ‚ฏใƒฉใ‚นๆ•ฐ - remap_mscoco_category: False - - train_dataloader: - type: DataLoader - dataset: - type: CocoDetection - img_folder: /data/yourdataset/train - ann_file: /data/yourdataset/train/train.json - return_masks: False - transforms: - type: Compose - ops: ~ - shuffle: True - num_workers: 4 - drop_last: True - collate_fn: - type: BatchImageCollateFunction - - val_dataloader: - type: DataLoader - dataset: - type: CocoDetection - img_folder: /data/yourdataset/val - ann_file: /data/yourdataset/val/ann.json - return_masks: False - transforms: - type: Compose - ops: ~ - shuffle: False - num_workers: 4 - drop_last: False - collate_fn: - type: BatchImageCollateFunction - ``` - -
- - -## ไฝฟ็”จๆ–นๆณ• -
- COCO2017 - - -1. ใƒขใƒ‡ใƒซใ‚’่จญๅฎšใ—ใพใ™ -```shell -export model=l # n s m l x -``` - -2. ใƒˆใƒฌใƒผใƒ‹ใƒณใ‚ฐ -```shell -CUDA_VISIBLE_DEVICES=0,1,2,3 torchrun --master_port=7777 --nproc_per_node=4 train.py -c configs/dfine/dfine_hgnetv2_${model}_coco.yml --use-amp --seed=0 -``` - - -3. ใƒ†ใ‚นใƒˆ -```shell -CUDA_VISIBLE_DEVICES=0,1,2,3 torchrun --master_port=7777 --nproc_per_node=4 train.py -c configs/dfine/dfine_hgnetv2_${model}_coco.yml --test-only -r model.pth -``` - - -4. ๅพฎ่ชฟๆ•ด -```shell -CUDA_VISIBLE_DEVICES=0,1,2,3 torchrun --master_port=7777 --nproc_per_node=4 train.py -c configs/dfine/dfine_hgnetv2_${model}_coco.yml --use-amp --seed=0 -t model.pth -``` -
- - -
- Objects365ใ‹ใ‚‰COCO2017ใธ - -1. ใƒขใƒ‡ใƒซใ‚’่จญๅฎšใ—ใพใ™ -```shell -export model=l # n s m l x -``` - -2. Objects365ใงใƒˆใƒฌใƒผใƒ‹ใƒณใ‚ฐ -```shell -CUDA_VISIBLE_DEVICES=0,1,2,3 torchrun --master_port=7777 --nproc_per_node=4 train.py -c configs/dfine/objects365/dfine_hgnetv2_${model}_obj365.yml --use-amp --seed=0 -``` - -3. COCO2017ใงๅพฎ่ชฟๆ•ด -```shell -CUDA_VISIBLE_DEVICES=0,1,2,3 torchrun --master_port=7777 --nproc_per_node=4 train.py -c configs/dfine/objects365/dfine_hgnetv2_${model}_obj2coco.yml --use-amp --seed=0 -t model.pth -``` - - -4. ใƒ†ใ‚นใƒˆ -```shell -CUDA_VISIBLE_DEVICES=0,1,2,3 torchrun --master_port=7777 --nproc_per_node=4 train.py -c configs/dfine/dfine_hgnetv2_${model}_coco.yml --test-only -r model.pth -``` -
- - -
- ใ‚ซใ‚นใ‚ฟใƒ ใƒ‡ใƒผใ‚ฟใ‚ปใƒƒใƒˆ - -1. ใƒขใƒ‡ใƒซใ‚’่จญๅฎšใ—ใพใ™ -```shell -export model=l # n s m l x -``` - -2. ใ‚ซใ‚นใ‚ฟใƒ ใƒ‡ใƒผใ‚ฟใ‚ปใƒƒใƒˆใงใƒˆใƒฌใƒผใƒ‹ใƒณใ‚ฐ -```shell -CUDA_VISIBLE_DEVICES=0,1,2,3 torchrun --master_port=7777 --nproc_per_node=4 train.py -c configs/dfine/custom/dfine_hgnetv2_${model}_custom.yml --use-amp --seed=0 -``` - -3. ใƒ†ใ‚นใƒˆ -```shell -CUDA_VISIBLE_DEVICES=0,1,2,3 torchrun --master_port=7777 --nproc_per_node=4 train.py -c configs/dfine/custom/dfine_hgnetv2_${model}_custom.yml --test-only -r model.pth -``` - -4. ใ‚ซใ‚นใ‚ฟใƒ ใƒ‡ใƒผใ‚ฟใ‚ปใƒƒใƒˆใงๅพฎ่ชฟๆ•ด -```shell -CUDA_VISIBLE_DEVICES=0,1,2,3 torchrun --master_port=7777 --nproc_per_node=4 train.py -c configs/dfine/custom/objects365/dfine_hgnetv2_${model}_obj2custom.yml --use-amp --seed=0 -t model.pth -``` - -5. **[ใ‚ชใƒ—ใ‚ทใƒงใƒณ]** ใ‚ฏใƒฉใ‚นใƒžใƒƒใƒ”ใƒณใ‚ฐใ‚’ๅค‰ๆ›ดใ—ใพใ™๏ผš - -Objects365ใฎไบ‹ๅ‰ใƒˆใƒฌใƒผใƒ‹ใƒณใ‚ฐๆธˆใฟใฎ้‡ใฟใ‚’ไฝฟ็”จใ—ใฆใ‚ซใ‚นใ‚ฟใƒ ใƒ‡ใƒผใ‚ฟใ‚ปใƒƒใƒˆใงใƒˆใƒฌใƒผใƒ‹ใƒณใ‚ฐใ™ใ‚‹ๅ ดๅˆใ€ไพ‹ใงใฏใƒ‡ใƒผใ‚ฟใ‚ปใƒƒใƒˆใซ `'Person'` ใจ `'Car'` ใ‚ฏใƒฉใ‚นใฎใฟใŒๅซใพใ‚Œใฆใ„ใ‚‹ใจไปฎๅฎšใ—ใฆใ„ใพใ™ใ€‚็‰นๅฎšใฎใ‚ฟใ‚นใ‚ฏใซๅฏพใ—ใฆๅŽๆŸใ‚’ๆ—ฉใ‚ใ‚‹ใŸใ‚ใซใ€`src/solver/_solver.py` ใฎ `self.obj365_ids` ใ‚’ไปฅไธ‹ใฎใ‚ˆใ†ใซๅค‰ๆ›ดใงใใพใ™๏ผš - -```python -self.obj365_ids = [0, 5] # Person, Cars -``` -ใ“ใ‚Œใ‚‰ใ‚’ใƒ‡ใƒผใ‚ฟใ‚ปใƒƒใƒˆใฎๅฏพๅฟœใ™ใ‚‹ใ‚ฏใƒฉใ‚นใซ็ฝฎใๆ›ใˆใ‚‹ใ“ใจใŒใงใใพใ™ใ€‚Objects365ใ‚ฏใƒฉใ‚นใจใใฎๅฏพๅฟœIDใฎใƒชใ‚นใƒˆ๏ผš -https://github.com/Peterande/D-FINE/blob/352a94ece291e26e1957df81277bef00fe88a8e3/src/solver/_solver.py#L330 - -ๆ–ฐใ—ใ„ใƒˆใƒฌใƒผใƒ‹ใƒณใ‚ฐใ‚ณใƒžใƒณใƒ‰๏ผš - -```shell -CUDA_VISIBLE_DEVICES=0,1,2,3 torchrun --master_port=7777 --nproc_per_node=4 train.py -c configs/dfine/custom/dfine_hgnetv2_${model}_custom.yml --use-amp --seed=0 -t model.pth -``` - -ใŸใ ใ—ใ€ใ‚ฏใƒฉใ‚นใƒžใƒƒใƒ”ใƒณใ‚ฐใ‚’ๅค‰ๆ›ดใ—ใŸใใชใ„ๅ ดๅˆใ€ไบ‹ๅ‰ใƒˆใƒฌใƒผใƒ‹ใƒณใ‚ฐๆธˆใฟใฎObjects365ใฎ้‡ใฟใฏๅค‰ๆ›ดใชใ—ใงใใฎใพใพไฝฟ็”จใงใใพใ™ใ€‚ใ‚ฏใƒฉใ‚นใƒžใƒƒใƒ”ใƒณใ‚ฐใฎๅค‰ๆ›ดใฏใ‚ชใƒ—ใ‚ทใƒงใƒณใงใ‚ใ‚Šใ€็‰นๅฎšใฎใ‚ฟใ‚นใ‚ฏใซๅฏพใ—ใฆๅŽๆŸใ‚’ๆ—ฉใ‚ใ‚‹ๅฏ่ƒฝๆ€งใŒใ‚ใ‚Šใพใ™ใ€‚ - - - -
- -
- ใƒใƒƒใƒใ‚ตใ‚คใ‚บใฎใ‚ซใ‚นใ‚ฟใƒžใ‚คใ‚บ - -ไพ‹ใˆใฐใ€COCO2017ใงD-FINE-Lใ‚’ใƒˆใƒฌใƒผใƒ‹ใƒณใ‚ฐใ™ใ‚‹้š›ใซใƒใƒƒใƒใ‚ตใ‚คใ‚บใ‚’2ๅ€ใซใ—ใŸใ„ๅ ดๅˆใ€ไปฅไธ‹ใฎๆ‰‹้ †ใซๅพ“ใฃใฆใใ ใ•ใ„๏ผš - -1. **[dataloader.yml](./configs/dfine/include/dataloader.yml) ใ‚’ไฟฎๆญฃใ—ใฆ `total_batch_size` ใ‚’ๅข—ใ‚„ใ—ใพใ™**๏ผš - - ```yaml - train_dataloader: - total_batch_size: 64 # ไปฅๅ‰ใฏ32ใ€ไปŠใฏ2ๅ€ - ``` - -2. **[dfine_hgnetv2_l_coco.yml](./configs/dfine/dfine_hgnetv2_l_coco.yml) ใ‚’ไฟฎๆญฃใ—ใพใ™**ใ€‚ไปฅไธ‹ใฎใ‚ˆใ†ใซไธป่ฆใชใƒ‘ใƒฉใƒกใƒผใ‚ฟใ‚’่ชฟๆ•ดใ—ใพใ™๏ผš - - ```yaml - optimizer: - type: AdamW - params: - - - params: '^(?=.*backbone)(?!.*norm|bn).*$' - lr: 0.000025 # 2ๅ€ใ€็ทšๅฝขใ‚นใ‚ฑใƒผใƒชใƒณใ‚ฐๆณ•ๅ‰‡ - - - params: '^(?=.*(?:encoder|decoder))(?=.*(?:norm|bn)).*$' - weight_decay: 0. - - lr: 0.0005 # 2ๅ€ใ€็ทšๅฝขใ‚นใ‚ฑใƒผใƒชใƒณใ‚ฐๆณ•ๅ‰‡ - betas: [0.9, 0.999] - weight_decay: 0.0001 # ใ‚ฐใƒชใƒƒใƒ‰ใ‚ตใƒผใƒใŒๅฟ…่ฆใงใ™ - - ema: # EMA่จญๅฎšใ‚’่ฟฝๅŠ  - decay: 0.9998 # 1 - (1 - decay) * 2 ใซใ‚ˆใฃใฆ่ชฟๆ•ด - warmups: 500 # ๅŠๅˆ† - - lr_warmup_scheduler: - warmup_duration: 250 # ๅŠๅˆ† - ``` - -
- - -
- ๅ…ฅๅŠ›ใ‚ตใ‚คใ‚บใฎใ‚ซใ‚นใ‚ฟใƒžใ‚คใ‚บ - -COCO2017ใง **D-FINE-L** ใ‚’320x320ใฎๅ…ฅๅŠ›ใ‚ตใ‚คใ‚บใงใƒˆใƒฌใƒผใƒ‹ใƒณใ‚ฐใ—ใŸใ„ๅ ดๅˆใ€ไปฅไธ‹ใฎๆ‰‹้ †ใซๅพ“ใฃใฆใใ ใ•ใ„๏ผš - -1. **[dataloader.yml](./configs/dfine/include/dataloader.yml) ใ‚’ไฟฎๆญฃใ—ใพใ™**๏ผš - - ```yaml - - train_dataloader: - dataset: - transforms: - ops: - - {type: Resize, size: [320, 320], } - collate_fn: - base_size: 320 - dataset: - transforms: - ops: - - {type: Resize, size: [320, 320], } - ``` - -2. **[dfine_hgnetv2.yml](./configs/dfine/include/dfine_hgnetv2.yml) ใ‚’ไฟฎๆญฃใ—ใพใ™**๏ผš - - ```yaml - eval_spatial_size: [320, 320] - ``` - -
- -## ใƒ„ใƒผใƒซ -
- ใƒ‡ใƒ—ใƒญใ‚ค - - -1. ใ‚ปใƒƒใƒˆใ‚ขใƒƒใƒ— -```shell -pip install onnx onnxsim -export model=l # n s m l x -``` - -2. onnxใฎใ‚จใ‚ฏใ‚นใƒใƒผใƒˆ -```shell -python tools/deployment/export_onnx.py --check -c configs/dfine/dfine_hgnetv2_${model}_coco.yml -r model.pth -``` - -3. [tensorrt](https://docs.nvidia.com/deeplearning/tensorrt/install-guide/index.html) ใฎใ‚จใ‚ฏใ‚นใƒใƒผใƒˆ -```shell -trtexec --onnx="model.onnx" --saveEngine="model.engine" --fp16 -``` - -
- -
- ๆŽจ่ซ–๏ผˆๅฏ่ฆ–ๅŒ–๏ผ‰ - - -1. ใ‚ปใƒƒใƒˆใ‚ขใƒƒใƒ— -```shell -pip install -r tools/inference/requirements.txt -export model=l # n s m l x -``` - - - -2. ๆŽจ่ซ– (onnxruntime / tensorrt / torch) - -็พๅœจใ€็”ปๅƒใจใƒ“ใƒ‡ใ‚ชใฎๆŽจ่ซ–ใŒใ‚ตใƒใƒผใƒˆใ•ใ‚Œใฆใ„ใพใ™ใ€‚ -```shell -python tools/inference/onnx_inf.py --onnx model.onnx --input image.jpg # video.mp4 -python tools/inference/trt_inf.py --trt model.engine --input image.jpg -python tools/inference/torch_inf.py -c configs/dfine/dfine_hgnetv2_${model}_coco.yml -r model.pth --input image.jpg --device cuda:0 -``` -
- -
- ใƒ™ใƒณใƒใƒžใƒผใ‚ฏ - -1. ใ‚ปใƒƒใƒˆใ‚ขใƒƒใƒ— -```shell -pip install -r tools/benchmark/requirements.txt -export model=l # n s m l x -``` - - -2. ใƒขใƒ‡ใƒซใฎFLOPsใ€MACsใ€ใŠใ‚ˆใณใƒ‘ใƒฉใƒกใƒผใ‚ฟๆ•ฐ -```shell -python tools/benchmark/get_info.py -c configs/dfine/dfine_hgnetv2_${model}_coco.yml -``` - -2. TensorRTใฎใƒฌใ‚คใƒ†ใƒณใ‚ท -```shell -python tools/benchmark/trt_benchmark.py --COCO_dir path/to/COCO2017 --engine_dir model.engine -``` -
- -
- Fiftyoneใฎๅฏ่ฆ–ๅŒ– - -1. ใ‚ปใƒƒใƒˆใ‚ขใƒƒใƒ— -```shell -pip install fiftyone -export model=l # n s m l x -``` -4. Voxel51 Fiftyoneใฎๅฏ่ฆ–ๅŒ– ([fiftyone](https://github.com/voxel51/fiftyone)) -```shell -python tools/visualization/fiftyone_vis.py -c configs/dfine/dfine_hgnetv2_${model}_coco.yml -r model.pth -``` -
- -
- ใใฎไป– - -1. ่‡ชๅ‹•ๅ†้–‹ใƒˆใƒฌใƒผใƒ‹ใƒณใ‚ฐ -```shell -bash reference/safe_training.sh -``` - -2. ใƒขใƒ‡ใƒซใฎ้‡ใฟใฎๅค‰ๆ› -```shell -python reference/convert_weight.py model.pth -``` -
- -## ๅ›ณใจๅฏ่ฆ–ๅŒ– - -
- FDRใจGO-LSD - -1. FDRใ‚’ๆญ่ผ‰ใ—ใŸD-FINEใฎๆฆ‚่ฆใ€‚ใ‚ˆใ‚Š็ดฐ็ฒ’ๅบฆใฎไธญ้–“่กจ็พใจใ—ใฆๆฉŸ่ƒฝใ™ใ‚‹็ขบ็Ž‡ๅˆ†ๅธƒใฏใ€ๆฎ‹ๅทฎ็š„ใซใƒ‡ใ‚ณใƒผใƒ€ๅฑคใซใ‚ˆใฃใฆ้€ๆฌกๆœ€้ฉๅŒ–ใ•ใ‚Œใพใ™ใ€‚ -ไธๅ‡ไธ€ใช้‡ใฟไป˜ใ‘้–ขๆ•ฐใŒ้ฉ็”จใ•ใ‚Œใ€ใ‚ˆใ‚Š็ดฐใ‹ใ„ไฝ็ฝฎ็‰นๅฎšใŒๅฏ่ƒฝใซใชใ‚Šใพใ™ใ€‚ - -

- ็ดฐ็ฒ’ๅบฆๅˆ†ๅธƒๆœ€้ฉๅŒ–ใƒ—ใƒญใ‚ปใ‚น -

- -2. GO-LSDใƒ—ใƒญใ‚ปใ‚นใฎๆฆ‚่ฆใ€‚ๆœ€็ต‚ๅฑคใฎๆœ€้ฉๅŒ–ใ•ใ‚ŒใŸๅˆ†ๅธƒใ‹ใ‚‰ใฎไฝ็ฝฎ็‰นๅฎš็Ÿฅ่ญ˜ใฏใ€ใƒ‡ใ‚ซใƒƒใƒ—ใƒชใƒณใ‚ฐใ•ใ‚ŒใŸ้‡ใฟไป˜ใ‘ๆˆฆ็•ฅใ‚’ไฝฟ็”จใ—ใฆDDFๆๅคฑใ‚’้€šใ˜ใฆๅ‰ใฎๅฑคใซ่’ธ็•™ใ•ใ‚Œใพใ™ใ€‚ - -

- GO-LSDใƒ—ใƒญใ‚ปใ‚น -

- -
- -
- ๅˆ†ๅธƒ - -ๅˆๆœŸใŠใ‚ˆใณๆœ€้ฉๅŒ–ใ•ใ‚ŒใŸๅขƒ็•Œใƒœใƒƒใ‚ฏใ‚นใจใ€ๆœช้‡ใฟไป˜ใ‘ใŠใ‚ˆใณ้‡ใฟไป˜ใ‘ใ•ใ‚ŒใŸๅˆ†ๅธƒใจใจใ‚‚ใซใ€ใ•ใพใ–ใพใชๆคœๅ‡บใ‚ทใƒŠใƒชใ‚ชใซใŠใ‘ใ‚‹FDRใฎๅฏ่ฆ–ๅŒ–ใ€‚ - -

- -

- -
- -
- ้›ฃใ—ใ„ใ‚ฑใƒผใ‚น - -ไปฅไธ‹ใฎๅฏ่ฆ–ๅŒ–ใฏใ€ใ•ใพใ–ใพใช่ค‡้›‘ใชๆคœๅ‡บใ‚ทใƒŠใƒชใ‚ชใซใŠใ‘ใ‚‹D-FINEใฎไบˆๆธฌใ‚’็คบใ—ใฆใ„ใพใ™ใ€‚ใ“ใ‚Œใ‚‰ใฎใ‚ทใƒŠใƒชใ‚ชใซใฏใ€้ฎ่”ฝใ€ไฝŽๅ…‰ๆกไปถใ€ใƒขใƒผใ‚ทใƒงใƒณใƒ–ใƒฉใƒผใ€่ขซๅ†™็•ŒๆทฑๅบฆๅŠนๆžœใ€ใŠใ‚ˆใณๅฏ†้›†ใ—ใŸใ‚ทใƒผใƒณใŒๅซใพใ‚Œใพใ™ใ€‚ใ“ใ‚Œใ‚‰ใฎ่ชฒ้กŒใซใ‚‚ใ‹ใ‹ใ‚ใ‚‰ใšใ€D-FINEใฏไธ€่ฒซใ—ใฆๆญฃ็ขบใชไฝ็ฝฎ็‰นๅฎš็ตๆžœใ‚’็”Ÿๆˆใ—ใพใ™ใ€‚ - -

- ่ค‡้›‘ใชใ‚ทใƒŠใƒชใ‚ชใซใŠใ‘ใ‚‹D-FINEใฎไบˆๆธฌ -

- -
- - - - - - - -## ๅผ•็”จ -ใ‚‚ใ—`D-FINE`ใ‚„ใใฎๆ–นๆณ•ใ‚’ใ‚ใชใŸใฎไป•ไบ‹ใงไฝฟ็”จใ™ใ‚‹ๅ ดๅˆใ€ไปฅไธ‹ใฎBibTeXใ‚จใƒณใƒˆใƒชใ‚’ๅผ•็”จใ—ใฆใใ ใ•ใ„๏ผš -
- bibtex - -```latex -@misc{peng2024dfine, - title={D-FINE: Redefine Regression Task in DETRs as Fine-grained Distribution Refinement}, - author={Yansong Peng and Hebei Li and Peixi Wu and Yueyi Zhang and Xiaoyan Sun and Feng Wu}, - year={2024}, - eprint={2410.13842}, - archivePrefix={arXiv}, - primaryClass={cs.CV} -} -``` -
- -## ่ฌ่พž -็งใŸใกใฎไป•ไบ‹ใฏ [RT-DETR](https://github.com/lyuwenyu/RT-DETR) ใซๅŸบใฅใ„ใฆใ„ใพใ™ใ€‚ -[RT-DETR](https://github.com/lyuwenyu/RT-DETR), [GFocal](https://github.com/implus/GFocal), [LD](https://github.com/HikariTJU/LD), ใŠใ‚ˆใณ [YOLOv9](https://github.com/WongKinYiu/yolov9) ใ‹ใ‚‰ใฎใ‚คใƒณใ‚นใƒ”ใƒฌใƒผใ‚ทใƒงใƒณใซๆ„Ÿ่ฌใ—ใพใ™ใ€‚ - -โœจ ่ฒข็Œฎใ‚’ๆญ“่ฟŽใ—ใ€่ณชๅ•ใŒใ‚ใ‚ŒใฐใŠๆฐ—่ปฝใซใŠๅ•ใ„ๅˆใ‚ใ›ใใ ใ•ใ„๏ผ โœจ diff --git a/vlm/detector/D-FINE.backup/configs/dataset/coco_detection.yml b/vlm/detector/D-FINE.backup/configs/dataset/coco_detection.yml deleted file mode 100644 index 670c3e3..0000000 --- a/vlm/detector/D-FINE.backup/configs/dataset/coco_detection.yml +++ /dev/null @@ -1,41 +0,0 @@ -task: detection - -evaluator: - type: CocoEvaluator - iou_types: ['bbox', ] - -num_classes: 80 -remap_mscoco_category: True - -train_dataloader: - type: DataLoader - dataset: - type: CocoDetection - img_folder: /data/COCO2017/train2017/ - ann_file: /data/COCO2017/annotations/instances_train2017.json - return_masks: False - transforms: - type: Compose - ops: ~ - shuffle: True - num_workers: 4 - drop_last: True - collate_fn: - type: BatchImageCollateFunction - - -val_dataloader: - type: DataLoader - dataset: - type: CocoDetection - img_folder: /data/COCO2017/val2017/ - ann_file: /data/COCO2017/annotations/instances_val2017.json - return_masks: False - transforms: - type: Compose - ops: ~ - shuffle: False - num_workers: 4 - drop_last: False - collate_fn: - type: BatchImageCollateFunction diff --git a/vlm/detector/D-FINE.backup/configs/dataset/crowdhuman_detection.yml b/vlm/detector/D-FINE.backup/configs/dataset/crowdhuman_detection.yml deleted file mode 100644 index 0a4bbad..0000000 --- a/vlm/detector/D-FINE.backup/configs/dataset/crowdhuman_detection.yml +++ /dev/null @@ -1,41 +0,0 @@ -task: detection - -evaluator: - type: CocoEvaluator - iou_types: ['bbox', ] - -num_classes: 1 # your dataset classes -remap_mscoco_category: False - -train_dataloader: - type: DataLoader - dataset: - type: CocoDetection - img_folder: /data/CrowdHuman/coco/CrowdHuman_train - ann_file: /data/CrowdHuman/coco/Chuman-train.json - return_masks: False - transforms: - type: Compose - ops: ~ - shuffle: True - num_workers: 4 - drop_last: True - collate_fn: - type: BatchImageCollateFunction - - -val_dataloader: - type: DataLoader - dataset: - type: CocoDetection - img_folder: /data/CrowdHuman/coco/CrowdHuman_val - ann_file: /data/CrowdHuman/coco/Chuman-val.json - return_masks: False - transforms: - type: Compose - ops: ~ - shuffle: False - num_workers: 4 - drop_last: False - collate_fn: - type: BatchImageCollateFunction diff --git a/vlm/detector/D-FINE.backup/configs/dataset/custom_detection.yml b/vlm/detector/D-FINE.backup/configs/dataset/custom_detection.yml deleted file mode 100644 index 35435ad..0000000 --- a/vlm/detector/D-FINE.backup/configs/dataset/custom_detection.yml +++ /dev/null @@ -1,41 +0,0 @@ -task: detection - -evaluator: - type: CocoEvaluator - iou_types: ['bbox', ] - -num_classes: 777 # your dataset classes -remap_mscoco_category: False - -train_dataloader: - type: DataLoader - dataset: - type: CocoDetection - img_folder: /data/yourdataset/train - ann_file: /data/yourdataset/train/train.json - return_masks: False - transforms: - type: Compose - ops: ~ - shuffle: True - num_workers: 4 - drop_last: True - collate_fn: - type: BatchImageCollateFunction - - -val_dataloader: - type: DataLoader - dataset: - type: CocoDetection - img_folder: /data/yourdataset/val - ann_file: /data/yourdataset/val/val.json - return_masks: False - transforms: - type: Compose - ops: ~ - shuffle: False - num_workers: 4 - drop_last: False - collate_fn: - type: BatchImageCollateFunction diff --git a/vlm/detector/D-FINE.backup/configs/dataset/obj365_detection.yml b/vlm/detector/D-FINE.backup/configs/dataset/obj365_detection.yml deleted file mode 100644 index e58fdbc..0000000 --- a/vlm/detector/D-FINE.backup/configs/dataset/obj365_detection.yml +++ /dev/null @@ -1,41 +0,0 @@ -task: detection - -evaluator: - type: CocoEvaluator - iou_types: ['bbox', ] - -num_classes: 366 -remap_mscoco_category: False - -train_dataloader: - type: DataLoader - dataset: - type: CocoDetection - img_folder: /data/Objects365/data/train - ann_file: /data/Objects365/data/train/new_zhiyuan_objv2_train_resized.json - return_masks: False - transforms: - type: Compose - ops: ~ - shuffle: True - num_workers: 4 - drop_last: True - collate_fn: - type: BatchImageCollateFunction - - -val_dataloader: - type: DataLoader - dataset: - type: CocoDetection - img_folder: /data/Objects365/data/val/ - ann_file: /data/Objects365/data/val/new_zhiyuan_objv2_val_resized.json - return_masks: False - transforms: - type: Compose - ops: ~ - shuffle: False - num_workers: 4 - drop_last: False - collate_fn: - type: BatchImageCollateFunction diff --git a/vlm/detector/D-FINE.backup/configs/dataset/voc_detection.yml b/vlm/detector/D-FINE.backup/configs/dataset/voc_detection.yml deleted file mode 100644 index 1f9ceeb..0000000 --- a/vlm/detector/D-FINE.backup/configs/dataset/voc_detection.yml +++ /dev/null @@ -1,40 +0,0 @@ -task: detection - -evaluator: - type: CocoEvaluator - iou_types: ['bbox', ] - -num_classes: 20 - -train_dataloader: - type: DataLoader - dataset: - type: VOCDetection - root: ./dataset/voc/ - ann_file: trainval.txt - label_file: label_list.txt - transforms: - type: Compose - ops: ~ - shuffle: True - num_workers: 4 - drop_last: True - collate_fn: - type: BatchImageCollateFunction - - -val_dataloader: - type: DataLoader - dataset: - type: VOCDetection - root: ./dataset/voc/ - ann_file: test.txt - label_file: label_list.txt - transforms: - type: Compose - ops: ~ - shuffle: False - num_workers: 4 - drop_last: False - collate_fn: - type: BatchImageCollateFunction diff --git a/vlm/detector/D-FINE.backup/configs/dfine/crowdhuman/dfine_hgnetv2_l_ch.yml b/vlm/detector/D-FINE.backup/configs/dfine/crowdhuman/dfine_hgnetv2_l_ch.yml deleted file mode 100644 index 761bcf8..0000000 --- a/vlm/detector/D-FINE.backup/configs/dfine/crowdhuman/dfine_hgnetv2_l_ch.yml +++ /dev/null @@ -1,44 +0,0 @@ -__include__: [ - '../../dataset/crowdhuman_detection.yml', - '../../runtime.yml', - '../include/dataloader.yml', - '../include/optimizer.yml', - '../include/dfine_hgnetv2.yml', -] - -output_dir: ./output/dfine_hgnetv2_l_crowdhuman - - -HGNetv2: - name: 'B4' - return_idx: [1, 2, 3] - freeze_stem_only: True - freeze_at: 0 - freeze_norm: True - -optimizer: - type: AdamW - params: - - - params: '^(?=.*backbone)(?!.*norm|bn).*$' - lr: 0.0000125 - - - params: '^(?=.*(?:encoder|decoder))(?=.*(?:norm|bn)).*$' - weight_decay: 0. - - lr: 0.00025 - betas: [0.9, 0.999] - weight_decay: 0.000125 - - -# Increase to search for the optimal ema -epochs: 140 -train_dataloader: - dataset: - transforms: - policy: - epoch: 120 - collate_fn: - stop_epoch: 120 - ema_restart_decay: 0.9999 - base_size_repeat: 4 diff --git a/vlm/detector/D-FINE.backup/configs/dfine/crowdhuman/dfine_hgnetv2_m_ch.yml b/vlm/detector/D-FINE.backup/configs/dfine/crowdhuman/dfine_hgnetv2_m_ch.yml deleted file mode 100644 index 48d37f1..0000000 --- a/vlm/detector/D-FINE.backup/configs/dfine/crowdhuman/dfine_hgnetv2_m_ch.yml +++ /dev/null @@ -1,60 +0,0 @@ -__include__: [ - '../../dataset/crowdhuman_detection.yml', - '../../runtime.yml', - '../include/dataloader.yml', - '../include/optimizer.yml', - '../include/dfine_hgnetv2.yml', -] - -output_dir: ./output/dfine_hgnetv2_m_crowdhuman - - -DFINE: - backbone: HGNetv2 - -HGNetv2: - name: 'B2' - return_idx: [1, 2, 3] - freeze_at: -1 - freeze_norm: False - use_lab: True - -DFINETransformer: - num_layers: 4 # 5 6 - eval_idx: -1 # -2 -3 - -HybridEncoder: - in_channels: [384, 768, 1536] - hidden_dim: 256 - depth_mult: 0.67 - -optimizer: - type: AdamW - params: - - - params: '^(?=.*backbone)(?!.*norm|bn).*$' - lr: 0.000025 - - - params: '^(?=.*backbone)(?=.*norm|bn).*$' - lr: 0.000025 - weight_decay: 0. - - - params: '^(?=.*(?:encoder|decoder))(?=.*(?:norm|bn|bias)).*$' - weight_decay: 0. - - lr: 0.00025 - betas: [0.9, 0.999] - weight_decay: 0.000125 - - -# Increase to search for the optimal ema -epochs: 220 -train_dataloader: - dataset: - transforms: - policy: - epoch: 200 - collate_fn: - stop_epoch: 200 - ema_restart_decay: 0.9999 - base_size_repeat: 6 diff --git a/vlm/detector/D-FINE.backup/configs/dfine/crowdhuman/dfine_hgnetv2_n_ch.yml b/vlm/detector/D-FINE.backup/configs/dfine/crowdhuman/dfine_hgnetv2_n_ch.yml deleted file mode 100644 index 51528bc..0000000 --- a/vlm/detector/D-FINE.backup/configs/dfine/crowdhuman/dfine_hgnetv2_n_ch.yml +++ /dev/null @@ -1,82 +0,0 @@ -__include__: [ - '../../dataset/crowdhuman_detection.yml', - '../../runtime.yml', - '../include/dataloader.yml', - '../include/optimizer.yml', - '../include/dfine_hgnetv2.yml', -] - -output_dir: ./output/dfine_hgnetv2_n_crowdhuman - - -DFINE: - backbone: HGNetv2 - -HGNetv2: - name: 'B0' - return_idx: [2, 3] - freeze_at: -1 - freeze_norm: False - use_lab: True - - -HybridEncoder: - in_channels: [512, 1024] - feat_strides: [16, 32] - - # intra - hidden_dim: 128 - use_encoder_idx: [1] - dim_feedforward: 512 - - # cross - expansion: 0.34 - depth_mult: 0.5 - - -DFINETransformer: - feat_channels: [128, 128] - feat_strides: [16, 32] - hidden_dim: 128 - dim_feedforward: 512 - num_levels: 2 - - num_layers: 3 - eval_idx: -1 - - num_points: [6, 6] - -optimizer: - type: AdamW - params: - - - params: '^(?=.*backbone)(?!.*norm|bn).*$' - lr: 0.0004 - - - params: '^(?=.*backbone)(?=.*norm|bn).*$' - lr: 0.0004 - weight_decay: 0. - - - params: '^(?=.*(?:encoder|decoder))(?=.*(?:norm|bn|bias)).*$' - weight_decay: 0. - - lr: 0.0008 - betas: [0.9, 0.999] - weight_decay: 0.0001 - - -# Increase to search for the optimal ema -epochs: 220 -train_dataloader: - total_batch_size: 128 - dataset: - transforms: - policy: - epoch: 200 - collate_fn: - stop_epoch: 200 - ema_restart_decay: 0.9999 - base_size_repeat: ~ - -val_dataloader: - total_batch_size: 256 diff --git a/vlm/detector/D-FINE.backup/configs/dfine/crowdhuman/dfine_hgnetv2_s_ch.yml b/vlm/detector/D-FINE.backup/configs/dfine/crowdhuman/dfine_hgnetv2_s_ch.yml deleted file mode 100644 index 55ab8b8..0000000 --- a/vlm/detector/D-FINE.backup/configs/dfine/crowdhuman/dfine_hgnetv2_s_ch.yml +++ /dev/null @@ -1,65 +0,0 @@ -__include__: [ - '../../dataset/crowdhuman_detection.yml', - '../../runtime.yml', - '../include/dataloader.yml', - '../include/optimizer.yml', - '../include/dfine_hgnetv2.yml', -] - -output_dir: ./output/dfine_hgnetv2_s_crowdhuman - - -DFINE: - backbone: HGNetv2 - -HGNetv2: - name: 'B0' - return_idx: [1, 2, 3] - freeze_at: -1 - freeze_norm: False - use_lab: True - -DFINETransformer: - num_layers: 3 # 4 5 6 - eval_idx: -1 # -2 -3 -4 - -HybridEncoder: - in_channels: [256, 512, 1024] - hidden_dim: 256 - depth_mult: 0.34 - expansion: 0.5 - -optimizer: - type: AdamW - params: - - - params: '^(?=.*backbone)(?!.*norm|bn).*$' - lr: 0.0002 - - - params: '^(?=.*backbone)(?=.*norm|bn).*$' - lr: 0.0002 - weight_decay: 0. - - - params: '^(?=.*(?:encoder|decoder))(?=.*(?:norm|bn|bias)).*$' - weight_decay: 0. - - lr: 0.0004 - betas: [0.9, 0.999] - weight_decay: 0.0001 - - -# Increase to search for the optimal ema -epochs: 220 -train_dataloader: - total_batch_size: 64 - dataset: - transforms: - policy: - epoch: 200 - collate_fn: - stop_epoch: 200 - ema_restart_decay: 0.9999 - base_size_repeat: 20 - -val_dataloader: - total_batch_size: 128 diff --git a/vlm/detector/D-FINE.backup/configs/dfine/crowdhuman/dfine_hgnetv2_x_ch.yml b/vlm/detector/D-FINE.backup/configs/dfine/crowdhuman/dfine_hgnetv2_x_ch.yml deleted file mode 100644 index 2ab49a0..0000000 --- a/vlm/detector/D-FINE.backup/configs/dfine/crowdhuman/dfine_hgnetv2_x_ch.yml +++ /dev/null @@ -1,55 +0,0 @@ -__include__: [ - '../../dataset/crowdhuman_detection.yml', - '../../runtime.yml', - '../include/dataloader.yml', - '../include/optimizer.yml', - '../include/dfine_hgnetv2.yml', -] - -output_dir: ./output/dfine_hgnetv2_x_crowdhuman - - -DFINE: - backbone: HGNetv2 - -HGNetv2: - name: 'B5' - return_idx: [1, 2, 3] - freeze_stem_only: True - freeze_at: 0 - freeze_norm: True - -HybridEncoder: - hidden_dim: 384 - dim_feedforward: 2048 - -DFINETransformer: - feat_channels: [384, 384, 384] - reg_scale: 8 - -optimizer: - type: AdamW - params: - - - params: '^(?=.*backbone)(?!.*norm|bn).*$' - lr: 0.0000025 - - - params: '^(?=.*(?:encoder|decoder))(?=.*(?:norm|bn)).*$' - weight_decay: 0. - - lr: 0.00025 - betas: [0.9, 0.999] - weight_decay: 0.000125 - - -# Increase to search for the optimal ema -epochs: 140 -train_dataloader: - dataset: - transforms: - policy: - epoch: 120 - collate_fn: - stop_epoch: 120 - ema_restart_decay: 0.9998 - base_size_repeat: 3 diff --git a/vlm/detector/D-FINE.backup/configs/dfine/custom/dfine_hgnetv2_l_custom.yml b/vlm/detector/D-FINE.backup/configs/dfine/custom/dfine_hgnetv2_l_custom.yml deleted file mode 100644 index 7526401..0000000 --- a/vlm/detector/D-FINE.backup/configs/dfine/custom/dfine_hgnetv2_l_custom.yml +++ /dev/null @@ -1,44 +0,0 @@ -__include__: [ - '../../dataset/custom_detection.yml', - '../../runtime.yml', - '../include/dataloader.yml', - '../include/optimizer.yml', - '../include/dfine_hgnetv2.yml', -] - -output_dir: ./output/dfine_hgnetv2_l_custom - - -HGNetv2: - name: 'B4' - return_idx: [1, 2, 3] - freeze_stem_only: True - freeze_at: 0 - freeze_norm: True - -optimizer: - type: AdamW - params: - - - params: '^(?=.*backbone)(?!.*norm|bn).*$' - lr: 0.0000125 - - - params: '^(?=.*(?:encoder|decoder))(?=.*(?:norm|bn)).*$' - weight_decay: 0. - - lr: 0.00025 - betas: [0.9, 0.999] - weight_decay: 0.000125 - - -# Increase to search for the optimal ema -epochs: 80 # 72 + 2n -train_dataloader: - dataset: - transforms: - policy: - epoch: 72 - collate_fn: - stop_epoch: 72 - ema_restart_decay: 0.9999 - base_size_repeat: 4 diff --git a/vlm/detector/D-FINE.backup/configs/dfine/custom/dfine_hgnetv2_m_custom.yml b/vlm/detector/D-FINE.backup/configs/dfine/custom/dfine_hgnetv2_m_custom.yml deleted file mode 100644 index bf31c1c..0000000 --- a/vlm/detector/D-FINE.backup/configs/dfine/custom/dfine_hgnetv2_m_custom.yml +++ /dev/null @@ -1,60 +0,0 @@ -__include__: [ - '../../dataset/custom_detection.yml', - '../../runtime.yml', - '../include/dataloader.yml', - '../include/optimizer.yml', - '../include/dfine_hgnetv2.yml', -] - -output_dir: ./output/dfine_hgnetv2_m_custom - - -DFINE: - backbone: HGNetv2 - -HGNetv2: - name: 'B2' - return_idx: [1, 2, 3] - freeze_at: -1 - freeze_norm: False - use_lab: True - -DFINETransformer: - num_layers: 4 # 5 6 - eval_idx: -1 # -2 -3 - -HybridEncoder: - in_channels: [384, 768, 1536] - hidden_dim: 256 - depth_mult: 0.67 - -optimizer: - type: AdamW - params: - - - params: '^(?=.*backbone)(?!.*norm|bn).*$' - lr: 0.000025 - - - params: '^(?=.*backbone)(?=.*norm|bn).*$' - lr: 0.000025 - weight_decay: 0. - - - params: '^(?=.*(?:encoder|decoder))(?=.*(?:norm|bn|bias)).*$' - weight_decay: 0. - - lr: 0.00025 - betas: [0.9, 0.999] - weight_decay: 0.000125 - - -# Increase to search for the optimal ema -epochs: 132 # 120 + 4n -train_dataloader: - dataset: - transforms: - policy: - epoch: 120 - collate_fn: - stop_epoch: 120 - ema_restart_decay: 0.9999 - base_size_repeat: 6 diff --git a/vlm/detector/D-FINE.backup/configs/dfine/custom/dfine_hgnetv2_n_custom.yml b/vlm/detector/D-FINE.backup/configs/dfine/custom/dfine_hgnetv2_n_custom.yml deleted file mode 100644 index 6da7702..0000000 --- a/vlm/detector/D-FINE.backup/configs/dfine/custom/dfine_hgnetv2_n_custom.yml +++ /dev/null @@ -1,82 +0,0 @@ -__include__: [ - '../../dataset/custom_detection.yml', - '../../runtime.yml', - '../include/dataloader.yml', - '../include/optimizer.yml', - '../include/dfine_hgnetv2.yml', -] - -output_dir: ./output/dfine_hgnetv2_n_custom - - -DFINE: - backbone: HGNetv2 - -HGNetv2: - name: 'B0' - return_idx: [2, 3] - freeze_at: -1 - freeze_norm: False - use_lab: True - - -HybridEncoder: - in_channels: [512, 1024] - feat_strides: [16, 32] - - # intra - hidden_dim: 128 - use_encoder_idx: [1] - dim_feedforward: 512 - - # cross - expansion: 0.34 - depth_mult: 0.5 - - -DFINETransformer: - feat_channels: [128, 128] - feat_strides: [16, 32] - hidden_dim: 128 - dim_feedforward: 512 - num_levels: 2 - - num_layers: 3 - eval_idx: -1 - - num_points: [6, 6] - -optimizer: - type: AdamW - params: - - - params: '^(?=.*backbone)(?!.*norm|bn).*$' - lr: 0.0004 - - - params: '^(?=.*backbone)(?=.*norm|bn).*$' - lr: 0.0004 - weight_decay: 0. - - - params: '^(?=.*(?:encoder|decoder))(?=.*(?:norm|bn|bias)).*$' - weight_decay: 0. - - lr: 0.0008 - betas: [0.9, 0.999] - weight_decay: 0.0001 - - -# Increase to search for the optimal ema -epochs: 220 -train_dataloader: - total_batch_size: 128 - dataset: - transforms: - policy: - epoch: 200 - collate_fn: - stop_epoch: 200 - ema_restart_decay: 0.9999 - base_size_repeat: ~ - -val_dataloader: - total_batch_size: 256 diff --git a/vlm/detector/D-FINE.backup/configs/dfine/custom/dfine_hgnetv2_s_custom.yml b/vlm/detector/D-FINE.backup/configs/dfine/custom/dfine_hgnetv2_s_custom.yml deleted file mode 100644 index 285cb77..0000000 --- a/vlm/detector/D-FINE.backup/configs/dfine/custom/dfine_hgnetv2_s_custom.yml +++ /dev/null @@ -1,65 +0,0 @@ -__include__: [ - '../../dataset/custom_detection.yml', - '../../runtime.yml', - '../include/dataloader.yml', - '../include/optimizer.yml', - '../include/dfine_hgnetv2.yml', -] - -output_dir: ./output/dfine_hgnetv2_s_custom - - -DFINE: - backbone: HGNetv2 - -HGNetv2: - name: 'B0' - return_idx: [1, 2, 3] - freeze_at: -1 - freeze_norm: False - use_lab: True - -DFINETransformer: - num_layers: 3 # 4 5 6 - eval_idx: -1 # -2 -3 -4 - -HybridEncoder: - in_channels: [256, 512, 1024] - hidden_dim: 256 - depth_mult: 0.34 - expansion: 0.5 - -optimizer: - type: AdamW - params: - - - params: '^(?=.*backbone)(?!.*norm|bn).*$' - lr: 0.0002 - - - params: '^(?=.*backbone)(?=.*norm|bn).*$' - lr: 0.0002 - weight_decay: 0. - - - params: '^(?=.*(?:encoder|decoder))(?=.*(?:norm|bn|bias)).*$' - weight_decay: 0. - - lr: 0.0004 - betas: [0.9, 0.999] - weight_decay: 0.0001 - - -# Increase to search for the optimal ema -epochs: 220 -train_dataloader: - total_batch_size: 64 - dataset: - transforms: - policy: - epoch: 200 - collate_fn: - stop_epoch: 200 - ema_restart_decay: 0.9999 - base_size_repeat: 20 - -val_dataloader: - total_batch_size: 128 diff --git a/vlm/detector/D-FINE.backup/configs/dfine/custom/dfine_hgnetv2_x_custom.yml b/vlm/detector/D-FINE.backup/configs/dfine/custom/dfine_hgnetv2_x_custom.yml deleted file mode 100644 index 2ba5646..0000000 --- a/vlm/detector/D-FINE.backup/configs/dfine/custom/dfine_hgnetv2_x_custom.yml +++ /dev/null @@ -1,55 +0,0 @@ -__include__: [ - '../../dataset/custom_detection.yml', - '../../runtime.yml', - '../include/dataloader.yml', - '../include/optimizer.yml', - '../include/dfine_hgnetv2.yml', -] - -output_dir: ./output/dfine_hgnetv2_x_custom - - -DFINE: - backbone: HGNetv2 - -HGNetv2: - name: 'B5' - return_idx: [1, 2, 3] - freeze_stem_only: True - freeze_at: 0 - freeze_norm: True - -HybridEncoder: - hidden_dim: 384 - dim_feedforward: 2048 - -DFINETransformer: - feat_channels: [384, 384, 384] - reg_scale: 8 - -optimizer: - type: AdamW - params: - - - params: '^(?=.*backbone)(?!.*norm|bn).*$' - lr: 0.0000025 - - - params: '^(?=.*(?:encoder|decoder))(?=.*(?:norm|bn)).*$' - weight_decay: 0. - - lr: 0.00025 - betas: [0.9, 0.999] - weight_decay: 0.000125 - - -# Increase to search for the optimal ema -epochs: 80 # 72 + 2n -train_dataloader: - dataset: - transforms: - policy: - epoch: 72 - collate_fn: - stop_epoch: 72 - ema_restart_decay: 0.9998 - base_size_repeat: 3 diff --git a/vlm/detector/D-FINE.backup/configs/dfine/custom/objects365/dfine_hgnetv2_l_obj2custom.yml b/vlm/detector/D-FINE.backup/configs/dfine/custom/objects365/dfine_hgnetv2_l_obj2custom.yml deleted file mode 100644 index ef6a95d..0000000 --- a/vlm/detector/D-FINE.backup/configs/dfine/custom/objects365/dfine_hgnetv2_l_obj2custom.yml +++ /dev/null @@ -1,53 +0,0 @@ -__include__: [ - '../../../dataset/custom_detection.yml', - '../../../runtime.yml', - '../../include/dataloader.yml', - '../../include/optimizer.yml', - '../../include/dfine_hgnetv2.yml', -] - -output_dir: ./output/dfine_hgnetv2_l_obj2custom - - -DFINE: - backbone: HGNetv2 - -HGNetv2: - name: 'B4' - return_idx: [1, 2, 3] - freeze_stem_only: True - freeze_at: 0 - freeze_norm: True - pretrained: False - -optimizer: - type: AdamW - params: - - - params: '^(?=.*backbone)(?!.*norm|bn).*$' - lr: 0.0000125 - - - params: '^(?=.*(?:encoder|decoder))(?=.*(?:norm|bn)).*$' - weight_decay: 0. - - lr: 0.00025 - betas: [0.9, 0.999] - weight_decay: 0.000125 - - -epochs: 36 # Early stop -train_dataloader: - dataset: - transforms: - policy: - epoch: 30 - collate_fn: - stop_epoch: 30 - ema_restart_decay: 0.9999 - base_size_repeat: 4 - -ema: - warmups: 0 - -lr_warmup_scheduler: - warmup_duration: 0 diff --git a/vlm/detector/D-FINE.backup/configs/dfine/custom/objects365/dfine_hgnetv2_m_obj2custom.yml b/vlm/detector/D-FINE.backup/configs/dfine/custom/objects365/dfine_hgnetv2_m_obj2custom.yml deleted file mode 100644 index f21fbbc..0000000 --- a/vlm/detector/D-FINE.backup/configs/dfine/custom/objects365/dfine_hgnetv2_m_obj2custom.yml +++ /dev/null @@ -1,66 +0,0 @@ -__include__: [ - '../../../dataset/custom_detection.yml', - '../../../runtime.yml', - '../../include/dataloader.yml', - '../../include/optimizer.yml', - '../../include/dfine_hgnetv2.yml', -] - -output_dir: ./output/dfine_hgnetv2_m_obj2custom - - -DFINE: - backbone: HGNetv2 - -HGNetv2: - name: 'B2' - return_idx: [1, 2, 3] - freeze_at: -1 - freeze_norm: False - use_lab: True - pretrained: False - -DFINETransformer: - num_layers: 4 # 5 6 - eval_idx: -1 # -2 -3 - -HybridEncoder: - in_channels: [384, 768, 1536] - hidden_dim: 256 - depth_mult: 0.67 - -optimizer: - type: AdamW - params: - - - params: '^(?=.*backbone)(?!.*norm|bn).*$' - lr: 0.000025 - - - params: '^(?=.*backbone)(?=.*norm|bn).*$' - lr: 0.000025 - weight_decay: 0. - - - params: '^(?=.*(?:encoder|decoder))(?=.*(?:norm|bn|bias)).*$' - weight_decay: 0. - - lr: 0.00025 - betas: [0.9, 0.999] - weight_decay: 0.000125 - - -epochs: 56 # Early stop -train_dataloader: - dataset: - transforms: - policy: - epoch: 48 - collate_fn: - stop_epoch: 48 - ema_restart_decay: 0.9999 - base_size_repeat: 6 - -ema: - warmups: 0 - -lr_warmup_scheduler: - warmup_duration: 0 diff --git a/vlm/detector/D-FINE.backup/configs/dfine/custom/objects365/dfine_hgnetv2_s_obj2custom.yml b/vlm/detector/D-FINE.backup/configs/dfine/custom/objects365/dfine_hgnetv2_s_obj2custom.yml deleted file mode 100644 index 9a74e34..0000000 --- a/vlm/detector/D-FINE.backup/configs/dfine/custom/objects365/dfine_hgnetv2_s_obj2custom.yml +++ /dev/null @@ -1,67 +0,0 @@ -__include__: [ - '../../../dataset/custom_detection.yml', - '../../../runtime.yml', - '../../include/dataloader.yml', - '../../include/optimizer.yml', - '../../include/dfine_hgnetv2.yml', -] - -output_dir: ./output/dfine_hgnetv2_s_obj2custom - - -DFINE: - backbone: HGNetv2 - -HGNetv2: - name: 'B0' - return_idx: [1, 2, 3] - freeze_at: -1 - freeze_norm: False - use_lab: True - pretrained: False - -DFINETransformer: - num_layers: 3 # 4 5 6 - eval_idx: -1 # -2 -3 -4 - -HybridEncoder: - in_channels: [256, 512, 1024] - hidden_dim: 256 - depth_mult: 0.34 - expansion: 0.5 - -optimizer: - type: AdamW - params: - - - params: '^(?=.*backbone)(?!.*norm|bn).*$' - lr: 0.000125 - - - params: '^(?=.*backbone)(?=.*norm|bn).*$' - lr: 0.000125 - weight_decay: 0. - - - params: '^(?=.*(?:encoder|decoder))(?=.*(?:norm|bn|bias)).*$' - weight_decay: 0. - - lr: 0.00025 - betas: [0.9, 0.999] - weight_decay: 0.000125 - - -epochs: 64 # Early stop -train_dataloader: - dataset: - transforms: - policy: - epoch: 56 - collate_fn: - stop_epoch: 56 - ema_restart_decay: 0.9999 - base_size_repeat: 10 - -ema: - warmups: 0 - -lr_warmup_scheduler: - warmup_duration: 0 diff --git a/vlm/detector/D-FINE.backup/configs/dfine/custom/objects365/dfine_hgnetv2_x_obj2custom.yml b/vlm/detector/D-FINE.backup/configs/dfine/custom/objects365/dfine_hgnetv2_x_obj2custom.yml deleted file mode 100644 index 717812f..0000000 --- a/vlm/detector/D-FINE.backup/configs/dfine/custom/objects365/dfine_hgnetv2_x_obj2custom.yml +++ /dev/null @@ -1,62 +0,0 @@ -__include__: [ - '../../../dataset/custom_detection.yml', - '../../../runtime.yml', - '../../include/dataloader.yml', - '../../include/optimizer.yml', - '../../include/dfine_hgnetv2.yml', -] - -output_dir: ./output/dfine_hgnetv2_x_obj2custom - - -DFINE: - backbone: HGNetv2 - -HGNetv2: - name: 'B5' - return_idx: [1, 2, 3] - freeze_stem_only: True - freeze_at: 0 - freeze_norm: True - pretrained: False - -HybridEncoder: - # intra - hidden_dim: 384 - dim_feedforward: 2048 - -DFINETransformer: - feat_channels: [384, 384, 384] - reg_scale: 8 - -optimizer: - type: AdamW - params: - - - params: '^(?=.*backbone)(?!.*norm|bn).*$' - lr: 0.0000025 - - - params: '^(?=.*(?:encoder|decoder))(?=.*(?:norm|bn)).*$' - weight_decay: 0. - - lr: 0.00025 - betas: [0.9, 0.999] - weight_decay: 0.000125 - - -epochs: 36 # Early stop -train_dataloader: - dataset: - transforms: - policy: - epoch: 30 - collate_fn: - stop_epoch: 30 - ema_restart_decay: 0.9999 - base_size_repeat: 3 - -ema: - warmups: 0 - -lr_warmup_scheduler: - warmup_duration: 0 diff --git a/vlm/detector/D-FINE.backup/configs/dfine/dfine_hgnetv2_l_coco.yml b/vlm/detector/D-FINE.backup/configs/dfine/dfine_hgnetv2_l_coco.yml deleted file mode 100644 index 718074c..0000000 --- a/vlm/detector/D-FINE.backup/configs/dfine/dfine_hgnetv2_l_coco.yml +++ /dev/null @@ -1,44 +0,0 @@ -__include__: [ - '../dataset/coco_detection.yml', - '../runtime.yml', - './include/dataloader.yml', - './include/optimizer.yml', - './include/dfine_hgnetv2.yml', -] - -output_dir: ./output/dfine_hgnetv2_l_coco - - -HGNetv2: - name: 'B4' - return_idx: [1, 2, 3] - freeze_stem_only: True - freeze_at: 0 - freeze_norm: True - -optimizer: - type: AdamW - params: - - - params: '^(?=.*backbone)(?!.*norm|bn).*$' - lr: 0.0000125 - - - params: '^(?=.*(?:encoder|decoder))(?=.*(?:norm|bn)).*$' - weight_decay: 0. - - lr: 0.00025 - betas: [0.9, 0.999] - weight_decay: 0.000125 - - -# Increase to search for the optimal ema -epochs: 80 # 72 + 2n -train_dataloader: - dataset: - transforms: - policy: - epoch: 72 - collate_fn: - stop_epoch: 72 - ema_restart_decay: 0.9999 - base_size_repeat: 4 diff --git a/vlm/detector/D-FINE.backup/configs/dfine/dfine_hgnetv2_m_coco.yml b/vlm/detector/D-FINE.backup/configs/dfine/dfine_hgnetv2_m_coco.yml deleted file mode 100644 index 9837eac..0000000 --- a/vlm/detector/D-FINE.backup/configs/dfine/dfine_hgnetv2_m_coco.yml +++ /dev/null @@ -1,60 +0,0 @@ -__include__: [ - '../dataset/coco_detection.yml', - '../runtime.yml', - './include/dataloader.yml', - './include/optimizer.yml', - './include/dfine_hgnetv2.yml', -] - -output_dir: ./output/dfine_hgnetv2_m_coco - - -DFINE: - backbone: HGNetv2 - -HGNetv2: - name: 'B2' - return_idx: [1, 2, 3] - freeze_at: -1 - freeze_norm: False - use_lab: True - -DFINETransformer: - num_layers: 4 # 5 6 - eval_idx: -1 # -2 -3 - -HybridEncoder: - in_channels: [384, 768, 1536] - hidden_dim: 256 - depth_mult: 0.67 - -optimizer: - type: AdamW - params: - - - params: '^(?=.*backbone)(?!.*norm|bn).*$' - lr: 0.00002 - - - params: '^(?=.*backbone)(?=.*norm|bn).*$' - lr: 0.00002 - weight_decay: 0. - - - params: '^(?=.*(?:encoder|decoder))(?=.*(?:norm|bn|bias)).*$' - weight_decay: 0. - - lr: 0.0002 - betas: [0.9, 0.999] - weight_decay: 0.0001 - - -# Increase to search for the optimal ema -epochs: 132 # 120 + 4n -train_dataloader: - dataset: - transforms: - policy: - epoch: 120 - collate_fn: - stop_epoch: 120 - ema_restart_decay: 0.9999 - base_size_repeat: 6 diff --git a/vlm/detector/D-FINE.backup/configs/dfine/dfine_hgnetv2_n_coco.yml b/vlm/detector/D-FINE.backup/configs/dfine/dfine_hgnetv2_n_coco.yml deleted file mode 100644 index f0c3a1a..0000000 --- a/vlm/detector/D-FINE.backup/configs/dfine/dfine_hgnetv2_n_coco.yml +++ /dev/null @@ -1,82 +0,0 @@ -__include__: [ - '../dataset/coco_detection.yml', - '../runtime.yml', - './include/dataloader.yml', - './include/optimizer.yml', - './include/dfine_hgnetv2.yml', -] - -output_dir: ./output/dfine_hgnetv2_n_coco - - -DFINE: - backbone: HGNetv2 - -HGNetv2: - name: 'B0' - return_idx: [2, 3] - freeze_at: -1 - freeze_norm: False - use_lab: True - - -HybridEncoder: - in_channels: [512, 1024] - feat_strides: [16, 32] - - # intra - hidden_dim: 128 - use_encoder_idx: [1] - dim_feedforward: 512 - - # cross - expansion: 0.34 - depth_mult: 0.5 - - -DFINETransformer: - feat_channels: [128, 128] - feat_strides: [16, 32] - hidden_dim: 128 - dim_feedforward: 512 - num_levels: 2 - - num_layers: 3 - eval_idx: -1 - - num_points: [6, 6] - -optimizer: - type: AdamW - params: - - - params: '^(?=.*backbone)(?!.*norm|bn).*$' - lr: 0.0004 - - - params: '^(?=.*backbone)(?=.*norm|bn).*$' - lr: 0.0004 - weight_decay: 0. - - - params: '^(?=.*(?:encoder|decoder))(?=.*(?:norm|bn|bias)).*$' - weight_decay: 0. - - lr: 0.0008 - betas: [0.9, 0.999] - weight_decay: 0.0001 - - -# Increase to search for the optimal ema -epochs: 160 # 148 + 4n -train_dataloader: - total_batch_size: 128 - dataset: - transforms: - policy: - epoch: 148 - collate_fn: - stop_epoch: 148 - ema_restart_decay: 0.9999 - base_size_repeat: ~ - -val_dataloader: - total_batch_size: 256 diff --git a/vlm/detector/D-FINE.backup/configs/dfine/dfine_hgnetv2_s_coco.yml b/vlm/detector/D-FINE.backup/configs/dfine/dfine_hgnetv2_s_coco.yml deleted file mode 100644 index e7aaa22..0000000 --- a/vlm/detector/D-FINE.backup/configs/dfine/dfine_hgnetv2_s_coco.yml +++ /dev/null @@ -1,61 +0,0 @@ -__include__: [ - '../dataset/coco_detection.yml', - '../runtime.yml', - './include/dataloader.yml', - './include/optimizer.yml', - './include/dfine_hgnetv2.yml', -] - -output_dir: ./output/dfine_hgnetv2_s_coco - - -DFINE: - backbone: HGNetv2 - -HGNetv2: - name: 'B0' - return_idx: [1, 2, 3] - freeze_at: -1 - freeze_norm: False - use_lab: True - -DFINETransformer: - num_layers: 3 # 4 5 6 - eval_idx: -1 # -2 -3 -4 - -HybridEncoder: - in_channels: [256, 512, 1024] - hidden_dim: 256 - depth_mult: 0.34 - expansion: 0.5 - -optimizer: - type: AdamW - params: - - - params: '^(?=.*backbone)(?!.*norm|bn).*$' - lr: 0.0001 - - - params: '^(?=.*backbone)(?=.*norm|bn).*$' - lr: 0.0001 - weight_decay: 0. - - - params: '^(?=.*(?:encoder|decoder))(?=.*(?:norm|bn|bias)).*$' - weight_decay: 0. - - lr: 0.0002 - betas: [0.9, 0.999] - weight_decay: 0.0001 - - -# Increase to search for the optimal ema -epochs: 132 # 120 + 4n -train_dataloader: - dataset: - transforms: - policy: - epoch: 120 - collate_fn: - stop_epoch: 120 - ema_restart_decay: 0.9999 - base_size_repeat: 20 diff --git a/vlm/detector/D-FINE.backup/configs/dfine/dfine_hgnetv2_x_coco.yml b/vlm/detector/D-FINE.backup/configs/dfine/dfine_hgnetv2_x_coco.yml deleted file mode 100644 index 38d976b..0000000 --- a/vlm/detector/D-FINE.backup/configs/dfine/dfine_hgnetv2_x_coco.yml +++ /dev/null @@ -1,56 +0,0 @@ -__include__: [ - '../dataset/coco_detection.yml', - '../runtime.yml', - './include/dataloader.yml', - './include/optimizer.yml', - './include/dfine_hgnetv2.yml', -] - -output_dir: ./output/dfine_hgnetv2_x_coco - - -DFINE: - backbone: HGNetv2 - -HGNetv2: - name: 'B5' - return_idx: [1, 2, 3] - freeze_stem_only: True - freeze_at: 0 - freeze_norm: True - -HybridEncoder: - # intra - hidden_dim: 384 - dim_feedforward: 2048 - -DFINETransformer: - feat_channels: [384, 384, 384] - reg_scale: 8 - -optimizer: - type: AdamW - params: - - - params: '^(?=.*backbone)(?!.*norm|bn).*$' - lr: 0.0000025 - - - params: '^(?=.*(?:encoder|decoder))(?=.*(?:norm|bn)).*$' - weight_decay: 0. - - lr: 0.00025 - betas: [0.9, 0.999] - weight_decay: 0.000125 - - -# Increase to search for the optimal ema -epochs: 80 # 72 + 2n -train_dataloader: - dataset: - transforms: - policy: - epoch: 72 - collate_fn: - stop_epoch: 72 - ema_restart_decay: 0.9998 - base_size_repeat: 3 diff --git a/vlm/detector/D-FINE.backup/configs/dfine/include/dataloader.yml b/vlm/detector/D-FINE.backup/configs/dfine/include/dataloader.yml deleted file mode 100644 index 3f7c67c..0000000 --- a/vlm/detector/D-FINE.backup/configs/dfine/include/dataloader.yml +++ /dev/null @@ -1,39 +0,0 @@ - -train_dataloader: - dataset: - transforms: - ops: - - {type: RandomPhotometricDistort, p: 0.5} - - {type: RandomZoomOut, fill: 0} - - {type: RandomIoUCrop, p: 0.8} - - {type: SanitizeBoundingBoxes, min_size: 1} - - {type: RandomHorizontalFlip} - - {type: Resize, size: [640, 640], } - - {type: SanitizeBoundingBoxes, min_size: 1} - - {type: ConvertPILImage, dtype: 'float32', scale: True} - - {type: ConvertBoxes, fmt: 'cxcywh', normalize: True} - policy: - name: stop_epoch - epoch: 72 # epoch in [71, ~) stop `ops` - ops: ['RandomPhotometricDistort', 'RandomZoomOut', 'RandomIoUCrop'] - - collate_fn: - type: BatchImageCollateFunction - base_size: 640 - base_size_repeat: 3 - stop_epoch: 72 # epoch in [72, ~) stop `multiscales` - - shuffle: True - total_batch_size: 32 # total batch size equals to 32 (4 * 8) - num_workers: 4 - - -val_dataloader: - dataset: - transforms: - ops: - - {type: Resize, size: [640, 640], } - - {type: ConvertPILImage, dtype: 'float32', scale: True} - shuffle: False - total_batch_size: 64 - num_workers: 4 diff --git a/vlm/detector/D-FINE.backup/configs/dfine/include/dfine_hgnetv2.yml b/vlm/detector/D-FINE.backup/configs/dfine/include/dfine_hgnetv2.yml deleted file mode 100644 index 91ad4a0..0000000 --- a/vlm/detector/D-FINE.backup/configs/dfine/include/dfine_hgnetv2.yml +++ /dev/null @@ -1,82 +0,0 @@ -task: detection - -model: DFINE -criterion: DFINECriterion -postprocessor: DFINEPostProcessor - -use_focal_loss: True -eval_spatial_size: [640, 640] # h w - -DFINE: - backbone: HGNetv2 - encoder: HybridEncoder - decoder: DFINETransformer - -HGNetv2: - pretrained: True - local_model_dir: weight/hgnetv2/ - -HybridEncoder: - in_channels: [512, 1024, 2048] - feat_strides: [8, 16, 32] - - # intra - hidden_dim: 256 - use_encoder_idx: [2] - num_encoder_layers: 1 - nhead: 8 - dim_feedforward: 1024 - dropout: 0. - enc_act: 'gelu' - - # cross - expansion: 1.0 - depth_mult: 1 - act: 'silu' - - -DFINETransformer: - feat_channels: [256, 256, 256] - feat_strides: [8, 16, 32] - hidden_dim: 256 - num_levels: 3 - - num_layers: 6 - eval_idx: -1 - num_queries: 300 - - num_denoising: 100 - label_noise_ratio: 0.5 - box_noise_scale: 1.0 - - # NEW - reg_max: 32 - reg_scale: 4 - - # Auxiliary decoder layers dimension scaling - # "eg. If num_layers: 6 eval_idx: -4, - # then layer 3, 4, 5 are auxiliary decoder layers." - layer_scale: 1 # 2 - - - num_points: [3, 6, 3] # [4, 4, 4] [3, 6, 3] - cross_attn_method: default # default, discrete - query_select_method: default # default, agnostic - - -DFINEPostProcessor: - num_top_queries: 300 - - -DFINECriterion: - weight_dict: {loss_vfl: 1, loss_bbox: 5, loss_giou: 2, loss_fgl: 0.15, loss_ddf: 1.5} - losses: ['vfl', 'boxes', 'local'] - alpha: 0.75 - gamma: 2.0 - reg_max: 32 - - matcher: - type: HungarianMatcher - weight_dict: {cost_class: 2, cost_bbox: 5, cost_giou: 2} - alpha: 0.25 - gamma: 2.0 diff --git a/vlm/detector/D-FINE.backup/configs/dfine/include/optimizer.yml b/vlm/detector/D-FINE.backup/configs/dfine/include/optimizer.yml deleted file mode 100644 index 8ec87c7..0000000 --- a/vlm/detector/D-FINE.backup/configs/dfine/include/optimizer.yml +++ /dev/null @@ -1,36 +0,0 @@ -use_amp: True -use_ema: True -ema: - type: ModelEMA - decay: 0.9999 - warmups: 1000 - start: 0 - - -epochs: 72 -clip_max_norm: 0.1 - - -optimizer: - type: AdamW - params: - - - params: '^(?=.*backbone)(?!.*norm).*$' - lr: 0.0000125 - - - params: '^(?=.*(?:encoder|decoder))(?=.*(?:norm|bn)).*$' - weight_decay: 0. - - lr: 0.00025 - betas: [0.9, 0.999] - weight_decay: 0.000125 - - -lr_scheduler: - type: MultiStepLR - milestones: [500] - gamma: 0.1 - -lr_warmup_scheduler: - type: LinearWarmup - warmup_duration: 500 diff --git a/vlm/detector/D-FINE.backup/configs/dfine/objects365/dfine_hgnetv2_l_obj2coco.yml b/vlm/detector/D-FINE.backup/configs/dfine/objects365/dfine_hgnetv2_l_obj2coco.yml deleted file mode 100644 index f4fda63..0000000 --- a/vlm/detector/D-FINE.backup/configs/dfine/objects365/dfine_hgnetv2_l_obj2coco.yml +++ /dev/null @@ -1,52 +0,0 @@ -__include__: [ - '../../dataset/coco_detection.yml', - '../../runtime.yml', - '../include/dataloader.yml', - '../include/optimizer.yml', - '../include/dfine_hgnetv2.yml', -] - -output_dir: ./output/dfine_hgnetv2_l_obj2coco - - -DFINE: - backbone: HGNetv2 - -HGNetv2: - name: 'B4' - return_idx: [1, 2, 3] - freeze_stem_only: True - freeze_at: 0 - freeze_norm: True - -optimizer: - type: AdamW - params: - - - params: '^(?=.*backbone)(?!.*norm|bn).*$' - lr: 0.0000125 - - - params: '^(?=.*(?:encoder|decoder))(?=.*(?:norm|bn)).*$' - weight_decay: 0. - - lr: 0.00025 - betas: [0.9, 0.999] - weight_decay: 0.000125 - - -epochs: 36 # Early stop -train_dataloader: - dataset: - transforms: - policy: - epoch: 30 - collate_fn: - stop_epoch: 30 - ema_restart_decay: 0.9999 - base_size_repeat: 4 - -ema: - warmups: 0 - -lr_warmup_scheduler: - warmup_duration: 0 diff --git a/vlm/detector/D-FINE.backup/configs/dfine/objects365/dfine_hgnetv2_l_obj365.yml b/vlm/detector/D-FINE.backup/configs/dfine/objects365/dfine_hgnetv2_l_obj365.yml deleted file mode 100644 index c7a2876..0000000 --- a/vlm/detector/D-FINE.backup/configs/dfine/objects365/dfine_hgnetv2_l_obj365.yml +++ /dev/null @@ -1,49 +0,0 @@ -__include__: [ - '../../dataset/obj365_detection.yml', - '../../runtime.yml', - '../include/dataloader.yml', - '../include/optimizer.yml', - '../include/dfine_hgnetv2.yml', -] - -output_dir: ./output/dfine_hgnetv2_l_obj365 - - -DFINE: - backbone: HGNetv2 - -HGNetv2: - name: 'B4' - return_idx: [1, 2, 3] - freeze_stem_only: True - freeze_at: 0 - freeze_norm: True - -optimizer: - type: AdamW - params: - - - params: '^(?=.*backbone)(?!.*norm|bn).*$' - lr: 0.0000125 - - - params: '^(?=.*(?:encoder|decoder))(?=.*(?:norm|bn)).*$' - weight_decay: 0. - - lr: 0.00025 - betas: [0.9, 0.999] - weight_decay: 0.000125 -# weight_decay: 0.00005 # Faster convergence (optional) - - -epochs: 24 # Early stop -train_dataloader: - dataset: - transforms: - policy: - epoch: 500 - collate_fn: - stop_epoch: 500 - base_size_repeat: 4 - -checkpoint_freq: 1 -print_freq: 1000 diff --git a/vlm/detector/D-FINE.backup/configs/dfine/objects365/dfine_hgnetv2_m_obj2coco.yml b/vlm/detector/D-FINE.backup/configs/dfine/objects365/dfine_hgnetv2_m_obj2coco.yml deleted file mode 100644 index fd77cca..0000000 --- a/vlm/detector/D-FINE.backup/configs/dfine/objects365/dfine_hgnetv2_m_obj2coco.yml +++ /dev/null @@ -1,65 +0,0 @@ -__include__: [ - '../../dataset/coco_detection.yml', - '../../runtime.yml', - '../include/dataloader.yml', - '../include/optimizer.yml', - '../include/dfine_hgnetv2.yml', -] - -output_dir: ./output/dfine_hgnetv2_m_obj2coco - - -DFINE: - backbone: HGNetv2 - -HGNetv2: - name: 'B2' - return_idx: [1, 2, 3] - freeze_at: -1 - freeze_norm: False - use_lab: True - -DFINETransformer: - num_layers: 4 # 5 6 - eval_idx: -1 # -2 -3 - -HybridEncoder: - in_channels: [384, 768, 1536] - hidden_dim: 256 - depth_mult: 0.67 - -optimizer: - type: AdamW - params: - - - params: '^(?=.*backbone)(?!.*norm|bn).*$' - lr: 0.000025 - - - params: '^(?=.*backbone)(?=.*norm|bn).*$' - lr: 0.000025 - weight_decay: 0. - - - params: '^(?=.*(?:encoder|decoder))(?=.*(?:norm|bn|bias)).*$' - weight_decay: 0. - - lr: 0.00025 - betas: [0.9, 0.999] - weight_decay: 0.000125 - - -epochs: 56 # Early stop -train_dataloader: - dataset: - transforms: - policy: - epoch: 48 - collate_fn: - stop_epoch: 48 - ema_restart_decay: 0.9999 - base_size_repeat: 6 - -ema: - warmups: 0 - -lr_warmup_scheduler: - warmup_duration: 0 diff --git a/vlm/detector/D-FINE.backup/configs/dfine/objects365/dfine_hgnetv2_m_obj365.yml b/vlm/detector/D-FINE.backup/configs/dfine/objects365/dfine_hgnetv2_m_obj365.yml deleted file mode 100644 index 108d4e6..0000000 --- a/vlm/detector/D-FINE.backup/configs/dfine/objects365/dfine_hgnetv2_m_obj365.yml +++ /dev/null @@ -1,62 +0,0 @@ -__include__: [ - '../../dataset/obj365_detection.yml', - '../../runtime.yml', - '../include/dataloader.yml', - '../include/optimizer.yml', - '../include/dfine_hgnetv2.yml', -] - -output_dir: .output/dfine_hgnetv2_s_obj365 - - -DFINE: - backbone: HGNetv2 - -HGNetv2: - name: 'B2' - return_idx: [1, 2, 3] - freeze_at: -1 - freeze_norm: False - use_lab: True - -DFINETransformer: - num_layers: 4 # 5 6 - eval_idx: -1 # -2 -3 - -HybridEncoder: - in_channels: [384, 768, 1536] - hidden_dim: 256 - depth_mult: 0.67 - -optimizer: - type: AdamW - params: - - - params: '^(?=.*backbone)(?!.*norm|bn).*$' - lr: 0.000025 - - - params: '^(?=.*backbone)(?=.*norm|bn).*$' - lr: 0.000025 - weight_decay: 0. - - - params: '^(?=.*(?:encoder|decoder))(?=.*(?:norm|bn|bias)).*$' - weight_decay: 0. - - lr: 0.00025 - betas: [0.9, 0.999] - weight_decay: 0.000125 -# weight_decay: 0.00005 # Faster convergence (optional) - - -epochs: 36 # Early stop -train_dataloader: - dataset: - transforms: - policy: - epoch: 500 - collate_fn: - stop_epoch: 500 - base_size_repeat: 6 - -checkpoint_freq: 1 -print_freq: 1000 diff --git a/vlm/detector/D-FINE.backup/configs/dfine/objects365/dfine_hgnetv2_n_obj2coco.yml b/vlm/detector/D-FINE.backup/configs/dfine/objects365/dfine_hgnetv2_n_obj2coco.yml deleted file mode 100644 index 6febf65..0000000 --- a/vlm/detector/D-FINE.backup/configs/dfine/objects365/dfine_hgnetv2_n_obj2coco.yml +++ /dev/null @@ -1,88 +0,0 @@ -__include__: [ - '../../dataset/coco_detection.yml', - '../../runtime.yml', - '../include/dataloader.yml', - '../include/optimizer.yml', - '../include/dfine_hgnetv2.yml', -] - -output_dir: ./output/dfine_hgnetv2_n_obj2coco - - -DFINE: - backbone: HGNetv2 - -HGNetv2: - name: 'B0' - return_idx: [2, 3] - freeze_at: -1 - freeze_norm: False - use_lab: True - - -HybridEncoder: - in_channels: [512, 1024] - feat_strides: [16, 32] - - # intra - hidden_dim: 128 - use_encoder_idx: [1] - dim_feedforward: 512 - - # cross - expansion: 0.34 - depth_mult: 0.5 - - -DFINETransformer: - feat_channels: [128, 128] - feat_strides: [16, 32] - hidden_dim: 128 - dim_feedforward: 512 - num_levels: 2 - - num_layers: 3 - eval_idx: -1 - - num_points: [6, 6] - -optimizer: - type: AdamW - params: - - - params: '^(?=.*backbone)(?!.*norm|bn).*$' - lr: 0.0004 - - - params: '^(?=.*backbone)(?=.*norm|bn).*$' - lr: 0.0004 - weight_decay: 0. - - - params: '^(?=.*(?:encoder|decoder))(?=.*(?:norm|bn|bias)).*$' - weight_decay: 0. - - lr: 0.0008 - betas: [0.9, 0.999] - weight_decay: 0.0001 - - - -epochs: 64 # Early stop -train_dataloader: - total_batch_size: 128 - dataset: - transforms: - policy: - epoch: 56 - collate_fn: - stop_epoch: 56 - ema_restart_decay: 0.9999 - base_size_repeat: ~ - -ema: - warmups: 0 - -lr_warmup_scheduler: - warmup_duration: 0 - -val_dataloader: - total_batch_size: 256 diff --git a/vlm/detector/D-FINE.backup/configs/dfine/objects365/dfine_hgnetv2_n_obj365.yml b/vlm/detector/D-FINE.backup/configs/dfine/objects365/dfine_hgnetv2_n_obj365.yml deleted file mode 100644 index b5f148a..0000000 --- a/vlm/detector/D-FINE.backup/configs/dfine/objects365/dfine_hgnetv2_n_obj365.yml +++ /dev/null @@ -1,84 +0,0 @@ -__include__: [ - '../../dataset/obj365_detection.yml', - '../../runtime.yml', - '../include/dataloader.yml', - '../include/optimizer.yml', - '../include/dfine_hgnetv2.yml', -] - -output_dir: ./output/dfine_hgnetv2_n_obj365 - - -DFINE: - backbone: HGNetv2 - -HGNetv2: - name: 'B0' - return_idx: [2, 3] - freeze_at: -1 - freeze_norm: False - use_lab: True - - -HybridEncoder: - in_channels: [512, 1024] - feat_strides: [16, 32] - - # intra - hidden_dim: 128 - use_encoder_idx: [1] - dim_feedforward: 512 - - # cross - expansion: 0.34 - depth_mult: 0.5 - - -DFINETransformer: - feat_channels: [128, 128] - feat_strides: [16, 32] - hidden_dim: 128 - dim_feedforward: 512 - num_levels: 2 - - num_layers: 3 - eval_idx: -1 - - num_points: [6, 6] - -optimizer: - type: AdamW - params: - - - params: '^(?=.*backbone)(?!.*norm|bn).*$' - lr: 0.0004 - - - params: '^(?=.*backbone)(?=.*norm|bn).*$' - lr: 0.0004 - weight_decay: 0. - - - params: '^(?=.*(?:encoder|decoder))(?=.*(?:norm|bn|bias)).*$' - weight_decay: 0. - - lr: 0.0008 - betas: [0.9, 0.999] - weight_decay: 0.0001 - - - -epochs: 48 # Early stop -train_dataloader: - total_batch_size: 128 - dataset: - transforms: - policy: - epoch: 500 - collate_fn: - stop_epoch: 500 - base_size_repeat: ~ - -checkpoint_freq: 1 -print_freq: 500 - -val_dataloader: - total_batch_size: 256 diff --git a/vlm/detector/D-FINE.backup/configs/dfine/objects365/dfine_hgnetv2_s_obj2coco.yml b/vlm/detector/D-FINE.backup/configs/dfine/objects365/dfine_hgnetv2_s_obj2coco.yml deleted file mode 100644 index 9fa01e2..0000000 --- a/vlm/detector/D-FINE.backup/configs/dfine/objects365/dfine_hgnetv2_s_obj2coco.yml +++ /dev/null @@ -1,66 +0,0 @@ -__include__: [ - '../../dataset/coco_detection.yml', - '../../runtime.yml', - '../include/dataloader.yml', - '../include/optimizer.yml', - '../include/dfine_hgnetv2.yml', -] - -output_dir: ./output/dfine_hgnetv2_s_obj2coco - - -DFINE: - backbone: HGNetv2 - -HGNetv2: - name: 'B0' - return_idx: [1, 2, 3] - freeze_at: -1 - freeze_norm: False - use_lab: True - -DFINETransformer: - num_layers: 3 # 4 5 6 - eval_idx: -1 # -2 -3 -4 - -HybridEncoder: - in_channels: [256, 512, 1024] - hidden_dim: 256 - depth_mult: 0.34 - expansion: 0.5 - -optimizer: - type: AdamW - params: - - - params: '^(?=.*backbone)(?!.*norm|bn).*$' - lr: 0.000125 - - - params: '^(?=.*backbone)(?=.*norm|bn).*$' - lr: 0.000125 - weight_decay: 0. - - - params: '^(?=.*(?:encoder|decoder))(?=.*(?:norm|bn|bias)).*$' - weight_decay: 0. - - lr: 0.00025 - betas: [0.9, 0.999] - weight_decay: 0.000125 - - -epochs: 64 # Early stop -train_dataloader: - dataset: - transforms: - policy: - epoch: 56 - collate_fn: - stop_epoch: 56 - ema_restart_decay: 0.9999 - base_size_repeat: 10 - -ema: - warmups: 0 - -lr_warmup_scheduler: - warmup_duration: 0 diff --git a/vlm/detector/D-FINE.backup/configs/dfine/objects365/dfine_hgnetv2_s_obj365.yml b/vlm/detector/D-FINE.backup/configs/dfine/objects365/dfine_hgnetv2_s_obj365.yml deleted file mode 100644 index 1e98a0d..0000000 --- a/vlm/detector/D-FINE.backup/configs/dfine/objects365/dfine_hgnetv2_s_obj365.yml +++ /dev/null @@ -1,63 +0,0 @@ -__include__: [ - '../../dataset/obj365_detection.yml', - '../../runtime.yml', - '../include/dataloader.yml', - '../include/optimizer.yml', - '../include/dfine_hgnetv2.yml', -] - -output_dir: ./output/dfine_hgnetv2_s_obj365 - - -DFINE: - backbone: HGNetv2 - -HGNetv2: - name: 'B0' - return_idx: [1, 2, 3] - freeze_at: -1 - freeze_norm: False - use_lab: True - -DFINETransformer: - num_layers: 3 # 4 5 6 - eval_idx: -1 # -2 -3 -4 - -HybridEncoder: - in_channels: [256, 512, 1024] - hidden_dim: 256 - depth_mult: 0.34 - expansion: 0.5 - -optimizer: - type: AdamW - params: - - - params: '^(?=.*backbone)(?!.*norm|bn).*$' - lr: 0.000125 - - - params: '^(?=.*backbone)(?=.*norm|bn).*$' - lr: 0.000125 - weight_decay: 0. - - - params: '^(?=.*(?:encoder|decoder))(?=.*(?:norm|bn|bias)).*$' - weight_decay: 0. - - lr: 0.00025 - betas: [0.9, 0.999] - weight_decay: 0.000125 -# weight_decay: 0.00005 # Faster convergence (optional) - - -epochs: 36 # Early stop -train_dataloader: - dataset: - transforms: - policy: - epoch: 500 - collate_fn: - stop_epoch: 500 - base_size_repeat: 20 - -checkpoint_freq: 1 -print_freq: 1000 diff --git a/vlm/detector/D-FINE.backup/configs/dfine/objects365/dfine_hgnetv2_x_obj2coco.yml b/vlm/detector/D-FINE.backup/configs/dfine/objects365/dfine_hgnetv2_x_obj2coco.yml deleted file mode 100644 index 738c75d..0000000 --- a/vlm/detector/D-FINE.backup/configs/dfine/objects365/dfine_hgnetv2_x_obj2coco.yml +++ /dev/null @@ -1,61 +0,0 @@ -__include__: [ - '../../dataset/coco_detection.yml', - '../../runtime.yml', - '../include/dataloader.yml', - '../include/optimizer.yml', - '../include/dfine_hgnetv2.yml', -] - -output_dir: ./output/dfine_hgnetv2_x_obj2coco - - -DFINE: - backbone: HGNetv2 - -HGNetv2: - name: 'B5' - return_idx: [1, 2, 3] - freeze_stem_only: True - freeze_at: 0 - freeze_norm: True - -HybridEncoder: - # intra - hidden_dim: 384 - dim_feedforward: 2048 - -DFINETransformer: - feat_channels: [384, 384, 384] - reg_scale: 8 - -optimizer: - type: AdamW - params: - - - params: '^(?=.*backbone)(?!.*norm|bn).*$' - lr: 0.0000025 - - - params: '^(?=.*(?:encoder|decoder))(?=.*(?:norm|bn)).*$' - weight_decay: 0. - - lr: 0.00025 - betas: [0.9, 0.999] - weight_decay: 0.000125 - - -epochs: 36 # Early stop -train_dataloader: - dataset: - transforms: - policy: - epoch: 30 - collate_fn: - stop_epoch: 30 - ema_restart_decay: 0.9999 - base_size_repeat: 3 - -ema: - warmups: 0 - -lr_warmup_scheduler: - warmup_duration: 0 diff --git a/vlm/detector/D-FINE.backup/configs/dfine/objects365/dfine_hgnetv2_x_obj365.yml b/vlm/detector/D-FINE.backup/configs/dfine/objects365/dfine_hgnetv2_x_obj365.yml deleted file mode 100644 index 0449d72..0000000 --- a/vlm/detector/D-FINE.backup/configs/dfine/objects365/dfine_hgnetv2_x_obj365.yml +++ /dev/null @@ -1,58 +0,0 @@ -__include__: [ - '../../dataset/obj365_detection.yml', - '../../runtime.yml', - '../include/dataloader.yml', - '../include/optimizer.yml', - '../include/dfine_hgnetv2.yml', -] - -output_dir: ./output/dfine_hgnetv2_x_obj365 - - -DFINE: - backbone: HGNetv2 - -HGNetv2: - name: 'B5' - return_idx: [1, 2, 3] - freeze_stem_only: True - freeze_at: 0 - freeze_norm: True - -HybridEncoder: - # intra - hidden_dim: 384 - dim_feedforward: 2048 - -DFINETransformer: - feat_channels: [384, 384, 384] - reg_scale: 8 - -optimizer: - type: AdamW - params: - - - params: '^(?=.*backbone)(?!.*norm|bn).*$' - lr: 0.0000025 - - - params: '^(?=.*(?:encoder|decoder))(?=.*(?:norm|bn)).*$' - weight_decay: 0. - - lr: 0.00025 - betas: [0.9, 0.999] - weight_decay: 0.000125 -# weight_decay: 0.00005 # Faster convergence (optional) - - -epochs: 24 # Early stop -train_dataloader: - dataset: - transforms: - policy: - epoch: 500 - collate_fn: - stop_epoch: 500 - base_size_repeat: 3 - -checkpoint_freq: 1 -print_freq: 1000 diff --git a/vlm/detector/D-FINE.backup/configs/runtime.yml b/vlm/detector/D-FINE.backup/configs/runtime.yml deleted file mode 100644 index b99af4a..0000000 --- a/vlm/detector/D-FINE.backup/configs/runtime.yml +++ /dev/null @@ -1,24 +0,0 @@ -print_freq: 100 -output_dir: './logs' -checkpoint_freq: 12 - - -sync_bn: True -find_unused_parameters: False - - -use_amp: False -scaler: - type: GradScaler - enabled: True - - -use_ema: False -ema: - type: ModelEMA - decay: 0.9999 - warmups: 1000 - -use_wandb: False -project_name: D-FINE # for wandb -exp_name: baseline # wandb experiment name diff --git a/vlm/detector/D-FINE.backup/reference/convert_weight.py b/vlm/detector/D-FINE.backup/reference/convert_weight.py deleted file mode 100644 index 6dd3207..0000000 --- a/vlm/detector/D-FINE.backup/reference/convert_weight.py +++ /dev/null @@ -1,30 +0,0 @@ -import argparse -import os - -import torch - - -def save_only_ema_weights(checkpoint_file): - """Extract and save only the EMA weights.""" - checkpoint = torch.load(checkpoint_file, map_location="cpu") - - weights = {} - if "ema" in checkpoint: - weights["model"] = checkpoint["ema"]["module"] - else: - raise ValueError("The checkpoint does not contain 'ema'.") - - dir_name, base_name = os.path.split(checkpoint_file) - name, ext = os.path.splitext(base_name) - output_file = os.path.join(dir_name, f"{name}_converted{ext}") - - torch.save(weights, output_file) - print(f"EMA weights saved to {output_file}") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Extract and save only EMA weights.") - parser.add_argument("checkpoint_file", type=str, help="Path to the input checkpoint file.") - - args = parser.parse_args() - save_only_ema_weights(args.checkpoint_file) diff --git a/vlm/detector/D-FINE.backup/reference/safe_training.sh b/vlm/detector/D-FINE.backup/reference/safe_training.sh deleted file mode 100644 index d3c752a..0000000 --- a/vlm/detector/D-FINE.backup/reference/safe_training.sh +++ /dev/null @@ -1,97 +0,0 @@ -#!/bin/bash - -# Function to display the menu for selecting model size -select_model_size() { - echo "Select model size:" - select size in s m l x; do - case $size in - s|m|l|x) - echo "You selected model size: $size" - MODEL_SIZE=$size - break - ;; - *) - echo "Invalid selection. Please try again." - ;; - esac - done -} - -# Function to display the menu for selecting task -select_task() { - echo "Select task:" - select task in obj365 obj2coco coco; do - case $task in - obj365|obj2coco|coco) - echo "You selected task: $task" - TASK=$task - break - ;; - *) - echo "Invalid selection. Please try again." - ;; - esac - done -} - -# Function to ask if the user wants to save logs to a txt file -ask_save_logs() { - while true; do - read -p "Do you want to save logs to a txt file? (y/n): " yn - case $yn in - [Yy]* ) - SAVE_LOGS=true - break - ;; - [Nn]* ) - SAVE_LOGS=false - break - ;; - * ) echo "Please answer yes or no.";; - esac - done -} - -# Call the functions to let the user select -select_model_size -select_task -ask_save_logs - -# Set config file and output directory based on selection -if [ "$TASK" = "coco" ]; then - CONFIG_FILE="configs/dfine/dfine_hgnetv2_${MODEL_SIZE}_${TASK}.yml" -else - CONFIG_FILE="configs/dfine/objects365/dfine_hgnetv2_${MODEL_SIZE}_${TASK}.yml" -fi - -OUTPUT_DIR="output/${MODEL_SIZE}_${TASK}" - -# Construct the training command -TRAIN_CMD="CUDA_VISIBLE_DEVICES=0,1,2,3 torchrun --master_port=7777 --nproc_per_node=4 train.py -c $CONFIG_FILE --use-amp --seed=0 --output-dir $OUTPUT_DIR" - -# Append log redirection if SAVE_LOGS is true -if [ "$SAVE_LOGS" = true ]; then - LOG_FILE="${MODEL_SIZE}_${TASK}.txt" - TRAIN_CMD="$TRAIN_CMD &> \"$LOG_FILE\" 2>&1 &" -else - TRAIN_CMD="$TRAIN_CMD &" -fi - -# Run the training command -eval $TRAIN_CMD -if [ $? -ne 0 ]; then - echo "First training failed, restarting with resume option..." - while true; do - RESUME_CMD="CUDA_VISIBLE_DEVICES=0,1,2,3 torchrun --master_port=7777 --nproc_per_node=4 train.py -c $CONFIG_FILE --use-amp --seed=0 --output-dir $OUTPUT_DIR -r ${OUTPUT_DIR}/last.pth" - if [ "$SAVE_LOGS" = true ]; then - LOG_FILE="${MODEL_SIZE}_${TASK}_2.txt" - RESUME_CMD="$RESUME_CMD &> \"$LOG_FILE\" 2>&1 &" - else - RESUME_CMD="$RESUME_CMD &" - fi - eval $RESUME_CMD - if [ $? -eq 0 ]; then - break - fi - done -fi diff --git a/vlm/detector/D-FINE.backup/requirements.txt b/vlm/detector/D-FINE.backup/requirements.txt deleted file mode 100644 index aadfb81..0000000 --- a/vlm/detector/D-FINE.backup/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -torch>=2.0.1 -torchvision>=0.15.2 -faster-coco-eval>=1.6.6 -PyYAML -tensorboard -scipy -calflops -transformers -loguru diff --git a/vlm/detector/D-FINE.backup/src/__init__.py b/vlm/detector/D-FINE.backup/src/__init__.py deleted file mode 100644 index 15e3d5a..0000000 --- a/vlm/detector/D-FINE.backup/src/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -""" -Copyright (c) 2024 The D-FINE Authors. All Rights Reserved. -""" - -# for register purpose -from . import data, nn, optim, zoo diff --git a/vlm/detector/D-FINE.backup/src/core/__init__.py b/vlm/detector/D-FINE.backup/src/core/__init__.py deleted file mode 100644 index a938d56..0000000 --- a/vlm/detector/D-FINE.backup/src/core/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -""" -Copied from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright(c) 2023 lyuwenyu. All Rights Reserved. -""" - -from ._config import BaseConfig -from .workspace import GLOBAL_CONFIG, create, register -from .yaml_config import YAMLConfig -from .yaml_utils import * diff --git a/vlm/detector/D-FINE.backup/src/core/_config.py b/vlm/detector/D-FINE.backup/src/core/_config.py deleted file mode 100644 index 3bef075..0000000 --- a/vlm/detector/D-FINE.backup/src/core/_config.py +++ /dev/null @@ -1,299 +0,0 @@ -""" -Copied from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright(c) 2023 lyuwenyu. All Rights Reserved. -""" - -from pathlib import Path -from typing import Callable, Dict, List - -import torch -import torch.nn as nn -from torch.cuda.amp.grad_scaler import GradScaler -from torch.optim import Optimizer -from torch.optim.lr_scheduler import LRScheduler -from torch.utils.data import DataLoader, Dataset -from torch.utils.tensorboard import SummaryWriter - -__all__ = [ - "BaseConfig", -] - - -class BaseConfig(object): - # TODO property - - def __init__(self) -> None: - super().__init__() - - self.task: str = None - - # instance / function - self._model: nn.Module = None - self._postprocessor: nn.Module = None - self._criterion: nn.Module = None - self._optimizer: Optimizer = None - self._lr_scheduler: LRScheduler = None - self._lr_warmup_scheduler: LRScheduler = None - self._train_dataloader: DataLoader = None - self._val_dataloader: DataLoader = None - self._ema: nn.Module = None - self._scaler: GradScaler = None - self._train_dataset: Dataset = None - self._val_dataset: Dataset = None - self._collate_fn: Callable = None - self._evaluator: Callable[[nn.Module, DataLoader, str],] = None - self._writer: SummaryWriter = None - - # dataset - self.num_workers: int = 0 - self.batch_size: int = None - self._train_batch_size: int = None - self._val_batch_size: int = None - self._train_shuffle: bool = None - self._val_shuffle: bool = None - - # runtime - self.resume: str = None - self.tuning: str = None - - self.epochs: int = None - self.last_epoch: int = -1 - - self.use_amp: bool = False - self.use_ema: bool = False - self.ema_decay: float = 0.9999 - self.ema_warmups: int = 2000 - self.sync_bn: bool = False - self.clip_max_norm: float = 0.0 - self.find_unused_parameters: bool = None - - self.seed: int = None - self.print_freq: int = None - self.checkpoint_freq: int = 1 - self.output_dir: str = None - self.summary_dir: str = None - self.device: str = "" - - @property - def model(self) -> nn.Module: - return self._model - - @model.setter - def model(self, m): - assert isinstance(m, nn.Module), f"{type(m)} != nn.Module, please check your model class" - self._model = m - - @property - def postprocessor(self) -> nn.Module: - return self._postprocessor - - @postprocessor.setter - def postprocessor(self, m): - assert isinstance(m, nn.Module), f"{type(m)} != nn.Module, please check your model class" - self._postprocessor = m - - @property - def criterion(self) -> nn.Module: - return self._criterion - - @criterion.setter - def criterion(self, m): - assert isinstance(m, nn.Module), f"{type(m)} != nn.Module, please check your model class" - self._criterion = m - - @property - def optimizer(self) -> Optimizer: - return self._optimizer - - @optimizer.setter - def optimizer(self, m): - assert isinstance( - m, Optimizer - ), f"{type(m)} != optim.Optimizer, please check your model class" - self._optimizer = m - - @property - def lr_scheduler(self) -> LRScheduler: - return self._lr_scheduler - - @lr_scheduler.setter - def lr_scheduler(self, m): - assert isinstance( - m, LRScheduler - ), f"{type(m)} != LRScheduler, please check your model class" - self._lr_scheduler = m - - @property - def lr_warmup_scheduler(self) -> LRScheduler: - return self._lr_warmup_scheduler - - @lr_warmup_scheduler.setter - def lr_warmup_scheduler(self, m): - self._lr_warmup_scheduler = m - - @property - def train_dataloader(self) -> DataLoader: - if self._train_dataloader is None and self.train_dataset is not None: - loader = DataLoader( - self.train_dataset, - batch_size=self.train_batch_size, - num_workers=self.num_workers, - collate_fn=self.collate_fn, - shuffle=self.train_shuffle, - ) - loader.shuffle = self.train_shuffle - self._train_dataloader = loader - - return self._train_dataloader - - @train_dataloader.setter - def train_dataloader(self, loader): - self._train_dataloader = loader - - @property - def val_dataloader(self) -> DataLoader: - if self._val_dataloader is None and self.val_dataset is not None: - loader = DataLoader( - self.val_dataset, - batch_size=self.val_batch_size, - num_workers=self.num_workers, - drop_last=False, - collate_fn=self.collate_fn, - shuffle=self.val_shuffle, - persistent_workers=True, - ) - loader.shuffle = self.val_shuffle - self._val_dataloader = loader - - return self._val_dataloader - - @val_dataloader.setter - def val_dataloader(self, loader): - self._val_dataloader = loader - - @property - def ema(self) -> nn.Module: - if self._ema is None and self.use_ema and self.model is not None: - from ..optim import ModelEMA - - self._ema = ModelEMA(self.model, self.ema_decay, self.ema_warmups) - return self._ema - - @ema.setter - def ema(self, obj): - self._ema = obj - - @property - def scaler(self) -> GradScaler: - if self._scaler is None and self.use_amp and torch.cuda.is_available(): - self._scaler = GradScaler() - return self._scaler - - @scaler.setter - def scaler(self, obj: GradScaler): - self._scaler = obj - - @property - def val_shuffle(self) -> bool: - if self._val_shuffle is None: - print("warning: set default val_shuffle=False") - return False - return self._val_shuffle - - @val_shuffle.setter - def val_shuffle(self, shuffle): - assert isinstance(shuffle, bool), "shuffle must be bool" - self._val_shuffle = shuffle - - @property - def train_shuffle(self) -> bool: - if self._train_shuffle is None: - print("warning: set default train_shuffle=True") - return True - return self._train_shuffle - - @train_shuffle.setter - def train_shuffle(self, shuffle): - assert isinstance(shuffle, bool), "shuffle must be bool" - self._train_shuffle = shuffle - - @property - def train_batch_size(self) -> int: - if self._train_batch_size is None and isinstance(self.batch_size, int): - print(f"warning: set train_batch_size=batch_size={self.batch_size}") - return self.batch_size - return self._train_batch_size - - @train_batch_size.setter - def train_batch_size(self, batch_size): - assert isinstance(batch_size, int), "batch_size must be int" - self._train_batch_size = batch_size - - @property - def val_batch_size(self) -> int: - if self._val_batch_size is None: - print(f"warning: set val_batch_size=batch_size={self.batch_size}") - return self.batch_size - return self._val_batch_size - - @val_batch_size.setter - def val_batch_size(self, batch_size): - assert isinstance(batch_size, int), "batch_size must be int" - self._val_batch_size = batch_size - - @property - def train_dataset(self) -> Dataset: - return self._train_dataset - - @train_dataset.setter - def train_dataset(self, dataset): - assert isinstance(dataset, Dataset), f"{type(dataset)} must be Dataset" - self._train_dataset = dataset - - @property - def val_dataset(self) -> Dataset: - return self._val_dataset - - @val_dataset.setter - def val_dataset(self, dataset): - assert isinstance(dataset, Dataset), f"{type(dataset)} must be Dataset" - self._val_dataset = dataset - - @property - def collate_fn(self) -> Callable: - return self._collate_fn - - @collate_fn.setter - def collate_fn(self, fn): - assert isinstance(fn, Callable), f"{type(fn)} must be Callable" - self._collate_fn = fn - - @property - def evaluator(self) -> Callable: - return self._evaluator - - @evaluator.setter - def evaluator(self, fn): - assert isinstance(fn, Callable), f"{type(fn)} must be Callable" - self._evaluator = fn - - @property - def writer(self) -> SummaryWriter: - if self._writer is None: - if self.summary_dir: - self._writer = SummaryWriter(self.summary_dir) - elif self.output_dir: - self._writer = SummaryWriter(Path(self.output_dir) / "summary") - return self._writer - - @writer.setter - def writer(self, m): - assert isinstance(m, SummaryWriter), f"{type(m)} must be SummaryWriter" - self._writer = m - - def __repr__(self): - s = "" - for k, v in self.__dict__.items(): - if not k.startswith("_"): - s += f"{k}: {v}\n" - return s diff --git a/vlm/detector/D-FINE.backup/src/core/workspace.py b/vlm/detector/D-FINE.backup/src/core/workspace.py deleted file mode 100644 index b99e317..0000000 --- a/vlm/detector/D-FINE.backup/src/core/workspace.py +++ /dev/null @@ -1,178 +0,0 @@ -""" -Copied from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright(c) 2023 lyuwenyu. All Rights Reserved. -""" - -import functools -import importlib -import inspect -from collections import defaultdict -from typing import Any, Dict, List, Optional - -GLOBAL_CONFIG = defaultdict(dict) - - -def register(dct: Any = GLOBAL_CONFIG, name=None, force=False): - """ - dct: - if dct is Dict, register foo into dct as key-value pair - if dct is Clas, register as modules attibute - force - whether force register. - """ - - def decorator(foo): - register_name = foo.__name__ if name is None else name - if not force: - if inspect.isclass(dct): - assert not hasattr(dct, foo.__name__), f"module {dct.__name__} has {foo.__name__}" - else: - assert foo.__name__ not in dct, f"{foo.__name__} has been already registered" - - if inspect.isfunction(foo): - - @functools.wraps(foo) - def wrap_func(*args, **kwargs): - return foo(*args, **kwargs) - - if isinstance(dct, dict): - dct[foo.__name__] = wrap_func - elif inspect.isclass(dct): - setattr(dct, foo.__name__, wrap_func) - else: - raise AttributeError("") - return wrap_func - - elif inspect.isclass(foo): - dct[register_name] = extract_schema(foo) - - else: - raise ValueError(f"Do not support {type(foo)} register") - - return foo - - return decorator - - -def extract_schema(module: type): - """ - Args: - module (type), - Return: - Dict, - """ - argspec = inspect.getfullargspec(module.__init__) - arg_names = [arg for arg in argspec.args if arg != "self"] - num_defualts = len(argspec.defaults) if argspec.defaults is not None else 0 - num_requires = len(arg_names) - num_defualts - - schame = dict() - schame["_name"] = module.__name__ - schame["_pymodule"] = importlib.import_module(module.__module__) - schame["_inject"] = getattr(module, "__inject__", []) - schame["_share"] = getattr(module, "__share__", []) - schame["_kwargs"] = {} - for i, name in enumerate(arg_names): - if name in schame["_share"]: - assert i >= num_requires, "share config must have default value." - value = argspec.defaults[i - num_requires] - - elif i >= num_requires: - value = argspec.defaults[i - num_requires] - - else: - value = None - - schame[name] = value - schame["_kwargs"][name] = value - - return schame - - -def create(type_or_name, global_cfg=GLOBAL_CONFIG, **kwargs): - """ """ - assert type(type_or_name) in (type, str), "create should be modules or name." - - name = type_or_name if isinstance(type_or_name, str) else type_or_name.__name__ - - if name in global_cfg: - if hasattr(global_cfg[name], "__dict__"): - return global_cfg[name] - else: - raise ValueError("The module {} is not registered".format(name)) - - cfg = global_cfg[name] - - if isinstance(cfg, dict) and "type" in cfg: - _cfg: dict = global_cfg[cfg["type"]] - # clean args - _keys = [k for k in _cfg.keys() if not k.startswith("_")] - for _arg in _keys: - del _cfg[_arg] - _cfg.update(_cfg["_kwargs"]) # restore default args - _cfg.update(cfg) # load config args - _cfg.update(kwargs) # TODO recive extra kwargs - name = _cfg.pop("type") # pop extra key `type` (from cfg) - - return create(name, global_cfg) - - module = getattr(cfg["_pymodule"], name) - module_kwargs = {} - module_kwargs.update(cfg) - - # shared var - for k in cfg["_share"]: - if k in global_cfg: - module_kwargs[k] = global_cfg[k] - else: - module_kwargs[k] = cfg[k] - - # inject - for k in cfg["_inject"]: - _k = cfg[k] - - if _k is None: - continue - - if isinstance(_k, str): - if _k not in global_cfg: - raise ValueError(f"Missing inject config of {_k}.") - - _cfg = global_cfg[_k] - - if isinstance(_cfg, dict): - module_kwargs[k] = create(_cfg["_name"], global_cfg) - else: - module_kwargs[k] = _cfg - - elif isinstance(_k, dict): - if "type" not in _k.keys(): - raise ValueError("Missing inject for `type` style.") - - _type = str(_k["type"]) - if _type not in global_cfg: - raise ValueError(f"Missing {_type} in inspect stage.") - - # TODO - _cfg: dict = global_cfg[_type] - # clean args - _keys = [k for k in _cfg.keys() if not k.startswith("_")] - for _arg in _keys: - del _cfg[_arg] - _cfg.update(_cfg["_kwargs"]) # restore default values - _cfg.update(_k) # load config args - name = _cfg.pop("type") # pop extra key (`type` from _k) - module_kwargs[k] = create(name, global_cfg) - - else: - raise ValueError(f"Inject does not support {_k}") - - # TODO hard code - module_kwargs = {k: v for k, v in module_kwargs.items() if not k.startswith("_")} - - # TODO for **kwargs - # extra_args = set(module_kwargs.keys()) - set(arg_names) - # if len(extra_args) > 0: - # raise RuntimeError(f'Error: unknown args {extra_args} for {module}') - - return module(**module_kwargs) diff --git a/vlm/detector/D-FINE.backup/src/core/yaml_config.py b/vlm/detector/D-FINE.backup/src/core/yaml_config.py deleted file mode 100644 index 0f7faf4..0000000 --- a/vlm/detector/D-FINE.backup/src/core/yaml_config.py +++ /dev/null @@ -1,187 +0,0 @@ -""" -Copied from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright(c) 2023 lyuwenyu. All Rights Reserved. -""" - -import copy -import re - -import torch -import torch.nn as nn -import torch.optim as optim -from torch.utils.data import DataLoader - -from ._config import BaseConfig -from .workspace import create -from .yaml_utils import load_config, merge_config, merge_dict - - -class YAMLConfig(BaseConfig): - def __init__(self, cfg_path: str, **kwargs) -> None: - super().__init__() - - cfg = load_config(cfg_path) - cfg = merge_dict(cfg, kwargs) - - self.yaml_cfg = copy.deepcopy(cfg) - - for k in super().__dict__: - if not k.startswith("_") and k in cfg: - self.__dict__[k] = cfg[k] - - @property - def global_cfg(self): - return merge_config(self.yaml_cfg, inplace=False, overwrite=False) - - @property - def model(self) -> torch.nn.Module: - if self._model is None and "model" in self.yaml_cfg: - self._model = create(self.yaml_cfg["model"], self.global_cfg) - return super().model - - @property - def postprocessor(self) -> torch.nn.Module: - if self._postprocessor is None and "postprocessor" in self.yaml_cfg: - self._postprocessor = create(self.yaml_cfg["postprocessor"], self.global_cfg) - return super().postprocessor - - @property - def criterion(self) -> torch.nn.Module: - if self._criterion is None and "criterion" in self.yaml_cfg: - self._criterion = create(self.yaml_cfg["criterion"], self.global_cfg) - return super().criterion - - @property - def optimizer(self) -> optim.Optimizer: - if self._optimizer is None and "optimizer" in self.yaml_cfg: - params = self.get_optim_params(self.yaml_cfg["optimizer"], self.model) - self._optimizer = create("optimizer", self.global_cfg, params=params) - return super().optimizer - - @property - def lr_scheduler(self) -> optim.lr_scheduler.LRScheduler: - if self._lr_scheduler is None and "lr_scheduler" in self.yaml_cfg: - self._lr_scheduler = create("lr_scheduler", self.global_cfg, optimizer=self.optimizer) - print(f"Initial lr: {self._lr_scheduler.get_last_lr()}") - return super().lr_scheduler - - @property - def lr_warmup_scheduler(self) -> optim.lr_scheduler.LRScheduler: - if self._lr_warmup_scheduler is None and "lr_warmup_scheduler" in self.yaml_cfg: - self._lr_warmup_scheduler = create( - "lr_warmup_scheduler", self.global_cfg, lr_scheduler=self.lr_scheduler - ) - return super().lr_warmup_scheduler - - @property - def train_dataloader(self) -> DataLoader: - if self._train_dataloader is None and "train_dataloader" in self.yaml_cfg: - self._train_dataloader = self.build_dataloader("train_dataloader") - return super().train_dataloader - - @property - def val_dataloader(self) -> DataLoader: - if self._val_dataloader is None and "val_dataloader" in self.yaml_cfg: - self._val_dataloader = self.build_dataloader("val_dataloader") - return super().val_dataloader - - @property - def ema(self) -> torch.nn.Module: - if self._ema is None and self.yaml_cfg.get("use_ema", False): - self._ema = create("ema", self.global_cfg, model=self.model) - return super().ema - - @property - def scaler(self): - if self._scaler is None and self.yaml_cfg.get("use_amp", False): - self._scaler = create("scaler", self.global_cfg) - return super().scaler - - @property - def evaluator(self): - if self._evaluator is None and "evaluator" in self.yaml_cfg: - if self.yaml_cfg["evaluator"]["type"] == "CocoEvaluator": - from ..data import get_coco_api_from_dataset - - base_ds = get_coco_api_from_dataset(self.val_dataloader.dataset) - self._evaluator = create("evaluator", self.global_cfg, coco_gt=base_ds) - else: - raise NotImplementedError(f"{self.yaml_cfg['evaluator']['type']}") - return super().evaluator - - @property - def use_wandb(self) -> bool: - return self.yaml_cfg.get("use_wandb", False) - - @staticmethod - def get_optim_params(cfg: dict, model: nn.Module): - """ - E.g.: - ^(?=.*a)(?=.*b).*$ means including a and b - ^(?=.*(?:a|b)).*$ means including a or b - ^(?=.*a)(?!.*b).*$ means including a, but not b - """ - assert "type" in cfg, "" - cfg = copy.deepcopy(cfg) - - if "params" not in cfg: - return model.parameters() - - assert isinstance(cfg["params"], list), "" - - param_groups = [] - visited = [] - for pg in cfg["params"]: - pattern = pg["params"] - params = { - k: v - for k, v in model.named_parameters() - if v.requires_grad and len(re.findall(pattern, k)) > 0 - } - pg["params"] = params.values() - param_groups.append(pg) - visited.extend(list(params.keys())) - # print(params.keys()) - - names = [k for k, v in model.named_parameters() if v.requires_grad] - - if len(visited) < len(names): - unseen = set(names) - set(visited) - params = {k: v for k, v in model.named_parameters() if v.requires_grad and k in unseen} - param_groups.append({"params": params.values()}) - visited.extend(list(params.keys())) - # print(params.keys()) - - assert len(visited) == len(names), "" - - return param_groups - - @staticmethod - def get_rank_batch_size(cfg): - """compute batch size for per rank if total_batch_size is provided.""" - assert ("total_batch_size" in cfg or "batch_size" in cfg) and not ( - "total_batch_size" in cfg and "batch_size" in cfg - ), "`batch_size` or `total_batch_size` should be choosed one" - - total_batch_size = cfg.get("total_batch_size", None) - if total_batch_size is None: - bs = cfg.get("batch_size") - else: - from ..misc import dist_utils - - assert ( - total_batch_size % dist_utils.get_world_size() == 0 - ), "total_batch_size should be divisible by world size" - bs = total_batch_size // dist_utils.get_world_size() - return bs - - def build_dataloader(self, name: str): - bs = self.get_rank_batch_size(self.yaml_cfg[name]) - global_cfg = self.global_cfg - if "total_batch_size" in global_cfg[name]: - # pop unexpected key for dataloader init - _ = global_cfg[name].pop("total_batch_size") - print(f"building {name} with batch_size={bs}...") - loader = create(name, global_cfg, batch_size=bs) - loader.shuffle = self.yaml_cfg[name].get("shuffle", False) - return loader diff --git a/vlm/detector/D-FINE.backup/src/core/yaml_utils.py b/vlm/detector/D-FINE.backup/src/core/yaml_utils.py deleted file mode 100644 index b77e071..0000000 --- a/vlm/detector/D-FINE.backup/src/core/yaml_utils.py +++ /dev/null @@ -1,126 +0,0 @@ -""" -Copied from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright(c) 2023 lyuwenyu. All Rights Reserved. -""" - -import copy -import os -from typing import Any, Dict, List, Optional - -import yaml - -from .workspace import GLOBAL_CONFIG - -__all__ = [ - "load_config", - "merge_config", - "merge_dict", - "parse_cli", -] - - -INCLUDE_KEY = "__include__" - - -def load_config(file_path, cfg=dict()): - """load config""" - _, ext = os.path.splitext(file_path) - assert ext in [".yml", ".yaml"], "only support yaml files" - - with open(file_path) as f: - file_cfg = yaml.load(f, Loader=yaml.Loader) - if file_cfg is None: - return {} - - if INCLUDE_KEY in file_cfg: - base_yamls = list(file_cfg[INCLUDE_KEY]) - for base_yaml in base_yamls: - if base_yaml.startswith("~"): - base_yaml = os.path.expanduser(base_yaml) - - if not base_yaml.startswith("/"): - base_yaml = os.path.join(os.path.dirname(file_path), base_yaml) - - with open(base_yaml) as f: - base_cfg = load_config(base_yaml, cfg) - merge_dict(cfg, base_cfg) - - return merge_dict(cfg, file_cfg) - - -def merge_dict(dct, another_dct, inplace=True) -> Dict: - """merge another_dct into dct""" - - def _merge(dct, another) -> Dict: - for k in another: - if k in dct and isinstance(dct[k], dict) and isinstance(another[k], dict): - _merge(dct[k], another[k]) - else: - dct[k] = another[k] - - return dct - - if not inplace: - dct = copy.deepcopy(dct) - - return _merge(dct, another_dct) - - -def dictify(s: str, v: Any) -> Dict: - if "." not in s: - return {s: v} - key, rest = s.split(".", 1) - return {key: dictify(rest, v)} - - -def parse_cli(nargs: List[str]) -> Dict: - """ - parse command-line arguments - convert `a.c=3 b=10` to `{'a': {'c': 3}, 'b': 10}` - """ - cfg = {} - if nargs is None or len(nargs) == 0: - return cfg - - for s in nargs: - s = s.strip() - k, v = s.split("=", 1) - d = dictify(k, yaml.load(v, Loader=yaml.Loader)) - cfg = merge_dict(cfg, d) - - return cfg - - -def merge_config(cfg, another_cfg=GLOBAL_CONFIG, inplace: bool = False, overwrite: bool = False): - """ - Merge another_cfg into cfg, return the merged config - - Example: - - cfg1 = load_config('./dfine_r18vd_6x_coco.yml') - cfg1 = merge_config(cfg, inplace=True) - - cfg2 = load_config('./dfine_r50vd_6x_coco.yml') - cfg2 = merge_config(cfg2, inplace=True) - - model1 = create(cfg1['model'], cfg1) - model2 = create(cfg2['model'], cfg2) - """ - - def _merge(dct, another): - for k in another: - if k not in dct: - dct[k] = another[k] - - elif isinstance(dct[k], dict) and isinstance(another[k], dict): - _merge(dct[k], another[k]) - - elif overwrite: - dct[k] = another[k] - - return cfg - - if not inplace: - cfg = copy.deepcopy(cfg) - - return _merge(cfg, another_cfg) diff --git a/vlm/detector/D-FINE.backup/src/misc/__init__.py b/vlm/detector/D-FINE.backup/src/misc/__init__.py deleted file mode 100644 index 3d1f955..0000000 --- a/vlm/detector/D-FINE.backup/src/misc/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -""" -Copied from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright(c) 2023 lyuwenyu. All Rights Reserved. -""" - -from .dist_utils import setup_print, setup_seed -from .logger import * -from .profiler_utils import stats -from .visualizer import * diff --git a/vlm/detector/D-FINE.backup/src/misc/box_ops.py b/vlm/detector/D-FINE.backup/src/misc/box_ops.py deleted file mode 100644 index 64dff0d..0000000 --- a/vlm/detector/D-FINE.backup/src/misc/box_ops.py +++ /dev/null @@ -1,106 +0,0 @@ -""" -Copied from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright(c) 2023 lyuwenyu. All Rights Reserved. -""" - -from typing import List, Tuple - -import torch -import torchvision -from torch import Tensor - - -def generalized_box_iou(boxes1: Tensor, boxes2: Tensor) -> Tensor: - assert (boxes1[:, 2:] >= boxes1[:, :2]).all() - assert (boxes2[:, 2:] >= boxes2[:, :2]).all() - return torchvision.ops.generalized_box_iou(boxes1, boxes2) - - -# elementwise -def elementwise_box_iou(boxes1: Tensor, boxes2: Tensor) -> Tensor: - """ - Args: - boxes1, [N, 4] - boxes2, [N, 4] - Returns: - iou, [N, ] - union, [N, ] - """ - area1 = torchvision.ops.box_area(boxes1) # [N, ] - area2 = torchvision.ops.box_area(boxes2) # [N, ] - lt = torch.max(boxes1[:, :2], boxes2[:, :2]) # [N, 2] - rb = torch.min(boxes1[:, 2:], boxes2[:, 2:]) # [N, 2] - wh = (rb - lt).clamp(min=0) # [N, 2] - inter = wh[:, 0] * wh[:, 1] # [N, ] - union = area1 + area2 - inter - iou = inter / union - return iou, union - - -def elementwise_generalized_box_iou(boxes1: Tensor, boxes2: Tensor) -> Tensor: - """ - Args: - boxes1, [N, 4] with [x1, y1, x2, y2] - boxes2, [N, 4] with [x1, y1, x2, y2] - Returns: - giou, [N, ] - """ - assert (boxes1[:, 2:] >= boxes1[:, :2]).all() - assert (boxes2[:, 2:] >= boxes2[:, :2]).all() - iou, union = elementwise_box_iou(boxes1, boxes2) - lt = torch.min(boxes1[:, :2], boxes2[:, :2]) # [N, 2] - rb = torch.max(boxes1[:, 2:], boxes2[:, 2:]) # [N, 2] - wh = (rb - lt).clamp(min=0) # [N, 2] - area = wh[:, 0] * wh[:, 1] - return iou - (area - union) / area - - -def check_point_inside_box(points: Tensor, boxes: Tensor, eps=1e-9) -> Tensor: - """ - Args: - points, [K, 2], (x, y) - boxes, [N, 4], (x1, y1, y2, y2) - Returns: - Tensor (bool), [K, N] - """ - x, y = [p.unsqueeze(-1) for p in points.unbind(-1)] - x1, y1, x2, y2 = [x.unsqueeze(0) for x in boxes.unbind(-1)] - - l = x - x1 - t = y - y1 - r = x2 - x - b = y2 - y - - ltrb = torch.stack([l, t, r, b], dim=-1) - mask = ltrb.min(dim=-1).values > eps - - return mask - - -def point_box_distance(points: Tensor, boxes: Tensor) -> Tensor: - """ - Args: - boxes, [N, 4], (x1, y1, x2, y2) - points, [N, 2], (x, y) - Returns: - Tensor (N, 4), (l, t, r, b) - """ - x1y1, x2y2 = torch.split(boxes, 2, dim=-1) - lt = points - x1y1 - rb = x2y2 - points - return torch.concat([lt, rb], dim=-1) - - -def point_distance_box(points: Tensor, distances: Tensor) -> Tensor: - """ - Args: - points (Tensor), [N, 2], (x, y) - distances (Tensor), [N, 4], (l, t, r, b) - Returns: - boxes (Tensor), (N, 4), (x1, y1, x2, y2) - """ - lt, rb = torch.split(distances, 2, dim=-1) - x1y1 = -lt + points - x2y2 = rb + points - boxes = torch.concat([x1y1, x2y2], dim=-1) - return boxes diff --git a/vlm/detector/D-FINE.backup/src/misc/dist_utils.py b/vlm/detector/D-FINE.backup/src/misc/dist_utils.py deleted file mode 100644 index 959ded6..0000000 --- a/vlm/detector/D-FINE.backup/src/misc/dist_utils.py +++ /dev/null @@ -1,281 +0,0 @@ -""" -reference -- https://github.com/pytorch/vision/blob/main/references/detection/utils.py -- https://github.com/facebookresearch/detr/blob/master/util/misc.py#L406 - -Copyright(c) 2023 lyuwenyu. All Rights Reserved. -""" - -import atexit -import os -import random -import time - -import numpy as np -import torch -import torch.backends.cudnn -import torch.distributed -import torch.nn as nn -from torch.distributed.fsdp import FullyShardedDataParallel as FSDP -from torch.nn.parallel import DataParallel as DP -from torch.nn.parallel import DistributedDataParallel as DDP -from torch.utils.data import DistributedSampler - -# from torch.utils.data.dataloader import DataLoader -from ..data import DataLoader - - -def setup_distributed( - print_rank: int = 0, - print_method: str = "builtin", - seed: int = None, -): - """ - env setup - args: - print_rank, - print_method, (builtin, rich) - seed, - """ - try: - # https://pytorch.org/docs/stable/elastic/run.html - RANK = int(os.getenv("RANK", -1)) - LOCAL_RANK = int(os.getenv("LOCAL_RANK", -1)) - WORLD_SIZE = int(os.getenv("WORLD_SIZE", 1)) - - # torch.distributed.init_process_group(backend=backend, init_method='env://') - torch.distributed.init_process_group(init_method="env://") - torch.distributed.barrier() - - rank = torch.distributed.get_rank() - torch.cuda.set_device(rank) - torch.cuda.empty_cache() - enabled_dist = True - if get_rank() == print_rank: - print("Initialized distributed mode...") - - except Exception: - enabled_dist = False - print("Not init distributed mode.") - - setup_print(get_rank() == print_rank, method=print_method) - if seed is not None: - setup_seed(seed) - - return enabled_dist - - -def setup_print(is_main, method="builtin"): - """This function disables printing when not in master process""" - import builtins as __builtin__ - - if method == "builtin": - builtin_print = __builtin__.print - - elif method == "rich": - import rich - - builtin_print = rich.print - - else: - raise AttributeError("") - - def print(*args, **kwargs): - force = kwargs.pop("force", False) - if is_main or force: - builtin_print(*args, **kwargs) - - __builtin__.print = print - - -def is_dist_available_and_initialized(): - if not torch.distributed.is_available(): - return False - if not torch.distributed.is_initialized(): - return False - return True - - -@atexit.register -def cleanup(): - """cleanup distributed environment""" - if is_dist_available_and_initialized(): - torch.distributed.barrier() - torch.distributed.destroy_process_group() - - -def get_rank(): - if not is_dist_available_and_initialized(): - return 0 - return torch.distributed.get_rank() - - -def get_world_size(): - if not is_dist_available_and_initialized(): - return 1 - return torch.distributed.get_world_size() - - -def is_main_process(): - return get_rank() == 0 - - -def save_on_master(*args, **kwargs): - if is_main_process(): - torch.save(*args, **kwargs) - - -def warp_model( - model: torch.nn.Module, - sync_bn: bool = False, - dist_mode: str = "ddp", - find_unused_parameters: bool = False, - compile: bool = False, - compile_mode: str = "reduce-overhead", - **kwargs, -): - if is_dist_available_and_initialized(): - rank = get_rank() - model = nn.SyncBatchNorm.convert_sync_batchnorm(model) if sync_bn else model - if dist_mode == "dp": - model = DP(model, device_ids=[rank], output_device=rank) - elif dist_mode == "ddp": - model = DDP( - model, - device_ids=[rank], - output_device=rank, - find_unused_parameters=find_unused_parameters, - ) - else: - raise AttributeError("") - - if compile: - model = torch.compile(model, mode=compile_mode) - - return model - - -def de_model(model): - return de_parallel(de_complie(model)) - - -def warp_loader(loader, shuffle=False): - if is_dist_available_and_initialized(): - sampler = DistributedSampler(loader.dataset, shuffle=shuffle) - loader = DataLoader( - loader.dataset, - loader.batch_size, - sampler=sampler, - drop_last=loader.drop_last, - collate_fn=loader.collate_fn, - pin_memory=loader.pin_memory, - num_workers=loader.num_workers, - ) - return loader - - -def is_parallel(model) -> bool: - # Returns True if model is of type DP or DDP - return type(model) in ( - torch.nn.parallel.DataParallel, - torch.nn.parallel.DistributedDataParallel, - ) - - -def de_parallel(model) -> nn.Module: - # De-parallelize a model: returns single-GPU model if model is of type DP or DDP - return model.module if is_parallel(model) else model - - -def reduce_dict(data, avg=True): - """ - Args - data dict: input, {k: v, ...} - avg bool: true - """ - world_size = get_world_size() - if world_size < 2: - return data - - with torch.no_grad(): - keys, values = [], [] - for k in sorted(data.keys()): - keys.append(k) - values.append(data[k]) - - values = torch.stack(values, dim=0) - torch.distributed.all_reduce(values) - - if avg is True: - values /= world_size - - return {k: v for k, v in zip(keys, values)} - - -def all_gather(data): - """ - Run all_gather on arbitrary picklable data (not necessarily tensors) - Args: - data: any picklable object - Returns: - list[data]: list of data gathered from each rank - """ - world_size = get_world_size() - if world_size == 1: - return [data] - data_list = [None] * world_size - torch.distributed.all_gather_object(data_list, data) - return data_list - - -def sync_time(): - """sync_time""" - if torch.cuda.is_available(): - torch.cuda.synchronize() - - return time.time() - - -def setup_seed(seed: int, deterministic=False): - """setup_seed for reproducibility - torch.manual_seed(3407) is all you need. https://arxiv.org/abs/2109.08203 - """ - seed = seed + get_rank() - random.seed(seed) - np.random.seed(seed) - torch.manual_seed(seed) - - if torch.cuda.is_available(): - torch.cuda.manual_seed_all(seed) - - # memory will be large when setting deterministic to True - if torch.backends.cudnn.is_available() and deterministic: - torch.backends.cudnn.deterministic = True - - -# for torch.compile -def check_compile(): - import warnings - - import torch - - gpu_ok = False - if torch.cuda.is_available(): - device_cap = torch.cuda.get_device_capability() - if device_cap in ((7, 0), (8, 0), (9, 0)): - gpu_ok = True - if not gpu_ok: - warnings.warn( - "GPU is not NVIDIA V100, A100, or H100. Speedup numbers may be lower " "than expected." - ) - return gpu_ok - - -def is_compile(model): - import torch._dynamo - - return type(model) in (torch._dynamo.OptimizedModule,) - - -def de_complie(model): - return model._orig_mod if is_compile(model) else model diff --git a/vlm/detector/D-FINE.backup/src/misc/lazy_loader.py b/vlm/detector/D-FINE.backup/src/misc/lazy_loader.py deleted file mode 100644 index 208fb12..0000000 --- a/vlm/detector/D-FINE.backup/src/misc/lazy_loader.py +++ /dev/null @@ -1,70 +0,0 @@ -""" -https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/util/lazy_loader.py -""" - -import importlib -import types - - -class LazyLoader(types.ModuleType): - """Lazily import a module, mainly to avoid pulling in large dependencies. - - `paddle`, and `ffmpeg` are examples of modules that are large and not always - needed, and this allows them to only be loaded when they are used. - """ - - # The lint error here is incorrect. - def __init__(self, local_name, parent_module_globals, name, warning=None): - self._local_name = local_name - self._parent_module_globals = parent_module_globals - self._warning = warning - - # These members allows doctest correctly process this module member without - # triggering self._load(). self._load() mutates parant_module_globals and - # triggers a dict mutated during iteration error from doctest.py. - # - for from_module() - self.__module__ = name.rsplit(".", 1)[0] - # - for is_routine() - self.__wrapped__ = None - - super(LazyLoader, self).__init__(name) - - def _load(self): - """Load the module and insert it into the parent's globals.""" - # Import the target module and insert it into the parent's namespace - module = importlib.import_module(self.__name__) - self._parent_module_globals[self._local_name] = module - - # Emit a warning if one was specified - if self._warning: - # logging.warning(self._warning) - # Make sure to only warn once. - self._warning = None - - # Update this object's dict so that if someone keeps a reference to the - # LazyLoader, lookups are efficient (__getattr__ is only called on lookups - # that fail). - self.__dict__.update(module.__dict__) - - return module - - def __getattr__(self, item): - module = self._load() - return getattr(module, item) - - def __repr__(self): - # Carefully to not trigger _load, since repr may be called in very - # sensitive places. - return f"" - - def __dir__(self): - module = self._load() - return dir(module) - - -# import paddle.nn as nn -# nn = LazyLoader("nn", globals(), "paddle.nn") - -# class M(nn.Layer): -# def __init__(self) -> None: -# super().__init__() diff --git a/vlm/detector/D-FINE.backup/src/misc/logger.py b/vlm/detector/D-FINE.backup/src/misc/logger.py deleted file mode 100644 index 0c1ca18..0000000 --- a/vlm/detector/D-FINE.backup/src/misc/logger.py +++ /dev/null @@ -1,255 +0,0 @@ -""" -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved -https://github.com/facebookresearch/detr/blob/main/util/misc.py -Mostly copy-paste from torchvision references. -""" - -import datetime -import pickle -import time -from collections import defaultdict, deque -from typing import Dict - -import torch -import torch.distributed as tdist - -from .dist_utils import get_world_size, is_dist_available_and_initialized - - -class SmoothedValue(object): - """Track a series of values and provide access to smoothed values over a - window or the global series average. - """ - - def __init__(self, window_size=20, fmt=None): - if fmt is None: - fmt = "{median:.4f} ({global_avg:.4f})" - self.deque = deque(maxlen=window_size) - self.total = 0.0 - self.count = 0 - self.fmt = fmt - - def update(self, value, n=1): - self.deque.append(value) - self.count += n - self.total += value * n - - def synchronize_between_processes(self): - """ - Warning: does not synchronize the deque! - """ - if not is_dist_available_and_initialized(): - return - t = torch.tensor([self.count, self.total], dtype=torch.float64, device="cuda") - tdist.barrier() - tdist.all_reduce(t) - t = t.tolist() - self.count = int(t[0]) - self.total = t[1] - - @property - def median(self): - d = torch.tensor(list(self.deque)) - return d.median().item() - - @property - def avg(self): - d = torch.tensor(list(self.deque), dtype=torch.float32) - return d.mean().item() - - @property - def global_avg(self): - return self.total / self.count - - @property - def max(self): - return max(self.deque) - - @property - def value(self): - return self.deque[-1] - - def __str__(self): - return self.fmt.format( - median=self.median, - avg=self.avg, - global_avg=self.global_avg, - max=self.max, - value=self.value, - ) - - -def all_gather(data): - """ - Run all_gather on arbitrary picklable data (not necessarily tensors) - Args: - data: any picklable object - Returns: - list[data]: list of data gathered from each rank - """ - world_size = get_world_size() - if world_size == 1: - return [data] - - # serialized to a Tensor - buffer = pickle.dumps(data) - storage = torch.ByteStorage.from_buffer(buffer) - tensor = torch.ByteTensor(storage).to("cuda") - - # obtain Tensor size of each rank - local_size = torch.tensor([tensor.numel()], device="cuda") - size_list = [torch.tensor([0], device="cuda") for _ in range(world_size)] - tdist.all_gather(size_list, local_size) - size_list = [int(size.item()) for size in size_list] - max_size = max(size_list) - - # receiving Tensor from all ranks - # we pad the tensor because torch all_gather does not support - # gathering tensors of different shapes - tensor_list = [] - for _ in size_list: - tensor_list.append(torch.empty((max_size,), dtype=torch.uint8, device="cuda")) - if local_size != max_size: - padding = torch.empty(size=(max_size - local_size,), dtype=torch.uint8, device="cuda") - tensor = torch.cat((tensor, padding), dim=0) - tdist.all_gather(tensor_list, tensor) - - data_list = [] - for size, tensor in zip(size_list, tensor_list): - buffer = tensor.cpu().numpy().tobytes()[:size] - data_list.append(pickle.loads(buffer)) - - return data_list - - -def reduce_dict(input_dict, average=True) -> Dict[str, torch.Tensor]: - """ - Args: - input_dict (dict): all the values will be reduced - average (bool): whether to do average or sum - Reduce the values in the dictionary from all processes so that all processes - have the averaged results. Returns a dict with the same fields as - input_dict, after reduction. - """ - world_size = get_world_size() - if world_size < 2: - return input_dict - with torch.no_grad(): - names = [] - values = [] - # sort the keys so that they are consistent across processes - for k in sorted(input_dict.keys()): - names.append(k) - values.append(input_dict[k]) - values = torch.stack(values, dim=0) - tdist.all_reduce(values) - if average: - values /= world_size - reduced_dict = {k: v for k, v in zip(names, values)} - return reduced_dict - - -class MetricLogger(object): - def __init__(self, delimiter="\t"): - self.meters = defaultdict(SmoothedValue) - self.delimiter = delimiter - - def update(self, **kwargs): - for k, v in kwargs.items(): - if isinstance(v, torch.Tensor): - v = v.item() - assert isinstance(v, (float, int)) - self.meters[k].update(v) - - def __getattr__(self, attr): - if attr in self.meters: - return self.meters[attr] - if attr in self.__dict__: - return self.__dict__[attr] - raise AttributeError("'{}' object has no attribute '{}'".format(type(self).__name__, attr)) - - def __str__(self): - loss_str = [] - for name, meter in self.meters.items(): - loss_str.append("{}: {}".format(name, str(meter))) - return self.delimiter.join(loss_str) - - def synchronize_between_processes(self): - for meter in self.meters.values(): - meter.synchronize_between_processes() - - def add_meter(self, name, meter): - self.meters[name] = meter - - def log_every(self, iterable, print_freq, header=None): - i = 0 - if not header: - header = "" - start_time = time.time() - end = time.time() - iter_time = SmoothedValue(fmt="{avg:.4f}") - data_time = SmoothedValue(fmt="{avg:.4f}") - space_fmt = ":" + str(len(str(len(iterable)))) + "d" - if torch.cuda.is_available(): - log_msg = self.delimiter.join( - [ - header, - "[{0" + space_fmt + "}/{1}]", - "eta: {eta}", - "{meters}", - "time: {time}", - "data: {data}", - "max mem: {memory:.0f}", - ] - ) - else: - log_msg = self.delimiter.join( - [ - header, - "[{0" + space_fmt + "}/{1}]", - "eta: {eta}", - "{meters}", - "time: {time}", - "data: {data}", - ] - ) - MB = 1024.0 * 1024.0 - for obj in iterable: - data_time.update(time.time() - end) - yield obj - iter_time.update(time.time() - end) - if i % print_freq == 0 or i == len(iterable) - 1: - eta_seconds = iter_time.global_avg * (len(iterable) - i) - eta_string = str(datetime.timedelta(seconds=int(eta_seconds))) - if torch.cuda.is_available(): - print( - log_msg.format( - i, - len(iterable), - eta=eta_string, - meters=str(self), - time=str(iter_time), - data=str(data_time), - memory=torch.cuda.max_memory_allocated() / MB, - ) - ) - else: - print( - log_msg.format( - i, - len(iterable), - eta=eta_string, - meters=str(self), - time=str(iter_time), - data=str(data_time), - ) - ) - i += 1 - end = time.time() - total_time = time.time() - start_time - total_time_str = str(datetime.timedelta(seconds=int(total_time))) - print( - "{} Total time: {} ({:.4f} s / it)".format( - header, total_time_str, total_time / len(iterable) - ) - ) diff --git a/vlm/detector/D-FINE.backup/src/misc/profiler_utils.py b/vlm/detector/D-FINE.backup/src/misc/profiler_utils.py deleted file mode 100644 index 84db32a..0000000 --- a/vlm/detector/D-FINE.backup/src/misc/profiler_utils.py +++ /dev/null @@ -1,30 +0,0 @@ -""" -Copyright (c) 2024 The D-FINE Authors. All Rights Reserved. -""" - -import copy -from typing import Tuple - -from calflops import calculate_flops - - -def stats( - cfg, - input_shape: Tuple = (1, 3, 640, 640), -) -> Tuple[int, dict]: - base_size = cfg.train_dataloader.collate_fn.base_size - input_shape = (1, 3, base_size, base_size) - - model_for_info = copy.deepcopy(cfg.model).deploy() - - flops, macs, _ = calculate_flops( - model=model_for_info, - input_shape=input_shape, - output_as_string=True, - output_precision=4, - print_detailed=False, - ) - params = sum(p.numel() for p in model_for_info.parameters()) - del model_for_info - - return params, {"Model FLOPs:%s MACs:%s Params:%s" % (flops, macs, params)} diff --git a/vlm/detector/D-FINE.backup/src/misc/visualizer.py b/vlm/detector/D-FINE.backup/src/misc/visualizer.py deleted file mode 100644 index fa82f7c..0000000 --- a/vlm/detector/D-FINE.backup/src/misc/visualizer.py +++ /dev/null @@ -1,121 +0,0 @@ -""" " -Copied from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright(c) 2023 lyuwenyu. All Rights Reserved. -""" - -import PIL -import numpy as np -import torch -import torch.utils.data -import torchvision -from typing import List, Dict - -torchvision.disable_beta_transforms_warning() - -__all__ = ["show_sample", "save_samples"] - -def save_samples(samples: torch.Tensor, targets: List[Dict], output_dir: str, split: str, normalized: bool, box_fmt: str): - ''' - normalized: whether the boxes are normalized to [0, 1] - box_fmt: 'xyxy', 'xywh', 'cxcywh', D-FINE uses 'cxcywh' for training, 'xyxy' for validation - ''' - from torchvision.transforms.functional import to_pil_image - from torchvision.ops import box_convert - from pathlib import Path - from PIL import ImageDraw, ImageFont - import os - - os.makedirs(Path(output_dir) / Path(f"{split}_samples"), exist_ok=True) - # Predefined colors (standard color names recognized by PIL) - BOX_COLORS = [ - "red", "blue", "green", "orange", "purple", - "cyan", "magenta", "yellow", "lime", "pink", - "teal", "lavender", "brown", "beige", "maroon", - "navy", "olive", "coral", "turquoise", "gold" - ] - - LABEL_TEXT_COLOR = "white" - - font = ImageFont.load_default() - font.size = 32 - - for i, (sample, target) in enumerate(zip(samples, targets)): - sample_visualization = sample.clone().cpu() - target_boxes = target["boxes"].clone().cpu() - target_labels = target["labels"].clone().cpu() - target_image_id = target["image_id"].item() - target_image_path = target["image_path"] - target_image_path_stem = Path(target_image_path).stem - - sample_visualization = to_pil_image(sample_visualization) - sample_visualization_w, sample_visualization_h = sample_visualization.size - - # normalized to pixel space - if normalized: - target_boxes[:, 0] = target_boxes[:, 0] * sample_visualization_w - target_boxes[:, 2] = target_boxes[:, 2] * sample_visualization_w - target_boxes[:, 1] = target_boxes[:, 1] * sample_visualization_h - target_boxes[:, 3] = target_boxes[:, 3] * sample_visualization_h - - # any box format -> xyxy - target_boxes = box_convert(target_boxes, in_fmt=box_fmt, out_fmt="xyxy") - - # clip to image size - target_boxes[:, 0] = torch.clamp(target_boxes[:, 0], 0, sample_visualization_w) - target_boxes[:, 1] = torch.clamp(target_boxes[:, 1], 0, sample_visualization_h) - target_boxes[:, 2] = torch.clamp(target_boxes[:, 2], 0, sample_visualization_w) - target_boxes[:, 3] = torch.clamp(target_boxes[:, 3], 0, sample_visualization_h) - - target_boxes = target_boxes.numpy().astype(np.int32) - target_labels = target_labels.numpy().astype(np.int32) - - draw = ImageDraw.Draw(sample_visualization) - - # draw target boxes - for box, label in zip(target_boxes, target_labels): - x1, y1, x2, y2 = box - - # Select color based on class ID - box_color = BOX_COLORS[int(label) % len(BOX_COLORS)] - - # Draw box (thick) - draw.rectangle([x1, y1, x2, y2], outline=box_color, width=3) - - label_text = f"{label}" - - # Measure text size - text_width, text_height = draw.textbbox((0, 0), label_text, font=font)[2:4] - - # Draw text background - padding = 2 - draw.rectangle( - [x1, y1 - text_height - padding * 2, x1 + text_width + padding * 2, y1], - fill=box_color - ) - - # Draw text (LABEL_TEXT_COLOR) - draw.text((x1 + padding, y1 - text_height - padding), label_text, - fill=LABEL_TEXT_COLOR, font=font) - - save_path = Path(output_dir) / f"{split}_samples" / f"{target_image_id}_{target_image_path_stem}.webp" - sample_visualization.save(save_path) - -def show_sample(sample): - """for coco dataset/dataloader""" - import matplotlib.pyplot as plt - from torchvision.transforms.v2 import functional as F - from torchvision.utils import draw_bounding_boxes - - image, target = sample - if isinstance(image, PIL.Image.Image): - image = F.to_image_tensor(image) - - image = F.convert_dtype(image, torch.uint8) - annotated_image = draw_bounding_boxes(image, target["boxes"], colors="yellow", width=3) - - fig, ax = plt.subplots() - ax.imshow(annotated_image.permute(1, 2, 0).numpy()) - ax.set(xticklabels=[], yticklabels=[], xticks=[], yticks=[]) - fig.tight_layout() - fig.show() - plt.show() diff --git a/vlm/detector/D-FINE.backup/src/nn/__init__.py b/vlm/detector/D-FINE.backup/src/nn/__init__.py deleted file mode 100644 index 4544b3c..0000000 --- a/vlm/detector/D-FINE.backup/src/nn/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -""" -Copied from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright(c) 2023 lyuwenyu. All Rights Reserved. -""" - -from .arch import * - -# -from .backbone import * -from .backbone import ( - FrozenBatchNorm2d, - freeze_batch_norm2d, - get_activation, -) -from .criterion import * -from .postprocessor import * diff --git a/vlm/detector/D-FINE.backup/src/nn/arch/__init__.py b/vlm/detector/D-FINE.backup/src/nn/arch/__init__.py deleted file mode 100644 index 29f908c..0000000 --- a/vlm/detector/D-FINE.backup/src/nn/arch/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -""" -Copied from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright(c) 2023 lyuwenyu. All Rights Reserved. -""" - -from .classification import ClassHead, Classification -from .yolo import YOLO diff --git a/vlm/detector/D-FINE.backup/src/nn/arch/classification.py b/vlm/detector/D-FINE.backup/src/nn/arch/classification.py deleted file mode 100644 index d287543..0000000 --- a/vlm/detector/D-FINE.backup/src/nn/arch/classification.py +++ /dev/null @@ -1,45 +0,0 @@ -""" -Copied from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright(c) 2023 lyuwenyu. All Rights Reserved. -""" - -import torch -import torch.nn as nn - -from ...core import register - -__all__ = ["Classification", "ClassHead"] - - -@register() -class Classification(torch.nn.Module): - __inject__ = ["backbone", "head"] - - def __init__(self, backbone: nn.Module, head: nn.Module = None): - super().__init__() - - self.backbone = backbone - self.head = head - - def forward(self, x): - x = self.backbone(x) - - if self.head is not None: - x = self.head(x) - - return x - - -@register() -class ClassHead(nn.Module): - def __init__(self, hidden_dim, num_classes): - super().__init__() - self.pool = nn.AdaptiveAvgPool2d(1) - self.proj = nn.Linear(hidden_dim, num_classes) - - def forward(self, x): - x = x[0] if isinstance(x, (list, tuple)) else x - x = self.pool(x) - x = x.reshape(x.shape[0], -1) - x = self.proj(x) - return x diff --git a/vlm/detector/D-FINE.backup/src/nn/arch/yolo.py b/vlm/detector/D-FINE.backup/src/nn/arch/yolo.py deleted file mode 100644 index c40a306..0000000 --- a/vlm/detector/D-FINE.backup/src/nn/arch/yolo.py +++ /dev/null @@ -1,42 +0,0 @@ -""" -Copied from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright(c) 2023 lyuwenyu. All Rights Reserved. -""" - -import torch - -from ...core import register - -__all__ = [ - "YOLO", -] - - -@register() -class YOLO(torch.nn.Module): - __inject__ = [ - "backbone", - "neck", - "head", - ] - - def __init__(self, backbone: torch.nn.Module, neck, head): - super().__init__() - self.backbone = backbone - self.neck = neck - self.head = head - - def forward(self, x, **kwargs): - x = self.backbone(x) - x = self.neck(x) - x = self.head(x) - return x - - def deploy( - self, - ): - self.eval() - for m in self.modules(): - if m is not self and hasattr(m, "deploy"): - m.deploy() - return self diff --git a/vlm/detector/D-FINE.backup/src/nn/backbone/__init__.py b/vlm/detector/D-FINE.backup/src/nn/backbone/__init__.py deleted file mode 100644 index 497d8df..0000000 --- a/vlm/detector/D-FINE.backup/src/nn/backbone/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -""" -Copied from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright(c) 2023 lyuwenyu. All Rights Reserved. -""" - -from .common import ( - FrozenBatchNorm2d, - freeze_batch_norm2d, - get_activation, -) -from .csp_darknet import CSPPAN, CSPDarkNet -from .csp_resnet import CSPResNet -from .hgnetv2 import HGNetv2 -from .presnet import PResNet -from .test_resnet import MResNet -from .timm_model import TimmModel -from .torchvision_model import TorchVisionModel diff --git a/vlm/detector/D-FINE.backup/src/nn/backbone/common.py b/vlm/detector/D-FINE.backup/src/nn/backbone/common.py deleted file mode 100644 index fcdb906..0000000 --- a/vlm/detector/D-FINE.backup/src/nn/backbone/common.py +++ /dev/null @@ -1,117 +0,0 @@ -""" -Copied from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright(c) 2023 lyuwenyu. All Rights Reserved. -""" - -import torch -import torch.nn as nn - - -class ConvNormLayer(nn.Module): - def __init__(self, ch_in, ch_out, kernel_size, stride, padding=None, bias=False, act=None): - super().__init__() - self.conv = nn.Conv2d( - ch_in, - ch_out, - kernel_size, - stride, - padding=(kernel_size - 1) // 2 if padding is None else padding, - bias=bias, - ) - self.norm = nn.BatchNorm2d(ch_out) - self.act = nn.Identity() if act is None else get_activation(act) - - def forward(self, x): - return self.act(self.norm(self.conv(x))) - - -class FrozenBatchNorm2d(nn.Module): - """copy and modified from https://github.com/facebookresearch/detr/blob/master/models/backbone.py - BatchNorm2d where the batch statistics and the affine parameters are fixed. - Copy-paste from torchvision.misc.ops with added eps before rqsrt, - without which any other models than torchvision.models.resnet[18,34,50,101] - produce nans. - """ - - def __init__(self, num_features, eps=1e-5): - super(FrozenBatchNorm2d, self).__init__() - n = num_features - self.register_buffer("weight", torch.ones(n)) - self.register_buffer("bias", torch.zeros(n)) - self.register_buffer("running_mean", torch.zeros(n)) - self.register_buffer("running_var", torch.ones(n)) - self.eps = eps - self.num_features = n - - def _load_from_state_dict( - self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs - ): - num_batches_tracked_key = prefix + "num_batches_tracked" - if num_batches_tracked_key in state_dict: - del state_dict[num_batches_tracked_key] - - super(FrozenBatchNorm2d, self)._load_from_state_dict( - state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs - ) - - def forward(self, x): - # move reshapes to the beginning - # to make it fuser-friendly - w = self.weight.reshape(1, -1, 1, 1) - b = self.bias.reshape(1, -1, 1, 1) - rv = self.running_var.reshape(1, -1, 1, 1) - rm = self.running_mean.reshape(1, -1, 1, 1) - scale = w * (rv + self.eps).rsqrt() - bias = b - rm * scale - return x * scale + bias - - def extra_repr(self): - return "{num_features}, eps={eps}".format(**self.__dict__) - - -def freeze_batch_norm2d(module: nn.Module) -> nn.Module: - if isinstance(module, nn.BatchNorm2d): - module = FrozenBatchNorm2d(module.num_features) - else: - for name, child in module.named_children(): - _child = freeze_batch_norm2d(child) - if _child is not child: - setattr(module, name, _child) - return module - - -def get_activation(act: str, inplace: bool = True): - """get activation""" - if act is None: - return nn.Identity() - - elif isinstance(act, nn.Module): - return act - - act = act.lower() - - if act == "silu" or act == "swish": - m = nn.SiLU() - - elif act == "relu": - m = nn.ReLU() - - elif act == "leaky_relu": - m = nn.LeakyReLU() - - elif act == "silu": - m = nn.SiLU() - - elif act == "gelu": - m = nn.GELU() - - elif act == "hardsigmoid": - m = nn.Hardsigmoid() - - else: - raise RuntimeError("") - - if hasattr(m, "inplace"): - m.inplace = inplace - - return m diff --git a/vlm/detector/D-FINE.backup/src/nn/backbone/csp_darknet.py b/vlm/detector/D-FINE.backup/src/nn/backbone/csp_darknet.py deleted file mode 100644 index f6efa01..0000000 --- a/vlm/detector/D-FINE.backup/src/nn/backbone/csp_darknet.py +++ /dev/null @@ -1,203 +0,0 @@ -""" -Copied from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright(c) 2023 lyuwenyu. All Rights Reserved. -""" - -import math -import warnings - -import torch -import torch.nn as nn -import torch.nn.functional as F - -from ...core import register -from .common import get_activation - - -def autopad(k, p=None): - if p is None: - p = k // 2 if isinstance(k, int) else [x // 2 for x in k] - return p - - -def make_divisible(c, d): - return math.ceil(c / d) * d - - -class Conv(nn.Module): - def __init__(self, cin, cout, k=1, s=1, p=None, g=1, act="silu") -> None: - super().__init__() - self.conv = nn.Conv2d(cin, cout, k, s, autopad(k, p), groups=g, bias=False) - self.bn = nn.BatchNorm2d(cout) - self.act = get_activation(act, inplace=True) - - def forward(self, x): - return self.act(self.bn(self.conv(x))) - - -class Bottleneck(nn.Module): - # Standard bottleneck - def __init__(self, c1, c2, shortcut=True, g=1, e=0.5, act="silu"): - super().__init__() - c_ = int(c2 * e) # hidden channels - self.cv1 = Conv(c1, c_, 1, 1, act=act) - self.cv2 = Conv(c_, c2, 3, 1, g=g, act=act) - self.add = shortcut and c1 == c2 - - def forward(self, x): - return x + self.cv2(self.cv1(x)) if self.add else self.cv2(self.cv1(x)) - - -class C3(nn.Module): - # CSP Bottleneck with 3 convolutions - def __init__( - self, c1, c2, n=1, shortcut=True, g=1, e=0.5, act="silu" - ): # ch_in, ch_out, number, shortcut, groups, expansion - super().__init__() - c_ = int(c2 * e) # hidden channels - self.cv1 = Conv(c1, c_, 1, 1, act=act) - self.cv2 = Conv(c1, c_, 1, 1, act=act) - self.m = nn.Sequential(*(Bottleneck(c_, c_, shortcut, g, e=1.0, act=act) for _ in range(n))) - self.cv3 = Conv(2 * c_, c2, 1, act=act) - - def forward(self, x): - return self.cv3(torch.cat((self.m(self.cv1(x)), self.cv2(x)), dim=1)) - - -class SPPF(nn.Module): - # Spatial Pyramid Pooling - Fast (SPPF) layer for YOLOv5 by Glenn Jocher - def __init__(self, c1, c2, k=5, act="silu"): # equivalent to SPP(k=(5, 9, 13)) - super().__init__() - c_ = c1 // 2 # hidden channels - self.cv1 = Conv(c1, c_, 1, 1, act=act) - self.cv2 = Conv(c_ * 4, c2, 1, 1, act=act) - self.m = nn.MaxPool2d(kernel_size=k, stride=1, padding=k // 2) - - def forward(self, x): - x = self.cv1(x) - with warnings.catch_warnings(): - warnings.simplefilter("ignore") # suppress torch 1.9.0 max_pool2d() warning - y1 = self.m(x) - y2 = self.m(y1) - return self.cv2(torch.cat([x, y1, y2, self.m(y2)], 1)) - - -@register() -class CSPDarkNet(nn.Module): - __share__ = ["depth_multi", "width_multi"] - - def __init__( - self, - in_channels=3, - width_multi=1.0, - depth_multi=1.0, - return_idx=[2, 3, -1], - act="silu", - ) -> None: - super().__init__() - - channels = [64, 128, 256, 512, 1024] - channels = [make_divisible(c * width_multi, 8) for c in channels] - - depths = [3, 6, 9, 3] - depths = [max(round(d * depth_multi), 1) for d in depths] - - self.layers = nn.ModuleList([Conv(in_channels, channels[0], 6, 2, 2, act=act)]) - for i, (c, d) in enumerate(zip(channels, depths), 1): - layer = nn.Sequential( - *[Conv(c, channels[i], 3, 2, act=act), C3(channels[i], channels[i], n=d, act=act)] - ) - self.layers.append(layer) - - self.layers.append(SPPF(channels[-1], channels[-1], k=5, act=act)) - - self.return_idx = return_idx - self.out_channels = [channels[i] for i in self.return_idx] - self.strides = [[2, 4, 8, 16, 32][i] for i in self.return_idx] - self.depths = depths - self.act = act - - def forward(self, x): - outputs = [] - for _, m in enumerate(self.layers): - x = m(x) - outputs.append(x) - - return [outputs[i] for i in self.return_idx] - - -@register() -class CSPPAN(nn.Module): - """ - P5 ---> 1x1 ---------------------------------> concat --> c3 --> det - | up | conv /2 - P4 ---> concat ---> c3 ---> 1x1 --> concat ---> c3 -----------> det - | up | conv /2 - P3 -----------------------> concat ---> c3 ---------------------> det - """ - - __share__ = [ - "depth_multi", - ] - - def __init__(self, in_channels=[256, 512, 1024], depth_multi=1.0, act="silu") -> None: - super().__init__() - depth = max(round(3 * depth_multi), 1) - - self.out_channels = in_channels - self.fpn_stems = nn.ModuleList( - [ - Conv(cin, cout, 1, 1, act=act) - for cin, cout in zip(in_channels[::-1], in_channels[::-1][1:]) - ] - ) - self.fpn_csps = nn.ModuleList( - [ - C3(cin, cout, depth, False, act=act) - for cin, cout in zip(in_channels[::-1], in_channels[::-1][1:]) - ] - ) - - self.pan_stems = nn.ModuleList([Conv(c, c, 3, 2, act=act) for c in in_channels[:-1]]) - self.pan_csps = nn.ModuleList([C3(c, c, depth, False, act=act) for c in in_channels[1:]]) - - def forward(self, feats): - fpn_feats = [] - for i, feat in enumerate(feats[::-1]): - if i == 0: - feat = self.fpn_stems[i](feat) - fpn_feats.append(feat) - else: - _feat = F.interpolate(fpn_feats[-1], scale_factor=2, mode="nearest") - feat = torch.concat([_feat, feat], dim=1) - feat = self.fpn_csps[i - 1](feat) - if i < len(self.fpn_stems): - feat = self.fpn_stems[i](feat) - fpn_feats.append(feat) - - pan_feats = [] - for i, feat in enumerate(fpn_feats[::-1]): - if i == 0: - pan_feats.append(feat) - else: - _feat = self.pan_stems[i - 1](pan_feats[-1]) - feat = torch.concat([_feat, feat], dim=1) - feat = self.pan_csps[i - 1](feat) - pan_feats.append(feat) - - return pan_feats - - -if __name__ == "__main__": - data = torch.rand(1, 3, 320, 640) - - width_multi = 0.75 - depth_multi = 0.33 - - m = CSPDarkNet(3, width_multi=width_multi, depth_multi=depth_multi, act="silu") - outputs = m(data) - print([o.shape for o in outputs]) - - m = CSPPAN(in_channels=m.out_channels, depth_multi=depth_multi, act="silu") - outputs = m(outputs) - print([o.shape for o in outputs]) diff --git a/vlm/detector/D-FINE.backup/src/nn/backbone/csp_resnet.py b/vlm/detector/D-FINE.backup/src/nn/backbone/csp_resnet.py deleted file mode 100644 index 17887c7..0000000 --- a/vlm/detector/D-FINE.backup/src/nn/backbone/csp_resnet.py +++ /dev/null @@ -1,302 +0,0 @@ -""" -https://github.com/PaddlePaddle/PaddleDetection/blob/release/2.6/ppdet/modeling/backbones/cspresnet.py - -Copyright(c) 2023 lyuwenyu. All Rights Reserved. -""" - -from collections import OrderedDict - -import torch -import torch.nn as nn -import torch.nn.functional as F - -from ...core import register -from .common import get_activation - -__all__ = ["CSPResNet"] - - -donwload_url = { - "s": "https://github.com/lyuwenyu/storage/releases/download/v0.1/CSPResNetb_s_pretrained_from_paddle.pth", - "m": "https://github.com/lyuwenyu/storage/releases/download/v0.1/CSPResNetb_m_pretrained_from_paddle.pth", - "l": "https://github.com/lyuwenyu/storage/releases/download/v0.1/CSPResNetb_l_pretrained_from_paddle.pth", - "x": "https://github.com/lyuwenyu/storage/releases/download/v0.1/CSPResNetb_x_pretrained_from_paddle.pth", -} - - -class ConvBNLayer(nn.Module): - def __init__(self, ch_in, ch_out, filter_size=3, stride=1, groups=1, padding=0, act=None): - super().__init__() - self.conv = nn.Conv2d( - ch_in, ch_out, filter_size, stride, padding, groups=groups, bias=False - ) - self.bn = nn.BatchNorm2d(ch_out) - self.act = get_activation(act) - - def forward(self, x: torch.Tensor) -> torch.Tensor: - x = self.conv(x) - x = self.bn(x) - x = self.act(x) - return x - - -class RepVggBlock(nn.Module): - def __init__(self, ch_in, ch_out, act="relu", alpha: bool = False): - super().__init__() - self.ch_in = ch_in - self.ch_out = ch_out - self.conv1 = ConvBNLayer(ch_in, ch_out, 3, stride=1, padding=1, act=None) - self.conv2 = ConvBNLayer(ch_in, ch_out, 1, stride=1, padding=0, act=None) - self.act = get_activation(act) - - if alpha: - self.alpha = nn.Parameter( - torch.ones( - 1, - ) - ) - else: - self.alpha = None - - def forward(self, x): - if hasattr(self, "conv"): - y = self.conv(x) - else: - if self.alpha: - y = self.conv1(x) + self.alpha * self.conv2(x) - else: - y = self.conv1(x) + self.conv2(x) - y = self.act(y) - return y - - def convert_to_deploy(self): - if not hasattr(self, "conv"): - self.conv = nn.Conv2d(self.ch_in, self.ch_out, 3, 1, padding=1) - - kernel, bias = self.get_equivalent_kernel_bias() - self.conv.weight.data = kernel - self.conv.bias.data = bias - - def get_equivalent_kernel_bias(self): - kernel3x3, bias3x3 = self._fuse_bn_tensor(self.conv1) - kernel1x1, bias1x1 = self._fuse_bn_tensor(self.conv2) - - if self.alpha: - return kernel3x3 + self.alpha * self._pad_1x1_to_3x3_tensor( - kernel1x1 - ), bias3x3 + self.alpha * bias1x1 - else: - return kernel3x3 + self._pad_1x1_to_3x3_tensor(kernel1x1), bias3x3 + bias1x1 - - def _pad_1x1_to_3x3_tensor(self, kernel1x1): - if kernel1x1 is None: - return 0 - else: - return F.pad(kernel1x1, [1, 1, 1, 1]) - - def _fuse_bn_tensor(self, branch: ConvBNLayer): - if branch is None: - return 0, 0 - kernel = branch.conv.weight - running_mean = branch.norm.running_mean - running_var = branch.norm.running_var - gamma = branch.norm.weight - beta = branch.norm.bias - eps = branch.norm.eps - std = (running_var + eps).sqrt() - t = (gamma / std).reshape(-1, 1, 1, 1) - return kernel * t, beta - running_mean * gamma / std - - -class BasicBlock(nn.Module): - def __init__(self, ch_in, ch_out, act="relu", shortcut=True, use_alpha=False): - super().__init__() - assert ch_in == ch_out - self.conv1 = ConvBNLayer(ch_in, ch_out, 3, stride=1, padding=1, act=act) - self.conv2 = RepVggBlock(ch_out, ch_out, act=act, alpha=use_alpha) - self.shortcut = shortcut - - def forward(self, x): - y = self.conv1(x) - y = self.conv2(y) - if self.shortcut: - return x + y - else: - return y - - -class EffectiveSELayer(nn.Module): - """Effective Squeeze-Excitation - From `CenterMask : Real-Time Anchor-Free Instance Segmentation` - https://arxiv.org/abs/1911.06667 - """ - - def __init__(self, channels, act="hardsigmoid"): - super(EffectiveSELayer, self).__init__() - self.fc = nn.Conv2d(channels, channels, kernel_size=1, padding=0) - self.act = get_activation(act) - - def forward(self, x: torch.Tensor): - x_se = x.mean((2, 3), keepdim=True) - x_se = self.fc(x_se) - x_se = self.act(x_se) - return x * x_se - - -class CSPResStage(nn.Module): - def __init__(self, block_fn, ch_in, ch_out, n, stride, act="relu", attn="eca", use_alpha=False): - super().__init__() - ch_mid = (ch_in + ch_out) // 2 - if stride == 2: - self.conv_down = ConvBNLayer(ch_in, ch_mid, 3, stride=2, padding=1, act=act) - else: - self.conv_down = None - self.conv1 = ConvBNLayer(ch_mid, ch_mid // 2, 1, act=act) - self.conv2 = ConvBNLayer(ch_mid, ch_mid // 2, 1, act=act) - self.blocks = nn.Sequential( - *[ - block_fn(ch_mid // 2, ch_mid // 2, act=act, shortcut=True, use_alpha=use_alpha) - for i in range(n) - ] - ) - if attn: - self.attn = EffectiveSELayer(ch_mid, act="hardsigmoid") - else: - self.attn = None - - self.conv3 = ConvBNLayer(ch_mid, ch_out, 1, act=act) - - def forward(self, x): - if self.conv_down is not None: - x = self.conv_down(x) - y1 = self.conv1(x) - y2 = self.blocks(self.conv2(x)) - y = torch.concat([y1, y2], dim=1) - if self.attn is not None: - y = self.attn(y) - y = self.conv3(y) - return y - - -@register() -class CSPResNet(nn.Module): - layers = [3, 6, 6, 3] - channels = [64, 128, 256, 512, 1024] - model_cfg = { - "s": { - "depth_mult": 0.33, - "width_mult": 0.50, - }, - "m": { - "depth_mult": 0.67, - "width_mult": 0.75, - }, - "l": { - "depth_mult": 1.00, - "width_mult": 1.00, - }, - "x": { - "depth_mult": 1.33, - "width_mult": 1.25, - }, - } - - def __init__( - self, - name: str, - act="silu", - return_idx=[1, 2, 3], - use_large_stem=True, - use_alpha=False, - pretrained=False, - ): - super().__init__() - depth_mult = self.model_cfg[name]["depth_mult"] - width_mult = self.model_cfg[name]["width_mult"] - - channels = [max(round(c * width_mult), 1) for c in self.channels] - layers = [max(round(l * depth_mult), 1) for l in self.layers] - act = get_activation(act) - - if use_large_stem: - self.stem = nn.Sequential( - OrderedDict( - [ - ( - "conv1", - ConvBNLayer(3, channels[0] // 2, 3, stride=2, padding=1, act=act), - ), - ( - "conv2", - ConvBNLayer( - channels[0] // 2, channels[0] // 2, 3, stride=1, padding=1, act=act - ), - ), - ( - "conv3", - ConvBNLayer( - channels[0] // 2, channels[0], 3, stride=1, padding=1, act=act - ), - ), - ] - ) - ) - else: - self.stem = nn.Sequential( - OrderedDict( - [ - ( - "conv1", - ConvBNLayer(3, channels[0] // 2, 3, stride=2, padding=1, act=act), - ), - ( - "conv2", - ConvBNLayer( - channels[0] // 2, channels[0], 3, stride=1, padding=1, act=act - ), - ), - ] - ) - ) - - n = len(channels) - 1 - self.stages = nn.Sequential( - OrderedDict( - [ - ( - str(i), - CSPResStage( - BasicBlock, - channels[i], - channels[i + 1], - layers[i], - 2, - act=act, - use_alpha=use_alpha, - ), - ) - for i in range(n) - ] - ) - ) - - self._out_channels = channels[1:] - self._out_strides = [4 * 2**i for i in range(n)] - self.return_idx = return_idx - - if pretrained: - if isinstance(pretrained, bool) or "http" in pretrained: - state = torch.hub.load_state_dict_from_url(donwload_url[name], map_location="cpu") - else: - state = torch.load(pretrained, map_location="cpu") - self.load_state_dict(state) - print(f"Load CSPResNet_{name} state_dict") - - def forward(self, x): - x = self.stem(x) - outs = [] - for idx, stage in enumerate(self.stages): - x = stage(x) - if idx in self.return_idx: - outs.append(x) - - return outs diff --git a/vlm/detector/D-FINE.backup/src/nn/backbone/hgnetv2.py b/vlm/detector/D-FINE.backup/src/nn/backbone/hgnetv2.py deleted file mode 100644 index 85d99d6..0000000 --- a/vlm/detector/D-FINE.backup/src/nn/backbone/hgnetv2.py +++ /dev/null @@ -1,579 +0,0 @@ -""" -reference -- https://github.com/PaddlePaddle/PaddleDetection/blob/develop/ppdet/modeling/backbones/hgnet_v2.py - -Copyright (c) 2024 The D-FINE Authors. All Rights Reserved. -""" - -import logging -import os - -import torch -import torch.nn as nn -import torch.nn.functional as F - -from ...core import register -from .common import FrozenBatchNorm2d - -# Constants for initialization -kaiming_normal_ = nn.init.kaiming_normal_ -zeros_ = nn.init.zeros_ -ones_ = nn.init.ones_ - -__all__ = ["HGNetv2"] - -def safe_barrier(): - if torch.distributed.is_available() and torch.distributed.is_initialized(): - torch.distributed.barrier() - else: - pass - -def safe_get_rank(): - if torch.distributed.is_available() and torch.distributed.is_initialized(): - return torch.distributed.get_rank() - else: - return 0 - -class LearnableAffineBlock(nn.Module): - def __init__(self, scale_value=1.0, bias_value=0.0): - super().__init__() - self.scale = nn.Parameter(torch.tensor([scale_value]), requires_grad=True) - self.bias = nn.Parameter(torch.tensor([bias_value]), requires_grad=True) - - def forward(self, x): - return self.scale * x + self.bias - - -class ConvBNAct(nn.Module): - def __init__( - self, - in_chs, - out_chs, - kernel_size, - stride=1, - groups=1, - padding="", - use_act=True, - use_lab=False, - ): - super().__init__() - self.use_act = use_act - self.use_lab = use_lab - if padding == "same": - self.conv = nn.Sequential( - nn.ZeroPad2d([0, 1, 0, 1]), - nn.Conv2d(in_chs, out_chs, kernel_size, stride, groups=groups, bias=False), - ) - else: - self.conv = nn.Conv2d( - in_chs, - out_chs, - kernel_size, - stride, - padding=(kernel_size - 1) // 2, - groups=groups, - bias=False, - ) - self.bn = nn.BatchNorm2d(out_chs) - if self.use_act: - self.act = nn.ReLU() - else: - self.act = nn.Identity() - if self.use_act and self.use_lab: - self.lab = LearnableAffineBlock() - else: - self.lab = nn.Identity() - - def forward(self, x): - x = self.conv(x) - x = self.bn(x) - x = self.act(x) - x = self.lab(x) - return x - - -class LightConvBNAct(nn.Module): - def __init__( - self, - in_chs, - out_chs, - kernel_size, - groups=1, - use_lab=False, - ): - super().__init__() - self.conv1 = ConvBNAct( - in_chs, - out_chs, - kernel_size=1, - use_act=False, - use_lab=use_lab, - ) - self.conv2 = ConvBNAct( - out_chs, - out_chs, - kernel_size=kernel_size, - groups=out_chs, - use_act=True, - use_lab=use_lab, - ) - - def forward(self, x): - x = self.conv1(x) - x = self.conv2(x) - return x - - -class StemBlock(nn.Module): - # for HGNetv2 - def __init__(self, in_chs, mid_chs, out_chs, use_lab=False): - super().__init__() - self.stem1 = ConvBNAct( - in_chs, - mid_chs, - kernel_size=3, - stride=2, - use_lab=use_lab, - ) - self.stem2a = ConvBNAct( - mid_chs, - mid_chs // 2, - kernel_size=2, - stride=1, - use_lab=use_lab, - ) - self.stem2b = ConvBNAct( - mid_chs // 2, - mid_chs, - kernel_size=2, - stride=1, - use_lab=use_lab, - ) - self.stem3 = ConvBNAct( - mid_chs * 2, - mid_chs, - kernel_size=3, - stride=2, - use_lab=use_lab, - ) - self.stem4 = ConvBNAct( - mid_chs, - out_chs, - kernel_size=1, - stride=1, - use_lab=use_lab, - ) - self.pool = nn.MaxPool2d(kernel_size=2, stride=1, ceil_mode=True) - - def forward(self, x): - x = self.stem1(x) - x = F.pad(x, (0, 1, 0, 1)) - x2 = self.stem2a(x) - x2 = F.pad(x2, (0, 1, 0, 1)) - x2 = self.stem2b(x2) - x1 = self.pool(x) - x = torch.cat([x1, x2], dim=1) - x = self.stem3(x) - x = self.stem4(x) - return x - - -class EseModule(nn.Module): - def __init__(self, chs): - super().__init__() - self.conv = nn.Conv2d( - chs, - chs, - kernel_size=1, - stride=1, - padding=0, - ) - self.sigmoid = nn.Sigmoid() - - def forward(self, x): - identity = x - x = x.mean((2, 3), keepdim=True) - x = self.conv(x) - x = self.sigmoid(x) - return torch.mul(identity, x) - - -class HG_Block(nn.Module): - def __init__( - self, - in_chs, - mid_chs, - out_chs, - layer_num, - kernel_size=3, - residual=False, - light_block=False, - use_lab=False, - agg="ese", - drop_path=0.0, - ): - super().__init__() - self.residual = residual - - self.layers = nn.ModuleList() - for i in range(layer_num): - if light_block: - self.layers.append( - LightConvBNAct( - in_chs if i == 0 else mid_chs, - mid_chs, - kernel_size=kernel_size, - use_lab=use_lab, - ) - ) - else: - self.layers.append( - ConvBNAct( - in_chs if i == 0 else mid_chs, - mid_chs, - kernel_size=kernel_size, - stride=1, - use_lab=use_lab, - ) - ) - - # feature aggregation - total_chs = in_chs + layer_num * mid_chs - if agg == "se": - aggregation_squeeze_conv = ConvBNAct( - total_chs, - out_chs // 2, - kernel_size=1, - stride=1, - use_lab=use_lab, - ) - aggregation_excitation_conv = ConvBNAct( - out_chs // 2, - out_chs, - kernel_size=1, - stride=1, - use_lab=use_lab, - ) - self.aggregation = nn.Sequential( - aggregation_squeeze_conv, - aggregation_excitation_conv, - ) - else: - aggregation_conv = ConvBNAct( - total_chs, - out_chs, - kernel_size=1, - stride=1, - use_lab=use_lab, - ) - att = EseModule(out_chs) - self.aggregation = nn.Sequential( - aggregation_conv, - att, - ) - - self.drop_path = nn.Dropout(drop_path) if drop_path else nn.Identity() - - def forward(self, x): - identity = x - output = [x] - for layer in self.layers: - x = layer(x) - output.append(x) - x = torch.cat(output, dim=1) - x = self.aggregation(x) - if self.residual: - x = self.drop_path(x) + identity - return x - - -class HG_Stage(nn.Module): - def __init__( - self, - in_chs, - mid_chs, - out_chs, - block_num, - layer_num, - downsample=True, - light_block=False, - kernel_size=3, - use_lab=False, - agg="se", - drop_path=0.0, - ): - super().__init__() - self.downsample = downsample - if downsample: - self.downsample = ConvBNAct( - in_chs, - in_chs, - kernel_size=3, - stride=2, - groups=in_chs, - use_act=False, - use_lab=use_lab, - ) - else: - self.downsample = nn.Identity() - - blocks_list = [] - for i in range(block_num): - blocks_list.append( - HG_Block( - in_chs if i == 0 else out_chs, - mid_chs, - out_chs, - layer_num, - residual=False if i == 0 else True, - kernel_size=kernel_size, - light_block=light_block, - use_lab=use_lab, - agg=agg, - drop_path=drop_path[i] if isinstance(drop_path, (list, tuple)) else drop_path, - ) - ) - self.blocks = nn.Sequential(*blocks_list) - - def forward(self, x): - x = self.downsample(x) - x = self.blocks(x) - return x - - -@register() -class HGNetv2(nn.Module): - """ - HGNetV2 - Args: - stem_channels: list. Number of channels for the stem block. - stage_type: str. The stage configuration of HGNet. such as the number of channels, stride, etc. - use_lab: boolean. Whether to use LearnableAffineBlock in network. - lr_mult_list: list. Control the learning rate of different stages. - Returns: - model: nn.Layer. Specific HGNetV2 model depends on args. - """ - - arch_configs = { - "B0": { - "stem_channels": [3, 16, 16], - "stage_config": { - # in_channels, mid_channels, out_channels, num_blocks, downsample, light_block, kernel_size, layer_num - "stage1": [16, 16, 64, 1, False, False, 3, 3], - "stage2": [64, 32, 256, 1, True, False, 3, 3], - "stage3": [256, 64, 512, 2, True, True, 5, 3], - "stage4": [512, 128, 1024, 1, True, True, 5, 3], - }, - "url": "https://github.com/Peterande/storage/releases/download/dfinev1.0/PPHGNetV2_B0_stage1.pth", - }, - "B1": { - "stem_channels": [3, 24, 32], - "stage_config": { - # in_channels, mid_channels, out_channels, num_blocks, downsample, light_block, kernel_size, layer_num - "stage1": [32, 32, 64, 1, False, False, 3, 3], - "stage2": [64, 48, 256, 1, True, False, 3, 3], - "stage3": [256, 96, 512, 2, True, True, 5, 3], - "stage4": [512, 192, 1024, 1, True, True, 5, 3], - }, - "url": "https://github.com/Peterande/storage/releases/download/dfinev1.0/PPHGNetV2_B1_stage1.pth", - }, - "B2": { - "stem_channels": [3, 24, 32], - "stage_config": { - # in_channels, mid_channels, out_channels, num_blocks, downsample, light_block, kernel_size, layer_num - "stage1": [32, 32, 96, 1, False, False, 3, 4], - "stage2": [96, 64, 384, 1, True, False, 3, 4], - "stage3": [384, 128, 768, 3, True, True, 5, 4], - "stage4": [768, 256, 1536, 1, True, True, 5, 4], - }, - "url": "https://github.com/Peterande/storage/releases/download/dfinev1.0/PPHGNetV2_B2_stage1.pth", - }, - "B3": { - "stem_channels": [3, 24, 32], - "stage_config": { - # in_channels, mid_channels, out_channels, num_blocks, downsample, light_block, kernel_size, layer_num - "stage1": [32, 32, 128, 1, False, False, 3, 5], - "stage2": [128, 64, 512, 1, True, False, 3, 5], - "stage3": [512, 128, 1024, 3, True, True, 5, 5], - "stage4": [1024, 256, 2048, 1, True, True, 5, 5], - }, - "url": "https://github.com/Peterande/storage/releases/download/dfinev1.0/PPHGNetV2_B3_stage1.pth", - }, - "B4": { - "stem_channels": [3, 32, 48], - "stage_config": { - # in_channels, mid_channels, out_channels, num_blocks, downsample, light_block, kernel_size, layer_num - "stage1": [48, 48, 128, 1, False, False, 3, 6], - "stage2": [128, 96, 512, 1, True, False, 3, 6], - "stage3": [512, 192, 1024, 3, True, True, 5, 6], - "stage4": [1024, 384, 2048, 1, True, True, 5, 6], - }, - "url": "https://github.com/Peterande/storage/releases/download/dfinev1.0/PPHGNetV2_B4_stage1.pth", - }, - "B5": { - "stem_channels": [3, 32, 64], - "stage_config": { - # in_channels, mid_channels, out_channels, num_blocks, downsample, light_block, kernel_size, layer_num - "stage1": [64, 64, 128, 1, False, False, 3, 6], - "stage2": [128, 128, 512, 2, True, False, 3, 6], - "stage3": [512, 256, 1024, 5, True, True, 5, 6], - "stage4": [1024, 512, 2048, 2, True, True, 5, 6], - }, - "url": "https://github.com/Peterande/storage/releases/download/dfinev1.0/PPHGNetV2_B5_stage1.pth", - }, - "B6": { - "stem_channels": [3, 48, 96], - "stage_config": { - # in_channels, mid_channels, out_channels, num_blocks, downsample, light_block, kernel_size, layer_num - "stage1": [96, 96, 192, 2, False, False, 3, 6], - "stage2": [192, 192, 512, 3, True, False, 3, 6], - "stage3": [512, 384, 1024, 6, True, True, 5, 6], - "stage4": [1024, 768, 2048, 3, True, True, 5, 6], - }, - "url": "https://github.com/Peterande/storage/releases/download/dfinev1.0/PPHGNetV2_B6_stage1.pth", - }, - } - - def __init__( - self, - name, - use_lab=False, - return_idx=[1, 2, 3], - freeze_stem_only=True, - freeze_at=0, - freeze_norm=True, - pretrained=True, - local_model_dir="weight/hgnetv2/", - ): - super().__init__() - self.use_lab = use_lab - self.return_idx = return_idx - - stem_channels = self.arch_configs[name]["stem_channels"] - stage_config = self.arch_configs[name]["stage_config"] - download_url = self.arch_configs[name]["url"] - - self._out_strides = [4, 8, 16, 32] - self._out_channels = [stage_config[k][2] for k in stage_config] - - # stem - self.stem = StemBlock( - in_chs=stem_channels[0], - mid_chs=stem_channels[1], - out_chs=stem_channels[2], - use_lab=use_lab, - ) - - # stages - self.stages = nn.ModuleList() - for i, k in enumerate(stage_config): - ( - in_channels, - mid_channels, - out_channels, - block_num, - downsample, - light_block, - kernel_size, - layer_num, - ) = stage_config[k] - self.stages.append( - HG_Stage( - in_channels, - mid_channels, - out_channels, - block_num, - layer_num, - downsample, - light_block, - kernel_size, - use_lab, - ) - ) - - if freeze_at >= 0: - self._freeze_parameters(self.stem) - if not freeze_stem_only: - for i in range(min(freeze_at + 1, len(self.stages))): - self._freeze_parameters(self.stages[i]) - - if freeze_norm: - self._freeze_norm(self) - - if pretrained: - RED, GREEN, RESET = "\033[91m", "\033[92m", "\033[0m" - try: - # If the file doesn't exist locally, download from the URL - if safe_get_rank() == 0: - print( - GREEN - + "If the pretrained HGNetV2 can't be downloaded automatically. Please check your network connection." - + RESET - ) - print( - GREEN - + "Please check your network connection. Or download the model manually from " - + RESET - + f"{download_url}" - + GREEN - + " to " - + RESET - + f"{local_model_dir}." - + RESET - ) - state = torch.hub.load_state_dict_from_url( - download_url, map_location="cpu", model_dir=local_model_dir - ) - print(f"Loaded stage1 {name} HGNetV2 from URL.") - - # Wait for rank 0 to download the model - safe_barrier() - - # All processes load the downloaded model - model_path = local_model_dir + "PPHGNetV2_" + name + "_stage1.pth" - state = torch.load(model_path, map_location="cpu") - - self.load_state_dict(state) - print(f"Loaded stage1 {name} HGNetV2 from URL.") - - except (Exception, KeyboardInterrupt) as e: - if safe_get_rank() == 0: - print(f"{str(e)}") - logging.error( - RED + "CRITICAL WARNING: Failed to load pretrained HGNetV2 model" + RESET - ) - logging.error( - GREEN - + "Please check your network connection. Or download the model manually from " - + RESET - + f"{download_url}" - + GREEN - + " to " - + RESET - + f"{local_model_dir}." - + RESET - ) - exit() - - def _freeze_norm(self, m: nn.Module): - if isinstance(m, nn.BatchNorm2d): - m = FrozenBatchNorm2d(m.num_features) - else: - for name, child in m.named_children(): - _child = self._freeze_norm(child) - if _child is not child: - setattr(m, name, _child) - return m - - def _freeze_parameters(self, m: nn.Module): - for p in m.parameters(): - p.requires_grad = False - - def forward(self, x): - x = self.stem(x) - outs = [] - for idx, stage in enumerate(self.stages): - x = stage(x) - if idx in self.return_idx: - outs.append(x) - return outs diff --git a/vlm/detector/D-FINE.backup/src/nn/backbone/presnet.py b/vlm/detector/D-FINE.backup/src/nn/backbone/presnet.py deleted file mode 100644 index 5543fe8..0000000 --- a/vlm/detector/D-FINE.backup/src/nn/backbone/presnet.py +++ /dev/null @@ -1,263 +0,0 @@ -""" -Copied from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright(c) 2023 lyuwenyu. All Rights Reserved. -""" - -from collections import OrderedDict - -import torch -import torch.nn as nn -import torch.nn.functional as F - -from ...core import register -from .common import FrozenBatchNorm2d, get_activation - -__all__ = ["PResNet"] - - -ResNet_cfg = { - 18: [2, 2, 2, 2], - 34: [3, 4, 6, 3], - 50: [3, 4, 6, 3], - 101: [3, 4, 23, 3], - # 152: [3, 8, 36, 3], -} - - -donwload_url = { - 18: "https://github.com/lyuwenyu/storage/releases/download/v0.1/ResNet18_vd_pretrained_from_paddle.pth", - 34: "https://github.com/lyuwenyu/storage/releases/download/v0.1/ResNet34_vd_pretrained_from_paddle.pth", - 50: "https://github.com/lyuwenyu/storage/releases/download/v0.1/ResNet50_vd_ssld_v2_pretrained_from_paddle.pth", - 101: "https://github.com/lyuwenyu/storage/releases/download/v0.1/ResNet101_vd_ssld_pretrained_from_paddle.pth", -} - - -class ConvNormLayer(nn.Module): - def __init__(self, ch_in, ch_out, kernel_size, stride, padding=None, bias=False, act=None): - super().__init__() - self.conv = nn.Conv2d( - ch_in, - ch_out, - kernel_size, - stride, - padding=(kernel_size - 1) // 2 if padding is None else padding, - bias=bias, - ) - self.norm = nn.BatchNorm2d(ch_out) - self.act = get_activation(act) - - def forward(self, x): - return self.act(self.norm(self.conv(x))) - - -class BasicBlock(nn.Module): - expansion = 1 - - def __init__(self, ch_in, ch_out, stride, shortcut, act="relu", variant="b"): - super().__init__() - - self.shortcut = shortcut - - if not shortcut: - if variant == "d" and stride == 2: - self.short = nn.Sequential( - OrderedDict( - [ - ("pool", nn.AvgPool2d(2, 2, 0, ceil_mode=True)), - ("conv", ConvNormLayer(ch_in, ch_out, 1, 1)), - ] - ) - ) - else: - self.short = ConvNormLayer(ch_in, ch_out, 1, stride) - - self.branch2a = ConvNormLayer(ch_in, ch_out, 3, stride, act=act) - self.branch2b = ConvNormLayer(ch_out, ch_out, 3, 1, act=None) - self.act = nn.Identity() if act is None else get_activation(act) - - def forward(self, x): - out = self.branch2a(x) - out = self.branch2b(out) - if self.shortcut: - short = x - else: - short = self.short(x) - - out = out + short - out = self.act(out) - - return out - - -class BottleNeck(nn.Module): - expansion = 4 - - def __init__(self, ch_in, ch_out, stride, shortcut, act="relu", variant="b"): - super().__init__() - - if variant == "a": - stride1, stride2 = stride, 1 - else: - stride1, stride2 = 1, stride - - width = ch_out - - self.branch2a = ConvNormLayer(ch_in, width, 1, stride1, act=act) - self.branch2b = ConvNormLayer(width, width, 3, stride2, act=act) - self.branch2c = ConvNormLayer(width, ch_out * self.expansion, 1, 1) - - self.shortcut = shortcut - if not shortcut: - if variant == "d" and stride == 2: - self.short = nn.Sequential( - OrderedDict( - [ - ("pool", nn.AvgPool2d(2, 2, 0, ceil_mode=True)), - ("conv", ConvNormLayer(ch_in, ch_out * self.expansion, 1, 1)), - ] - ) - ) - else: - self.short = ConvNormLayer(ch_in, ch_out * self.expansion, 1, stride) - - self.act = nn.Identity() if act is None else get_activation(act) - - def forward(self, x): - out = self.branch2a(x) - out = self.branch2b(out) - out = self.branch2c(out) - - if self.shortcut: - short = x - else: - short = self.short(x) - - out = out + short - out = self.act(out) - - return out - - -class Blocks(nn.Module): - def __init__(self, block, ch_in, ch_out, count, stage_num, act="relu", variant="b"): - super().__init__() - - self.blocks = nn.ModuleList() - for i in range(count): - self.blocks.append( - block( - ch_in, - ch_out, - stride=2 if i == 0 and stage_num != 2 else 1, - shortcut=False if i == 0 else True, - variant=variant, - act=act, - ) - ) - - if i == 0: - ch_in = ch_out * block.expansion - - def forward(self, x): - out = x - for block in self.blocks: - out = block(out) - return out - - -@register() -class PResNet(nn.Module): - def __init__( - self, - depth, - variant="d", - num_stages=4, - return_idx=[0, 1, 2, 3], - act="relu", - freeze_at=-1, - freeze_norm=True, - pretrained=False, - ): - super().__init__() - - block_nums = ResNet_cfg[depth] - ch_in = 64 - if variant in ["c", "d"]: - conv_def = [ - [3, ch_in // 2, 3, 2, "conv1_1"], - [ch_in // 2, ch_in // 2, 3, 1, "conv1_2"], - [ch_in // 2, ch_in, 3, 1, "conv1_3"], - ] - else: - conv_def = [[3, ch_in, 7, 2, "conv1_1"]] - - self.conv1 = nn.Sequential( - OrderedDict( - [ - (name, ConvNormLayer(cin, cout, k, s, act=act)) - for cin, cout, k, s, name in conv_def - ] - ) - ) - - ch_out_list = [64, 128, 256, 512] - block = BottleNeck if depth >= 50 else BasicBlock - - _out_channels = [block.expansion * v for v in ch_out_list] - _out_strides = [4, 8, 16, 32] - - self.res_layers = nn.ModuleList() - for i in range(num_stages): - stage_num = i + 2 - self.res_layers.append( - Blocks( - block, ch_in, ch_out_list[i], block_nums[i], stage_num, act=act, variant=variant - ) - ) - ch_in = _out_channels[i] - - self.return_idx = return_idx - self.out_channels = [_out_channels[_i] for _i in return_idx] - self.out_strides = [_out_strides[_i] for _i in return_idx] - - if freeze_at >= 0: - self._freeze_parameters(self.conv1) - for i in range(min(freeze_at, num_stages)): - self._freeze_parameters(self.res_layers[i]) - - if freeze_norm: - self._freeze_norm(self) - - if pretrained: - if isinstance(pretrained, bool) or "http" in pretrained: - state = torch.hub.load_state_dict_from_url( - donwload_url[depth], map_location="cpu", model_dir="weight" - ) - else: - state = torch.load(pretrained, map_location="cpu") - self.load_state_dict(state) - print(f"Load PResNet{depth} state_dict") - - def _freeze_parameters(self, m: nn.Module): - for p in m.parameters(): - p.requires_grad = False - - def _freeze_norm(self, m: nn.Module): - if isinstance(m, nn.BatchNorm2d): - m = FrozenBatchNorm2d(m.num_features) - else: - for name, child in m.named_children(): - _child = self._freeze_norm(child) - if _child is not child: - setattr(m, name, _child) - return m - - def forward(self, x): - conv1 = self.conv1(x) - x = F.max_pool2d(conv1, kernel_size=3, stride=2, padding=1) - outs = [] - for idx, stage in enumerate(self.res_layers): - x = stage(x) - if idx in self.return_idx: - outs.append(x) - return outs diff --git a/vlm/detector/D-FINE.backup/src/nn/backbone/test_resnet.py b/vlm/detector/D-FINE.backup/src/nn/backbone/test_resnet.py deleted file mode 100644 index ab877ff..0000000 --- a/vlm/detector/D-FINE.backup/src/nn/backbone/test_resnet.py +++ /dev/null @@ -1,83 +0,0 @@ -from collections import OrderedDict - -import torch -import torch.nn as nn -import torch.nn.functional as F - -from ...core import register - - -class BasicBlock(nn.Module): - expansion = 1 - - def __init__(self, in_planes, planes, stride=1): - super(BasicBlock, self).__init__() - - self.conv1 = nn.Conv2d( - in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False - ) - self.bn1 = nn.BatchNorm2d(planes) - - self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False) - self.bn2 = nn.BatchNorm2d(planes) - - self.shortcut = nn.Sequential() - if stride != 1 or in_planes != self.expansion * planes: - self.shortcut = nn.Sequential( - nn.Conv2d( - in_planes, self.expansion * planes, kernel_size=1, stride=stride, bias=False - ), - nn.BatchNorm2d(self.expansion * planes), - ) - - def forward(self, x): - out = F.relu(self.bn1(self.conv1(x))) - out = self.bn2(self.conv2(out)) - out += self.shortcut(x) - out = F.relu(out) - return out - - -class _ResNet(nn.Module): - def __init__(self, block, num_blocks, num_classes=10): - super().__init__() - self.in_planes = 64 - - self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False) - self.bn1 = nn.BatchNorm2d(64) - - self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1) - self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2) - self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2) - self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2) - - self.linear = nn.Linear(512 * block.expansion, num_classes) - - def _make_layer(self, block, planes, num_blocks, stride): - strides = [stride] + [1] * (num_blocks - 1) - layers = [] - for stride in strides: - layers.append(block(self.in_planes, planes, stride)) - self.in_planes = planes * block.expansion - return nn.Sequential(*layers) - - def forward(self, x): - out = F.relu(self.bn1(self.conv1(x))) - out = self.layer1(out) - out = self.layer2(out) - out = self.layer3(out) - out = self.layer4(out) - out = F.avg_pool2d(out, 4) - out = out.view(out.size(0), -1) - out = self.linear(out) - return out - - -@register() -class MResNet(nn.Module): - def __init__(self, num_classes=10, num_blocks=[2, 2, 2, 2]) -> None: - super().__init__() - self.model = _ResNet(BasicBlock, num_blocks, num_classes) - - def forward(self, x): - return self.model(x) diff --git a/vlm/detector/D-FINE.backup/src/nn/backbone/timm_model.py b/vlm/detector/D-FINE.backup/src/nn/backbone/timm_model.py deleted file mode 100644 index d2258a5..0000000 --- a/vlm/detector/D-FINE.backup/src/nn/backbone/timm_model.py +++ /dev/null @@ -1,66 +0,0 @@ -"""Copyright(c) 2023 lyuwenyu. All Rights Reserved. - -https://towardsdatascience.com/getting-started-with-pytorch-image-models-timm-a-practitioners-guide-4e77b4bf9055#0583 -""" - -import torch -from torchvision.models.feature_extraction import create_feature_extractor, get_graph_node_names - -from ...core import register -from .utils import IntermediateLayerGetter - - -@register() -class TimmModel(torch.nn.Module): - def __init__( - self, name, return_layers, pretrained=False, exportable=True, features_only=True, **kwargs - ) -> None: - super().__init__() - - import timm - - model = timm.create_model( - name, - pretrained=pretrained, - exportable=exportable, - features_only=features_only, - **kwargs, - ) - # nodes, _ = get_graph_node_names(model) - # print(nodes) - # features = {'': ''} - # model = create_feature_extractor(model, return_nodes=features) - - assert set(return_layers).issubset( - model.feature_info.module_name() - ), f"return_layers should be a subset of {model.feature_info.module_name()}" - - # self.model = model - self.model = IntermediateLayerGetter(model, return_layers) - - return_idx = [model.feature_info.module_name().index(name) for name in return_layers] - self.strides = [model.feature_info.reduction()[i] for i in return_idx] - self.channels = [model.feature_info.channels()[i] for i in return_idx] - self.return_idx = return_idx - self.return_layers = return_layers - - def forward(self, x: torch.Tensor): - outputs = self.model(x) - # outputs = [outputs[i] for i in self.return_idx] - return outputs - - -if __name__ == "__main__": - model = TimmModel(name="resnet34", return_layers=["layer2", "layer3"]) - data = torch.rand(1, 3, 640, 640) - outputs = model(data) - - for output in outputs: - print(output.shape) - - """ - model: - type: TimmModel - name: resnet34 - return_layers: ['layer2', 'layer4'] - """ diff --git a/vlm/detector/D-FINE.backup/src/nn/backbone/torchvision_model.py b/vlm/detector/D-FINE.backup/src/nn/backbone/torchvision_model.py deleted file mode 100644 index dd207d9..0000000 --- a/vlm/detector/D-FINE.backup/src/nn/backbone/torchvision_model.py +++ /dev/null @@ -1,50 +0,0 @@ -""" -Copied from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright(c) 2023 lyuwenyu. All Rights Reserved. -""" - -import torch -import torchvision - -from ...core import register -from .utils import IntermediateLayerGetter - -__all__ = ["TorchVisionModel"] - - -@register() -class TorchVisionModel(torch.nn.Module): - def __init__(self, name, return_layers, weights=None, **kwargs) -> None: - super().__init__() - - if weights is not None: - weights = getattr(torchvision.models.get_model_weights(name), weights) - - model = torchvision.models.get_model(name, weights=weights, **kwargs) - - # TODO hard code. - if hasattr(model, "features"): - model = IntermediateLayerGetter(model.features, return_layers) - else: - model = IntermediateLayerGetter(model, return_layers) - - self.model = model - - def forward(self, x): - return self.model(x) - - -# TorchVisionModel('swin_t', return_layers=['5', '7']) -# TorchVisionModel('resnet34', return_layers=['layer2','layer3', 'layer4']) - -# TorchVisionModel: -# name: swin_t -# return_layers: ['5', '7'] -# weights: DEFAULT - - -# model: -# type: TorchVisionModel -# name: resnet34 -# return_layers: ['layer2','layer3', 'layer4'] -# weights: DEFAULT diff --git a/vlm/detector/D-FINE.backup/src/nn/backbone/utils.py b/vlm/detector/D-FINE.backup/src/nn/backbone/utils.py deleted file mode 100644 index 58efeb3..0000000 --- a/vlm/detector/D-FINE.backup/src/nn/backbone/utils.py +++ /dev/null @@ -1,56 +0,0 @@ -""" -https://github.com/pytorch/vision/blob/main/torchvision/models/_utils.py - -Copyright(c) 2023 lyuwenyu. All Rights Reserved. -""" - -from collections import OrderedDict -from typing import Dict, List - -import torch.nn as nn - - -class IntermediateLayerGetter(nn.ModuleDict): - """ - Module wrapper that returns intermediate layers from a model - - It has a strong assumption that the modules have been registered - into the model in the same order as they are used. - This means that one should **not** reuse the same nn.Module - twice in the forward if you want this to work. - - Additionally, it is only able to query submodules that are directly - assigned to the model. So if `model` is passed, `model.feature1` can - be returned, but not `model.feature1.layer2`. - """ - - _version = 3 - - def __init__(self, model: nn.Module, return_layers: List[str]) -> None: - if not set(return_layers).issubset([name for name, _ in model.named_children()]): - raise ValueError( - "return_layers are not present in model. {}".format( - [name for name, _ in model.named_children()] - ) - ) - orig_return_layers = return_layers - return_layers = {str(k): str(k) for k in return_layers} - layers = OrderedDict() - for name, module in model.named_children(): - layers[name] = module - if name in return_layers: - del return_layers[name] - if not return_layers: - break - - super().__init__(layers) - self.return_layers = orig_return_layers - - def forward(self, x): - outputs = [] - for name, module in self.items(): - x = module(x) - if name in self.return_layers: - outputs.append(x) - - return outputs diff --git a/vlm/detector/D-FINE.backup/src/nn/criterion/__init__.py b/vlm/detector/D-FINE.backup/src/nn/criterion/__init__.py deleted file mode 100644 index da74acf..0000000 --- a/vlm/detector/D-FINE.backup/src/nn/criterion/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -""" -Copied from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright(c) 2023 lyuwenyu. All Rights Reserved. -""" - -import torch.nn as nn - -from ...core import register -from .det_criterion import DetCriterion - -CrossEntropyLoss = register()(nn.CrossEntropyLoss) diff --git a/vlm/detector/D-FINE.backup/src/nn/criterion/det_criterion.py b/vlm/detector/D-FINE.backup/src/nn/criterion/det_criterion.py deleted file mode 100644 index d5dbde5..0000000 --- a/vlm/detector/D-FINE.backup/src/nn/criterion/det_criterion.py +++ /dev/null @@ -1,188 +0,0 @@ -""" -Copied from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright(c) 2023 lyuwenyu. All Rights Reserved. -""" - -import torch -import torch.distributed -import torch.nn.functional as F -import torchvision - -from ...core import register -from ...misc import box_ops, dist_utils - - -@register() -class DetCriterion(torch.nn.Module): - """Default Detection Criterion""" - - __share__ = ["num_classes"] - __inject__ = ["matcher"] - - def __init__( - self, - losses, - weight_dict, - num_classes=80, - alpha=0.75, - gamma=2.0, - box_fmt="cxcywh", - matcher=None, - ): - """ - Args: - losses (list[str]): requested losses, support ['boxes', 'vfl', 'focal'] - weight_dict (dict[str, float)]: corresponding losses weight, including - ['loss_bbox', 'loss_giou', 'loss_vfl', 'loss_focal'] - box_fmt (str): in box format, 'cxcywh' or 'xyxy' - matcher (Matcher): matcher used to match source to target - """ - super().__init__() - self.losses = losses - self.weight_dict = weight_dict - self.alpha = alpha - self.gamma = gamma - self.num_classes = num_classes - self.box_fmt = box_fmt - assert matcher is not None, "" - self.matcher = matcher - - def forward(self, outputs, targets, **kwargs): - """ - Args: - outputs: Dict[Tensor], 'pred_boxes', 'pred_logits', 'meta'. - targets, List[Dict[str, Tensor]], len(targets) == batch_size. - kwargs, store other information such as current epoch id. - Return: - losses, Dict[str, Tensor] - """ - matched = self.matcher(outputs, targets) - values = matched["values"] - indices = matched["indices"] - num_boxes = self._get_positive_nums(indices) - - # Compute all the requested losses - losses = {} - for loss in self.losses: - l_dict = self.get_loss(loss, outputs, targets, indices, num_boxes) - l_dict = {k: l_dict[k] * self.weight_dict[k] for k in l_dict if k in self.weight_dict} - losses.update(l_dict) - return losses - - def _get_src_permutation_idx(self, indices): - # permute predictions following indices - batch_idx = torch.cat([torch.full_like(src, i) for i, (src, _) in enumerate(indices)]) - src_idx = torch.cat([src for (src, _) in indices]) - return batch_idx, src_idx - - def _get_tgt_permutation_idx(self, indices): - # permute targets following indices - batch_idx = torch.cat([torch.full_like(tgt, i) for i, (_, tgt) in enumerate(indices)]) - tgt_idx = torch.cat([tgt for (_, tgt) in indices]) - return batch_idx, tgt_idx - - def _get_positive_nums(self, indices): - # number of positive samples - num_pos = sum(len(i) for (i, _) in indices) - num_pos = torch.as_tensor([num_pos], dtype=torch.float32, device=indices[0][0].device) - if dist_utils.is_dist_available_and_initialized(): - torch.distributed.all_reduce(num_pos) - num_pos = torch.clamp(num_pos / dist_utils.get_world_size(), min=1).item() - return num_pos - - def loss_labels_focal(self, outputs, targets, indices, num_boxes): - assert "pred_logits" in outputs - src_logits = outputs["pred_logits"] - - idx = self._get_src_permutation_idx(indices) - target_classes_o = torch.cat([t["labels"][j] for t, (_, j) in zip(targets, indices)]) - target_classes = torch.full( - src_logits.shape[:2], self.num_classes, dtype=torch.int64, device=src_logits.device - ) - target_classes[idx] = target_classes_o - - target = F.one_hot(target_classes, num_classes=self.num_classes + 1)[..., :-1].to( - src_logits.dtype - ) - loss = torchvision.ops.sigmoid_focal_loss( - src_logits, target, self.alpha, self.gamma, reduction="none" - ) - loss = loss.sum() / num_boxes - return {"loss_focal": loss} - - def loss_labels_vfl(self, outputs, targets, indices, num_boxes): - assert "pred_boxes" in outputs - idx = self._get_src_permutation_idx(indices) - - src_boxes = outputs["pred_boxes"][idx] - target_boxes = torch.cat([t["boxes"][j] for t, (_, j) in zip(targets, indices)], dim=0) - - src_boxes = torchvision.ops.box_convert(src_boxes, in_fmt=self.box_fmt, out_fmt="xyxy") - target_boxes = torchvision.ops.box_convert( - target_boxes, in_fmt=self.box_fmt, out_fmt="xyxy" - ) - iou, _ = box_ops.elementwise_box_iou(src_boxes.detach(), target_boxes) - - src_logits: torch.Tensor = outputs["pred_logits"] - target_classes_o = torch.cat([t["labels"][j] for t, (_, j) in zip(targets, indices)]) - target_classes = torch.full( - src_logits.shape[:2], self.num_classes, dtype=torch.int64, device=src_logits.device - ) - target_classes[idx] = target_classes_o - target = F.one_hot(target_classes, num_classes=self.num_classes + 1)[..., :-1] - - target_score_o = torch.zeros_like(target_classes, dtype=src_logits.dtype) - target_score_o[idx] = iou.to(src_logits.dtype) - target_score = target_score_o.unsqueeze(-1) * target - - src_score = F.sigmoid(src_logits.detach()) - weight = self.alpha * src_score.pow(self.gamma) * (1 - target) + target_score - - loss = F.binary_cross_entropy_with_logits( - src_logits, target_score, weight=weight, reduction="none" - ) - loss = loss.sum() / num_boxes - return {"loss_vfl": loss} - - def loss_boxes(self, outputs, targets, indices, num_boxes): - assert "pred_boxes" in outputs - idx = self._get_src_permutation_idx(indices) - src_boxes = outputs["pred_boxes"][idx] - target_boxes = torch.cat([t["boxes"][i] for t, (_, i) in zip(targets, indices)], dim=0) - - losses = {} - loss_bbox = F.l1_loss(src_boxes, target_boxes, reduction="none") - losses["loss_bbox"] = loss_bbox.sum() / num_boxes - - src_boxes = torchvision.ops.box_convert(src_boxes, in_fmt=self.box_fmt, out_fmt="xyxy") - target_boxes = torchvision.ops.box_convert( - target_boxes, in_fmt=self.box_fmt, out_fmt="xyxy" - ) - loss_giou = 1 - box_ops.elementwise_generalized_box_iou(src_boxes, target_boxes) - losses["loss_giou"] = loss_giou.sum() / num_boxes - return losses - - def loss_boxes_giou(self, outputs, targets, indices, num_boxes): - assert "pred_boxes" in outputs - idx = self._get_src_permutation_idx(indices) - src_boxes = outputs["pred_boxes"][idx] - target_boxes = torch.cat([t["boxes"][i] for t, (_, i) in zip(targets, indices)], dim=0) - - losses = {} - src_boxes = torchvision.ops.box_convert(src_boxes, in_fmt=self.box_fmt, out_fmt="xyxy") - target_boxes = torchvision.ops.box_convert( - target_boxes, in_fmt=self.box_fmt, out_fmt="xyxy" - ) - loss_giou = 1 - box_ops.elementwise_generalized_box_iou(src_boxes, target_boxes) - losses["loss_giou"] = loss_giou.sum() / num_boxes - return losses - - def get_loss(self, loss, outputs, targets, indices, num_boxes, **kwargs): - loss_map = { - "boxes": self.loss_boxes, - "giou": self.loss_boxes_giou, - "vfl": self.loss_labels_vfl, - "focal": self.loss_labels_focal, - } - assert loss in loss_map, f"do you really want to compute {loss} loss?" - return loss_map[loss](outputs, targets, indices, num_boxes, **kwargs) diff --git a/vlm/detector/D-FINE.backup/src/nn/postprocessor/__init__.py b/vlm/detector/D-FINE.backup/src/nn/postprocessor/__init__.py deleted file mode 100644 index 0949998..0000000 --- a/vlm/detector/D-FINE.backup/src/nn/postprocessor/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -""" -Copied from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright(c) 2023 lyuwenyu. All Rights Reserved. -""" - -from .nms_postprocessor import DetNMSPostProcessor diff --git a/vlm/detector/D-FINE.backup/src/nn/postprocessor/box_revert.py b/vlm/detector/D-FINE.backup/src/nn/postprocessor/box_revert.py deleted file mode 100644 index c40f0c1..0000000 --- a/vlm/detector/D-FINE.backup/src/nn/postprocessor/box_revert.py +++ /dev/null @@ -1,66 +0,0 @@ -""" -Copied from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright(c) 2023 lyuwenyu. All Rights Reserved. -""" - -from enum import Enum - -import torch -import torchvision -from torch import Tensor - - -class BoxProcessFormat(Enum): - """Box process format - - Available formats are - * ``RESIZE`` - * ``RESIZE_KEEP_RATIO`` - * ``RESIZE_KEEP_RATIO_PADDING`` - """ - - RESIZE = 1 - RESIZE_KEEP_RATIO = 2 - RESIZE_KEEP_RATIO_PADDING = 3 - - -def box_revert( - boxes: Tensor, - orig_sizes: Tensor = None, - eval_sizes: Tensor = None, - inpt_sizes: Tensor = None, - inpt_padding: Tensor = None, - normalized: bool = True, - in_fmt: str = "cxcywh", - out_fmt: str = "xyxy", - process_fmt=BoxProcessFormat.RESIZE, -) -> Tensor: - """ - Args: - boxes(Tensor), [N, :, 4], (x1, y1, x2, y2), pred boxes. - inpt_sizes(Tensor), [N, 2], (w, h). input sizes. - orig_sizes(Tensor), [N, 2], (w, h). origin sizes. - inpt_padding (Tensor), [N, 2], (w_pad, h_pad, ...). - (inpt_sizes + inpt_padding) == eval_sizes - """ - assert in_fmt in ("cxcywh", "xyxy"), "" - - if normalized and eval_sizes is not None: - boxes = boxes * eval_sizes.repeat(1, 2).unsqueeze(1) - - if inpt_padding is not None: - if in_fmt == "xyxy": - boxes -= inpt_padding[:, :2].repeat(1, 2).unsqueeze(1) - elif in_fmt == "cxcywh": - boxes[..., :2] -= inpt_padding[:, :2].repeat(1, 2).unsqueeze(1) - - if orig_sizes is not None: - orig_sizes = orig_sizes.repeat(1, 2).unsqueeze(1) - if inpt_sizes is not None: - inpt_sizes = inpt_sizes.repeat(1, 2).unsqueeze(1) - boxes = boxes * (orig_sizes / inpt_sizes) - else: - boxes = boxes * orig_sizes - - boxes = torchvision.ops.box_convert(boxes, in_fmt=in_fmt, out_fmt=out_fmt) - return boxes diff --git a/vlm/detector/D-FINE.backup/src/nn/postprocessor/detr_postprocessor.py b/vlm/detector/D-FINE.backup/src/nn/postprocessor/detr_postprocessor.py deleted file mode 100644 index 6cccf72..0000000 --- a/vlm/detector/D-FINE.backup/src/nn/postprocessor/detr_postprocessor.py +++ /dev/null @@ -1,86 +0,0 @@ -""" -Copied from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright(c) 2023 lyuwenyu. All Rights Reserved. -""" - -import torch -import torch.nn as nn -import torch.nn.functional as F -import torchvision - -__all__ = ["DetDETRPostProcessor"] - -from .box_revert import BoxProcessFormat, box_revert - - -def mod(a, b): - out = a - a // b * b - return out - - -class DetDETRPostProcessor(nn.Module): - def __init__( - self, - num_classes=80, - use_focal_loss=True, - num_top_queries=300, - box_process_format=BoxProcessFormat.RESIZE, - ) -> None: - super().__init__() - self.use_focal_loss = use_focal_loss - self.num_top_queries = num_top_queries - self.num_classes = int(num_classes) - self.box_process_format = box_process_format - self.deploy_mode = False - - def extra_repr(self) -> str: - return f"use_focal_loss={self.use_focal_loss}, num_classes={self.num_classes}, num_top_queries={self.num_top_queries}" - - def forward(self, outputs, **kwargs): - logits, boxes = outputs["pred_logits"], outputs["pred_boxes"] - - if self.use_focal_loss: - scores = F.sigmoid(logits) - scores, index = torch.topk(scores.flatten(1), self.num_top_queries, dim=-1) - labels = index % self.num_classes - # labels = mod(index, self.num_classes) # for tensorrt - index = index // self.num_classes - boxes = boxes.gather(dim=1, index=index.unsqueeze(-1).repeat(1, 1, boxes.shape[-1])) - - else: - scores = F.softmax(logits)[:, :, :-1] - scores, labels = scores.max(dim=-1) - if scores.shape[1] > self.num_top_queries: - scores, index = torch.topk(scores, self.num_top_queries, dim=-1) - labels = torch.gather(labels, dim=1, index=index) - boxes = torch.gather( - boxes, dim=1, index=index.unsqueeze(-1).tile(1, 1, boxes.shape[-1]) - ) - - if kwargs is not None: - boxes = box_revert( - boxes, - in_fmt="cxcywh", - out_fmt="xyxy", - process_fmt=self.box_process_format, - normalized=True, - **kwargs, - ) - - # TODO for onnx export - if self.deploy_mode: - return labels, boxes, scores - - results = [] - for lab, box, sco in zip(labels, boxes, scores): - result = dict(labels=lab, boxes=box, scores=sco) - results.append(result) - - return results - - def deploy( - self, - ): - self.eval() - self.deploy_mode = True - return self diff --git a/vlm/detector/D-FINE.backup/src/nn/postprocessor/nms_postprocessor.py b/vlm/detector/D-FINE.backup/src/nn/postprocessor/nms_postprocessor.py deleted file mode 100644 index 260e08c..0000000 --- a/vlm/detector/D-FINE.backup/src/nn/postprocessor/nms_postprocessor.py +++ /dev/null @@ -1,86 +0,0 @@ -""" -Copied from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright(c) 2023 lyuwenyu. All Rights Reserved. -""" - -from typing import Dict - -import torch -import torch.distributed -import torch.nn.functional as F -import torchvision -from torch import Tensor - -from ...core import register - -__all__ = [ - "DetNMSPostProcessor", -] - - -@register() -class DetNMSPostProcessor(torch.nn.Module): - def __init__( - self, - iou_threshold=0.7, - score_threshold=0.01, - keep_topk=300, - box_fmt="cxcywh", - logit_fmt="sigmoid", - ) -> None: - super().__init__() - self.iou_threshold = iou_threshold - self.score_threshold = score_threshold - self.keep_topk = keep_topk - self.box_fmt = box_fmt.lower() - self.logit_fmt = logit_fmt.lower() - self.logit_func = getattr(F, self.logit_fmt, None) - self.deploy_mode = False - - def forward(self, outputs: Dict[str, Tensor], orig_target_sizes: Tensor): - logits, boxes = outputs["pred_logits"], outputs["pred_boxes"] - pred_boxes = torchvision.ops.box_convert(boxes, in_fmt=self.box_fmt, out_fmt="xyxy") - pred_boxes *= orig_target_sizes.repeat(1, 2).unsqueeze(1) - - values, pred_labels = torch.max(logits, dim=-1) - - if self.logit_func: - pred_scores = self.logit_func(values) - else: - pred_scores = values - - # TODO for onnx export - if self.deploy_mode: - blobs = { - "pred_labels": pred_labels, - "pred_boxes": pred_boxes, - "pred_scores": pred_scores, - } - return blobs - - results = [] - for i in range(logits.shape[0]): - score_keep = pred_scores[i] > self.score_threshold - pred_box = pred_boxes[i][score_keep] - pred_label = pred_labels[i][score_keep] - pred_score = pred_scores[i][score_keep] - - keep = torchvision.ops.batched_nms(pred_box, pred_score, pred_label, self.iou_threshold) - keep = keep[: self.keep_topk] - - blob = { - "labels": pred_label[keep], - "boxes": pred_box[keep], - "scores": pred_score[keep], - } - - results.append(blob) - - return results - - def deploy( - self, - ): - self.eval() - self.deploy_mode = True - return self diff --git a/vlm/detector/D-FINE.backup/src/optim/__init__.py b/vlm/detector/D-FINE.backup/src/optim/__init__.py deleted file mode 100644 index fbc06c0..0000000 --- a/vlm/detector/D-FINE.backup/src/optim/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -""" -Copied from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright(c) 2023 lyuwenyu. All Rights Reserved. -""" - -from .amp import * -from .ema import * -from .optim import * -from .warmup import * diff --git a/vlm/detector/D-FINE.backup/src/optim/amp.py b/vlm/detector/D-FINE.backup/src/optim/amp.py deleted file mode 100644 index 5392899..0000000 --- a/vlm/detector/D-FINE.backup/src/optim/amp.py +++ /dev/null @@ -1,12 +0,0 @@ -""" -Copied from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright(c) 2023 lyuwenyu. All Rights Reserved. -""" - -import torch.cuda.amp as amp - -from ..core import register - -__all__ = ["GradScaler"] - -GradScaler = register()(amp.grad_scaler.GradScaler) diff --git a/vlm/detector/D-FINE.backup/src/optim/ema.py b/vlm/detector/D-FINE.backup/src/optim/ema.py deleted file mode 100644 index 9e508b6..0000000 --- a/vlm/detector/D-FINE.backup/src/optim/ema.py +++ /dev/null @@ -1,108 +0,0 @@ -""" -D-FINE: Redefine Regression Task of DETRs as Fine-grained Distribution Refinement -Copyright (c) 2024 The D-FINE Authors. All Rights Reserved. ---------------------------------------------------------------------------------- -Modified from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright (c) 2023 lyuwenyu. All Rights Reserved. -""" - -import math -from copy import deepcopy - -import torch -import torch.nn as nn - -from ..core import register -from ..misc import dist_utils - -__all__ = ["ModelEMA"] - - -@register() -class ModelEMA(object): - """ - Model Exponential Moving Average from https://github.com/rwightman/pytorch-image-models - Keep a moving average of everything in the model state_dict (parameters and buffers). - This is intended to allow functionality like - https://www.tensorflow.org/api_docs/python/tf/train/ExponentialMovingAverage - A smoothed version of the weights is necessary for some training schemes to perform well. - This class is sensitive where it is initialized in the sequence of model init, - GPU assignment and distributed training wrappers. - """ - - def __init__( - self, model: nn.Module, decay: float = 0.9999, warmups: int = 1000, start: int = 0 - ): - super().__init__() - - self.module = deepcopy(dist_utils.de_parallel(model)).eval() - # if next(model.parameters()).device.type != 'cpu': - # self.module.half() # FP16 EMA - - self.decay = decay - self.warmups = warmups - self.before_start = 0 - self.start = start - self.updates = 0 # number of EMA updates - if warmups == 0: - self.decay_fn = lambda x: decay - else: - self.decay_fn = lambda x: decay * ( - 1 - math.exp(-x / warmups) - ) # decay exponential ramp (to help early epochs) - - for p in self.module.parameters(): - p.requires_grad_(False) - - def update(self, model: nn.Module): - if self.before_start < self.start: - self.before_start += 1 - return - # Update EMA parameters - with torch.no_grad(): - self.updates += 1 - d = self.decay_fn(self.updates) - msd = dist_utils.de_parallel(model).state_dict() - for k, v in self.module.state_dict().items(): - if v.dtype.is_floating_point: - v *= d - v += (1 - d) * msd[k].detach() - - def to(self, *args, **kwargs): - self.module = self.module.to(*args, **kwargs) - return self - - def state_dict( - self, - ): - return dict(module=self.module.state_dict(), updates=self.updates) - - def load_state_dict(self, state, strict=True): - self.module.load_state_dict(state["module"], strict=strict) - if "updates" in state: - self.updates = state["updates"] - - def forwad( - self, - ): - raise RuntimeError("ema...") - - def extra_repr(self) -> str: - return f"decay={self.decay}, warmups={self.warmups}" - - -class ExponentialMovingAverage(torch.optim.swa_utils.AveragedModel): - """Maintains moving averages of model parameters using an exponential decay. - ``ema_avg = decay * avg_model_param + (1 - decay) * model_param`` - `torch.optim.swa_utils.AveragedModel `_ - is used to compute the EMA. - """ - - def __init__(self, model, decay, device="cpu", use_buffers=True): - self.decay_fn = lambda x: decay * (1 - math.exp(-x / 2000)) - - def ema_avg(avg_model_param, model_param, num_averaged): - decay = self.decay_fn(num_averaged) - return decay * avg_model_param + (1 - decay) * model_param - - super().__init__(model, device, ema_avg, use_buffers=use_buffers) diff --git a/vlm/detector/D-FINE.backup/src/optim/optim.py b/vlm/detector/D-FINE.backup/src/optim/optim.py deleted file mode 100644 index cec3030..0000000 --- a/vlm/detector/D-FINE.backup/src/optim/optim.py +++ /dev/null @@ -1,22 +0,0 @@ -""" -Copied from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright(c) 2023 lyuwenyu. All Rights Reserved. -""" - -import torch.optim as optim -import torch.optim.lr_scheduler as lr_scheduler - -from ..core import register - -__all__ = ["AdamW", "SGD", "Adam", "MultiStepLR", "CosineAnnealingLR", "OneCycleLR", "LambdaLR"] - - -SGD = register()(optim.SGD) -Adam = register()(optim.Adam) -AdamW = register()(optim.AdamW) - - -MultiStepLR = register()(lr_scheduler.MultiStepLR) -CosineAnnealingLR = register()(lr_scheduler.CosineAnnealingLR) -OneCycleLR = register()(lr_scheduler.OneCycleLR) -LambdaLR = register()(lr_scheduler.LambdaLR) diff --git a/vlm/detector/D-FINE.backup/src/optim/warmup.py b/vlm/detector/D-FINE.backup/src/optim/warmup.py deleted file mode 100644 index cd475ef..0000000 --- a/vlm/detector/D-FINE.backup/src/optim/warmup.py +++ /dev/null @@ -1,56 +0,0 @@ -""" -Copied from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright(c) 2023 lyuwenyu. All Rights Reserved. -""" - -from torch.optim.lr_scheduler import LRScheduler - -from ..core import register - - -class Warmup(object): - def __init__( - self, lr_scheduler: LRScheduler, warmup_duration: int, last_step: int = -1 - ) -> None: - self.lr_scheduler = lr_scheduler - self.warmup_end_values = [pg["lr"] for pg in lr_scheduler.optimizer.param_groups] - self.last_step = last_step - self.warmup_duration = warmup_duration - self.step() - - def state_dict(self): - return {k: v for k, v in self.__dict__.items() if k != "lr_scheduler"} - - def load_state_dict(self, state_dict): - self.__dict__.update(state_dict) - - def get_warmup_factor(self, step, **kwargs): - raise NotImplementedError - - def step( - self, - ): - self.last_step += 1 - if self.last_step >= self.warmup_duration: - return - factor = self.get_warmup_factor(self.last_step) - for i, pg in enumerate(self.lr_scheduler.optimizer.param_groups): - pg["lr"] = factor * self.warmup_end_values[i] - - def finished( - self, - ): - if self.last_step >= self.warmup_duration: - return True - return False - - -@register() -class LinearWarmup(Warmup): - def __init__( - self, lr_scheduler: LRScheduler, warmup_duration: int, last_step: int = -1 - ) -> None: - super().__init__(lr_scheduler, warmup_duration, last_step) - - def get_warmup_factor(self, step): - return min(1.0, (step + 1) / self.warmup_duration) diff --git a/vlm/detector/D-FINE.backup/src/solver/__init__.py b/vlm/detector/D-FINE.backup/src/solver/__init__.py deleted file mode 100644 index 440182a..0000000 --- a/vlm/detector/D-FINE.backup/src/solver/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -""" -Copied from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright(c) 2023 lyuwenyu. All Rights Reserved. -""" - -from typing import Dict - -from ._solver import BaseSolver -from .clas_solver import ClasSolver -from .det_solver import DetSolver - -TASKS: Dict[str, BaseSolver] = { - "classification": ClasSolver, - "detection": DetSolver, -} diff --git a/vlm/detector/D-FINE.backup/src/solver/_solver.py b/vlm/detector/D-FINE.backup/src/solver/_solver.py deleted file mode 100644 index 3120437..0000000 --- a/vlm/detector/D-FINE.backup/src/solver/_solver.py +++ /dev/null @@ -1,783 +0,0 @@ -import atexit -from datetime import datetime -from pathlib import Path -from typing import Dict - -import torch -import torch.nn as nn - -from ..core import BaseConfig -from ..misc import dist_utils - - -def to(m: nn.Module, device: str): - if m is None: - return None - return m.to(device) - - -def remove_module_prefix(state_dict): - new_state_dict = {} - for k, v in state_dict.items(): - if k.startswith("module."): - new_state_dict[k[7:]] = v - else: - new_state_dict[k] = v - return new_state_dict - - -class BaseSolver(object): - def __init__(self, cfg: BaseConfig) -> None: - self.cfg = cfg - self.obj365_ids = [ - 0, - 46, - 5, - 58, - 114, - 55, - 116, - 65, - 21, - 40, - 176, - 127, - 249, - 24, - 56, - 139, - 92, - 78, - 99, - 96, - 144, - 295, - 178, - 180, - 38, - 39, - 13, - 43, - 120, - 219, - 148, - 173, - 165, - 154, - 137, - 113, - 145, - 146, - 204, - 8, - 35, - 10, - 88, - 84, - 93, - 26, - 112, - 82, - 265, - 104, - 141, - 152, - 234, - 143, - 150, - 97, - 2, - 50, - 25, - 75, - 98, - 153, - 37, - 73, - 115, - 132, - 106, - 61, - 163, - 134, - 277, - 81, - 133, - 18, - 94, - 30, - 169, - 70, - 328, - 226, - ] - - def _setup(self): - """Avoid instantiating unnecessary classes""" - cfg = self.cfg - if cfg.device: - device = torch.device(cfg.device) - else: - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - - self.model = cfg.model - - # NOTE: Must load_tuning_state before EMA instance building - if self.cfg.tuning: - print(f"Tuning checkpoint from {self.cfg.tuning}") - self.load_tuning_state(self.cfg.tuning) - - self.model = dist_utils.warp_model( - self.model.to(device), - sync_bn=cfg.sync_bn, - find_unused_parameters=cfg.find_unused_parameters, - ) - - self.criterion = self.to(cfg.criterion, device) - self.postprocessor = self.to(cfg.postprocessor, device) - - self.ema = self.to(cfg.ema, device) - self.scaler = cfg.scaler - - self.device = device - self.last_epoch = self.cfg.last_epoch - - self.output_dir = Path(cfg.output_dir) - self.output_dir.mkdir(parents=True, exist_ok=True) - self.writer = cfg.writer - - if self.writer: - atexit.register(self.writer.close) - if dist_utils.is_main_process(): - self.writer.add_text("config", "{:s}".format(cfg.__repr__()), 0) - self.use_wandb = self.cfg.use_wandb - if self.use_wandb: - try: - import wandb - self.use_wandb = True - except ImportError: - self.use_wandb = False - - def cleanup(self): - if self.writer: - atexit.register(self.writer.close) - - def train(self): - self._setup() - self.optimizer = self.cfg.optimizer - self.lr_scheduler = self.cfg.lr_scheduler - self.lr_warmup_scheduler = self.cfg.lr_warmup_scheduler - - self.train_dataloader = dist_utils.warp_loader( - self.cfg.train_dataloader, shuffle=self.cfg.train_dataloader.shuffle - ) - self.val_dataloader = dist_utils.warp_loader( - self.cfg.val_dataloader, shuffle=self.cfg.val_dataloader.shuffle - ) - - self.evaluator = self.cfg.evaluator - - # NOTE: Instantiating order - if self.cfg.resume: - print(f"Resume checkpoint from {self.cfg.resume}") - self.load_resume_state(self.cfg.resume) - - def eval(self): - self._setup() - - self.val_dataloader = dist_utils.warp_loader( - self.cfg.val_dataloader, shuffle=self.cfg.val_dataloader.shuffle - ) - - self.evaluator = self.cfg.evaluator - - if self.cfg.resume: - print(f"Resume checkpoint from {self.cfg.resume}") - self.load_resume_state(self.cfg.resume) - - def to(self, module, device): - return module.to(device) if hasattr(module, "to") else module - - def state_dict(self): - """State dict, train/eval""" - state = {} - state["date"] = datetime.now().isoformat() - - # For resume - state["last_epoch"] = self.last_epoch - - for k, v in self.__dict__.items(): - if hasattr(v, "state_dict"): - v = dist_utils.de_parallel(v) - state[k] = v.state_dict() - - return state - - def load_state_dict(self, state): - """Load state dict, train/eval""" - if "last_epoch" in state: - self.last_epoch = state["last_epoch"] - print("Load last_epoch") - - for k, v in self.__dict__.items(): - if hasattr(v, "load_state_dict") and k in state: - v = dist_utils.de_parallel(v) - v.load_state_dict(state[k]) - print(f"Load {k}.state_dict") - - if hasattr(v, "load_state_dict") and k not in state: - if k == "ema": - model = getattr(self, "model", None) - if model is not None: - ema = dist_utils.de_parallel(v) - model_state_dict = remove_module_prefix(model.state_dict()) - ema.load_state_dict({"module": model_state_dict}) - print(f"Load {k}.state_dict from model.state_dict") - else: - print(f"Not load {k}.state_dict") - - def load_resume_state(self, path: str): - """Load resume""" - if path.startswith("http"): - state = torch.hub.load_state_dict_from_url(path, map_location="cpu") - else: - state = torch.load(path, map_location="cpu") - - # state['model'] = remove_module_prefix(state['model']) - self.load_state_dict(state) - - def load_tuning_state(self, path: str): - """Load model for tuning and adjust mismatched head parameters""" - if path.startswith("http"): - state = torch.hub.load_state_dict_from_url(path, map_location="cpu") - else: - state = torch.load(path, map_location="cpu") - - module = dist_utils.de_parallel(self.model) - - # Load the appropriate state dict - if "ema" in state: - pretrain_state_dict = state["ema"]["module"] - else: - pretrain_state_dict = state["model"] - - # Adjust head parameters between datasets - try: - adjusted_state_dict = self._adjust_head_parameters( - module.state_dict(), pretrain_state_dict - ) - stat, infos = self._matched_state(module.state_dict(), adjusted_state_dict) - except Exception: - stat, infos = self._matched_state(module.state_dict(), pretrain_state_dict) - - module.load_state_dict(stat, strict=False) - print(f"Load model.state_dict, {infos}") - - @staticmethod - def _matched_state(state: Dict[str, torch.Tensor], params: Dict[str, torch.Tensor]): - missed_list = [] - unmatched_list = [] - matched_state = {} - for k, v in state.items(): - if k in params: - if v.shape == params[k].shape: - matched_state[k] = params[k] - else: - unmatched_list.append(k) - else: - missed_list.append(k) - - return matched_state, {"missed": missed_list, "unmatched": unmatched_list} - - def _adjust_head_parameters(self, cur_state_dict, pretrain_state_dict): - """Adjust head parameters between datasets.""" - # List of parameters to adjust - if ( - pretrain_state_dict["decoder.denoising_class_embed.weight"].size() - != cur_state_dict["decoder.denoising_class_embed.weight"].size() - ): - del pretrain_state_dict["decoder.denoising_class_embed.weight"] - - head_param_names = ["decoder.enc_score_head.weight", "decoder.enc_score_head.bias"] - for i in range(8): - head_param_names.append(f"decoder.dec_score_head.{i}.weight") - head_param_names.append(f"decoder.dec_score_head.{i}.bias") - - adjusted_params = [] - - for param_name in head_param_names: - if param_name in cur_state_dict and param_name in pretrain_state_dict: - cur_tensor = cur_state_dict[param_name] - pretrain_tensor = pretrain_state_dict[param_name] - adjusted_tensor = self.map_class_weights(cur_tensor, pretrain_tensor) - if adjusted_tensor is not None: - pretrain_state_dict[param_name] = adjusted_tensor - adjusted_params.append(param_name) - else: - print(f"Cannot adjust parameter '{param_name}' due to size mismatch.") - - return pretrain_state_dict - - def map_class_weights(self, cur_tensor, pretrain_tensor): - """Map class weights from pretrain model to current model based on class IDs.""" - if pretrain_tensor.size() == cur_tensor.size(): - return pretrain_tensor - - adjusted_tensor = cur_tensor.clone() - adjusted_tensor.requires_grad = False - - if pretrain_tensor.size() > cur_tensor.size(): - for coco_id, obj_id in enumerate(self.obj365_ids): - adjusted_tensor[coco_id] = pretrain_tensor[obj_id + 1] - else: - for coco_id, obj_id in enumerate(self.obj365_ids): - adjusted_tensor[obj_id + 1] = pretrain_tensor[coco_id] - - return adjusted_tensor - - def fit(self): - raise NotImplementedError("") - - def val(self): - raise NotImplementedError("") - - -# obj365_classes = [ -# 'Person', 'Sneakers', 'Chair', 'Other Shoes', 'Hat', 'Car', 'Lamp', 'Glasses', -# 'Bottle', 'Desk', 'Cup', 'Street Lights', 'Cabinet/shelf', 'Handbag/Satchel', -# 'Bracelet', 'Plate', 'Picture/Frame', 'Helmet', 'Book', 'Gloves', 'Storage box', -# 'Boat', 'Leather Shoes', 'Flower', 'Bench', 'Potted Plant', 'Bowl/Basin', 'Flag', -# 'Pillow', 'Boots', 'Vase', 'Microphone', 'Necklace', 'Ring', 'SUV', 'Wine Glass', -# 'Belt', 'Moniter/TV', 'Backpack', 'Umbrella', 'Traffic Light', 'Speaker', 'Watch', -# 'Tie', 'Trash bin Can', 'Slippers', 'Bicycle', 'Stool', 'Barrel/bucket', 'Van', -# 'Couch', 'Sandals', 'Bakset', 'Drum', 'Pen/Pencil', 'Bus', 'Wild Bird', 'High Heels', -# 'Motorcycle', 'Guitar', 'Carpet', 'Cell Phone', 'Bread', 'Camera', 'Canned', 'Truck', -# 'Traffic cone', 'Cymbal', 'Lifesaver', 'Towel', 'Stuffed Toy', 'Candle', 'Sailboat', -# 'Laptop', 'Awning', 'Bed', 'Faucet', 'Tent', 'Horse', 'Mirror', 'Power outlet', -# 'Sink', 'Apple', 'Air Conditioner', 'Knife', 'Hockey Stick', 'Paddle', 'Pickup Truck', -# 'Fork', 'Traffic Sign', 'Ballon', 'Tripod', 'Dog', 'Spoon', 'Clock', 'Pot', 'Cow', -# 'Cake', 'Dinning Table', 'Sheep', 'Hanger', 'Blackboard/Whiteboard', 'Napkin', -# 'Other Fish', 'Orange/Tangerine', 'Toiletry', 'Keyboard', 'Tomato', 'Lantern', -# 'Machinery Vehicle', 'Fan', 'Green Vegetables', 'Banana', 'Baseball Glove', -# 'Airplane', 'Mouse', 'Train', 'Pumpkin', 'Soccer', 'Skiboard', 'Luggage', 'Nightstand', -# 'Tea pot', 'Telephone', 'Trolley', 'Head Phone', 'Sports Car', 'Stop Sign', 'Dessert', -# 'Scooter', 'Stroller', 'Crane', 'Remote', 'Refrigerator', 'Oven', 'Lemon', 'Duck', -# 'Baseball Bat', 'Surveillance Camera', 'Cat', 'Jug', 'Broccoli', 'Piano', 'Pizza', -# 'Elephant', 'Skateboard', 'Surfboard', 'Gun', 'Skating and Skiing shoes', 'Gas stove', -# 'Donut', 'Bow Tie', 'Carrot', 'Toilet', 'Kite', 'Strawberry', 'Other Balls', 'Shovel', -# 'Pepper', 'Computer Box', 'Toilet Paper', 'Cleaning Products', 'Chopsticks', 'Microwave', -# 'Pigeon', 'Baseball', 'Cutting/chopping Board', 'Coffee Table', 'Side Table', 'Scissors', -# 'Marker', 'Pie', 'Ladder', 'Snowboard', 'Cookies', 'Radiator', 'Fire Hydrant', 'Basketball', -# 'Zebra', 'Grape', 'Giraffe', 'Potato', 'Sausage', 'Tricycle', 'Violin', 'Egg', -# 'Fire Extinguisher', 'Candy', 'Fire Truck', 'Billards', 'Converter', 'Bathtub', -# 'Wheelchair', 'Golf Club', 'Briefcase', 'Cucumber', 'Cigar/Cigarette ', 'Paint Brush', -# 'Pear', 'Heavy Truck', 'Hamburger', 'Extractor', 'Extention Cord', 'Tong', -# 'Tennis Racket', 'Folder', 'American Football', 'earphone', 'Mask', 'Kettle', -# 'Tennis', 'Ship', 'Swing', 'Coffee Machine', 'Slide', 'Carriage', 'Onion', -# 'Green beans', 'Projector', 'Frisbee', 'Washing Machine/Drying Machine', 'Chicken', -# 'Printer', 'Watermelon', 'Saxophone', 'Tissue', 'Toothbrush', 'Ice cream', -# 'Hotair ballon', 'Cello', 'French Fries', 'Scale', 'Trophy', 'Cabbage', 'Hot dog', -# 'Blender', 'Peach', 'Rice', 'Wallet/Purse', 'Volleyball', 'Deer', 'Goose', 'Tape', -# 'Tablet', 'Cosmetics', 'Trumpet', 'Pineapple', 'Golf Ball', 'Ambulance', 'Parking meter', -# 'Mango', 'Key', 'Hurdle', 'Fishing Rod', 'Medal', 'Flute', 'Brush', 'Penguin', -# 'Megaphone', 'Corn', 'Lettuce', 'Garlic', 'Swan', 'Helicopter', 'Green Onion', -# 'Sandwich', 'Nuts', 'Speed Limit Sign', 'Induction Cooker', 'Broom', 'Trombone', -# 'Plum', 'Rickshaw', 'Goldfish', 'Kiwi fruit', 'Router/modem', 'Poker Card', 'Toaster', -# 'Shrimp', 'Sushi', 'Cheese', 'Notepaper', 'Cherry', 'Pliers', 'CD', 'Pasta', 'Hammer', -# 'Cue', 'Avocado', 'Hamimelon', 'Flask', 'Mushroon', 'Screwdriver', 'Soap', 'Recorder', -# 'Bear', 'Eggplant', 'Board Eraser', 'Coconut', 'Tape Measur/ Ruler', 'Pig', -# 'Showerhead', 'Globe', 'Chips', 'Steak', 'Crosswalk Sign', 'Stapler', 'Campel', -# 'Formula 1 ', 'Pomegranate', 'Dishwasher', 'Crab', 'Hoverboard', 'Meat ball', -# 'Rice Cooker', 'Tuba', 'Calculator', 'Papaya', 'Antelope', 'Parrot', 'Seal', -# 'Buttefly', 'Dumbbell', 'Donkey', 'Lion', 'Urinal', 'Dolphin', 'Electric Drill', -# 'Hair Dryer', 'Egg tart', 'Jellyfish', 'Treadmill', 'Lighter', 'Grapefruit', -# 'Game board', 'Mop', 'Radish', 'Baozi', 'Target', 'French', 'Spring Rolls', 'Monkey', -# 'Rabbit', 'Pencil Case', 'Yak', 'Red Cabbage', 'Binoculars', 'Asparagus', 'Barbell', -# 'Scallop', 'Noddles', 'Comb', 'Dumpling', 'Oyster', 'Table Teniis paddle', -# 'Cosmetics Brush/Eyeliner Pencil', 'Chainsaw', 'Eraser', 'Lobster', 'Durian', 'Okra', -# 'Lipstick', 'Cosmetics Mirror', 'Curling', 'Table Tennis ' -# ] - -# coco_classes = ['person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', -# 'train', 'truck', 'boat', 'traffic light', 'fire hydrant', -# 'stop sign', 'parking meter', 'bench', 'wild bird', 'cat', 'dog', -# 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', -# 'backpack', 'umbrella', 'handbag/satchel', 'tie', 'luggage', 'frisbee', -# 'skating and skiing shoes', 'snowboard', 'baseball', 'kite', 'baseball bat', -# 'baseball glove', 'skateboard', 'surfboard', 'tennis racket', -# 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl/basin', -# 'banana', 'apple', 'sandwich', 'orange/tangerine', 'broccoli', 'carrot', -# 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch', -# 'potted plant', 'bed', 'dinning table', 'toilet', 'moniter/tv', 'laptop', -# 'mouse', 'remote', 'keyboard', 'cell phone', 'microwave', -# 'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', -# 'vase', 'scissors', 'stuffed toy', 'hair dryer', 'toothbrush'] - - -# obj365_classes = [ -# (0, 'Person'), -# (1, 'Sneakers'), -# (2, 'Chair'), -# (3, 'Other Shoes'), -# (4, 'Hat'), -# (5, 'Car'), -# (6, 'Lamp'), -# (7, 'Glasses'), -# (8, 'Bottle'), -# (9, 'Desk'), -# (10, 'Cup'), -# (11, 'Street Lights'), -# (12, 'Cabinet/shelf'), -# (13, 'Handbag/Satchel'), -# (14, 'Bracelet'), -# (15, 'Plate'), -# (16, 'Picture/Frame'), -# (17, 'Helmet'), -# (18, 'Book'), -# (19, 'Gloves'), -# (20, 'Storage box'), -# (21, 'Boat'), -# (22, 'Leather Shoes'), -# (23, 'Flower'), -# (24, 'Bench'), -# (25, 'Potted Plant'), -# (26, 'Bowl/Basin'), -# (27, 'Flag'), -# (28, 'Pillow'), -# (29, 'Boots'), -# (30, 'Vase'), -# (31, 'Microphone'), -# (32, 'Necklace'), -# (33, 'Ring'), -# (34, 'SUV'), -# (35, 'Wine Glass'), -# (36, 'Belt'), -# (37, 'Monitor/TV'), -# (38, 'Backpack'), -# (39, 'Umbrella'), -# (40, 'Traffic Light'), -# (41, 'Speaker'), -# (42, 'Watch'), -# (43, 'Tie'), -# (44, 'Trash bin Can'), -# (45, 'Slippers'), -# (46, 'Bicycle'), -# (47, 'Stool'), -# (48, 'Barrel/bucket'), -# (49, 'Van'), -# (50, 'Couch'), -# (51, 'Sandals'), -# (52, 'Basket'), -# (53, 'Drum'), -# (54, 'Pen/Pencil'), -# (55, 'Bus'), -# (56, 'Wild Bird'), -# (57, 'High Heels'), -# (58, 'Motorcycle'), -# (59, 'Guitar'), -# (60, 'Carpet'), -# (61, 'Cell Phone'), -# (62, 'Bread'), -# (63, 'Camera'), -# (64, 'Canned'), -# (65, 'Truck'), -# (66, 'Traffic cone'), -# (67, 'Cymbal'), -# (68, 'Lifesaver'), -# (69, 'Towel'), -# (70, 'Stuffed Toy'), -# (71, 'Candle'), -# (72, 'Sailboat'), -# (73, 'Laptop'), -# (74, 'Awning'), -# (75, 'Bed'), -# (76, 'Faucet'), -# (77, 'Tent'), -# (78, 'Horse'), -# (79, 'Mirror'), -# (80, 'Power outlet'), -# (81, 'Sink'), -# (82, 'Apple'), -# (83, 'Air Conditioner'), -# (84, 'Knife'), -# (85, 'Hockey Stick'), -# (86, 'Paddle'), -# (87, 'Pickup Truck'), -# (88, 'Fork'), -# (89, 'Traffic Sign'), -# (90, 'Balloon'), -# (91, 'Tripod'), -# (92, 'Dog'), -# (93, 'Spoon'), -# (94, 'Clock'), -# (95, 'Pot'), -# (96, 'Cow'), -# (97, 'Cake'), -# (98, 'Dining Table'), -# (99, 'Sheep'), -# (100, 'Hanger'), -# (101, 'Blackboard/Whiteboard'), -# (102, 'Napkin'), -# (103, 'Other Fish'), -# (104, 'Orange/Tangerine'), -# (105, 'Toiletry'), -# (106, 'Keyboard'), -# (107, 'Tomato'), -# (108, 'Lantern'), -# (109, 'Machinery Vehicle'), -# (110, 'Fan'), -# (111, 'Green Vegetables'), -# (112, 'Banana'), -# (113, 'Baseball Glove'), -# (114, 'Airplane'), -# (115, 'Mouse'), -# (116, 'Train'), -# (117, 'Pumpkin'), -# (118, 'Soccer'), -# (119, 'Skiboard'), -# (120, 'Luggage'), -# (121, 'Nightstand'), -# (122, 'Tea pot'), -# (123, 'Telephone'), -# (124, 'Trolley'), -# (125, 'Head Phone'), -# (126, 'Sports Car'), -# (127, 'Stop Sign'), -# (128, 'Dessert'), -# (129, 'Scooter'), -# (130, 'Stroller'), -# (131, 'Crane'), -# (132, 'Remote'), -# (133, 'Refrigerator'), -# (134, 'Oven'), -# (135, 'Lemon'), -# (136, 'Duck'), -# (137, 'Baseball Bat'), -# (138, 'Surveillance Camera'), -# (139, 'Cat'), -# (140, 'Jug'), -# (141, 'Broccoli'), -# (142, 'Piano'), -# (143, 'Pizza'), -# (144, 'Elephant'), -# (145, 'Skateboard'), -# (146, 'Surfboard'), -# (147, 'Gun'), -# (148, 'Skating and Skiing Shoes'), -# (149, 'Gas Stove'), -# (150, 'Donut'), -# (151, 'Bow Tie'), -# (152, 'Carrot'), -# (153, 'Toilet'), -# (154, 'Kite'), -# (155, 'Strawberry'), -# (156, 'Other Balls'), -# (157, 'Shovel'), -# (158, 'Pepper'), -# (159, 'Computer Box'), -# (160, 'Toilet Paper'), -# (161, 'Cleaning Products'), -# (162, 'Chopsticks'), -# (163, 'Microwave'), -# (164, 'Pigeon'), -# (165, 'Baseball'), -# (166, 'Cutting/chopping Board'), -# (167, 'Coffee Table'), -# (168, 'Side Table'), -# (169, 'Scissors'), -# (170, 'Marker'), -# (171, 'Pie'), -# (172, 'Ladder'), -# (173, 'Snowboard'), -# (174, 'Cookies'), -# (175, 'Radiator'), -# (176, 'Fire Hydrant'), -# (177, 'Basketball'), -# (178, 'Zebra'), -# (179, 'Grape'), -# (180, 'Giraffe'), -# (181, 'Potato'), -# (182, 'Sausage'), -# (183, 'Tricycle'), -# (184, 'Violin'), -# (185, 'Egg'), -# (186, 'Fire Extinguisher'), -# (187, 'Candy'), -# (188, 'Fire Truck'), -# (189, 'Billiards'), -# (190, 'Converter'), -# (191, 'Bathtub'), -# (192, 'Wheelchair'), -# (193, 'Golf Club'), -# (194, 'Briefcase'), -# (195, 'Cucumber'), -# (196, 'Cigar/Cigarette'), -# (197, 'Paint Brush'), -# (198, 'Pear'), -# (199, 'Heavy Truck'), -# (200, 'Hamburger'), -# (201, 'Extractor'), -# (202, 'Extension Cord'), -# (203, 'Tong'), -# (204, 'Tennis Racket'), -# (205, 'Folder'), -# (206, 'American Football'), -# (207, 'Earphone'), -# (208, 'Mask'), -# (209, 'Kettle'), -# (210, 'Tennis'), -# (211, 'Ship'), -# (212, 'Swing'), -# (213, 'Coffee Machine'), -# (214, 'Slide'), -# (215, 'Carriage'), -# (216, 'Onion'), -# (217, 'Green Beans'), -# (218, 'Projector'), -# (219, 'Frisbee'), -# (220, 'Washing Machine/Drying Machine'), -# (221, 'Chicken'), -# (222, 'Printer'), -# (223, 'Watermelon'), -# (224, 'Saxophone'), -# (225, 'Tissue'), -# (226, 'Toothbrush'), -# (227, 'Ice Cream'), -# (228, 'Hot Air Balloon'), -# (229, 'Cello'), -# (230, 'French Fries'), -# (231, 'Scale'), -# (232, 'Trophy'), -# (233, 'Cabbage'), -# (234, 'Hot Dog'), -# (235, 'Blender'), -# (236, 'Peach'), -# (237, 'Rice'), -# (238, 'Wallet/Purse'), -# (239, 'Volleyball'), -# (240, 'Deer'), -# (241, 'Goose'), -# (242, 'Tape'), -# (243, 'Tablet'), -# (244, 'Cosmetics'), -# (245, 'Trumpet'), -# (246, 'Pineapple'), -# (247, 'Golf Ball'), -# (248, 'Ambulance'), -# (249, 'Parking Meter'), -# (250, 'Mango'), -# (251, 'Key'), -# (252, 'Hurdle'), -# (253, 'Fishing Rod'), -# (254, 'Medal'), -# (255, 'Flute'), -# (256, 'Brush'), -# (257, 'Penguin'), -# (258, 'Megaphone'), -# (259, 'Corn'), -# (260, 'Lettuce'), -# (261, 'Garlic'), -# (262, 'Swan'), -# (263, 'Helicopter'), -# (264, 'Green Onion'), -# (265, 'Sandwich'), -# (266, 'Nuts'), -# (267, 'Speed Limit Sign'), -# (268, 'Induction Cooker'), -# (269, 'Broom'), -# (270, 'Trombone'), -# (271, 'Plum'), -# (272, 'Rickshaw'), -# (273, 'Goldfish'), -# (274, 'Kiwi Fruit'), -# (275, 'Router/Modem'), -# (276, 'Poker Card'), -# (277, 'Toaster'), -# (278, 'Shrimp'), -# (279, 'Sushi'), -# (280, 'Cheese'), -# (281, 'Notepaper'), -# (282, 'Cherry'), -# (283, 'Pliers'), -# (284, 'CD'), -# (285, 'Pasta'), -# (286, 'Hammer'), -# (287, 'Cue'), -# (288, 'Avocado'), -# (289, 'Hami Melon'), -# (290, 'Flask'), -# (291, 'Mushroom'), -# (292, 'Screwdriver'), -# (293, 'Soap'), -# (294, 'Recorder'), -# (295, 'Bear'), -# (296, 'Eggplant'), -# (297, 'Board Eraser'), -# (298, 'Coconut'), -# (299, 'Tape Measure/Ruler'), -# (300, 'Pig'), -# (301, 'Showerhead'), -# (302, 'Globe'), -# (303, 'Chips'), -# (304, 'Steak'), -# (305, 'Crosswalk Sign'), -# (306, 'Stapler'), -# (307, 'Camel'), -# (308, 'Formula 1'), -# (309, 'Pomegranate'), -# (310, 'Dishwasher'), -# (311, 'Crab'), -# (312, 'Hoverboard'), -# (313, 'Meatball'), -# (314, 'Rice Cooker'), -# (315, 'Tuba'), -# (316, 'Calculator'), -# (317, 'Papaya'), -# (318, 'Antelope'), -# (319, 'Parrot'), -# (320, 'Seal'), -# (321, 'Butterfly'), -# (322, 'Dumbbell'), -# (323, 'Donkey'), -# (324, 'Lion'), -# (325, 'Urinal'), -# (326, 'Dolphin'), -# (327, 'Electric Drill'), -# (328, 'Hair Dryer'), -# (329, 'Egg Tart'), -# (330, 'Jellyfish'), -# (331, 'Treadmill'), -# (332, 'Lighter'), -# (333, 'Grapefruit'), -# (334, 'Game Board'), -# (335, 'Mop'), -# (336, 'Radish'), -# (337, 'Baozi'), -# (338, 'Target'), -# (339, 'French'), -# (340, 'Spring Rolls'), -# (341, 'Monkey'), -# (342, 'Rabbit'), -# (343, 'Pencil Case'), -# (344, 'Yak'), -# (345, 'Red Cabbage'), -# (346, 'Binoculars'), -# (347, 'Asparagus'), -# (348, 'Barbell'), -# (349, 'Scallop'), -# (350, 'Noodles'), -# (351, 'Comb'), -# (352, 'Dumpling'), -# (353, 'Oyster'), -# (354, 'Table Tennis Paddle'), -# (355, 'Cosmetics Brush/Eyeliner Pencil'), -# (356, 'Chainsaw'), -# (357, 'Eraser'), -# (358, 'Lobster'), -# (359, 'Durian'), -# (360, 'Okra'), -# (361, 'Lipstick'), -# (362, 'Cosmetics Mirror'), -# (363, 'Curling'), -# (364, 'Table Tennis') -# ] diff --git a/vlm/detector/D-FINE.backup/src/solver/clas_engine.py b/vlm/detector/D-FINE.backup/src/solver/clas_engine.py deleted file mode 100644 index fc5a680..0000000 --- a/vlm/detector/D-FINE.backup/src/solver/clas_engine.py +++ /dev/null @@ -1,74 +0,0 @@ -""" -Copied from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright(c) 2023 lyuwenyu. All Rights Reserved. -""" - -import torch -import torch.nn as nn - -from ..misc import MetricLogger, SmoothedValue, reduce_dict - - -def train_one_epoch( - model: nn.Module, criterion: nn.Module, dataloader, optimizer, ema, epoch, device -): - """ """ - model.train() - - metric_logger = MetricLogger(delimiter=" ") - metric_logger.add_meter("lr", SmoothedValue(window_size=1, fmt="{value:.6f}")) - print_freq = 100 - header = "Epoch: [{}]".format(epoch) - - for imgs, labels in metric_logger.log_every(dataloader, print_freq, header): - imgs = imgs.to(device) - labels = labels.to(device) - - preds = model(imgs) - loss: torch.Tensor = criterion(preds, labels, epoch) - - optimizer.zero_grad() - loss.backward() - optimizer.step() - - if ema is not None: - ema.update(model) - - loss_reduced_values = {k: v.item() for k, v in reduce_dict({"loss": loss}).items()} - metric_logger.update(**loss_reduced_values) - metric_logger.update(lr=optimizer.param_groups[0]["lr"]) - - metric_logger.synchronize_between_processes() - print("Averaged stats:", metric_logger) - - stats = {k: meter.global_avg for k, meter in metric_logger.meters.items()} - return stats - - -@torch.no_grad() -def evaluate(model, criterion, dataloader, device): - model.eval() - - metric_logger = MetricLogger(delimiter=" ") - # metric_logger.add_meter('acc', SmoothedValue(window_size=1, fmt='{global_avg:.4f}')) - # metric_logger.add_meter('loss', SmoothedValue(window_size=1, fmt='{value:.2f}')) - metric_logger.add_meter("acc", SmoothedValue(window_size=1)) - metric_logger.add_meter("loss", SmoothedValue(window_size=1)) - - header = "Test:" - for imgs, labels in metric_logger.log_every(dataloader, 10, header): - imgs, labels = imgs.to(device), labels.to(device) - preds = model(imgs) - - acc = (preds.argmax(dim=-1) == labels).sum() / preds.shape[0] - loss = criterion(preds, labels) - - dict_reduced = reduce_dict({"acc": acc, "loss": loss}) - reduced_values = {k: v.item() for k, v in dict_reduced.items()} - metric_logger.update(**reduced_values) - - metric_logger.synchronize_between_processes() - print("Averaged stats:", metric_logger) - - stats = {k: meter.global_avg for k, meter in metric_logger.meters.items()} - return stats diff --git a/vlm/detector/D-FINE.backup/src/solver/clas_solver.py b/vlm/detector/D-FINE.backup/src/solver/clas_solver.py deleted file mode 100644 index aefb3d0..0000000 --- a/vlm/detector/D-FINE.backup/src/solver/clas_solver.py +++ /dev/null @@ -1,75 +0,0 @@ -""" -Copied from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright(c) 2023 lyuwenyu. All Rights Reserved. -""" - -import datetime -import json -import time -from pathlib import Path - -import torch -import torch.nn as nn - -from ..misc import dist_utils -from ._solver import BaseSolver -from .clas_engine import evaluate, train_one_epoch - - -class ClasSolver(BaseSolver): - def fit( - self, - ): - print("Start training") - self.train() - args = self.cfg - - n_parameters = sum(p.numel() for p in self.model.parameters() if p.requires_grad) - print("Number of params:", n_parameters) - - output_dir = Path(args.output_dir) - output_dir.mkdir(exist_ok=True) - - start_time = time.time() - start_epoch = self.last_epoch + 1 - for epoch in range(start_epoch, args.epochs): - if dist_utils.is_dist_available_and_initialized(): - self.train_dataloader.sampler.set_epoch(epoch) - - train_stats = train_one_epoch( - self.model, - self.criterion, - self.train_dataloader, - self.optimizer, - self.ema, - epoch=epoch, - device=self.device, - ) - self.lr_scheduler.step() - self.last_epoch += 1 - - if output_dir: - checkpoint_paths = [output_dir / "checkpoint.pth"] - # extra checkpoint before LR drop and every 100 epochs - if (epoch + 1) % args.checkpoint_freq == 0: - checkpoint_paths.append(output_dir / f"checkpoint{epoch:04}.pth") - for checkpoint_path in checkpoint_paths: - dist_utils.save_on_master(self.state_dict(epoch), checkpoint_path) - - module = self.ema.module if self.ema else self.model - test_stats = evaluate(module, self.criterion, self.val_dataloader, self.device) - - log_stats = { - **{f"train_{k}": v for k, v in train_stats.items()}, - **{f"test_{k}": v for k, v in test_stats.items()}, - "epoch": epoch, - "n_parameters": n_parameters, - } - - if output_dir and dist_utils.is_main_process(): - with (output_dir / "log.txt").open("a") as f: - f.write(json.dumps(log_stats) + "\n") - - total_time = time.time() - start_time - total_time_str = str(datetime.timedelta(seconds=int(total_time))) - print("Training time {}".format(total_time_str)) diff --git a/vlm/detector/D-FINE.backup/src/solver/det_engine.py b/vlm/detector/D-FINE.backup/src/solver/det_engine.py deleted file mode 100644 index b877b09..0000000 --- a/vlm/detector/D-FINE.backup/src/solver/det_engine.py +++ /dev/null @@ -1,259 +0,0 @@ -""" -D-FINE: Redefine Regression Task of DETRs as Fine-grained Distribution Refinement -Copyright (c) 2024 The D-FINE Authors. All Rights Reserved. ---------------------------------------------------------------------------------- -Modified from DETR (https://github.com/facebookresearch/detr/blob/main/engine.py) -Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -""" - -import math -import sys -from typing import Dict, Iterable, List - -import numpy as np -import torch -import torch.amp -from torch.cuda.amp.grad_scaler import GradScaler -from torch.utils.tensorboard import SummaryWriter - -from ..data import CocoEvaluator -from ..data.dataset import mscoco_category2label -from ..misc import MetricLogger, SmoothedValue, dist_utils, save_samples -from ..optim import ModelEMA, Warmup -from .validator import Validator, scale_boxes - - -def train_one_epoch( - model: torch.nn.Module, - criterion: torch.nn.Module, - data_loader: Iterable, - optimizer: torch.optim.Optimizer, - device: torch.device, - epoch: int, - use_wandb: bool, - max_norm: float = 0, - **kwargs, -): - if use_wandb: - import wandb - - model.train() - criterion.train() - metric_logger = MetricLogger(delimiter=" ") - metric_logger.add_meter("lr", SmoothedValue(window_size=1, fmt="{value:.6f}")) - - epochs = kwargs.get("epochs", None) - header = "Epoch: [{}]".format(epoch) if epochs is None else "Epoch: [{}/{}]".format(epoch, epochs) - - print_freq = kwargs.get("print_freq", 10) - writer: SummaryWriter = kwargs.get("writer", None) - - ema: ModelEMA = kwargs.get("ema", None) - scaler: GradScaler = kwargs.get("scaler", None) - lr_warmup_scheduler: Warmup = kwargs.get("lr_warmup_scheduler", None) - losses = [] - - output_dir = kwargs.get("output_dir", None) - num_visualization_sample_batch = kwargs.get("num_visualization_sample_batch", 1) - - for i, (samples, targets) in enumerate( - metric_logger.log_every(data_loader, print_freq, header) - ): - global_step = epoch * len(data_loader) + i - metas = dict(epoch=epoch, step=i, global_step=global_step, epoch_step=len(data_loader)) - - if global_step < num_visualization_sample_batch and output_dir is not None and dist_utils.is_main_process(): - save_samples(samples, targets, output_dir, "train", normalized=True, box_fmt="cxcywh") - - samples = samples.to(device) - targets = [{k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in t.items()} for t in targets] - - if scaler is not None: - with torch.autocast(device_type=str(device), cache_enabled=True): - outputs = model(samples, targets=targets) - - if torch.isnan(outputs["pred_boxes"]).any() or torch.isinf(outputs["pred_boxes"]).any(): - print(outputs["pred_boxes"]) - state = model.state_dict() - new_state = {} - for key, value in model.state_dict().items(): - # Replace 'module' with 'model' in each key - new_key = key.replace("module.", "") - # Add the updated key-value pair to the state dictionary - state[new_key] = value - new_state["model"] = state - dist_utils.save_on_master(new_state, "./NaN.pth") - - with torch.autocast(device_type=str(device), enabled=False): - loss_dict = criterion(outputs, targets, **metas) - - loss = sum(loss_dict.values()) - scaler.scale(loss).backward() - - if max_norm > 0: - scaler.unscale_(optimizer) - torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm) - - scaler.step(optimizer) - scaler.update() - optimizer.zero_grad() - - else: - outputs = model(samples, targets=targets) - loss_dict = criterion(outputs, targets, **metas) - - loss: torch.Tensor = sum(loss_dict.values()) - optimizer.zero_grad() - loss.backward() - - if max_norm > 0: - torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm) - - optimizer.step() - - # ema - if ema is not None: - ema.update(model) - - if lr_warmup_scheduler is not None: - lr_warmup_scheduler.step() - - loss_dict_reduced = dist_utils.reduce_dict(loss_dict) - loss_value = sum(loss_dict_reduced.values()) - losses.append(loss_value.detach().cpu().numpy()) - - if not math.isfinite(loss_value): - print("Loss is {}, stopping training".format(loss_value)) - print(loss_dict_reduced) - sys.exit(1) - - metric_logger.update(loss=loss_value, **loss_dict_reduced) - metric_logger.update(lr=optimizer.param_groups[0]["lr"]) - - if writer and dist_utils.is_main_process() and global_step % 10 == 0: - writer.add_scalar("Loss/total", loss_value.item(), global_step) - for j, pg in enumerate(optimizer.param_groups): - writer.add_scalar(f"Lr/pg_{j}", pg["lr"], global_step) - for k, v in loss_dict_reduced.items(): - writer.add_scalar(f"Loss/{k}", v.item(), global_step) - - if use_wandb: - wandb.log( - {"lr": optimizer.param_groups[0]["lr"], "epoch": epoch, "train/loss": np.mean(losses)} - ) - # gather the stats from all processes - metric_logger.synchronize_between_processes() - print("Averaged stats:", metric_logger) - return {k: meter.global_avg for k, meter in metric_logger.meters.items()} - - -@torch.no_grad() -def evaluate( - model: torch.nn.Module, - criterion: torch.nn.Module, - postprocessor, - data_loader, - coco_evaluator: CocoEvaluator, - device, - epoch: int, - use_wandb: bool, - **kwargs, -): - if use_wandb: - import wandb - - model.eval() - criterion.eval() - coco_evaluator.cleanup() - - metric_logger = MetricLogger(delimiter=" ") - # metric_logger.add_meter('class_error', SmoothedValue(window_size=1, fmt='{value:.2f}')) - header = "Test:" - - # iou_types = tuple(k for k in ('segm', 'bbox') if k in postprocessor.keys()) - iou_types = coco_evaluator.iou_types - # coco_evaluator = CocoEvaluator(base_ds, iou_types) - # coco_evaluator.coco_eval[iou_types[0]].params.iouThrs = [0, 0.1, 0.5, 0.75] - - gt: List[Dict[str, torch.Tensor]] = [] - preds: List[Dict[str, torch.Tensor]] = [] - - output_dir = kwargs.get("output_dir", None) - num_visualization_sample_batch = kwargs.get("num_visualization_sample_batch", 1) - - for i, (samples, targets) in enumerate(metric_logger.log_every(data_loader, 10, header)): - global_step = epoch * len(data_loader) + i - - if global_step < num_visualization_sample_batch and output_dir is not None and dist_utils.is_main_process(): - save_samples(samples, targets, output_dir, "val", normalized=False, box_fmt="xyxy") - - samples = samples.to(device) - targets = [{k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in t.items()} for t in targets] - - outputs = model(samples) - # with torch.autocast(device_type=str(device)): - # outputs = model(samples) - - # TODO (lyuwenyu), fix dataset converted using `convert_to_coco_api`? - orig_target_sizes = torch.stack([t["orig_size"] for t in targets], dim=0) - # orig_target_sizes = torch.tensor([[samples.shape[-1], samples.shape[-2]]], device=samples.device) - - results = postprocessor(outputs, orig_target_sizes) - - # if 'segm' in postprocessor.keys(): - # target_sizes = torch.stack([t["size"] for t in targets], dim=0) - # results = postprocessor['segm'](results, outputs, orig_target_sizes, target_sizes) - - res = {target["image_id"].item(): output for target, output in zip(targets, results)} - if coco_evaluator is not None: - coco_evaluator.update(res) - - # validator format for metrics - for idx, (target, result) in enumerate(zip(targets, results)): - gt.append( - { - "boxes": scale_boxes( # from model input size to original img size - target["boxes"], - (target["orig_size"][1], target["orig_size"][0]), - (samples[idx].shape[-1], samples[idx].shape[-2]), - ), - "labels": target["labels"], - } - ) - labels = ( - torch.tensor([mscoco_category2label[int(x.item())] for x in result["labels"].flatten()]) - .to(result["labels"].device) - .reshape(result["labels"].shape) - ) if postprocessor.remap_mscoco_category else result["labels"] - preds.append( - {"boxes": result["boxes"], "labels": labels, "scores": result["scores"]} - ) - - # Conf matrix, F1, Precision, Recall, box IoU - metrics = Validator(gt, preds).compute_metrics() - print("Metrics:", metrics) - if use_wandb: - metrics = {f"metrics/{k}": v for k, v in metrics.items()} - metrics["epoch"] = epoch - wandb.log(metrics) - - # gather the stats from all processes - metric_logger.synchronize_between_processes() - print("Averaged stats:", metric_logger) - if coco_evaluator is not None: - coco_evaluator.synchronize_between_processes() - - # accumulate predictions from all images - if coco_evaluator is not None: - coco_evaluator.accumulate() - coco_evaluator.summarize() - - stats = {} - # stats = {k: meter.global_avg for k, meter in metric_logger.meters.items()} - if coco_evaluator is not None: - if "bbox" in iou_types: - stats["coco_eval_bbox"] = coco_evaluator.coco_eval["bbox"].stats.tolist() - if "segm" in iou_types: - stats["coco_eval_masks"] = coco_evaluator.coco_eval["segm"].stats.tolist() - - return stats, coco_evaluator diff --git a/vlm/detector/D-FINE.backup/src/solver/det_solver.py b/vlm/detector/D-FINE.backup/src/solver/det_solver.py deleted file mode 100644 index 96b9376..0000000 --- a/vlm/detector/D-FINE.backup/src/solver/det_solver.py +++ /dev/null @@ -1,228 +0,0 @@ -""" -D-FINE: Redefine Regression Task of DETRs as Fine-grained Distribution Refinement -Copyright (c) 2024 The D-FINE Authors. All Rights Reserved. ---------------------------------------------------------------------------------- -Modified from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright (c) 2023 lyuwenyu. All Rights Reserved. -""" - -import datetime -import json -import time - -import torch - -from ..misc import dist_utils, stats -from ._solver import BaseSolver -from .det_engine import evaluate, train_one_epoch - - -class DetSolver(BaseSolver): - def fit(self): - self.train() - args = self.cfg - metric_names = ["AP50:95", "AP50", "AP75", "APsmall", "APmedium", "APlarge"] - - if self.use_wandb: - import wandb - - wandb.init( - project=args.yaml_cfg["project_name"], - name=args.yaml_cfg["exp_name"], - config=args.yaml_cfg, - ) - wandb.watch(self.model) - - n_parameters, model_stats = stats(self.cfg) - print(model_stats) - print("-" * 42 + "Start training" + "-" * 43) - top1 = 0 - best_stat = { - "epoch": -1, - } - if self.last_epoch > 0: - module = self.ema.module if self.ema else self.model - test_stats, coco_evaluator = evaluate( - module, - self.criterion, - self.postprocessor, - self.val_dataloader, - self.evaluator, - self.device, - self.last_epoch, - self.use_wandb - ) - for k in test_stats: - best_stat["epoch"] = self.last_epoch - best_stat[k] = test_stats[k][0] - top1 = test_stats[k][0] - print(f"best_stat: {best_stat}") - - best_stat_print = best_stat.copy() - start_time = time.time() - start_epoch = self.last_epoch + 1 - for epoch in range(start_epoch, args.epochs): - self.train_dataloader.set_epoch(epoch) - # self.train_dataloader.dataset.set_epoch(epoch) - if dist_utils.is_dist_available_and_initialized(): - self.train_dataloader.sampler.set_epoch(epoch) - - if epoch == self.train_dataloader.collate_fn.stop_epoch: - self.load_resume_state(str(self.output_dir / "best_stg1.pth")) - if self.ema: - self.ema.decay = self.train_dataloader.collate_fn.ema_restart_decay - print(f"Refresh EMA at epoch {epoch} with decay {self.ema.decay}") - - train_stats = train_one_epoch( - self.model, - self.criterion, - self.train_dataloader, - self.optimizer, - self.device, - epoch, - epochs=args.epochs, - max_norm=args.clip_max_norm, - print_freq=args.print_freq, - ema=self.ema, - scaler=self.scaler, - lr_warmup_scheduler=self.lr_warmup_scheduler, - writer=self.writer, - use_wandb=self.use_wandb, - output_dir=self.output_dir, - ) - - if self.lr_warmup_scheduler is None or self.lr_warmup_scheduler.finished(): - self.lr_scheduler.step() - - self.last_epoch += 1 - - if self.output_dir and epoch < self.train_dataloader.collate_fn.stop_epoch: - checkpoint_paths = [self.output_dir / "last.pth"] - # extra checkpoint before LR drop and every 100 epochs - if (epoch + 1) % args.checkpoint_freq == 0: - checkpoint_paths.append(self.output_dir / f"checkpoint{epoch:04}.pth") - for checkpoint_path in checkpoint_paths: - dist_utils.save_on_master(self.state_dict(), checkpoint_path) - - module = self.ema.module if self.ema else self.model - test_stats, coco_evaluator = evaluate( - module, - self.criterion, - self.postprocessor, - self.val_dataloader, - self.evaluator, - self.device, - epoch, - self.use_wandb, - output_dir=self.output_dir, - ) - - # TODO - for k in test_stats: - if self.writer and dist_utils.is_main_process(): - for i, v in enumerate(test_stats[k]): - self.writer.add_scalar(f"Test/{k}_{i}".format(k), v, epoch) - - if k in best_stat: - best_stat["epoch"] = ( - epoch if test_stats[k][0] > best_stat[k] else best_stat["epoch"] - ) - best_stat[k] = max(best_stat[k], test_stats[k][0]) - else: - best_stat["epoch"] = epoch - best_stat[k] = test_stats[k][0] - - if best_stat[k] > top1: - best_stat_print["epoch"] = epoch - top1 = best_stat[k] - if self.output_dir: - if epoch >= self.train_dataloader.collate_fn.stop_epoch: - dist_utils.save_on_master( - self.state_dict(), self.output_dir / "best_stg2.pth" - ) - else: - dist_utils.save_on_master( - self.state_dict(), self.output_dir / "best_stg1.pth" - ) - - best_stat_print[k] = max(best_stat[k], top1) - print(f"best_stat: {best_stat_print}") # global best - - if best_stat["epoch"] == epoch and self.output_dir: - if epoch >= self.train_dataloader.collate_fn.stop_epoch: - if test_stats[k][0] > top1: - top1 = test_stats[k][0] - dist_utils.save_on_master( - self.state_dict(), self.output_dir / "best_stg2.pth" - ) - else: - top1 = max(test_stats[k][0], top1) - dist_utils.save_on_master( - self.state_dict(), self.output_dir / "best_stg1.pth" - ) - - elif epoch >= self.train_dataloader.collate_fn.stop_epoch: - best_stat = { - "epoch": -1, - } - if self.ema: - self.ema.decay -= 0.0001 - self.load_resume_state(str(self.output_dir / "best_stg1.pth")) - print(f"Refresh EMA at epoch {epoch} with decay {self.ema.decay}") - - log_stats = { - **{f"train_{k}": v for k, v in train_stats.items()}, - **{f"test_{k}": v for k, v in test_stats.items()}, - "epoch": epoch, - "n_parameters": n_parameters, - } - - if self.use_wandb: - wandb_logs = {} - for idx, metric_name in enumerate(metric_names): - wandb_logs[f"metrics/{metric_name}"] = test_stats["coco_eval_bbox"][idx] - wandb_logs["epoch"] = epoch - wandb.log(wandb_logs) - - if self.output_dir and dist_utils.is_main_process(): - with (self.output_dir / "log.txt").open("a") as f: - f.write(json.dumps(log_stats) + "\n") - - # for evaluation logs - if coco_evaluator is not None: - (self.output_dir / "eval").mkdir(exist_ok=True) - if "bbox" in coco_evaluator.coco_eval: - filenames = ["latest.pth"] - if epoch % 50 == 0: - filenames.append(f"{epoch:03}.pth") - for name in filenames: - torch.save( - coco_evaluator.coco_eval["bbox"].eval, - self.output_dir / "eval" / name, - ) - - total_time = time.time() - start_time - total_time_str = str(datetime.timedelta(seconds=int(total_time))) - print("Training time {}".format(total_time_str)) - - def val(self): - self.eval() - - module = self.ema.module if self.ema else self.model - test_stats, coco_evaluator = evaluate( - module, - self.criterion, - self.postprocessor, - self.val_dataloader, - self.evaluator, - self.device, - epoch=-1, - use_wandb=False, - ) - - if self.output_dir: - dist_utils.save_on_master( - coco_evaluator.coco_eval["bbox"].eval, self.output_dir / "eval.pth" - ) - - return diff --git a/vlm/detector/D-FINE.backup/src/solver/validator.py b/vlm/detector/D-FINE.backup/src/solver/validator.py deleted file mode 100644 index e38308c..0000000 --- a/vlm/detector/D-FINE.backup/src/solver/validator.py +++ /dev/null @@ -1,347 +0,0 @@ -import copy -from collections import defaultdict -from pathlib import Path -from typing import Dict, List - -import matplotlib.pyplot as plt -import numpy as np -import torch -from loguru import logger -from torchvision.ops import box_iou - - -class Validator: - def __init__( - self, - gt: List[Dict[str, torch.Tensor]], - preds: List[Dict[str, torch.Tensor]], - conf_thresh=0.5, - iou_thresh=0.5, - ) -> None: - """ - Format example: - gt = [{'labels': tensor([0]), 'boxes': tensor([[561.0, 297.0, 661.0, 359.0]])}, ...] - len(gt) is the number of images - bboxes are in format [x1, y1, x2, y2], absolute values - """ - self.gt = gt - self.preds = preds - self.conf_thresh = conf_thresh - self.iou_thresh = iou_thresh - self.thresholds = np.arange(0.2, 1.0, 0.05) - self.conf_matrix = None - - def compute_metrics(self, extended=False) -> Dict[str, float]: - filtered_preds = filter_preds(copy.deepcopy(self.preds), self.conf_thresh) - metrics = self._compute_main_metrics(filtered_preds) - if not extended: - metrics.pop("extended_metrics", None) - return metrics - - def _compute_main_metrics(self, preds): - ( - self.metrics_per_class, - self.conf_matrix, - self.class_to_idx, - ) = self._compute_metrics_and_confusion_matrix(preds) - tps, fps, fns = 0, 0, 0 - ious = [] - extended_metrics = {} - for key, value in self.metrics_per_class.items(): - tps += value["TPs"] - fps += value["FPs"] - fns += value["FNs"] - ious.extend(value["IoUs"]) - - extended_metrics[f"precision_{key}"] = ( - value["TPs"] / (value["TPs"] + value["FPs"]) - if value["TPs"] + value["FPs"] > 0 - else 0 - ) - extended_metrics[f"recall_{key}"] = ( - value["TPs"] / (value["TPs"] + value["FNs"]) - if value["TPs"] + value["FNs"] > 0 - else 0 - ) - - extended_metrics[f"iou_{key}"] = np.mean(value["IoUs"]) - - precision = tps / (tps + fps) if (tps + fps) > 0 else 0 - recall = tps / (tps + fns) if (tps + fns) > 0 else 0 - f1 = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0 - iou = np.mean(ious).item() if ious else 0 - return { - "f1": f1, - "precision": precision, - "recall": recall, - "iou": iou, - "TPs": tps, - "FPs": fps, - "FNs": fns, - "extended_metrics": extended_metrics, - } - - def _compute_matrix_multi_class(self, preds): - metrics_per_class = defaultdict(lambda: {"TPs": 0, "FPs": 0, "FNs": 0, "IoUs": []}) - for pred, gt in zip(preds, self.gt): - pred_boxes = pred["boxes"] - pred_labels = pred["labels"] - gt_boxes = gt["boxes"] - gt_labels = gt["labels"] - - # isolate each class - labels = torch.unique(torch.cat([pred_labels, gt_labels])) - for label in labels: - pred_cl_boxes = pred_boxes[pred_labels == label] # filter by bool mask - gt_cl_boxes = gt_boxes[gt_labels == label] - - n_preds = len(pred_cl_boxes) - n_gts = len(gt_cl_boxes) - if not (n_preds or n_gts): - continue - if not n_preds: - metrics_per_class[label.item()]["FNs"] += n_gts - metrics_per_class[label.item()]["IoUs"].extend([0] * n_gts) - continue - if not n_gts: - metrics_per_class[label.item()]["FPs"] += n_preds - metrics_per_class[label.item()]["IoUs"].extend([0] * n_preds) - continue - - ious = box_iou(pred_cl_boxes, gt_cl_boxes) # matrix of all IoUs - ious_mask = ious >= self.iou_thresh - - # indeces of boxes that have IoU >= threshold - pred_indices, gt_indices = torch.nonzero(ious_mask, as_tuple=True) - - if not pred_indices.numel(): # no predicts matched gts - metrics_per_class[label.item()]["FNs"] += n_gts - metrics_per_class[label.item()]["IoUs"].extend([0] * n_gts) - metrics_per_class[label.item()]["FPs"] += n_preds - metrics_per_class[label.item()]["IoUs"].extend([0] * n_preds) - continue - - iou_values = ious[pred_indices, gt_indices] - - # sorting by IoU to match hgihest scores first - sorted_indices = torch.argsort(-iou_values) - pred_indices = pred_indices[sorted_indices] - gt_indices = gt_indices[sorted_indices] - iou_values = iou_values[sorted_indices] - - matched_preds = set() - matched_gts = set() - for pred_idx, gt_idx, iou in zip(pred_indices, gt_indices, iou_values): - if gt_idx.item() not in matched_gts and pred_idx.item() not in matched_preds: - matched_preds.add(pred_idx.item()) - matched_gts.add(gt_idx.item()) - metrics_per_class[label.item()]["TPs"] += 1 - metrics_per_class[label.item()]["IoUs"].append(iou.item()) - - unmatched_preds = set(range(n_preds)) - matched_preds - unmatched_gts = set(range(n_gts)) - matched_gts - metrics_per_class[label.item()]["FPs"] += len(unmatched_preds) - metrics_per_class[label.item()]["IoUs"].extend([0] * len(unmatched_preds)) - metrics_per_class[label.item()]["FNs"] += len(unmatched_gts) - metrics_per_class[label.item()]["IoUs"].extend([0] * len(unmatched_gts)) - return metrics_per_class - - def _compute_metrics_and_confusion_matrix(self, preds): - # Initialize per-class metrics - metrics_per_class = defaultdict(lambda: {"TPs": 0, "FPs": 0, "FNs": 0, "IoUs": []}) - - # Collect all class IDs - all_classes = set() - for pred in preds: - all_classes.update(pred["labels"].tolist()) - for gt in self.gt: - all_classes.update(gt["labels"].tolist()) - all_classes = sorted(list(all_classes)) - class_to_idx = {cls_id: idx for idx, cls_id in enumerate(all_classes)} - n_classes = len(all_classes) - conf_matrix = np.zeros((n_classes + 1, n_classes + 1), dtype=int) # +1 for background class - - for pred, gt in zip(preds, self.gt): - pred_boxes = pred["boxes"] - pred_labels = pred["labels"] - gt_boxes = gt["boxes"] - gt_labels = gt["labels"] - - n_preds = len(pred_boxes) - n_gts = len(gt_boxes) - - if n_preds == 0 and n_gts == 0: - continue - - ious = box_iou(pred_boxes, gt_boxes) if n_preds > 0 and n_gts > 0 else torch.tensor([]) - # Assign matches between preds and gts - matched_pred_indices = set() - matched_gt_indices = set() - - if ious.numel() > 0: - # For each pred box, find the gt box with highest IoU - ious_mask = ious >= self.iou_thresh - pred_indices, gt_indices = torch.nonzero(ious_mask, as_tuple=True) - iou_values = ious[pred_indices, gt_indices] - - # Sorting by IoU to match highest scores first - sorted_indices = torch.argsort(-iou_values) - pred_indices = pred_indices[sorted_indices] - gt_indices = gt_indices[sorted_indices] - iou_values = iou_values[sorted_indices] - - for pred_idx, gt_idx, iou in zip(pred_indices, gt_indices, iou_values): - if ( - pred_idx.item() in matched_pred_indices - or gt_idx.item() in matched_gt_indices - ): - continue - matched_pred_indices.add(pred_idx.item()) - matched_gt_indices.add(gt_idx.item()) - - pred_label = pred_labels[pred_idx].item() - gt_label = gt_labels[gt_idx].item() - - pred_cls_idx = class_to_idx[pred_label] - gt_cls_idx = class_to_idx[gt_label] - - # Update confusion matrix - conf_matrix[gt_cls_idx, pred_cls_idx] += 1 - - # Update per-class metrics - if pred_label == gt_label: - metrics_per_class[gt_label]["TPs"] += 1 - metrics_per_class[gt_label]["IoUs"].append(iou.item()) - else: - # Misclassification - metrics_per_class[gt_label]["FNs"] += 1 - metrics_per_class[pred_label]["FPs"] += 1 - metrics_per_class[gt_label]["IoUs"].append(0) - metrics_per_class[pred_label]["IoUs"].append(0) - - # Unmatched predictions (False Positives) - unmatched_pred_indices = set(range(n_preds)) - matched_pred_indices - for pred_idx in unmatched_pred_indices: - pred_label = pred_labels[pred_idx].item() - pred_cls_idx = class_to_idx[pred_label] - # Update confusion matrix: background row - conf_matrix[n_classes, pred_cls_idx] += 1 - # Update per-class metrics - metrics_per_class[pred_label]["FPs"] += 1 - metrics_per_class[pred_label]["IoUs"].append(0) - - # Unmatched ground truths (False Negatives) - unmatched_gt_indices = set(range(n_gts)) - matched_gt_indices - for gt_idx in unmatched_gt_indices: - gt_label = gt_labels[gt_idx].item() - gt_cls_idx = class_to_idx[gt_label] - # Update confusion matrix: background column - conf_matrix[gt_cls_idx, n_classes] += 1 - # Update per-class metrics - metrics_per_class[gt_label]["FNs"] += 1 - metrics_per_class[gt_label]["IoUs"].append(0) - - return metrics_per_class, conf_matrix, class_to_idx - - def save_plots(self, path_to_save) -> None: - path_to_save = Path(path_to_save) - path_to_save.mkdir(parents=True, exist_ok=True) - - if self.conf_matrix is not None: - class_labels = [str(cls_id) for cls_id in self.class_to_idx.keys()] + ["background"] - - plt.figure(figsize=(10, 8)) - plt.imshow(self.conf_matrix, interpolation="nearest", cmap=plt.cm.Blues) - plt.title("Confusion Matrix") - plt.colorbar() - tick_marks = np.arange(len(class_labels)) - plt.xticks(tick_marks, class_labels, rotation=45) - plt.yticks(tick_marks, class_labels) - - # Add labels to each cell - thresh = self.conf_matrix.max() / 2.0 - for i in range(self.conf_matrix.shape[0]): - for j in range(self.conf_matrix.shape[1]): - plt.text( - j, - i, - format(self.conf_matrix[i, j], "d"), - horizontalalignment="center", - color="white" if self.conf_matrix[i, j] > thresh else "black", - ) - - plt.ylabel("True label") - plt.xlabel("Predicted label") - plt.tight_layout() - plt.savefig(path_to_save / "confusion_matrix.png") - plt.close() - - thresholds = self.thresholds - precisions, recalls, f1_scores = [], [], [] - - # Store the original predictions to reset after each threshold - original_preds = copy.deepcopy(self.preds) - - for threshold in thresholds: - # Filter predictions based on the current threshold - filtered_preds = filter_preds(copy.deepcopy(original_preds), threshold) - # Compute metrics with the filtered predictions - metrics = self._compute_main_metrics(filtered_preds) - precisions.append(metrics["precision"]) - recalls.append(metrics["recall"]) - f1_scores.append(metrics["f1"]) - - # Plot Precision and Recall vs Threshold - plt.figure() - plt.plot(thresholds, precisions, label="Precision", marker="o") - plt.plot(thresholds, recalls, label="Recall", marker="o") - plt.xlabel("Threshold") - plt.ylabel("Value") - plt.title("Precision and Recall vs Threshold") - plt.legend() - plt.grid(True) - plt.savefig(path_to_save / "precision_recall_vs_threshold.png") - plt.close() - - # Plot F1 Score vs Threshold - plt.figure() - plt.plot(thresholds, f1_scores, label="F1 Score", marker="o") - plt.xlabel("Threshold") - plt.ylabel("F1 Score") - plt.title("F1 Score vs Threshold") - plt.grid(True) - plt.savefig(path_to_save / "f1_score_vs_threshold.png") - plt.close() - - # Find the best threshold based on F1 Score (last occurence) - best_idx = len(f1_scores) - np.argmax(f1_scores[::-1]) - 1 - best_threshold = thresholds[best_idx] - best_f1 = f1_scores[best_idx] - - logger.info( - f"Best Threshold: {round(best_threshold, 2)} with F1 Score: {round(best_f1, 3)}" - ) - - -def filter_preds(preds, conf_thresh): - for pred in preds: - keep_idxs = pred["scores"] >= conf_thresh - pred["scores"] = pred["scores"][keep_idxs] - pred["boxes"] = pred["boxes"][keep_idxs] - pred["labels"] = pred["labels"][keep_idxs] - return preds - - -def scale_boxes(boxes, orig_shape, resized_shape): - """ - boxes in format: [x1, y1, x2, y2], absolute values - orig_shape: [height, width] - resized_shape: [height, width] - """ - scale_x = orig_shape[1] / resized_shape[1] - scale_y = orig_shape[0] / resized_shape[0] - boxes[:, 0] *= scale_x - boxes[:, 2] *= scale_x - boxes[:, 1] *= scale_y - boxes[:, 3] *= scale_y - return boxes diff --git a/vlm/detector/D-FINE.backup/src/zoo/__init__.py b/vlm/detector/D-FINE.backup/src/zoo/__init__.py deleted file mode 100644 index de38c3a..0000000 --- a/vlm/detector/D-FINE.backup/src/zoo/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -""" -Copied from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright(c) 2023 lyuwenyu. All Rights Reserved. -""" - -from . import dfine diff --git a/vlm/detector/D-FINE.backup/src/zoo/dfine/__init__.py b/vlm/detector/D-FINE.backup/src/zoo/dfine/__init__.py deleted file mode 100644 index f61af2a..0000000 --- a/vlm/detector/D-FINE.backup/src/zoo/dfine/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -""" -Copied from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright(c) 2023 lyuwenyu. All Rights Reserved. -""" - -from .dfine import DFINE -from .dfine_criterion import DFINECriterion -from .dfine_decoder import DFINETransformer -from .hybrid_encoder import HybridEncoder -from .matcher import HungarianMatcher -from .postprocessor import DFINEPostProcessor diff --git a/vlm/detector/D-FINE.backup/src/zoo/dfine/blog.md b/vlm/detector/D-FINE.backup/src/zoo/dfine/blog.md deleted file mode 100644 index 24abf57..0000000 --- a/vlm/detector/D-FINE.backup/src/zoo/dfine/blog.md +++ /dev/null @@ -1,90 +0,0 @@ -English Blog | [ไธญๆ–‡ๅšๅฎข](blog_cn.md) - -## ๐Ÿ”ฅ Revolutionizing Real-Time Object Detection: D-FINE vs. YOLO and Other DETR Models - -In the rapidly evolving field of real-time object detection, **D-FINE** emerges as a revolutionary approach that significantly surpasses existing models like **YOLOv10**, **YOLO11**, and **RT-DETR v1/v2/v3**, raising the performance ceiling for real-time object detection. After pretraining on the large-scale dataset Objects365, **D-FINE** far exceeds its competitor **LW-DETR**, achieving up to **59.3%** AP on the COCO dataset while maintaining excellent frame rates, parameter counts, and computational complexity. This positions **D-FINE** as a leader in the realm of real-time object detection, laying the groundwork for future research advancements. - -Currently, all code, weights, logs, compilation tools, and the FiftyOne visualization tool for **D-FINE** have been fully open-sourced, thanks to the codebase provided by **RT-DETR**. This includes pretraining tutorials, custom dataset tutorials, and more. We will continue to update with improvement insights and tuning strategies. We welcome everyone to raise issues and collectively promote the **D-FINE** series. We also hope you can leave a โญ; it's the best encouragement for us. - -**GitHub Repo**: https://github.com/Peterande/D-FINE - -**ArXiv Paper**: https://arxiv.org/abs/2410.13842 - ---- - -### ๐Ÿ” Exploring the Key Innovations Behind D-FINE - -**D-FINE** redefines the regression task in DETR-based object detectors as **FDR**, and based on this, develops a performance-enhancing self-distillation mechanism **GO-LSD**. Below is a brief introduction to **FDR** and **GO-LSD**: - -#### **FDR (Fine-grained Distribution Refinement)** Decouples the Bounding Box Generation Process: - -1. **Initial Box Prediction**: Similar to traditional DETR methods, the decoder of **D-FINE** transforms object queries into several initial bounding boxes in the first layer. These boxes do not need to be highly accurate and serve only as an initialization. -2. **Fine-Grained Distribution Optimization**: Unlike traditional methods that directly decode new bounding boxes, **D-FINE** generates four sets of probability distributions based on these initial bounding boxes in the decoder layers and iteratively optimizes these distributions layer by layer. These distributions essentially act as a "fine-grained intermediate representation" of the detection boxes. Coupled with a carefully designed weighting function **W(n)**, **D-FINE** can adjust the initial bounding boxes by fine-tuning these representations, allowing for subtle modifications or significant shifts of the edges (top, bottom, left, right). The specific process is illustrated in the figure: - -

- Fine-grained Distribution Refinement Process -

- -For readability, we will not elaborate on the mathematical formulas and the Fine-Grained Localization (FGL) Loss that aids optimization here. Interested readers can refer to the original paper for derivations. - -The main advantages of redefining the bounding box regression task as **FDR** are: - -1. **Simplified Supervision**: While optimizing detection boxes using traditional L1 loss and IoU loss, the "residual" between labels and predictions can be additionally used to constrain these intermediate probability distributions. This allows each decoder layer to more effectively focus on and address the localization errors it currently faces. As the number of layers increases, their optimization objectives become progressively simpler, thereby simplifying the overall optimization process. - -2. **Robustness in Complex Scenarios**: The values of these probability distributions inherently represent the confidence level of fine-tuning for each edge. This enables **D-FINE** to independently model the uncertainty of each edge at different network depths, thereby exhibiting stronger robustness in complex real-world scenarios such as occlusion, motion blur, and low-light conditions, compared to directly regressing four fixed values. - -3. **Flexible Optimization Mechanism**: The probability distributions are transformed into final bounding box offsets through a weighted sum. The carefully designed weighting function ensures fine-grained adjustments when the initial box is accurate and provides significant corrections when necessary. - -4. **Research Potential and Scalability**: By transforming the regression task into a probability distribution prediction problem consistent with classification tasks, **FDR** not only enhances compatibility with other tasks but also enables object detection models to benefit from innovations in areas such as knowledge distillation, multi-task learning, and distribution optimization, opening new avenues for future research. - ---- - -#### **GO-LSD (Global Optimal Localization Self-Distillation)** Integrates Knowledge Distillation into FDR-Based Detectors Seamlessly - -Based on the above, object detectors equipped with the **FDR** framework satisfy the following two points: - -1. **Ability to Achieve Knowledge Transfer**: As Hinton mentioned in the paper *"Distilling the Knowledge in a Neural Network"*, probabilities are "knowledge." The network's output becomes probability distributions, and these distributions carry localization knowledge. By calculating the KLD loss, this "knowledge" can be transferred from deeper layers to shallower layers. This is something that traditional fixed box representations (Dirac ฮด functions) cannot achieve. - -2. **Consistent Optimization Objectives**: Since each decoder layer in the **FDR** framework shares a common goal: reducing the residual between the initial bounding box and the ground truth bounding box, the precise probability distributions generated by the final layer can serve as the ultimate target for each preceding layer and guide them through distillation. - -Thus, based on **FDR**, we propose **GO-LSD (Global Optimal Localization Self-Distillation)**. By implementing localization knowledge distillation between network layers, we further extend the capabilities of **D-FINE**. The specific process is illustrated in the figure: - -

- GO-LSD Process -

- -Similarly, for readability, we will not elaborate on the mathematical formulas and the Decoupled Distillation Focal (DDF) Loss that aids optimization here. Interested readers can refer to the original paper for derivations. - -This results in a synergistic win-win effect: as training progresses, the predictions of the final layer become increasingly accurate, and its generated soft labels can better help the preceding layers improve prediction accuracy. Conversely, the earlier layers learn to localize accurately more quickly, simplifying the optimization tasks of the deeper layers and further enhancing overall accuracy. - ---- - -### Visualization of D-FINE Predictions - -The following visualization showcases **D-FINE**'s predictions in various complex detection scenarios. These scenarios include occlusion, low-light conditions, motion blur, depth-of-field effects, and densely populated scenes. Despite these challenges, **D-FINE** still produces accurate localization results. - -

- D-FINE Predictions in Complex Scenarios -

- -Additionally, the visualization below shows the prediction results of the first layer and the last layer, the corresponding distributions of the four edges, and the weighted distributions. It can be seen that the localization of the predicted boxes becomes more precise as the distributions are optimized. - -

- -

- ---- - -### Frequently Asked Questions - -#### Question 1: Will FDR and GO-LSD increase the inference cost? - -**Answer**: No, FDR and the original prediction have almost no difference in speed, parameter count, and computational complexity, making it a seamless replacement. - -#### Question 2: Will FDR and GO-LSD increase the training cost? - -**Answer**: The increase in training cost mainly comes from how to generate the labels of the distributions. We have optimized this process, keeping the increase in training time and memory consumption to 6% and 2%, respectively, making it almost negligible. - -#### Question 3: Why is D-FINE faster and more lightweight than the RT-DETR series? - -**Answer**: Directly applying FDR and GO-LSD will significantly improve performance but will not make the network faster or lighter. Therefore, we performed a series of lightweight optimizations on RT-DETR. These adjustments led to a performance drop, but our methods compensated for these losses, achieving a perfect balance of speed, parameters, computational complexity, and performance. diff --git a/vlm/detector/D-FINE.backup/src/zoo/dfine/blog_cn.md b/vlm/detector/D-FINE.backup/src/zoo/dfine/blog_cn.md deleted file mode 100644 index 04fd5df..0000000 --- a/vlm/detector/D-FINE.backup/src/zoo/dfine/blog_cn.md +++ /dev/null @@ -1,90 +0,0 @@ -[English Blog](blog.md) | ไธญๆ–‡ๅšๅฎข - -## ๐Ÿ”ฅ ้ฉๆ–ฐๅฎžๆ—ถ็›ฎๆ ‡ๆฃ€ๆต‹๏ผšD-FINE ไธŽ YOLO ๅ’Œๅ…ถไป– DETR ๆจกๅž‹็š„ๅฏนๆฏ” - -ๅœจๅฟซ้€Ÿๅ‘ๅฑ•็š„ๅฎžๆ—ถ็›ฎๆ ‡ๆฃ€ๆต‹้ข†ๅŸŸ๏ผŒ**D-FINE** ไฝœไธบไธ€้กน้ฉๅ‘ฝๆ€ง็š„ๆ–นๆณ•๏ผŒๆ˜พ่‘—่ถ…่ถŠไบ†็Žฐๆœ‰ๆจกๅž‹๏ผˆๅฆ‚ **YOLOv10**ใ€**YOLO11** ๅŠ **RT-DETR v1/v2/v3**๏ผ‰๏ผŒๆๅ‡ไบ†ๅฎžๆ—ถ็›ฎๆ ‡ๆฃ€ๆต‹็š„ๆ€ง่ƒฝไธŠ้™ใ€‚็ป่ฟ‡ๅคง่ง„ๆจกๆ•ฐๆฎ้›† Objects365 ็š„้ข„่ฎญ็ปƒ๏ผŒ**D-FINE** ่ฟœ่ถ…ๅ…ถ็ซžไบ‰ๅฏนๆ‰‹ **LW-DETR**๏ผŒๅœจ COCO ๆ•ฐๆฎ้›†ไธŠๅฎž็Žฐไบ†้ซ˜่พพ **59.3%** ็š„ AP๏ผŒๅŒๆ—ถไฟๆŒไบ†ๅ“่ถŠ็š„ๅธง็Ž‡ใ€ๅ‚ๆ•ฐ้‡ๅ’Œ่ฎก็ฎ—ๅคๆ‚ๅบฆใ€‚่ฟ™ไฝฟๅพ— **D-FINE** ๆˆไธบๅฎžๆ—ถ็›ฎๆ ‡ๆฃ€ๆต‹้ข†ๅŸŸ็š„ไฝผไฝผ่€…๏ผŒไธบๆœชๆฅ็š„็ ”็ฉถๅฅ ๅฎšไบ†ๅŸบ็ก€ใ€‚ - -็›ฎๅ‰๏ผŒD-FINE ็š„ๆ‰€ๆœ‰ไปฃ็ ใ€ๆƒ้‡ใ€ๆ—ฅๅฟ—ใ€็ผ–่ฏ‘ๅทฅๅ…ท๏ผŒไปฅๅŠ FiftyOne ๅฏ่ง†ๅŒ–ๅทฅๅ…ทๅทฒ็ปๅ…จ้ƒจๅผ€ๆบ๏ผŒๆ„Ÿ่ฐข RT-DETR ๆไพ›็š„ codebaseใ€‚ๅ…ถไธญ่ฟ˜ๅŒ…ๆ‹ฌไบ†้ข„่ฎญ็ปƒๆ•™็จ‹ใ€่‡ชๅฎšไน‰ๆ•ฐๆฎ้›†ๆ•™็จ‹็ญ‰ใ€‚ไน‹ๅŽ่ฟ˜ไผš้™†็ปญๆ›ดๆ–ฐไธ€ไบ›ๆ”น่ฟ›ๅฟƒๅพ—ๅ’Œ่ฐƒๅ‚ๆ”ป็•ฅ๏ผŒๆฌข่ฟŽๅคงๅฎถๅคšๆ issue๏ผŒๅ…ฑๅŒๅฐ† D-FINE ็ณปๅˆ—ๅ‘ๆ‰ฌๅ…‰ๅคงใ€‚ๅŒๆ—ถๅธŒๆœ›ๆ‚จ่ƒฝ้šๆ‰‹็•™ไธ‹ไธ€้ข— โญ๏ผŒ่ฟ™ๆ˜ฏๅฏนๆˆ‘ไปฌๆœ€ๅฅฝ็š„้ผ“ๅŠฑใ€‚ - -**Github Repo**: https://github.com/Peterande/D-FINE - -**Arxiv Paper**: https://arxiv.org/abs/2410.13842 - ---- - -### ๐Ÿ” ๆŽข็ดข D-FINE ่ƒŒๅŽ็š„ๅ…ณ้”ฎๅˆ›ๆ–ฐ - -**D-FINE** ๅฐ†ๅŸบไบŽ DETR ็š„็›ฎๆ ‡ๆฃ€ๆต‹ๅ™จไธญ็š„ๅ›žๅฝ’ไปปๅŠก้‡ๆ–ฐๅฎšไน‰ไธบ FDR๏ผŒๅนถๅœจๆญคๅŸบ็ก€ไธŠๅผ€ๅ‘ๅ‡บไบ†ๆ— ๆ„Ÿๆๅ‡ๆ€ง่ƒฝ็š„่‡ช่’ธ้ฆๆœบๅˆถ GO-LSDใ€‚ไธ‹้ขๅฏน FDR ๅ’Œ GO-LSD ่ฟ›่กŒ็ฎ€่ฆไป‹็ป๏ผš - -#### FDR (Fine-grained Distribution Refinement) ๅฐ†ๆฃ€ๆต‹ๆก†็š„็”Ÿๆˆ่ฟ‡็จ‹ๆ‹†่งฃไธบ๏ผš - -1. **ๅˆๅง‹ๆก†้ข„ๆต‹**๏ผšไธŽไผ ็ปŸ DETR ๆ–นๆณ•็ฑปไผผ๏ผŒ**D-FINE** ็š„่งฃ็ ๅ™จ (decoder) ไผšๅœจ็ฌฌไธ€ๅฑ‚ๅฐ† object queries ่ฝฌๅ˜ไธบ่‹ฅๅนฒไธชๅˆๅง‹็š„่พน็•Œๆก†๏ผŒ่ฟ™ไบ›ๆก†ไธ้œ€่ฆ็‰นๅˆซ็ฒพๅ‡†๏ผŒไป…ไฝœไธบไธ€็งๅˆๅง‹ๅŒ–ใ€‚ -2. **็ป†็ฒ’ๅบฆ็š„ๅˆ†ๅธƒไผ˜ๅŒ–**๏ผš**D-FINE** ่งฃ็ ๅฑ‚ไธไผšๅƒไผ ็ปŸๆ–นๆณ•้‚ฃๆ ท็›ดๆŽฅ่งฃ็ ๅ‡บๆ–ฐ็š„่พน็•Œๆก†๏ผŒ่€Œๆ˜ฏๅŸบไบŽ่ฟ™ไบ›ๅˆๅง‹ๅŒ–็š„่พน็•Œๆก†๏ผŒ็”Ÿๆˆๅ››็ป„ๆฆ‚็Ž‡ๅˆ†ๅธƒ๏ผ›ๅนถ่ฟญไปฃๅœฐๅฏน่ฟ™ๅ››็ป„ๆฆ‚็Ž‡ๅˆ†ๅธƒ่ฟ›่กŒ้€ๅฑ‚ไผ˜ๅŒ–ใ€‚่ฟ™ไบ›ๅˆ†ๅธƒๆœฌ่ดจไธŠๆ˜ฏไฝœไธบๆฃ€ๆต‹ๆก†็š„ไธ€็งโ€œ็ป†็ฒ’ๅบฆไธญ้—ด่กจๅพโ€๏ผ›้…ๅˆ็ฒพๅฟƒ่ฎพ่ฎก็š„ๅŠ ๆƒๅ‡ฝๆ•ฐ W(n)๏ผŒ**D-FINE** ่ƒฝๅคŸ้€š่ฟ‡ๅพฎ่ฐƒ่ฟ™ไบ›่กจๅพๆฅๅฎž็Žฐๅฏนๅˆๅง‹่พน็•Œๆก†็š„่ฐƒๆ•ด๏ผŒๅŒ…ๅซๅฏนๅ…ถไธŠไธ‹ๅทฆๅณ่พน็ผ˜่ฟ›่กŒ็ป†ๅพฎ็š„ๅฐๅน…ๅบฆไฟฎๆญฃไบฆๆˆ–ๆ˜ฏๅคงๅน…ๅบฆ็š„ๆฌ็งป๏ผŒๅ…ทไฝ“็š„ๆต็จ‹ๅฆ‚ๅ›พ๏ผš - -

- ็ฒพ็ป†ๅˆ†ๅธƒไผ˜ๅŒ–่ฟ‡็จ‹ -

- -ไธบไบ†ๆ–นไพฟ้˜…่ฏป๏ผŒๆˆ‘ไปฌไธๅœจๆญค่ต˜่ฟฐๆ•ฐๅญฆๅ…ฌๅผๅŠๅธฎๅŠฉไผ˜ๅŒ–็š„ๆŸๅคฑๅ‡ฝๆ•ฐ Fine-Grained Localization (FGL) Loss๏ผŒๆœ‰ๅ…ด่ถฃ็š„ๅฏไปฅๆ นๆฎๅŽŸๆ–‡ๆŽจๅฏผใ€‚ - -ๅฐ†่พน็•Œๆก†ๅ›žๅฝ’ไปปๅŠก้‡ๆ–ฐๅฎšไน‰ไธบ FDR ็š„ไธป่ฆไผ˜ๅŠฟๅœจไบŽ๏ผš - -1. **็ฎ€ๅŒ–็š„็›‘็ฃ**๏ผšๅœจไฝฟ็”จไผ ็ปŸ็š„ L1 ๆŸๅคฑใ€IoU ๆŸๅคฑไผ˜ๅŒ–ๆฃ€ๆต‹ๆก†็š„ๅŒๆ—ถ๏ผŒๅฏไปฅ้ขๅค–็”จๆ ‡็ญพๅ’Œ้ข„ๆต‹็ป“ๆžœไน‹้—ด็š„โ€œๆฎ‹ๅทฎโ€ๆฅ็บฆๆŸ่ฟ™ไบ›ไธญ้—ดๆ€็š„ๆฆ‚็Ž‡ๅˆ†ๅธƒใ€‚่ฟ™ไฝฟๆฏไธช่งฃ็ ๅฑ‚ (decoder layer) ่ƒฝๅคŸๆ›ดๆœ‰ๆ•ˆๅœฐๅ…ณๆณจๅนถ่งฃๅ†ณๅ…ถๅฝ“ๅ‰้ขไธด็š„ๅฎšไฝ่ฏฏๅทฎ๏ผŒ้š็€ๅฑ‚ๆ•ฐๅŠ ๆทฑ๏ผŒๅ…ถไผ˜ๅŒ–็›ฎๆ ‡ๅ˜ๅพ—่ถŠๆฅ่ถŠ็ฎ€ๅ•๏ผŒไปŽ่€Œ็ฎ€ๅŒ–ไบ†ๆ•ดไฝ“ไผ˜ๅŒ–่ฟ‡็จ‹ใ€‚ - -2. **ๅคๆ‚ๅœบๆ™ฏไธ‹็š„้ฒๆฃ’ๆ€ง**๏ผš่ฟ™ไบ›ๆฆ‚็Ž‡ๅˆ†ๅธƒ็š„ๅ€ผๆœฌ่ดจไธŠไปฃ่กจไบ†ๅฏนๆฏไธช่พน็•Œโ€œๅพฎ่ฐƒโ€็š„่‡ชไฟก็จ‹ๅบฆใ€‚่ฟ™ไฝฟ **D-FINE** ่ƒฝๅคŸๅœจไธๅŒ็ฝ‘็ปœๆทฑๅบฆ็‹ฌ็ซ‹ๅปบๆจกๆฏไธช่พน็•Œ็š„ไธ็กฎๅฎšๆ€ง๏ผŒไปŽ่€Œๅœจ้ฎๆŒกใ€่ฟๅŠจๆจก็ณŠๅ’ŒไฝŽๅ…‰็…ง็ญ‰ๅคๆ‚็š„ๅฎž้™…ๅœบๆ™ฏไธ‹่กจ็Žฐๅ‡บๆ›ดๅผบ็š„้ฒๆฃ’ๆ€ง๏ผŒ็›ธๆฏ”็›ดๆŽฅๅ›žๅฝ’ๅ››ไธชๅ›บๅฎšๅ€ผ่ฆๆ›ดไธบ็จณๅฅใ€‚ - -3. **็ตๆดป็š„ไผ˜ๅŒ–ๆœบๅˆถ**๏ผšๆฆ‚็Ž‡ๅˆ†ๅธƒ้€š่ฟ‡ๅŠ ๆƒๆฑ‚ๅ’Œ่ฝฌๅŒ–ไธบๆœ€็ปˆ็š„่พน็•Œๆก†ๅ็งปๅ€ผใ€‚็ฒพๅฟƒ่ฎพ่ฎก็š„ๅŠ ๆƒๅ‡ฝๆ•ฐ็กฎไฟๅœจๅˆๅง‹ๆก†ๅ‡†็กฎๆ—ถ่ฟ›่กŒ็ป†ๅพฎ่ฐƒๆ•ด๏ผŒ่€Œๅœจๅฟ…่ฆๆ—ถๅˆ™ๆไพ›่พƒๅคง็š„ไฟฎๆญฃใ€‚ - -4. **็ ”็ฉถๆฝœๅŠ›ไธŽๅฏๆ‰ฉๅฑ•ๆ€ง**๏ผšFDR ้€š่ฟ‡ๅฐ†ๅ›žๅฝ’ไปปๅŠก่ฝฌๅ˜ไธบๅŒๅˆ†็ฑปไปปๅŠกไธ€่‡ด็š„ๆฆ‚็Ž‡ๅˆ†ๅธƒ้ข„ๆต‹้—ฎ้ข˜๏ผŒไธไป…ๆ้ซ˜ไบ†ไธŽๅ…ถไป–ไปปๅŠก็š„ๅ…ผๅฎนๆ€ง๏ผŒ่ฟ˜ไฝฟๅพ—็›ฎๆ ‡ๆฃ€ๆต‹ๆจกๅž‹ๅฏไปฅๅ—็›ŠไบŽ็Ÿฅ่ฏ†่’ธ้ฆใ€ๅคšไปปๅŠกๅญฆไน ๅ’Œๅˆ†ๅธƒไผ˜ๅŒ–็ญ‰ๆ›ดๅคš้ข†ๅŸŸ็š„ๅˆ›ๆ–ฐ๏ผŒไธบๆœชๆฅ็š„็ ”็ฉถๆ‰“ๅผ€ไบ†ๆ–ฐ็š„ๅคง้—จใ€‚ - ---- - -#### GO-LSD (Global Optimal Localization Self-Distillation) ๅฐ†็Ÿฅ่ฏ†่’ธ้ฆๆ— ็—›ๅบ”็”จๅˆฐ FDR ๆก†ๆžถๆฃ€ๆต‹ๅ™จ - -ๆ นๆฎไธŠๆ–‡๏ผŒๆญ่ฝฝ FDR ๆก†ๆžถ็š„็›ฎๆ ‡ๆฃ€ๆต‹ๅ™จๆปก่ถณไบ†ไปฅไธ‹ไธค็‚น๏ผš - -1. **่ƒฝๅคŸๅฎž็Žฐ็Ÿฅ่ฏ†ไผ ้€’**๏ผšHinton ๆ—ฉๅœจ *"Distilling the Knowledge in a Neural Network"* ไธ€ๆ–‡ไธญๅฐฑ่ฏด่ฟ‡๏ผšๆฆ‚็Ž‡ๅณโ€œ็Ÿฅ่ฏ†โ€๏ผ›็ฝ‘็ปœ่พ“ๅ‡บๅ˜ๆˆไบ†ๆฆ‚็Ž‡ๅˆ†ๅธƒ๏ผŒ่€Œๆฆ‚็Ž‡ๅˆ†ๅธƒๆบๅธฆๅฎšไฝ็Ÿฅ่ฏ† (Localization Knowledge)๏ผŒ่€Œ้€š่ฟ‡่ฎก็ฎ— KLD ๆŸๅคฑ๏ผŒๅฏไปฅๅฐ†่ฟ™ไบ›โ€œ็Ÿฅ่ฏ†โ€ไปŽๆทฑๅฑ‚ไผ ้€’ๅˆฐๆต…ๅฑ‚ใ€‚่ฟ™ๆ˜ฏไผ ็ปŸๅ›บๅฎšๆก†่กจ็คบ๏ผˆ็‹„ๆ‹‰ๅ…‹ ฮด ๅ‡ฝๆ•ฐ๏ผ‰ๆ— ๆณ•ๅฎž็Žฐ็š„ใ€‚ - -2. **ไธ€่‡ด็š„ไผ˜ๅŒ–็›ฎๆ ‡**๏ผš็”ฑไบŽ FDR ๆžถๆž„ไธญๆฏไธ€ไธช่งฃ็ ๅฑ‚้ƒฝๅ…ฑไบซไธ€ไธชๅ…ฑๅŒ็›ฎๆ ‡๏ผšๅ‡ๅฐ‘ๅˆๅง‹่พน็•Œๆก†ไธŽ็œŸๅฎž่พน็•Œๆก†ไน‹้—ด็š„ๆฎ‹ๅทฎ๏ผ›ๅ› ๆญคๆœ€ๅŽไธ€ๅฑ‚็”Ÿๆˆ็š„็ฒพ็กฎๆฆ‚็Ž‡ๅˆ†ๅธƒๅฏไปฅไฝœไธบๅ‰้ขๆฏไธ€ๅฑ‚็š„ๆœ€็ปˆ็›ฎๆ ‡๏ผŒๅนถ้€š่ฟ‡่’ธ้ฆๅผ•ๅฏผๅ‰ๅ‡ ๅฑ‚ใ€‚ - -ไบŽๆ˜ฏ๏ผŒๅŸบไบŽ FDR๏ผŒๆˆ‘ไปฌๆๅ‡บไบ† GO-LSD๏ผˆๅ…จๅฑ€ๆœ€ไผ˜ๅฎšไฝ่‡ช่’ธ้ฆ๏ผ‰ใ€‚้€š่ฟ‡ๅœจ็ฝ‘็ปœๅฑ‚้—ดๅฎž็Žฐๅฎšไฝ็Ÿฅ่ฏ†่’ธ้ฆ๏ผŒ่ฟ›ไธ€ๆญฅๆ‰ฉๅฑ•ไบ† **D-FINE** ็š„่ƒฝๅŠ›๏ผŒๅ…ทไฝ“ๆต็จ‹ๅฆ‚ๅ›พ๏ผš - -

- GO-LSD่ฟ‡็จ‹ -

- -ๅŒๆ ท็š„๏ผŒไธบไบ†ๆ–นไพฟ้˜…่ฏป๏ผŒๆˆ‘ไปฌไธๅœจๆญค่ต˜่ฟฐๆ•ฐๅญฆๅ…ฌๅผๅŠๅธฎๅŠฉไผ˜ๅŒ–็š„ๆŸๅคฑๅ‡ฝๆ•ฐ Decoupled Distillation Focal (DDF) Loss๏ผŒๆœ‰ๅ…ด่ถฃ็š„ๅฏไปฅๆ นๆฎๅŽŸๆ–‡ๆŽจๅฏผใ€‚ - -่ฟ™ไบง็”Ÿไบ†ไธ€็งๅŒ่ตข็š„ๅๅŒๆ•ˆๅบ”๏ผš้š็€่ฎญ็ปƒ็š„่ฟ›่กŒ๏ผŒๆœ€ๅŽไธ€ๅฑ‚็š„้ข„ๆต‹ๅ˜ๅพ—่ถŠๆฅ่ถŠๅ‡†็กฎ๏ผŒๅ…ถ็”Ÿๆˆ็š„่ฝฏๆ ‡็ญพ่ƒฝๅคŸๆ›ดๅฅฝๅœฐๅธฎๅŠฉๅ‰ๅ‡ ๅฑ‚ๆ้ซ˜้ข„ๆต‹ๅ‡†็กฎๆ€งใ€‚ๅ่ฟ‡ๆฅ๏ผŒๅ‰ๅ‡ ๅฑ‚ๅญฆไผšๆ›ดๅฟซๅœฐๅฎšไฝๅˆฐๅ‡†็กฎไฝ็ฝฎ๏ผŒ็ฎ€ๅŒ–ไบ†ๆทฑๅฑ‚็š„ไผ˜ๅŒ–ไปปๅŠก๏ผŒ่ฟ›ไธ€ๆญฅๆ้ซ˜ไบ†ๆ•ดไฝ“ๅ‡†็กฎๆ€งใ€‚ - ---- - -### D-FINE ้ข„ๆต‹็š„ๅฏ่ง†ๅŒ– - -ไปฅไธ‹ๅฏ่ง†ๅŒ–ๅฑ•็คบไบ† **D-FINE** ๅœจๅ„็งๅคๆ‚ๆฃ€ๆต‹ๅœบๆ™ฏไธญ็š„้ข„ๆต‹็ป“ๆžœใ€‚่ฟ™ไบ›ๅœบๆ™ฏๅŒ…ๆ‹ฌ้ฎๆŒกใ€ไฝŽๅ…‰็…งใ€่ฟๅŠจๆจก็ณŠใ€ๆ™ฏๆทฑๆ•ˆๆžœๅ’Œๅฏ†้›†ๅœบๆ™ฏใ€‚ๅฐฝ็ฎก้ขๅฏน่ฟ™ไบ›ๆŒ‘ๆˆ˜๏ผŒ**D-FINE** ไพ็„ถ่ƒฝๅคŸไบง็”Ÿๅ‡†็กฎ็š„ๅฎšไฝ็ป“ๆžœใ€‚ - -

- D-FINEๅœจๅคๆ‚ๅœบๆ™ฏไธญ็š„้ข„ๆต‹ -

- -ๅŒๆ—ถไธ‹้ข็ป™ๅ‡บ็š„ๅฏ่ง†ๅŒ–็ป“ๆžœๅฑ•็คบไบ†็ฌฌไธ€ๅฑ‚ๅ’Œๆœ€ๅŽไธ€ๅฑ‚็š„้ข„ๆต‹็ป“ๆžœใ€ๅฏนๅบ”ๅ››ๆก่พน็š„ๅˆ†ๅธƒใ€ไปฅๅŠๅŠ ๆƒๅŽ็š„ๅˆ†ๅธƒใ€‚ๅฏไปฅ็œ‹ๅˆฐ๏ผŒ้ข„ๆต‹ๆก†็š„ๅฎšไฝไผš้š็€ๅˆ†ๅธƒ็š„ไผ˜ๅŒ–่€Œๅ˜ๅพ—ๆ›ดๅŠ ็ฒพๅ‡†ใ€‚ - -

- -

- ---- - -### ๅธธ่ง้—ฎ้ข˜่งฃ็ญ” - -#### ้—ฎ้ข˜1๏ผšFDR ๅ’Œ GO-LSD ไผšๅธฆๆฅๆ›ดๅคš็š„ๆŽจ็†ๆˆๆœฌๅ—๏ผŸ - -**ๅ›ž็ญ”**๏ผšๅนถไธไผš๏ผŒFDR ๅ’ŒๅŽŸๅง‹็š„้ข„ๆต‹ๅ‡ ไนŽๆฒกๆœ‰ๅœจ้€Ÿๅบฆใ€ๅ‚ๆ•ฐ้‡ๅ’Œ่ฎก็ฎ—ๅคๆ‚ๅบฆไธŠ็š„ไปปไฝ•ๅŒบๅˆซ๏ผŒๅฎŒๅ…จๆ˜ฏๆ— ๆ„Ÿๆ›ฟๆขใ€‚ - -#### ้—ฎ้ข˜2๏ผšFDR ๅ’Œ GO-LSD ไผšๅธฆๆฅๆ›ดๅคš็š„่ฎญ็ปƒๆˆๆœฌๅ—๏ผŸ - -**ๅ›ž็ญ”**๏ผš่ฎญ็ปƒๆˆๆœฌ็š„ๅขžๅŠ ไธป่ฆๆฅๆบไบŽๅฆ‚ไฝ•็”Ÿๆˆๅˆ†ๅธƒ็š„ๆ ‡็ญพใ€‚ๆˆ‘ไปฌๅทฒ็ปๅฏน่ฏฅ่ฟ‡็จ‹่ฟ›่กŒไบ†ไผ˜ๅŒ–๏ผŒๅฐ†้ขๅค–่ฎญ็ปƒๆ—ถ้•ฟๅ’Œๆ˜พๅญ˜ๅ ็”จๆŽงๅˆถๅœจไบ† 6% ๅ’Œ 2%๏ผŒๅ‡ ไนŽๆ— ๆ„Ÿใ€‚ - -#### ้—ฎ้ข˜3๏ผšD-FINE ไธบไป€ไนˆไผšๆฏ” RT-DETR ็ณปๅˆ—ๆ›ดๅฟซใ€ๆ›ด่ฝป้‡๏ผŸ - -**ๅ›ž็ญ”**๏ผš็›ดๆŽฅๅบ”็”จ FDR ๅ’Œ GO-LSD ๅชไผšๆ˜พ่‘—ๆ้ซ˜ๆ€ง่ƒฝ๏ผŒๅนถไธไผš่ฎฉ็ฝ‘็ปœๆ›ดๅฟซใ€ๆ›ด่ฝปใ€‚ๆ‰€ไปฅๆˆ‘ไปฌๅฏน RT-DETR ่ฟ›่กŒไบ†ไธ€็ณปๅˆ—็š„่ฝป้‡ๅŒ–ๅค„็†๏ผŒ่ฟ™ไบ›ๅค„็†ๅธฆๆฅไบ†ๆ€ง่ƒฝ็š„ไธ‹้™๏ผŒไฝ†ๆˆ‘ไปฌ็š„ๆ–นๆณ•ๅผฅ่กฅไบ†่ฟ™ไบ›ๆŸๅคฑ๏ผŒๅฎž็Žฐไบ†้€Ÿๅบฆ-ๅ‚ๆ•ฐ-่ฎก็ฎ—้‡-ๆ€ง่ƒฝ็š„ๅฎŒ็พŽๅนณ่กกใ€‚ diff --git a/vlm/detector/D-FINE.backup/src/zoo/dfine/box_ops.py b/vlm/detector/D-FINE.backup/src/zoo/dfine/box_ops.py deleted file mode 100644 index 1e2acba..0000000 --- a/vlm/detector/D-FINE.backup/src/zoo/dfine/box_ops.py +++ /dev/null @@ -1,93 +0,0 @@ -""" -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved -https://github.com/facebookresearch/detr/blob/main/util/box_ops.py -""" - -import torch -from torch import Tensor -from torchvision.ops.boxes import box_area - - -def box_cxcywh_to_xyxy(x): - x_c, y_c, w, h = x.unbind(-1) - b = [ - (x_c - 0.5 * w.clamp(min=0.0)), - (y_c - 0.5 * h.clamp(min=0.0)), - (x_c + 0.5 * w.clamp(min=0.0)), - (y_c + 0.5 * h.clamp(min=0.0)), - ] - return torch.stack(b, dim=-1) - - -def box_xyxy_to_cxcywh(x: Tensor) -> Tensor: - x0, y0, x1, y1 = x.unbind(-1) - b = [(x0 + x1) / 2, (y0 + y1) / 2, (x1 - x0), (y1 - y0)] - return torch.stack(b, dim=-1) - - -# modified from torchvision to also return the union -def box_iou(boxes1: Tensor, boxes2: Tensor): - area1 = box_area(boxes1) - area2 = box_area(boxes2) - - lt = torch.max(boxes1[:, None, :2], boxes2[:, :2]) # [N,M,2] - rb = torch.min(boxes1[:, None, 2:], boxes2[:, 2:]) # [N,M,2] - - wh = (rb - lt).clamp(min=0) # [N,M,2] - inter = wh[:, :, 0] * wh[:, :, 1] # [N,M] - - union = area1[:, None] + area2 - inter - - iou = inter / union - return iou, union - - -def generalized_box_iou(boxes1, boxes2): - """ - Generalized IoU from https://giou.stanford.edu/ - - The boxes should be in [x0, y0, x1, y1] format - - Returns a [N, M] pairwise matrix, where N = len(boxes1) - and M = len(boxes2) - """ - # degenerate boxes gives inf / nan results - # so do an early check - assert (boxes1[:, 2:] >= boxes1[:, :2]).all() - assert (boxes2[:, 2:] >= boxes2[:, :2]).all() - iou, union = box_iou(boxes1, boxes2) - - lt = torch.min(boxes1[:, None, :2], boxes2[:, :2]) - rb = torch.max(boxes1[:, None, 2:], boxes2[:, 2:]) - - wh = (rb - lt).clamp(min=0) # [N,M,2] - area = wh[:, :, 0] * wh[:, :, 1] - - return iou - (area - union) / area - - -def masks_to_boxes(masks): - """Compute the bounding boxes around the provided masks - - The masks should be in format [N, H, W] where N is the number of masks, (H, W) are the spatial dimensions. - - Returns a [N, 4] tensors, with the boxes in xyxy format - """ - if masks.numel() == 0: - return torch.zeros((0, 4), device=masks.device) - - h, w = masks.shape[-2:] - - y = torch.arange(0, h, dtype=torch.float) - x = torch.arange(0, w, dtype=torch.float) - y, x = torch.meshgrid(y, x) - - x_mask = masks * x.unsqueeze(0) - x_max = x_mask.flatten(1).max(-1)[0] - x_min = x_mask.masked_fill(~(masks.bool()), 1e8).flatten(1).min(-1)[0] - - y_mask = masks * y.unsqueeze(0) - y_max = y_mask.flatten(1).max(-1)[0] - y_min = y_mask.masked_fill(~(masks.bool()), 1e8).flatten(1).min(-1)[0] - - return torch.stack([x_min, y_min, x_max, y_max], 1) diff --git a/vlm/detector/D-FINE.backup/src/zoo/dfine/denoising.py b/vlm/detector/D-FINE.backup/src/zoo/dfine/denoising.py deleted file mode 100644 index 4d2409a..0000000 --- a/vlm/detector/D-FINE.backup/src/zoo/dfine/denoising.py +++ /dev/null @@ -1,121 +0,0 @@ -"""Copyright(c) 2023 lyuwenyu. All Rights Reserved. -Modifications Copyright (c) 2024 The D-FINE Authors. All Rights Reserved. -""" - -import torch - -from .box_ops import box_cxcywh_to_xyxy, box_xyxy_to_cxcywh -from .utils import inverse_sigmoid - - -def get_contrastive_denoising_training_group( - targets, - num_classes, - num_queries, - class_embed, - num_denoising=100, - label_noise_ratio=0.5, - box_noise_scale=1.0, -): - """cnd""" - if num_denoising <= 0: - return None, None, None, None - - num_gts = [len(t["labels"]) for t in targets] - device = targets[0]["labels"].device - - max_gt_num = max(num_gts) - if max_gt_num == 0: - dn_meta = {"dn_positive_idx": None, "dn_num_group": 0, "dn_num_split": [0, num_queries]} - return None, None, None, dn_meta - - num_group = num_denoising // max_gt_num - num_group = 1 if num_group == 0 else num_group - # pad gt to max_num of a batch - bs = len(num_gts) - - input_query_class = torch.full([bs, max_gt_num], num_classes, dtype=torch.int32, device=device) - input_query_bbox = torch.zeros([bs, max_gt_num, 4], device=device) - pad_gt_mask = torch.zeros([bs, max_gt_num], dtype=torch.bool, device=device) - - for i in range(bs): - num_gt = num_gts[i] - if num_gt > 0: - input_query_class[i, :num_gt] = targets[i]["labels"] - input_query_bbox[i, :num_gt] = targets[i]["boxes"] - pad_gt_mask[i, :num_gt] = 1 - # each group has positive and negative queries. - input_query_class = input_query_class.tile([1, 2 * num_group]) - input_query_bbox = input_query_bbox.tile([1, 2 * num_group, 1]) - pad_gt_mask = pad_gt_mask.tile([1, 2 * num_group]) - # positive and negative mask - negative_gt_mask = torch.zeros([bs, max_gt_num * 2, 1], device=device) - negative_gt_mask[:, max_gt_num:] = 1 - negative_gt_mask = negative_gt_mask.tile([1, num_group, 1]) - positive_gt_mask = 1 - negative_gt_mask - # contrastive denoising training positive index - positive_gt_mask = positive_gt_mask.squeeze(-1) * pad_gt_mask - dn_positive_idx = torch.nonzero(positive_gt_mask)[:, 1] - dn_positive_idx = torch.split(dn_positive_idx, [n * num_group for n in num_gts]) - # total denoising queries - num_denoising = int(max_gt_num * 2 * num_group) - - if label_noise_ratio > 0: - mask = torch.rand_like(input_query_class, dtype=torch.float) < (label_noise_ratio * 0.5) - # randomly put a new one here - new_label = torch.randint_like(mask, 0, num_classes, dtype=input_query_class.dtype) - input_query_class = torch.where(mask & pad_gt_mask, new_label, input_query_class) - - if box_noise_scale > 0: - known_bbox = box_cxcywh_to_xyxy(input_query_bbox) - diff = torch.tile(input_query_bbox[..., 2:] * 0.5, [1, 1, 2]) * box_noise_scale - rand_sign = torch.randint_like(input_query_bbox, 0, 2) * 2.0 - 1.0 - rand_part = torch.rand_like(input_query_bbox) - rand_part = (rand_part + 1.0) * negative_gt_mask + rand_part * (1 - negative_gt_mask) - # shrink_mask = torch.zeros_like(rand_sign) - # shrink_mask[:, :, :2] = (rand_sign[:, :, :2] == 1) # rand_sign == 1 โ†’ (x1, y1) โ†˜ โ†’ smaller bbox - # shrink_mask[:, :, 2:] = (rand_sign[:, :, 2:] == -1) # rand_sign == -1 โ†’ (x2, y2) โ†– โ†’ smaller bbox - # mask = rand_part > (upper_bound / (upper_bound+1)) - # # this is to make sure the dn bbox can be reversed to the original bbox by dfine head. - # rand_sign = torch.where((shrink_mask * (1 - negative_gt_mask) * mask).bool(), \ - # rand_sign * upper_bound / (upper_bound+1) / rand_part, rand_sign) - known_bbox += rand_sign * rand_part * diff - known_bbox = torch.clip(known_bbox, min=0.0, max=1.0) - input_query_bbox = box_xyxy_to_cxcywh(known_bbox) - input_query_bbox[input_query_bbox < 0] *= -1 - input_query_bbox_unact = inverse_sigmoid(input_query_bbox) - - input_query_logits = class_embed(input_query_class) - - tgt_size = num_denoising + num_queries - attn_mask = torch.full([tgt_size, tgt_size], False, dtype=torch.bool, device=device) - # match query cannot see the reconstruction - attn_mask[num_denoising:, :num_denoising] = True - - # reconstruct cannot see each other - for i in range(num_group): - if i == 0: - attn_mask[ - max_gt_num * 2 * i : max_gt_num * 2 * (i + 1), - max_gt_num * 2 * (i + 1) : num_denoising, - ] = True - if i == num_group - 1: - attn_mask[max_gt_num * 2 * i : max_gt_num * 2 * (i + 1), : max_gt_num * i * 2] = True - else: - attn_mask[ - max_gt_num * 2 * i : max_gt_num * 2 * (i + 1), - max_gt_num * 2 * (i + 1) : num_denoising, - ] = True - attn_mask[max_gt_num * 2 * i : max_gt_num * 2 * (i + 1), : max_gt_num * 2 * i] = True - - dn_meta = { - "dn_positive_idx": dn_positive_idx, - "dn_num_group": num_group, - "dn_num_split": [num_denoising, num_queries], - } - - # print(input_query_class.shape) # torch.Size([4, 196, 256]) - # print(input_query_bbox.shape) # torch.Size([4, 196, 4]) - # print(attn_mask.shape) # torch.Size([496, 496]) - - return input_query_logits, input_query_bbox_unact, attn_mask, dn_meta diff --git a/vlm/detector/D-FINE.backup/src/zoo/dfine/dfine.py b/vlm/detector/D-FINE.backup/src/zoo/dfine/dfine.py deleted file mode 100644 index b83bd99..0000000 --- a/vlm/detector/D-FINE.backup/src/zoo/dfine/dfine.py +++ /dev/null @@ -1,47 +0,0 @@ -""" -Copyright (c) 2024 The D-FINE Authors. All Rights Reserved. -""" - -import torch.nn as nn - -from ...core import register - -__all__ = [ - "DFINE", -] - - -@register() -class DFINE(nn.Module): - __inject__ = [ - "backbone", - "encoder", - "decoder", - ] - - def __init__( - self, - backbone: nn.Module, - encoder: nn.Module, - decoder: nn.Module, - ): - super().__init__() - self.backbone = backbone - self.decoder = decoder - self.encoder = encoder - - def forward(self, x, targets=None): - x = self.backbone(x) - x = self.encoder(x) - x = self.decoder(x, targets) - - return x - - def deploy( - self, - ): - self.eval() - for m in self.modules(): - if hasattr(m, "convert_to_deploy"): - m.convert_to_deploy() - return self diff --git a/vlm/detector/D-FINE.backup/src/zoo/dfine/dfine_criterion.py b/vlm/detector/D-FINE.backup/src/zoo/dfine/dfine_criterion.py deleted file mode 100644 index 954d42b..0000000 --- a/vlm/detector/D-FINE.backup/src/zoo/dfine/dfine_criterion.py +++ /dev/null @@ -1,525 +0,0 @@ -""" -D-FINE: Redefine Regression Task of DETRs as Fine-grained Distribution Refinement -Copyright (c) 2024 The D-FINE Authors. All Rights Reserved. ---------------------------------------------------------------------------------- -Modified from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright (c) 2023 lyuwenyu. All Rights Reserved. -""" - -import copy - -import torch -import torch.distributed -import torch.nn as nn -import torch.nn.functional as F -import torchvision - -from ...core import register -from ...misc.dist_utils import get_world_size, is_dist_available_and_initialized -from .box_ops import box_cxcywh_to_xyxy, box_iou, generalized_box_iou -from .dfine_utils import bbox2distance - - -@register() -class DFINECriterion(nn.Module): - """This class computes the loss for D-FINE.""" - - __share__ = [ - "num_classes", - ] - __inject__ = [ - "matcher", - ] - - def __init__( - self, - matcher, - weight_dict, - losses, - alpha=0.2, - gamma=2.0, - num_classes=80, - reg_max=32, - boxes_weight_format=None, - share_matched_indices=False, - ): - """Create the criterion. - Parameters: - matcher: module able to compute a matching between targets and proposals. - weight_dict: dict containing as key the names of the losses and as values their relative weight. - losses: list of all the losses to be applied. See get_loss for list of available losses. - num_classes: number of object categories, omitting the special no-object category. - reg_max (int): Max number of the discrete bins in D-FINE. - boxes_weight_format: format for boxes weight (iou, ). - """ - super().__init__() - self.num_classes = num_classes - self.matcher = matcher - self.weight_dict = weight_dict - self.losses = losses - self.boxes_weight_format = boxes_weight_format - self.share_matched_indices = share_matched_indices - self.alpha = alpha - self.gamma = gamma - self.fgl_targets, self.fgl_targets_dn = None, None - self.own_targets, self.own_targets_dn = None, None - self.reg_max = reg_max - self.num_pos, self.num_neg = None, None - - def loss_labels_focal(self, outputs, targets, indices, num_boxes): - assert "pred_logits" in outputs - src_logits = outputs["pred_logits"] - idx = self._get_src_permutation_idx(indices) - target_classes_o = torch.cat([t["labels"][J] for t, (_, J) in zip(targets, indices)]) - target_classes = torch.full( - src_logits.shape[:2], self.num_classes, dtype=torch.int64, device=src_logits.device - ) - target_classes[idx] = target_classes_o - target = F.one_hot(target_classes, num_classes=self.num_classes + 1)[..., :-1] - loss = torchvision.ops.sigmoid_focal_loss( - src_logits, target, self.alpha, self.gamma, reduction="none" - ) - loss = loss.mean(1).sum() * src_logits.shape[1] / num_boxes - - return {"loss_focal": loss} - - def loss_labels_vfl(self, outputs, targets, indices, num_boxes, values=None): - assert "pred_boxes" in outputs - idx = self._get_src_permutation_idx(indices) - if values is None: - src_boxes = outputs["pred_boxes"][idx] - target_boxes = torch.cat([t["boxes"][i] for t, (_, i) in zip(targets, indices)], dim=0) - ious, _ = box_iou(box_cxcywh_to_xyxy(src_boxes), box_cxcywh_to_xyxy(target_boxes)) - ious = torch.diag(ious).detach() - else: - ious = values - - src_logits = outputs["pred_logits"] - target_classes_o = torch.cat([t["labels"][J] for t, (_, J) in zip(targets, indices)]) - target_classes = torch.full( - src_logits.shape[:2], self.num_classes, dtype=torch.int64, device=src_logits.device - ) - target_classes[idx] = target_classes_o - target = F.one_hot(target_classes, num_classes=self.num_classes + 1)[..., :-1] - - target_score_o = torch.zeros_like(target_classes, dtype=src_logits.dtype) - target_score_o[idx] = ious.to(target_score_o.dtype) - target_score = target_score_o.unsqueeze(-1) * target - - pred_score = F.sigmoid(src_logits).detach() - weight = self.alpha * pred_score.pow(self.gamma) * (1 - target) + target_score - - loss = F.binary_cross_entropy_with_logits( - src_logits, target_score, weight=weight, reduction="none" - ) - loss = loss.mean(1).sum() * src_logits.shape[1] / num_boxes - return {"loss_vfl": loss} - - def loss_boxes(self, outputs, targets, indices, num_boxes, boxes_weight=None): - """Compute the losses related to the bounding boxes, the L1 regression loss and the GIoU loss - targets dicts must contain the key "boxes" containing a tensor of dim [nb_target_boxes, 4] - The target boxes are expected in format (center_x, center_y, w, h), normalized by the image size. - """ - assert "pred_boxes" in outputs - idx = self._get_src_permutation_idx(indices) - src_boxes = outputs["pred_boxes"][idx] - target_boxes = torch.cat([t["boxes"][i] for t, (_, i) in zip(targets, indices)], dim=0) - losses = {} - loss_bbox = F.l1_loss(src_boxes, target_boxes, reduction="none") - losses["loss_bbox"] = loss_bbox.sum() / num_boxes - - loss_giou = 1 - torch.diag( - generalized_box_iou(box_cxcywh_to_xyxy(src_boxes), box_cxcywh_to_xyxy(target_boxes)) - ) - loss_giou = loss_giou if boxes_weight is None else loss_giou * boxes_weight - losses["loss_giou"] = loss_giou.sum() / num_boxes - - return losses - - def loss_local(self, outputs, targets, indices, num_boxes, T=5): - """Compute Fine-Grained Localization (FGL) Loss - and Decoupled Distillation Focal (DDF) Loss.""" - - losses = {} - if "pred_corners" in outputs: - idx = self._get_src_permutation_idx(indices) - target_boxes = torch.cat([t["boxes"][i] for t, (_, i) in zip(targets, indices)], dim=0) - - pred_corners = outputs["pred_corners"][idx].reshape(-1, (self.reg_max + 1)) - ref_points = outputs["ref_points"][idx].detach() - with torch.no_grad(): - if self.fgl_targets_dn is None and "is_dn" in outputs: - self.fgl_targets_dn = bbox2distance( - ref_points, - box_cxcywh_to_xyxy(target_boxes), - self.reg_max, - outputs["reg_scale"], - outputs["up"], - ) - if self.fgl_targets is None and "is_dn" not in outputs: - self.fgl_targets = bbox2distance( - ref_points, - box_cxcywh_to_xyxy(target_boxes), - self.reg_max, - outputs["reg_scale"], - outputs["up"], - ) - - target_corners, weight_right, weight_left = ( - self.fgl_targets_dn if "is_dn" in outputs else self.fgl_targets - ) - - ious = torch.diag( - box_iou( - box_cxcywh_to_xyxy(outputs["pred_boxes"][idx]), box_cxcywh_to_xyxy(target_boxes) - )[0] - ) - weight_targets = ious.unsqueeze(-1).repeat(1, 1, 4).reshape(-1).detach() - - losses["loss_fgl"] = self.unimodal_distribution_focal_loss( - pred_corners, - target_corners, - weight_right, - weight_left, - weight_targets, - avg_factor=num_boxes, - ) - - if "teacher_corners" in outputs: - pred_corners = outputs["pred_corners"].reshape(-1, (self.reg_max + 1)) - target_corners = outputs["teacher_corners"].reshape(-1, (self.reg_max + 1)) - if torch.equal(pred_corners, target_corners): - losses["loss_ddf"] = pred_corners.sum() * 0 - else: - weight_targets_local = outputs["teacher_logits"].sigmoid().max(dim=-1)[0] - - mask = torch.zeros_like(weight_targets_local, dtype=torch.bool) - mask[idx] = True - mask = mask.unsqueeze(-1).repeat(1, 1, 4).reshape(-1) - - weight_targets_local[idx] = ious.reshape_as(weight_targets_local[idx]).to( - weight_targets_local.dtype - ) - weight_targets_local = ( - weight_targets_local.unsqueeze(-1).repeat(1, 1, 4).reshape(-1).detach() - ) - - loss_match_local = ( - weight_targets_local - * (T**2) - * ( - nn.KLDivLoss(reduction="none")( - F.log_softmax(pred_corners / T, dim=1), - F.softmax(target_corners.detach() / T, dim=1), - ) - ).sum(-1) - ) - if "is_dn" not in outputs: - batch_scale = ( - 8 / outputs["pred_boxes"].shape[0] - ) # Avoid the influence of batch size per GPU - self.num_pos, self.num_neg = ( - (mask.sum() * batch_scale) ** 0.5, - ((~mask).sum() * batch_scale) ** 0.5, - ) - loss_match_local1 = loss_match_local[mask].mean() if mask.any() else 0 - loss_match_local2 = loss_match_local[~mask].mean() if (~mask).any() else 0 - losses["loss_ddf"] = ( - loss_match_local1 * self.num_pos + loss_match_local2 * self.num_neg - ) / (self.num_pos + self.num_neg) - - return losses - - def _get_src_permutation_idx(self, indices): - # permute predictions following indices - batch_idx = torch.cat([torch.full_like(src, i) for i, (src, _) in enumerate(indices)]) - src_idx = torch.cat([src for (src, _) in indices]) - return batch_idx, src_idx - - def _get_tgt_permutation_idx(self, indices): - # permute targets following indices - batch_idx = torch.cat([torch.full_like(tgt, i) for i, (_, tgt) in enumerate(indices)]) - tgt_idx = torch.cat([tgt for (_, tgt) in indices]) - return batch_idx, tgt_idx - - def _get_go_indices(self, indices, indices_aux_list): - """Get a matching union set across all decoder layers.""" - results = [] - for indices_aux in indices_aux_list: - indices = [ - (torch.cat([idx1[0], idx2[0]]), torch.cat([idx1[1], idx2[1]])) - for idx1, idx2 in zip(indices.copy(), indices_aux.copy()) - ] - - for ind in [torch.cat([idx[0][:, None], idx[1][:, None]], 1) for idx in indices]: - unique, counts = torch.unique(ind, return_counts=True, dim=0) - count_sort_indices = torch.argsort(counts, descending=True) - unique_sorted = unique[count_sort_indices] - column_to_row = {} - for idx in unique_sorted: - row_idx, col_idx = idx[0].item(), idx[1].item() - if row_idx not in column_to_row: - column_to_row[row_idx] = col_idx - final_rows = torch.tensor(list(column_to_row.keys()), device=ind.device) - final_cols = torch.tensor(list(column_to_row.values()), device=ind.device) - results.append((final_rows.long(), final_cols.long())) - return results - - def _clear_cache(self): - self.fgl_targets, self.fgl_targets_dn = None, None - self.own_targets, self.own_targets_dn = None, None - self.num_pos, self.num_neg = None, None - - def get_loss(self, loss, outputs, targets, indices, num_boxes, **kwargs): - loss_map = { - "boxes": self.loss_boxes, - "focal": self.loss_labels_focal, - "vfl": self.loss_labels_vfl, - "local": self.loss_local, - } - assert loss in loss_map, f"do you really want to compute {loss} loss?" - return loss_map[loss](outputs, targets, indices, num_boxes, **kwargs) - - def forward(self, outputs, targets, **kwargs): - """This performs the loss computation. - Parameters: - outputs: dict of tensors, see the output specification of the model for the format - targets: list of dicts, such that len(targets) == batch_size. - The expected keys in each dict depends on the losses applied, see each loss' doc - """ - outputs_without_aux = {k: v for k, v in outputs.items() if "aux" not in k} - - # Retrieve the matching between the outputs of the last layer and the targets - indices = self.matcher(outputs_without_aux, targets)["indices"] - self._clear_cache() - - # Get the matching union set across all decoder layers. - if "aux_outputs" in outputs: - indices_aux_list, cached_indices, cached_indices_enc = [], [], [] - for i, aux_outputs in enumerate(outputs["aux_outputs"] + [outputs["pre_outputs"]]): - indices_aux = self.matcher(aux_outputs, targets)["indices"] - cached_indices.append(indices_aux) - indices_aux_list.append(indices_aux) - for i, aux_outputs in enumerate(outputs["enc_aux_outputs"]): - indices_enc = self.matcher(aux_outputs, targets)["indices"] - cached_indices_enc.append(indices_enc) - indices_aux_list.append(indices_enc) - indices_go = self._get_go_indices(indices, indices_aux_list) - - num_boxes_go = sum(len(x[0]) for x in indices_go) - num_boxes_go = torch.as_tensor( - [num_boxes_go], dtype=torch.float, device=next(iter(outputs.values())).device - ) - if is_dist_available_and_initialized(): - torch.distributed.all_reduce(num_boxes_go) - num_boxes_go = torch.clamp(num_boxes_go / get_world_size(), min=1).item() - else: - assert "aux_outputs" in outputs, "" - - # Compute the average number of target boxes accross all nodes, for normalization purposes - num_boxes = sum(len(t["labels"]) for t in targets) - num_boxes = torch.as_tensor( - [num_boxes], dtype=torch.float, device=next(iter(outputs.values())).device - ) - if is_dist_available_and_initialized(): - torch.distributed.all_reduce(num_boxes) - num_boxes = torch.clamp(num_boxes / get_world_size(), min=1).item() - - # Compute all the requested losses - losses = {} - for loss in self.losses: - indices_in = indices_go if loss in ["boxes", "local"] else indices - num_boxes_in = num_boxes_go if loss in ["boxes", "local"] else num_boxes - meta = self.get_loss_meta_info(loss, outputs, targets, indices_in) - l_dict = self.get_loss(loss, outputs, targets, indices_in, num_boxes_in, **meta) - l_dict = {k: l_dict[k] * self.weight_dict[k] for k in l_dict if k in self.weight_dict} - losses.update(l_dict) - - # In case of auxiliary losses, we repeat this process with the output of each intermediate layer. - if "aux_outputs" in outputs: - for i, aux_outputs in enumerate(outputs["aux_outputs"]): - aux_outputs["up"], aux_outputs["reg_scale"] = outputs["up"], outputs["reg_scale"] - for loss in self.losses: - indices_in = indices_go if loss in ["boxes", "local"] else cached_indices[i] - num_boxes_in = num_boxes_go if loss in ["boxes", "local"] else num_boxes - meta = self.get_loss_meta_info(loss, aux_outputs, targets, indices_in) - l_dict = self.get_loss( - loss, aux_outputs, targets, indices_in, num_boxes_in, **meta - ) - - l_dict = { - k: l_dict[k] * self.weight_dict[k] for k in l_dict if k in self.weight_dict - } - l_dict = {k + f"_aux_{i}": v for k, v in l_dict.items()} - losses.update(l_dict) - - # In case of auxiliary traditional head output at first decoder layer. - if "pre_outputs" in outputs: - aux_outputs = outputs["pre_outputs"] - for loss in self.losses: - indices_in = indices_go if loss in ["boxes", "local"] else cached_indices[-1] - num_boxes_in = num_boxes_go if loss in ["boxes", "local"] else num_boxes - meta = self.get_loss_meta_info(loss, aux_outputs, targets, indices_in) - l_dict = self.get_loss(loss, aux_outputs, targets, indices_in, num_boxes_in, **meta) - - l_dict = { - k: l_dict[k] * self.weight_dict[k] for k in l_dict if k in self.weight_dict - } - l_dict = {k + "_pre": v for k, v in l_dict.items()} - losses.update(l_dict) - - # In case of encoder auxiliary losses. - if "enc_aux_outputs" in outputs: - assert "enc_meta" in outputs, "" - class_agnostic = outputs["enc_meta"]["class_agnostic"] - if class_agnostic: - orig_num_classes = self.num_classes - self.num_classes = 1 - enc_targets = copy.deepcopy(targets) - for t in enc_targets: - t["labels"] = torch.zeros_like(t["labels"]) - else: - enc_targets = targets - - for i, aux_outputs in enumerate(outputs["enc_aux_outputs"]): - for loss in self.losses: - indices_in = indices_go if loss == "boxes" else cached_indices_enc[i] - num_boxes_in = num_boxes_go if loss == "boxes" else num_boxes - meta = self.get_loss_meta_info(loss, aux_outputs, enc_targets, indices_in) - l_dict = self.get_loss( - loss, aux_outputs, enc_targets, indices_in, num_boxes_in, **meta - ) - l_dict = { - k: l_dict[k] * self.weight_dict[k] for k in l_dict if k in self.weight_dict - } - l_dict = {k + f"_enc_{i}": v for k, v in l_dict.items()} - losses.update(l_dict) - - if class_agnostic: - self.num_classes = orig_num_classes - - # In case of cdn auxiliary losses. For dfine - if "dn_outputs" in outputs: - assert "dn_meta" in outputs, "" - indices_dn = self.get_cdn_matched_indices(outputs["dn_meta"], targets) - dn_num_boxes = num_boxes * outputs["dn_meta"]["dn_num_group"] - dn_num_boxes = dn_num_boxes if dn_num_boxes > 0 else 1 - - for i, aux_outputs in enumerate(outputs["dn_outputs"]): - aux_outputs["is_dn"] = True - aux_outputs["up"], aux_outputs["reg_scale"] = outputs["up"], outputs["reg_scale"] - for loss in self.losses: - meta = self.get_loss_meta_info(loss, aux_outputs, targets, indices_dn) - l_dict = self.get_loss( - loss, aux_outputs, targets, indices_dn, dn_num_boxes, **meta - ) - l_dict = { - k: l_dict[k] * self.weight_dict[k] for k in l_dict if k in self.weight_dict - } - l_dict = {k + f"_dn_{i}": v for k, v in l_dict.items()} - losses.update(l_dict) - - # In case of auxiliary traditional head output at first decoder layer. - if "dn_pre_outputs" in outputs: - aux_outputs = outputs["dn_pre_outputs"] - for loss in self.losses: - meta = self.get_loss_meta_info(loss, aux_outputs, targets, indices_dn) - l_dict = self.get_loss( - loss, aux_outputs, targets, indices_dn, dn_num_boxes, **meta - ) - l_dict = { - k: l_dict[k] * self.weight_dict[k] for k in l_dict if k in self.weight_dict - } - l_dict = {k + "_dn_pre": v for k, v in l_dict.items()} - losses.update(l_dict) - - # For debugging Objects365 pre-train. - losses = {k: torch.nan_to_num(v, nan=0.0) for k, v in losses.items()} - return losses - - def get_loss_meta_info(self, loss, outputs, targets, indices): - if self.boxes_weight_format is None: - return {} - - src_boxes = outputs["pred_boxes"][self._get_src_permutation_idx(indices)] - target_boxes = torch.cat([t["boxes"][j] for t, (_, j) in zip(targets, indices)], dim=0) - - if self.boxes_weight_format == "iou": - iou, _ = box_iou( - box_cxcywh_to_xyxy(src_boxes.detach()), box_cxcywh_to_xyxy(target_boxes) - ) - iou = torch.diag(iou) - elif self.boxes_weight_format == "giou": - iou = torch.diag( - generalized_box_iou( - box_cxcywh_to_xyxy(src_boxes.detach()), box_cxcywh_to_xyxy(target_boxes) - ) - ) - else: - raise AttributeError() - - if loss in ("boxes",): - meta = {"boxes_weight": iou} - elif loss in ("vfl",): - meta = {"values": iou} - else: - meta = {} - - return meta - - @staticmethod - def get_cdn_matched_indices(dn_meta, targets): - """get_cdn_matched_indices""" - dn_positive_idx, dn_num_group = dn_meta["dn_positive_idx"], dn_meta["dn_num_group"] - num_gts = [len(t["labels"]) for t in targets] - device = targets[0]["labels"].device - - dn_match_indices = [] - for i, num_gt in enumerate(num_gts): - if num_gt > 0: - gt_idx = torch.arange(num_gt, dtype=torch.int64, device=device) - gt_idx = gt_idx.tile(dn_num_group) - assert len(dn_positive_idx[i]) == len(gt_idx) - dn_match_indices.append((dn_positive_idx[i], gt_idx)) - else: - dn_match_indices.append( - ( - torch.zeros(0, dtype=torch.int64, device=device), - torch.zeros(0, dtype=torch.int64, device=device), - ) - ) - - return dn_match_indices - - def feature_loss_function(self, fea, target_fea): - loss = (fea - target_fea) ** 2 * ((fea > 0) | (target_fea > 0)).float() - return torch.abs(loss) - - def unimodal_distribution_focal_loss( - self, pred, label, weight_right, weight_left, weight=None, reduction="sum", avg_factor=None - ): - dis_left = label.long() - dis_right = dis_left + 1 - - loss = F.cross_entropy(pred, dis_left, reduction="none") * weight_left.reshape( - -1 - ) + F.cross_entropy(pred, dis_right, reduction="none") * weight_right.reshape(-1) - - if weight is not None: - weight = weight.float() - loss = loss * weight - - if avg_factor is not None: - loss = loss.sum() / avg_factor - elif reduction == "mean": - loss = loss.mean() - elif reduction == "sum": - loss = loss.sum() - - return loss - - def get_gradual_steps(self, outputs): - num_layers = len(outputs["aux_outputs"]) + 1 if "aux_outputs" in outputs else 1 - step = 0.5 / (num_layers - 1) - opt_list = [0.5 + step * i for i in range(num_layers)] if num_layers > 1 else [1] - return opt_list diff --git a/vlm/detector/D-FINE.backup/src/zoo/dfine/dfine_decoder.py b/vlm/detector/D-FINE.backup/src/zoo/dfine/dfine_decoder.py deleted file mode 100644 index d7c77f7..0000000 --- a/vlm/detector/D-FINE.backup/src/zoo/dfine/dfine_decoder.py +++ /dev/null @@ -1,959 +0,0 @@ -""" -D-FINE: Redefine Regression Task of DETRs as Fine-grained Distribution Refinement -Copyright (c) 2024 The D-FINE Authors. All Rights Reserved. ---------------------------------------------------------------------------------- -Modified from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright (c) 2023 lyuwenyu. All Rights Reserved. -""" - -import copy -import functools -import math -from collections import OrderedDict -from typing import List - -import torch -import torch.nn as nn -import torch.nn.functional as F -import torch.nn.init as init - -from ...core import register -from .denoising import get_contrastive_denoising_training_group -from .dfine_utils import distance2bbox, weighting_function -from .utils import ( - bias_init_with_prob, - deformable_attention_core_func_v2, - get_activation, - inverse_sigmoid, -) - -__all__ = ["DFINETransformer"] - - -class MLP(nn.Module): - def __init__(self, input_dim, hidden_dim, output_dim, num_layers, act="relu"): - super().__init__() - self.num_layers = num_layers - h = [hidden_dim] * (num_layers - 1) - self.layers = nn.ModuleList( - nn.Linear(n, k) for n, k in zip([input_dim] + h, h + [output_dim]) - ) - self.act = get_activation(act) - - def forward(self, x): - for i, layer in enumerate(self.layers): - x = self.act(layer(x)) if i < self.num_layers - 1 else layer(x) - return x - - -class MSDeformableAttention(nn.Module): - def __init__( - self, - embed_dim=256, - num_heads=8, - num_levels=4, - num_points=4, - method="default", - offset_scale=0.5, - ): - """Multi-Scale Deformable Attention""" - super(MSDeformableAttention, self).__init__() - self.embed_dim = embed_dim - self.num_heads = num_heads - self.num_levels = num_levels - self.offset_scale = offset_scale - - if isinstance(num_points, list): - assert len(num_points) == num_levels, "" - num_points_list = num_points - else: - num_points_list = [num_points for _ in range(num_levels)] - - self.num_points_list = num_points_list - - num_points_scale = [1 / n for n in num_points_list for _ in range(n)] - self.register_buffer( - "num_points_scale", torch.tensor(num_points_scale, dtype=torch.float32) - ) - - self.total_points = num_heads * sum(num_points_list) - self.method = method - - self.head_dim = embed_dim // num_heads - assert ( - self.head_dim * num_heads == self.embed_dim - ), "embed_dim must be divisible by num_heads" - - self.sampling_offsets = nn.Linear(embed_dim, self.total_points * 2) - self.attention_weights = nn.Linear(embed_dim, self.total_points) - - self.ms_deformable_attn_core = functools.partial( - deformable_attention_core_func_v2, method=self.method - ) - - self._reset_parameters() - - if method == "discrete": - for p in self.sampling_offsets.parameters(): - p.requires_grad = False - - def _reset_parameters(self): - # sampling_offsets - init.constant_(self.sampling_offsets.weight, 0) - thetas = torch.arange(self.num_heads, dtype=torch.float32) * ( - 2.0 * math.pi / self.num_heads - ) - grid_init = torch.stack([thetas.cos(), thetas.sin()], -1) - grid_init = grid_init / grid_init.abs().max(-1, keepdim=True).values - grid_init = grid_init.reshape(self.num_heads, 1, 2).tile([1, sum(self.num_points_list), 1]) - scaling = torch.concat([torch.arange(1, n + 1) for n in self.num_points_list]).reshape( - 1, -1, 1 - ) - grid_init *= scaling - self.sampling_offsets.bias.data[...] = grid_init.flatten() - - # attention_weights - init.constant_(self.attention_weights.weight, 0) - init.constant_(self.attention_weights.bias, 0) - - def forward( - self, - query: torch.Tensor, - reference_points: torch.Tensor, - value: torch.Tensor, - value_spatial_shapes: List[int], - ): - """ - Args: - query (Tensor): [bs, query_length, C] - reference_points (Tensor): [bs, query_length, n_levels, 2], range in [0, 1], top-left (0,0), - bottom-right (1, 1), including padding area - value (Tensor): [bs, value_length, C] - value_spatial_shapes (List): [n_levels, 2], [(H_0, W_0), (H_1, W_1), ..., (H_{L-1}, W_{L-1})] - - Returns: - output (Tensor): [bs, Length_{query}, C] - """ - bs, Len_q = query.shape[:2] - - sampling_offsets: torch.Tensor = self.sampling_offsets(query) - sampling_offsets = sampling_offsets.reshape( - bs, Len_q, self.num_heads, sum(self.num_points_list), 2 - ) - - attention_weights = self.attention_weights(query).reshape( - bs, Len_q, self.num_heads, sum(self.num_points_list) - ) - attention_weights = F.softmax(attention_weights, dim=-1) - - if reference_points.shape[-1] == 2: - offset_normalizer = torch.tensor(value_spatial_shapes) - offset_normalizer = offset_normalizer.flip([1]).reshape(1, 1, 1, self.num_levels, 1, 2) - sampling_locations = ( - reference_points.reshape(bs, Len_q, 1, self.num_levels, 1, 2) - + sampling_offsets / offset_normalizer - ) - elif reference_points.shape[-1] == 4: - # reference_points [8, 480, None, 1, 4] - # sampling_offsets [8, 480, 8, 12, 2] - num_points_scale = self.num_points_scale.to(dtype=query.dtype).unsqueeze(-1) - offset = ( - sampling_offsets - * num_points_scale - * reference_points[:, :, None, :, 2:] - * self.offset_scale - ) - sampling_locations = reference_points[:, :, None, :, :2] + offset - else: - raise ValueError( - "Last dim of reference_points must be 2 or 4, but get {} instead.".format( - reference_points.shape[-1] - ) - ) - - output = self.ms_deformable_attn_core( - value, value_spatial_shapes, sampling_locations, attention_weights, self.num_points_list - ) - - return output - - -class TransformerDecoderLayer(nn.Module): - def __init__( - self, - d_model=256, - n_head=8, - dim_feedforward=1024, - dropout=0.0, - activation="relu", - n_levels=4, - n_points=4, - cross_attn_method="default", - layer_scale=None, - ): - super(TransformerDecoderLayer, self).__init__() - if layer_scale is not None: - dim_feedforward = round(layer_scale * dim_feedforward) - d_model = round(layer_scale * d_model) - - # self attention - self.self_attn = nn.MultiheadAttention(d_model, n_head, dropout=dropout, batch_first=True) - self.dropout1 = nn.Dropout(dropout) - self.norm1 = nn.LayerNorm(d_model) - - # cross attention - self.cross_attn = MSDeformableAttention( - d_model, n_head, n_levels, n_points, method=cross_attn_method - ) - self.dropout2 = nn.Dropout(dropout) - - # gate - self.gateway = Gate(d_model) - - # ffn - self.linear1 = nn.Linear(d_model, dim_feedforward) - self.activation = get_activation(activation) - self.dropout3 = nn.Dropout(dropout) - self.linear2 = nn.Linear(dim_feedforward, d_model) - self.dropout4 = nn.Dropout(dropout) - self.norm3 = nn.LayerNorm(d_model) - - self._reset_parameters() - - def _reset_parameters(self): - init.xavier_uniform_(self.linear1.weight) - init.xavier_uniform_(self.linear2.weight) - - def with_pos_embed(self, tensor, pos): - return tensor if pos is None else tensor + pos - - def forward_ffn(self, tgt): - return self.linear2(self.dropout3(self.activation(self.linear1(tgt)))) - - def forward( - self, target, reference_points, value, spatial_shapes, attn_mask=None, query_pos_embed=None - ): - # self attention - q = k = self.with_pos_embed(target, query_pos_embed) - - target2, _ = self.self_attn(q, k, value=target, attn_mask=attn_mask) - target = target + self.dropout1(target2) - target = self.norm1(target) - - # cross attention - target2 = self.cross_attn( - self.with_pos_embed(target, query_pos_embed), reference_points, value, spatial_shapes - ) - - target = self.gateway(target, self.dropout2(target2)) - - # ffn - target2 = self.forward_ffn(target) - target = target + self.dropout4(target2) - target = self.norm3(target.clamp(min=-65504, max=65504)) - - return target - - -class Gate(nn.Module): - def __init__(self, d_model): - super(Gate, self).__init__() - self.gate = nn.Linear(2 * d_model, 2 * d_model) - bias = bias_init_with_prob(0.5) - init.constant_(self.gate.bias, bias) - init.constant_(self.gate.weight, 0) - self.norm = nn.LayerNorm(d_model) - - def forward(self, x1, x2): - gate_input = torch.cat([x1, x2], dim=-1) - gates = torch.sigmoid(self.gate(gate_input)) - gate1, gate2 = gates.chunk(2, dim=-1) - return self.norm(gate1 * x1 + gate2 * x2) - - -class Integral(nn.Module): - """ - A static layer that calculates integral results from a distribution. - - This layer computes the target location using the formula: `sum{Pr(n) * W(n)}`, - where Pr(n) is the softmax probability vector representing the discrete - distribution, and W(n) is the non-uniform Weighting Function. - - Args: - reg_max (int): Max number of the discrete bins. Default is 32. - It can be adjusted based on the dataset or task requirements. - """ - - def __init__(self, reg_max=32): - super(Integral, self).__init__() - self.reg_max = reg_max - - def forward(self, x, project): - shape = x.shape - x = F.softmax(x.reshape(-1, self.reg_max + 1), dim=1) - x = F.linear(x, project.to(x.device)).reshape(-1, 4) - return x.reshape(list(shape[:-1]) + [-1]) - - -class LQE(nn.Module): - def __init__(self, k, hidden_dim, num_layers, reg_max): - super(LQE, self).__init__() - self.k = k - self.reg_max = reg_max - self.reg_conf = MLP(4 * (k + 1), hidden_dim, 1, num_layers) - init.constant_(self.reg_conf.layers[-1].bias, 0) - init.constant_(self.reg_conf.layers[-1].weight, 0) - - def forward(self, scores, pred_corners): - B, L, _ = pred_corners.size() - prob = F.softmax(pred_corners.reshape(B, L, 4, self.reg_max + 1), dim=-1) - prob_topk, _ = prob.topk(self.k, dim=-1) - stat = torch.cat([prob_topk, prob_topk.mean(dim=-1, keepdim=True)], dim=-1) - quality_score = self.reg_conf(stat.reshape(B, L, -1)) - return scores + quality_score - - -class TransformerDecoder(nn.Module): - """ - Transformer Decoder implementing Fine-grained Distribution Refinement (FDR). - - This decoder refines object detection predictions through iterative updates across multiple layers, - utilizing attention mechanisms, location quality estimators, and distribution refinement techniques - to improve bounding box accuracy and robustness. - """ - - def __init__( - self, - hidden_dim, - decoder_layer, - decoder_layer_wide, - num_layers, - num_head, - reg_max, - reg_scale, - up, - eval_idx=-1, - layer_scale=2, - ): - super(TransformerDecoder, self).__init__() - self.hidden_dim = hidden_dim - self.num_layers = num_layers - self.layer_scale = layer_scale - self.num_head = num_head - self.eval_idx = eval_idx if eval_idx >= 0 else num_layers + eval_idx - self.up, self.reg_scale, self.reg_max = up, reg_scale, reg_max - self.layers = nn.ModuleList( - [copy.deepcopy(decoder_layer) for _ in range(self.eval_idx + 1)] - + [copy.deepcopy(decoder_layer_wide) for _ in range(num_layers - self.eval_idx - 1)] - ) - self.lqe_layers = nn.ModuleList( - [copy.deepcopy(LQE(4, 64, 2, reg_max)) for _ in range(num_layers)] - ) - - def value_op(self, memory, value_proj, value_scale, memory_mask, memory_spatial_shapes): - """ - Preprocess values for MSDeformableAttention. - """ - value = value_proj(memory) if value_proj is not None else memory - value = F.interpolate(memory, size=value_scale) if value_scale is not None else value - if memory_mask is not None: - value = value * memory_mask.to(value.dtype).unsqueeze(-1) - value = value.reshape(value.shape[0], value.shape[1], self.num_head, -1) - split_shape = [h * w for h, w in memory_spatial_shapes] - return value.permute(0, 2, 3, 1).split(split_shape, dim=-1) - - def convert_to_deploy(self): - self.project = weighting_function(self.reg_max, self.up, self.reg_scale, deploy=True) - self.layers = self.layers[: self.eval_idx + 1] - self.lqe_layers = nn.ModuleList( - [nn.Identity()] * (self.eval_idx) + [self.lqe_layers[self.eval_idx]] - ) - - def forward( - self, - target, - ref_points_unact, - memory, - spatial_shapes, - bbox_head, - score_head, - query_pos_head, - pre_bbox_head, - integral, - up, - reg_scale, - attn_mask=None, - memory_mask=None, - dn_meta=None, - ): - output = target - output_detach = pred_corners_undetach = 0 - value = self.value_op(memory, None, None, memory_mask, spatial_shapes) - - dec_out_bboxes = [] - dec_out_logits = [] - dec_out_pred_corners = [] - dec_out_refs = [] - if not hasattr(self, "project"): - project = weighting_function(self.reg_max, up, reg_scale) - else: - project = self.project - - ref_points_detach = F.sigmoid(ref_points_unact) - - for i, layer in enumerate(self.layers): - ref_points_input = ref_points_detach.unsqueeze(2) - query_pos_embed = query_pos_head(ref_points_detach).clamp(min=-10, max=10) - - # TODO Adjust scale if needed for detachable wider layers - if i >= self.eval_idx + 1 and self.layer_scale > 1: - query_pos_embed = F.interpolate(query_pos_embed, scale_factor=self.layer_scale) - value = self.value_op( - memory, None, query_pos_embed.shape[-1], memory_mask, spatial_shapes - ) - output = F.interpolate(output, size=query_pos_embed.shape[-1]) - output_detach = output.detach() - - output = layer( - output, ref_points_input, value, spatial_shapes, attn_mask, query_pos_embed - ) - - if i == 0: - # Initial bounding box predictions with inverse sigmoid refinement - pre_bboxes = F.sigmoid(pre_bbox_head(output) + inverse_sigmoid(ref_points_detach)) - pre_scores = score_head[0](output) - ref_points_initial = pre_bboxes.detach() - - # Refine bounding box corners using FDR, integrating previous layer's corrections - pred_corners = bbox_head[i](output + output_detach) + pred_corners_undetach - inter_ref_bbox = distance2bbox( - ref_points_initial, integral(pred_corners, project), reg_scale - ) - - if self.training or i == self.eval_idx: - scores = score_head[i](output) - # Lqe does not affect the performance here. - scores = self.lqe_layers[i](scores, pred_corners) - dec_out_logits.append(scores) - dec_out_bboxes.append(inter_ref_bbox) - dec_out_pred_corners.append(pred_corners) - dec_out_refs.append(ref_points_initial) - - if not self.training: - break - - pred_corners_undetach = pred_corners - ref_points_detach = inter_ref_bbox.detach() - output_detach = output.detach() - - return ( - torch.stack(dec_out_bboxes), - torch.stack(dec_out_logits), - torch.stack(dec_out_pred_corners), - torch.stack(dec_out_refs), - pre_bboxes, - pre_scores, - ) - - -@register() -class DFINETransformer(nn.Module): - __share__ = ["num_classes", "eval_spatial_size"] - - def __init__( - self, - num_classes=80, - hidden_dim=256, - num_queries=300, - feat_channels=[512, 1024, 2048], - feat_strides=[8, 16, 32], - num_levels=3, - num_points=4, - nhead=8, - num_layers=6, - dim_feedforward=1024, - dropout=0.0, - activation="relu", - num_denoising=100, - label_noise_ratio=0.5, - box_noise_scale=1.0, - learn_query_content=False, - eval_spatial_size=None, - eval_idx=-1, - eps=1e-2, - aux_loss=True, - cross_attn_method="default", - query_select_method="default", - reg_max=32, - reg_scale=4.0, - layer_scale=1, - ): - super().__init__() - assert len(feat_channels) <= num_levels - assert len(feat_strides) == len(feat_channels) - - for _ in range(num_levels - len(feat_strides)): - feat_strides.append(feat_strides[-1] * 2) - - self.hidden_dim = hidden_dim - scaled_dim = round(layer_scale * hidden_dim) - self.nhead = nhead - self.feat_strides = feat_strides - self.num_levels = num_levels - self.num_classes = num_classes - self.num_queries = num_queries - self.eps = eps - self.num_layers = num_layers - self.eval_spatial_size = eval_spatial_size - self.aux_loss = aux_loss - self.reg_max = reg_max - - assert query_select_method in ("default", "one2many", "agnostic"), "" - assert cross_attn_method in ("default", "discrete"), "" - self.cross_attn_method = cross_attn_method - self.query_select_method = query_select_method - - # backbone feature projection - self._build_input_proj_layer(feat_channels) - - # Transformer module - self.up = nn.Parameter(torch.tensor([0.5]), requires_grad=False) - self.reg_scale = nn.Parameter(torch.tensor([reg_scale]), requires_grad=False) - decoder_layer = TransformerDecoderLayer( - hidden_dim, - nhead, - dim_feedforward, - dropout, - activation, - num_levels, - num_points, - cross_attn_method=cross_attn_method, - ) - decoder_layer_wide = TransformerDecoderLayer( - hidden_dim, - nhead, - dim_feedforward, - dropout, - activation, - num_levels, - num_points, - cross_attn_method=cross_attn_method, - layer_scale=layer_scale, - ) - self.decoder = TransformerDecoder( - hidden_dim, - decoder_layer, - decoder_layer_wide, - num_layers, - nhead, - reg_max, - self.reg_scale, - self.up, - eval_idx, - layer_scale, - ) - # denoising - self.num_denoising = num_denoising - self.label_noise_ratio = label_noise_ratio - self.box_noise_scale = box_noise_scale - if num_denoising > 0: - self.denoising_class_embed = nn.Embedding( - num_classes + 1, hidden_dim, padding_idx=num_classes - ) - init.normal_(self.denoising_class_embed.weight[:-1]) - - # decoder embedding - self.learn_query_content = learn_query_content - if learn_query_content: - self.tgt_embed = nn.Embedding(num_queries, hidden_dim) - self.query_pos_head = MLP(4, 2 * hidden_dim, hidden_dim, 2) - - # if num_select_queries != self.num_queries: - # layer = TransformerEncoderLayer(hidden_dim, nhead, dim_feedforward, activation='gelu') - # self.encoder = TransformerEncoder(layer, 1) - - self.enc_output = nn.Sequential( - OrderedDict( - [ - ("proj", nn.Linear(hidden_dim, hidden_dim)), - ( - "norm", - nn.LayerNorm( - hidden_dim, - ), - ), - ] - ) - ) - - if query_select_method == "agnostic": - self.enc_score_head = nn.Linear(hidden_dim, 1) - else: - self.enc_score_head = nn.Linear(hidden_dim, num_classes) - - self.enc_bbox_head = MLP(hidden_dim, hidden_dim, 4, 3) - - # decoder head - self.eval_idx = eval_idx if eval_idx >= 0 else num_layers + eval_idx - self.dec_score_head = nn.ModuleList( - [nn.Linear(hidden_dim, num_classes) for _ in range(self.eval_idx + 1)] - + [nn.Linear(scaled_dim, num_classes) for _ in range(num_layers - self.eval_idx - 1)] - ) - self.pre_bbox_head = MLP(hidden_dim, hidden_dim, 4, 3) - self.dec_bbox_head = nn.ModuleList( - [ - MLP(hidden_dim, hidden_dim, 4 * (self.reg_max + 1), 3) - for _ in range(self.eval_idx + 1) - ] - + [ - MLP(scaled_dim, scaled_dim, 4 * (self.reg_max + 1), 3) - for _ in range(num_layers - self.eval_idx - 1) - ] - ) - self.integral = Integral(self.reg_max) - - # init encoder output anchors and valid_mask - if self.eval_spatial_size: - anchors, valid_mask = self._generate_anchors() - self.register_buffer("anchors", anchors) - self.register_buffer("valid_mask", valid_mask) - # init encoder output anchors and valid_mask - if self.eval_spatial_size: - self.anchors, self.valid_mask = self._generate_anchors() - - self._reset_parameters(feat_channels) - - def convert_to_deploy(self): - self.dec_score_head = nn.ModuleList( - [nn.Identity()] * (self.eval_idx) + [self.dec_score_head[self.eval_idx]] - ) - self.dec_bbox_head = nn.ModuleList( - [ - self.dec_bbox_head[i] if i <= self.eval_idx else nn.Identity() - for i in range(len(self.dec_bbox_head)) - ] - ) - - def _reset_parameters(self, feat_channels): - bias = bias_init_with_prob(0.01) - init.constant_(self.enc_score_head.bias, bias) - init.constant_(self.enc_bbox_head.layers[-1].weight, 0) - init.constant_(self.enc_bbox_head.layers[-1].bias, 0) - - init.constant_(self.pre_bbox_head.layers[-1].weight, 0) - init.constant_(self.pre_bbox_head.layers[-1].bias, 0) - - for cls_, reg_ in zip(self.dec_score_head, self.dec_bbox_head): - init.constant_(cls_.bias, bias) - if hasattr(reg_, "layers"): - init.constant_(reg_.layers[-1].weight, 0) - init.constant_(reg_.layers[-1].bias, 0) - - init.xavier_uniform_(self.enc_output[0].weight) - if self.learn_query_content: - init.xavier_uniform_(self.tgt_embed.weight) - init.xavier_uniform_(self.query_pos_head.layers[0].weight) - init.xavier_uniform_(self.query_pos_head.layers[1].weight) - for m, in_channels in zip(self.input_proj, feat_channels): - if in_channels != self.hidden_dim: - init.xavier_uniform_(m[0].weight) - - def _build_input_proj_layer(self, feat_channels): - self.input_proj = nn.ModuleList() - for in_channels in feat_channels: - if in_channels == self.hidden_dim: - self.input_proj.append(nn.Identity()) - else: - self.input_proj.append( - nn.Sequential( - OrderedDict( - [ - ("conv", nn.Conv2d(in_channels, self.hidden_dim, 1, bias=False)), - ( - "norm", - nn.BatchNorm2d( - self.hidden_dim, - ), - ), - ] - ) - ) - ) - - in_channels = feat_channels[-1] - - for _ in range(self.num_levels - len(feat_channels)): - if in_channels == self.hidden_dim: - self.input_proj.append(nn.Identity()) - else: - self.input_proj.append( - nn.Sequential( - OrderedDict( - [ - ( - "conv", - nn.Conv2d( - in_channels, self.hidden_dim, 3, 2, padding=1, bias=False - ), - ), - ("norm", nn.BatchNorm2d(self.hidden_dim)), - ] - ) - ) - ) - in_channels = self.hidden_dim - - def _get_encoder_input(self, feats: List[torch.Tensor]): - # get projection features - proj_feats = [self.input_proj[i](feat) for i, feat in enumerate(feats)] - if self.num_levels > len(proj_feats): - len_srcs = len(proj_feats) - for i in range(len_srcs, self.num_levels): - if i == len_srcs: - proj_feats.append(self.input_proj[i](feats[-1])) - else: - proj_feats.append(self.input_proj[i](proj_feats[-1])) - - # get encoder inputs - feat_flatten = [] - spatial_shapes = [] - for i, feat in enumerate(proj_feats): - _, _, h, w = feat.shape - # [b, c, h, w] -> [b, h*w, c] - feat_flatten.append(feat.flatten(2).permute(0, 2, 1)) - # [num_levels, 2] - spatial_shapes.append([h, w]) - - # [b, l, c] - feat_flatten = torch.concat(feat_flatten, 1) - return feat_flatten, spatial_shapes - - def _generate_anchors( - self, spatial_shapes=None, grid_size=0.05, dtype=torch.float32, device="cpu" - ): - if spatial_shapes is None: - spatial_shapes = [] - eval_h, eval_w = self.eval_spatial_size - for s in self.feat_strides: - spatial_shapes.append([int(eval_h / s), int(eval_w / s)]) - - anchors = [] - for lvl, (h, w) in enumerate(spatial_shapes): - grid_y, grid_x = torch.meshgrid(torch.arange(h), torch.arange(w), indexing="ij") - grid_xy = torch.stack([grid_x, grid_y], dim=-1) - grid_xy = (grid_xy.unsqueeze(0) + 0.5) / torch.tensor([w, h], dtype=dtype) - wh = torch.ones_like(grid_xy) * grid_size * (2.0**lvl) - lvl_anchors = torch.concat([grid_xy, wh], dim=-1).reshape(-1, h * w, 4) - anchors.append(lvl_anchors) - - anchors = torch.concat(anchors, dim=1).to(device) - valid_mask = ((anchors > self.eps) * (anchors < 1 - self.eps)).all(-1, keepdim=True) - anchors = torch.log(anchors / (1 - anchors)) - anchors = torch.where(valid_mask, anchors, torch.inf) - - return anchors, valid_mask - - def _get_decoder_input( - self, memory: torch.Tensor, spatial_shapes, denoising_logits=None, denoising_bbox_unact=None - ): - # prepare input for decoder - if self.training or self.eval_spatial_size is None: - anchors, valid_mask = self._generate_anchors(spatial_shapes, device=memory.device) - else: - anchors = self.anchors - valid_mask = self.valid_mask - if memory.shape[0] > 1: - anchors = anchors.repeat(memory.shape[0], 1, 1) - - # memory = torch.where(valid_mask, memory, 0) - # TODO fix type error for onnx export - memory = valid_mask.to(memory.dtype) * memory - - output_memory: torch.Tensor = self.enc_output(memory) - enc_outputs_logits: torch.Tensor = self.enc_score_head(output_memory) - - enc_topk_bboxes_list, enc_topk_logits_list = [], [] - enc_topk_memory, enc_topk_logits, enc_topk_anchors = self._select_topk( - output_memory, enc_outputs_logits, anchors, self.num_queries - ) - - enc_topk_bbox_unact: torch.Tensor = self.enc_bbox_head(enc_topk_memory) + enc_topk_anchors - - if self.training: - enc_topk_bboxes = F.sigmoid(enc_topk_bbox_unact) - enc_topk_bboxes_list.append(enc_topk_bboxes) - enc_topk_logits_list.append(enc_topk_logits) - - # if self.num_select_queries != self.num_queries: - # raise NotImplementedError('') - - if self.learn_query_content: - content = self.tgt_embed.weight.unsqueeze(0).tile([memory.shape[0], 1, 1]) - else: - content = enc_topk_memory.detach() - - enc_topk_bbox_unact = enc_topk_bbox_unact.detach() - - if denoising_bbox_unact is not None: - enc_topk_bbox_unact = torch.concat([denoising_bbox_unact, enc_topk_bbox_unact], dim=1) - content = torch.concat([denoising_logits, content], dim=1) - - return content, enc_topk_bbox_unact, enc_topk_bboxes_list, enc_topk_logits_list - - def _select_topk( - self, - memory: torch.Tensor, - outputs_logits: torch.Tensor, - outputs_anchors_unact: torch.Tensor, - topk: int, - ): - if self.query_select_method == "default": - _, topk_ind = torch.topk(outputs_logits.max(-1).values, topk, dim=-1) - - elif self.query_select_method == "one2many": - _, topk_ind = torch.topk(outputs_logits.flatten(1), topk, dim=-1) - topk_ind = topk_ind // self.num_classes - - elif self.query_select_method == "agnostic": - _, topk_ind = torch.topk(outputs_logits.squeeze(-1), topk, dim=-1) - - topk_ind: torch.Tensor - - topk_anchors = outputs_anchors_unact.gather( - dim=1, index=topk_ind.unsqueeze(-1).repeat(1, 1, outputs_anchors_unact.shape[-1]) - ) - - topk_logits = ( - outputs_logits.gather( - dim=1, index=topk_ind.unsqueeze(-1).repeat(1, 1, outputs_logits.shape[-1]) - ) - if self.training - else None - ) - - topk_memory = memory.gather( - dim=1, index=topk_ind.unsqueeze(-1).repeat(1, 1, memory.shape[-1]) - ) - - return topk_memory, topk_logits, topk_anchors - - def forward(self, feats, targets=None): - # input projection and embedding - memory, spatial_shapes = self._get_encoder_input(feats) - - # prepare denoising training - if self.training and self.num_denoising > 0: - denoising_logits, denoising_bbox_unact, attn_mask, dn_meta = ( - get_contrastive_denoising_training_group( - targets, - self.num_classes, - self.num_queries, - self.denoising_class_embed, - num_denoising=self.num_denoising, - label_noise_ratio=self.label_noise_ratio, - box_noise_scale=1.0, - ) - ) - else: - denoising_logits, denoising_bbox_unact, attn_mask, dn_meta = None, None, None, None - - init_ref_contents, init_ref_points_unact, enc_topk_bboxes_list, enc_topk_logits_list = ( - self._get_decoder_input(memory, spatial_shapes, denoising_logits, denoising_bbox_unact) - ) - - # decoder - out_bboxes, out_logits, out_corners, out_refs, pre_bboxes, pre_logits = self.decoder( - init_ref_contents, - init_ref_points_unact, - memory, - spatial_shapes, - self.dec_bbox_head, - self.dec_score_head, - self.query_pos_head, - self.pre_bbox_head, - self.integral, - self.up, - self.reg_scale, - attn_mask=attn_mask, - dn_meta=dn_meta, - ) - - if self.training and dn_meta is not None: - dn_pre_logits, pre_logits = torch.split(pre_logits, dn_meta["dn_num_split"], dim=1) - dn_pre_bboxes, pre_bboxes = torch.split(pre_bboxes, dn_meta["dn_num_split"], dim=1) - dn_out_bboxes, out_bboxes = torch.split(out_bboxes, dn_meta["dn_num_split"], dim=2) - dn_out_logits, out_logits = torch.split(out_logits, dn_meta["dn_num_split"], dim=2) - - dn_out_corners, out_corners = torch.split(out_corners, dn_meta["dn_num_split"], dim=2) - dn_out_refs, out_refs = torch.split(out_refs, dn_meta["dn_num_split"], dim=2) - - if self.training: - out = { - "pred_logits": out_logits[-1], - "pred_boxes": out_bboxes[-1], - "pred_corners": out_corners[-1], - "ref_points": out_refs[-1], - "up": self.up, - "reg_scale": self.reg_scale, - } - else: - out = {"pred_logits": out_logits[-1], "pred_boxes": out_bboxes[-1]} - - if self.training and self.aux_loss: - out["aux_outputs"] = self._set_aux_loss2( - out_logits[:-1], - out_bboxes[:-1], - out_corners[:-1], - out_refs[:-1], - out_corners[-1], - out_logits[-1], - ) - out["enc_aux_outputs"] = self._set_aux_loss(enc_topk_logits_list, enc_topk_bboxes_list) - out["pre_outputs"] = {"pred_logits": pre_logits, "pred_boxes": pre_bboxes} - out["enc_meta"] = {"class_agnostic": self.query_select_method == "agnostic"} - - if dn_meta is not None: - out["dn_outputs"] = self._set_aux_loss2( - dn_out_logits, - dn_out_bboxes, - dn_out_corners, - dn_out_refs, - dn_out_corners[-1], - dn_out_logits[-1], - ) - out["dn_pre_outputs"] = {"pred_logits": dn_pre_logits, "pred_boxes": dn_pre_bboxes} - out["dn_meta"] = dn_meta - - return out - - @torch.jit.unused - def _set_aux_loss(self, outputs_class, outputs_coord): - # this is a workaround to make torchscript happy, as torchscript - # doesn't support dictionary with non-homogeneous values, such - # as a dict having both a Tensor and a list. - return [{"pred_logits": a, "pred_boxes": b} for a, b in zip(outputs_class, outputs_coord)] - - @torch.jit.unused - def _set_aux_loss2( - self, - outputs_class, - outputs_coord, - outputs_corners, - outputs_ref, - teacher_corners=None, - teacher_logits=None, - ): - # this is a workaround to make torchscript happy, as torchscript - # doesn't support dictionary with non-homogeneous values, such - # as a dict having both a Tensor and a list. - return [ - { - "pred_logits": a, - "pred_boxes": b, - "pred_corners": c, - "ref_points": d, - "teacher_corners": teacher_corners, - "teacher_logits": teacher_logits, - } - for a, b, c, d in zip(outputs_class, outputs_coord, outputs_corners, outputs_ref) - ] diff --git a/vlm/detector/D-FINE.backup/src/zoo/dfine/dfine_utils.py b/vlm/detector/D-FINE.backup/src/zoo/dfine/dfine_utils.py deleted file mode 100644 index 512d161..0000000 --- a/vlm/detector/D-FINE.backup/src/zoo/dfine/dfine_utils.py +++ /dev/null @@ -1,169 +0,0 @@ -""" -Copyright (c) 2024 The D-FINE Authors. All Rights Reserved. -""" - -import torch - -from .box_ops import box_xyxy_to_cxcywh - - -def weighting_function(reg_max, up, reg_scale, deploy=False): - """ - Generates the non-uniform Weighting Function W(n) for bounding box regression. - - Args: - reg_max (int): Max number of the discrete bins. - up (Tensor): Controls upper bounds of the sequence, - where maximum offset is ยฑup * H / W. - reg_scale (float): Controls the curvature of the Weighting Function. - Larger values result in flatter weights near the central axis W(reg_max/2)=0 - and steeper weights at both ends. - deploy (bool): If True, uses deployment mode settings. - - Returns: - Tensor: Sequence of Weighting Function. - """ - if deploy: - upper_bound1 = (abs(up[0]) * abs(reg_scale)).item() - upper_bound2 = (abs(up[0]) * abs(reg_scale) * 2).item() - step = (upper_bound1 + 1) ** (2 / (reg_max - 2)) - left_values = [-((step) ** i) + 1 for i in range(reg_max // 2 - 1, 0, -1)] - right_values = [(step) ** i - 1 for i in range(1, reg_max // 2)] - values = ( - [-upper_bound2] - + left_values - + [torch.zeros_like(up[0][None])] - + right_values - + [upper_bound2] - ) - return torch.tensor(values, dtype=up.dtype, device=up.device) - else: - upper_bound1 = abs(up[0]) * abs(reg_scale) - upper_bound2 = abs(up[0]) * abs(reg_scale) * 2 - step = (upper_bound1 + 1) ** (2 / (reg_max - 2)) - left_values = [-((step) ** i) + 1 for i in range(reg_max // 2 - 1, 0, -1)] - right_values = [(step) ** i - 1 for i in range(1, reg_max // 2)] - values = ( - [-upper_bound2] - + left_values - + [torch.zeros_like(up[0][None])] - + right_values - + [upper_bound2] - ) - return torch.cat(values, 0) - - -def translate_gt(gt, reg_max, reg_scale, up): - """ - Decodes bounding box ground truth (GT) values into distribution-based GT representations. - - This function maps continuous GT values into discrete distribution bins, which can be used - for regression tasks in object detection models. It calculates the indices of the closest - bins to each GT value and assigns interpolation weights to these bins based on their proximity - to the GT value. - - Args: - gt (Tensor): Ground truth bounding box values, shape (N, ). - reg_max (int): Maximum number of discrete bins for the distribution. - reg_scale (float): Controls the curvature of the Weighting Function. - up (Tensor): Controls the upper bounds of the Weighting Function. - - Returns: - Tuple[Tensor, Tensor, Tensor]: - - indices (Tensor): Index of the left bin closest to each GT value, shape (N, ). - - weight_right (Tensor): Weight assigned to the right bin, shape (N, ). - - weight_left (Tensor): Weight assigned to the left bin, shape (N, ). - """ - gt = gt.reshape(-1) - function_values = weighting_function(reg_max, up, reg_scale) - - # Find the closest left-side indices for each value - diffs = function_values.unsqueeze(0) - gt.unsqueeze(1) - mask = diffs <= 0 - closest_left_indices = torch.sum(mask, dim=1) - 1 - - # Calculate the weights for the interpolation - indices = closest_left_indices.float() - - weight_right = torch.zeros_like(indices) - weight_left = torch.zeros_like(indices) - - valid_idx_mask = (indices >= 0) & (indices < reg_max) - valid_indices = indices[valid_idx_mask].long() - - # Obtain distances - left_values = function_values[valid_indices] - right_values = function_values[valid_indices + 1] - - left_diffs = torch.abs(gt[valid_idx_mask] - left_values) - right_diffs = torch.abs(right_values - gt[valid_idx_mask]) - - # Valid weights - weight_right[valid_idx_mask] = left_diffs / (left_diffs + right_diffs) - weight_left[valid_idx_mask] = 1.0 - weight_right[valid_idx_mask] - - # Invalid weights (out of range) - invalid_idx_mask_neg = indices < 0 - weight_right[invalid_idx_mask_neg] = 0.0 - weight_left[invalid_idx_mask_neg] = 1.0 - indices[invalid_idx_mask_neg] = 0.0 - - invalid_idx_mask_pos = indices >= reg_max - weight_right[invalid_idx_mask_pos] = 1.0 - weight_left[invalid_idx_mask_pos] = 0.0 - indices[invalid_idx_mask_pos] = reg_max - 0.1 - - return indices, weight_right, weight_left - - -def distance2bbox(points, distance, reg_scale): - """ - Decodes edge-distances into bounding box coordinates. - - Args: - points (Tensor): (B, N, 4) or (N, 4) format, representing [x, y, w, h], - where (x, y) is the center and (w, h) are width and height. - distance (Tensor): (B, N, 4) or (N, 4), representing distances from the - point to the left, top, right, and bottom boundaries. - - reg_scale (float): Controls the curvature of the Weighting Function. - - Returns: - Tensor: Bounding boxes in (N, 4) or (B, N, 4) format [cx, cy, w, h]. - """ - reg_scale = abs(reg_scale) - x1 = points[..., 0] - (0.5 * reg_scale + distance[..., 0]) * (points[..., 2] / reg_scale) - y1 = points[..., 1] - (0.5 * reg_scale + distance[..., 1]) * (points[..., 3] / reg_scale) - x2 = points[..., 0] + (0.5 * reg_scale + distance[..., 2]) * (points[..., 2] / reg_scale) - y2 = points[..., 1] + (0.5 * reg_scale + distance[..., 3]) * (points[..., 3] / reg_scale) - - bboxes = torch.stack([x1, y1, x2, y2], -1) - - return box_xyxy_to_cxcywh(bboxes) - - -def bbox2distance(points, bbox, reg_max, reg_scale, up, eps=0.1): - """ - Converts bounding box coordinates to distances from a reference point. - - Args: - points (Tensor): (n, 4) [x, y, w, h], where (x, y) is the center. - bbox (Tensor): (n, 4) bounding boxes in "xyxy" format. - reg_max (float): Maximum bin value. - reg_scale (float): Controling curvarture of W(n). - up (Tensor): Controling upper bounds of W(n). - eps (float): Small value to ensure target < reg_max. - - Returns: - Tensor: Decoded distances. - """ - reg_scale = abs(reg_scale) - left = (points[:, 0] - bbox[:, 0]) / (points[..., 2] / reg_scale + 1e-16) - 0.5 * reg_scale - top = (points[:, 1] - bbox[:, 1]) / (points[..., 3] / reg_scale + 1e-16) - 0.5 * reg_scale - right = (bbox[:, 2] - points[:, 0]) / (points[..., 2] / reg_scale + 1e-16) - 0.5 * reg_scale - bottom = (bbox[:, 3] - points[:, 1]) / (points[..., 3] / reg_scale + 1e-16) - 0.5 * reg_scale - four_lens = torch.stack([left, top, right, bottom], -1) - four_lens, weight_right, weight_left = translate_gt(four_lens, reg_max, reg_scale, up) - if reg_max is not None: - four_lens = four_lens.clamp(min=0, max=reg_max - eps) - return four_lens.reshape(-1).detach(), weight_right.detach(), weight_left.detach() diff --git a/vlm/detector/D-FINE.backup/src/zoo/dfine/hybrid_encoder.py b/vlm/detector/D-FINE.backup/src/zoo/dfine/hybrid_encoder.py deleted file mode 100644 index 77af3fb..0000000 --- a/vlm/detector/D-FINE.backup/src/zoo/dfine/hybrid_encoder.py +++ /dev/null @@ -1,488 +0,0 @@ -""" -D-FINE: Redefine Regression Task of DETRs as Fine-grained Distribution Refinement -Copyright (c) 2024 The D-FINE Authors. All Rights Reserved. ---------------------------------------------------------------------------------- -Modified from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright (c) 2023 lyuwenyu. All Rights Reserved. -""" - -import copy -from collections import OrderedDict - -import torch -import torch.nn as nn -import torch.nn.functional as F - -from ...core import register -from .utils import get_activation - -__all__ = ["HybridEncoder"] - - -class ConvNormLayer_fuse(nn.Module): - def __init__(self, ch_in, ch_out, kernel_size, stride, g=1, padding=None, bias=False, act=None): - super().__init__() - padding = (kernel_size - 1) // 2 if padding is None else padding - self.conv = nn.Conv2d( - ch_in, ch_out, kernel_size, stride, groups=g, padding=padding, bias=bias - ) - self.norm = nn.BatchNorm2d(ch_out) - self.act = nn.Identity() if act is None else get_activation(act) - self.ch_in, self.ch_out, self.kernel_size, self.stride, self.g, self.padding, self.bias = ( - ch_in, - ch_out, - kernel_size, - stride, - g, - padding, - bias, - ) - - def forward(self, x): - if hasattr(self, "conv_bn_fused"): - y = self.conv_bn_fused(x) - else: - y = self.norm(self.conv(x)) - return self.act(y) - - def convert_to_deploy(self): - if not hasattr(self, "conv_bn_fused"): - self.conv_bn_fused = nn.Conv2d( - self.ch_in, - self.ch_out, - self.kernel_size, - self.stride, - groups=self.g, - padding=self.padding, - bias=True, - ) - - kernel, bias = self.get_equivalent_kernel_bias() - self.conv_bn_fused.weight.data = kernel - self.conv_bn_fused.bias.data = bias - self.__delattr__("conv") - self.__delattr__("norm") - - def get_equivalent_kernel_bias(self): - kernel3x3, bias3x3 = self._fuse_bn_tensor() - - return kernel3x3, bias3x3 - - def _fuse_bn_tensor(self): - kernel = self.conv.weight - running_mean = self.norm.running_mean - running_var = self.norm.running_var - gamma = self.norm.weight - beta = self.norm.bias - eps = self.norm.eps - std = (running_var + eps).sqrt() - t = (gamma / std).reshape(-1, 1, 1, 1) - return kernel * t, beta - running_mean * gamma / std - - -class ConvNormLayer(nn.Module): - def __init__(self, ch_in, ch_out, kernel_size, stride, g=1, padding=None, bias=False, act=None): - super().__init__() - padding = (kernel_size - 1) // 2 if padding is None else padding - self.conv = nn.Conv2d( - ch_in, ch_out, kernel_size, stride, groups=g, padding=padding, bias=bias - ) - self.norm = nn.BatchNorm2d(ch_out) - self.act = nn.Identity() if act is None else get_activation(act) - - def forward(self, x): - return self.act(self.norm(self.conv(x))) - - -class SCDown(nn.Module): - def __init__(self, c1, c2, k, s): - super().__init__() - self.cv1 = ConvNormLayer_fuse(c1, c2, 1, 1) - self.cv2 = ConvNormLayer_fuse(c2, c2, k, s, c2) - - def forward(self, x): - return self.cv2(self.cv1(x)) - - -class VGGBlock(nn.Module): - def __init__(self, ch_in, ch_out, act="relu"): - super().__init__() - self.ch_in = ch_in - self.ch_out = ch_out - self.conv1 = ConvNormLayer(ch_in, ch_out, 3, 1, padding=1, act=None) - self.conv2 = ConvNormLayer(ch_in, ch_out, 1, 1, padding=0, act=None) - self.act = nn.Identity() if act is None else act - - def forward(self, x): - if hasattr(self, "conv"): - y = self.conv(x) - else: - y = self.conv1(x) + self.conv2(x) - - return self.act(y) - - def convert_to_deploy(self): - if not hasattr(self, "conv"): - self.conv = nn.Conv2d(self.ch_in, self.ch_out, 3, 1, padding=1) - - kernel, bias = self.get_equivalent_kernel_bias() - self.conv.weight.data = kernel - self.conv.bias.data = bias - self.__delattr__("conv1") - self.__delattr__("conv2") - - def get_equivalent_kernel_bias(self): - kernel3x3, bias3x3 = self._fuse_bn_tensor(self.conv1) - kernel1x1, bias1x1 = self._fuse_bn_tensor(self.conv2) - - return kernel3x3 + self._pad_1x1_to_3x3_tensor(kernel1x1), bias3x3 + bias1x1 - - def _pad_1x1_to_3x3_tensor(self, kernel1x1): - if kernel1x1 is None: - return 0 - else: - return F.pad(kernel1x1, [1, 1, 1, 1]) - - def _fuse_bn_tensor(self, branch: ConvNormLayer): - if branch is None: - return 0, 0 - kernel = branch.conv.weight - running_mean = branch.norm.running_mean - running_var = branch.norm.running_var - gamma = branch.norm.weight - beta = branch.norm.bias - eps = branch.norm.eps - std = (running_var + eps).sqrt() - t = (gamma / std).reshape(-1, 1, 1, 1) - return kernel * t, beta - running_mean * gamma / std - - -class ELAN(nn.Module): - # csp-elan - def __init__(self, c1, c2, c3, c4, n=2, bias=False, act="silu", bottletype=VGGBlock): - super().__init__() - self.c = c3 - self.cv1 = ConvNormLayer_fuse(c1, c3, 1, 1, bias=bias, act=act) - self.cv2 = nn.Sequential( - bottletype(c3 // 2, c4, act=get_activation(act)), - ConvNormLayer_fuse(c4, c4, 3, 1, bias=bias, act=act), - ) - self.cv3 = nn.Sequential( - bottletype(c4, c4, act=get_activation(act)), - ConvNormLayer_fuse(c4, c4, 3, 1, bias=bias, act=act), - ) - self.cv4 = ConvNormLayer_fuse(c3 + (2 * c4), c2, 1, 1, bias=bias, act=act) - - def forward(self, x): - # y = [self.cv1(x)] - y = list(self.cv1(x).chunk(2, 1)) - y.extend((m(y[-1])) for m in [self.cv2, self.cv3]) - return self.cv4(torch.cat(y, 1)) - - -class RepNCSPELAN4(nn.Module): - # csp-elan - def __init__(self, c1, c2, c3, c4, n=3, bias=False, act="silu"): - super().__init__() - self.c = c3 // 2 - self.cv1 = ConvNormLayer_fuse(c1, c3, 1, 1, bias=bias, act=act) - self.cv2 = nn.Sequential( - CSPLayer(c3 // 2, c4, n, 1, bias=bias, act=act, bottletype=VGGBlock), - ConvNormLayer_fuse(c4, c4, 3, 1, bias=bias, act=act), - ) - self.cv3 = nn.Sequential( - CSPLayer(c4, c4, n, 1, bias=bias, act=act, bottletype=VGGBlock), - ConvNormLayer_fuse(c4, c4, 3, 1, bias=bias, act=act), - ) - self.cv4 = ConvNormLayer_fuse(c3 + (2 * c4), c2, 1, 1, bias=bias, act=act) - - def forward_chunk(self, x): - y = list(self.cv1(x).chunk(2, 1)) - y.extend((m(y[-1])) for m in [self.cv2, self.cv3]) - return self.cv4(torch.cat(y, 1)) - - def forward(self, x): - y = list(self.cv1(x).split((self.c, self.c), 1)) - y.extend(m(y[-1]) for m in [self.cv2, self.cv3]) - return self.cv4(torch.cat(y, 1)) - - -class CSPLayer(nn.Module): - def __init__( - self, - in_channels, - out_channels, - num_blocks=3, - expansion=1.0, - bias=False, - act="silu", - bottletype=VGGBlock, - ): - super(CSPLayer, self).__init__() - hidden_channels = int(out_channels * expansion) - self.conv1 = ConvNormLayer_fuse(in_channels, hidden_channels, 1, 1, bias=bias, act=act) - self.conv2 = ConvNormLayer_fuse(in_channels, hidden_channels, 1, 1, bias=bias, act=act) - self.bottlenecks = nn.Sequential( - *[ - bottletype(hidden_channels, hidden_channels, act=get_activation(act)) - for _ in range(num_blocks) - ] - ) - if hidden_channels != out_channels: - self.conv3 = ConvNormLayer_fuse(hidden_channels, out_channels, 1, 1, bias=bias, act=act) - else: - self.conv3 = nn.Identity() - - def forward(self, x): - x_1 = self.conv1(x) - x_1 = self.bottlenecks(x_1) - x_2 = self.conv2(x) - return self.conv3(x_1 + x_2) - - -# transformer -class TransformerEncoderLayer(nn.Module): - def __init__( - self, - d_model, - nhead, - dim_feedforward=2048, - dropout=0.1, - activation="relu", - normalize_before=False, - ): - super().__init__() - self.normalize_before = normalize_before - - self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout, batch_first=True) - - self.linear1 = nn.Linear(d_model, dim_feedforward) - self.dropout = nn.Dropout(dropout) - self.linear2 = nn.Linear(dim_feedforward, d_model) - - self.norm1 = nn.LayerNorm(d_model) - self.norm2 = nn.LayerNorm(d_model) - self.dropout1 = nn.Dropout(dropout) - self.dropout2 = nn.Dropout(dropout) - self.activation = get_activation(activation) - - @staticmethod - def with_pos_embed(tensor, pos_embed): - return tensor if pos_embed is None else tensor + pos_embed - - def forward(self, src, src_mask=None, pos_embed=None) -> torch.Tensor: - residual = src - if self.normalize_before: - src = self.norm1(src) - q = k = self.with_pos_embed(src, pos_embed) - src, _ = self.self_attn(q, k, value=src, attn_mask=src_mask) - - src = residual + self.dropout1(src) - if not self.normalize_before: - src = self.norm1(src) - - residual = src - if self.normalize_before: - src = self.norm2(src) - src = self.linear2(self.dropout(self.activation(self.linear1(src)))) - src = residual + self.dropout2(src) - if not self.normalize_before: - src = self.norm2(src) - return src - - -class TransformerEncoder(nn.Module): - def __init__(self, encoder_layer, num_layers, norm=None): - super(TransformerEncoder, self).__init__() - self.layers = nn.ModuleList([copy.deepcopy(encoder_layer) for _ in range(num_layers)]) - self.num_layers = num_layers - self.norm = norm - - def forward(self, src, src_mask=None, pos_embed=None) -> torch.Tensor: - output = src - for layer in self.layers: - output = layer(output, src_mask=src_mask, pos_embed=pos_embed) - - if self.norm is not None: - output = self.norm(output) - - return output - - -@register() -class HybridEncoder(nn.Module): - __share__ = [ - "eval_spatial_size", - ] - - def __init__( - self, - in_channels=[512, 1024, 2048], - feat_strides=[8, 16, 32], - hidden_dim=256, - nhead=8, - dim_feedforward=1024, - dropout=0.0, - enc_act="gelu", - use_encoder_idx=[2], - num_encoder_layers=1, - pe_temperature=10000, - expansion=1.0, - depth_mult=1.0, - act="silu", - eval_spatial_size=None, - ): - super().__init__() - self.in_channels = in_channels - self.feat_strides = feat_strides - self.hidden_dim = hidden_dim - self.use_encoder_idx = use_encoder_idx - self.num_encoder_layers = num_encoder_layers - self.pe_temperature = pe_temperature - self.eval_spatial_size = eval_spatial_size - self.out_channels = [hidden_dim for _ in range(len(in_channels))] - self.out_strides = feat_strides - - # channel projection - self.input_proj = nn.ModuleList() - for in_channel in in_channels: - proj = nn.Sequential( - OrderedDict( - [ - ("conv", nn.Conv2d(in_channel, hidden_dim, kernel_size=1, bias=False)), - ("norm", nn.BatchNorm2d(hidden_dim)), - ] - ) - ) - - self.input_proj.append(proj) - - # encoder transformer - encoder_layer = TransformerEncoderLayer( - hidden_dim, - nhead=nhead, - dim_feedforward=dim_feedforward, - dropout=dropout, - activation=enc_act, - ) - - self.encoder = nn.ModuleList( - [ - TransformerEncoder(copy.deepcopy(encoder_layer), num_encoder_layers) - for _ in range(len(use_encoder_idx)) - ] - ) - - # top-down fpn - self.lateral_convs = nn.ModuleList() - self.fpn_blocks = nn.ModuleList() - for _ in range(len(in_channels) - 1, 0, -1): - self.lateral_convs.append(ConvNormLayer_fuse(hidden_dim, hidden_dim, 1, 1)) - self.fpn_blocks.append( - RepNCSPELAN4( - hidden_dim * 2, - hidden_dim, - hidden_dim * 2, - round(expansion * hidden_dim // 2), - round(3 * depth_mult), - ) - # CSPLayer(hidden_dim * 2, hidden_dim, round(3 * depth_mult), act=act, expansion=expansion, bottletype=VGGBlock) - ) - - # bottom-up pan - self.downsample_convs = nn.ModuleList() - self.pan_blocks = nn.ModuleList() - for _ in range(len(in_channels) - 1): - self.downsample_convs.append( - nn.Sequential( - SCDown(hidden_dim, hidden_dim, 3, 2), - ) - ) - self.pan_blocks.append( - RepNCSPELAN4( - hidden_dim * 2, - hidden_dim, - hidden_dim * 2, - round(expansion * hidden_dim // 2), - round(3 * depth_mult), - ) - # CSPLayer(hidden_dim * 2, hidden_dim, round(3 * depth_mult), act=act, expansion=expansion, bottletype=VGGBlock) - ) - - self._reset_parameters() - - def _reset_parameters(self): - if self.eval_spatial_size: - for idx in self.use_encoder_idx: - stride = self.feat_strides[idx] - pos_embed = self.build_2d_sincos_position_embedding( - self.eval_spatial_size[1] // stride, - self.eval_spatial_size[0] // stride, - self.hidden_dim, - self.pe_temperature, - ) - setattr(self, f"pos_embed{idx}", pos_embed) - # self.register_buffer(f'pos_embed{idx}', pos_embed) - - @staticmethod - def build_2d_sincos_position_embedding(w, h, embed_dim=256, temperature=10000.0): - """ """ - grid_w = torch.arange(int(w), dtype=torch.float32) - grid_h = torch.arange(int(h), dtype=torch.float32) - grid_w, grid_h = torch.meshgrid(grid_w, grid_h, indexing="ij") - assert ( - embed_dim % 4 == 0 - ), "Embed dimension must be divisible by 4 for 2D sin-cos position embedding" - pos_dim = embed_dim // 4 - omega = torch.arange(pos_dim, dtype=torch.float32) / pos_dim - omega = 1.0 / (temperature**omega) - - out_w = grid_w.flatten()[..., None] @ omega[None] - out_h = grid_h.flatten()[..., None] @ omega[None] - - return torch.concat([out_w.sin(), out_w.cos(), out_h.sin(), out_h.cos()], dim=1)[None, :, :] - - def forward(self, feats): - assert len(feats) == len(self.in_channels) - proj_feats = [self.input_proj[i](feat) for i, feat in enumerate(feats)] - - # encoder - if self.num_encoder_layers > 0: - for i, enc_ind in enumerate(self.use_encoder_idx): - h, w = proj_feats[enc_ind].shape[2:] - # flatten [B, C, H, W] to [B, HxW, C] - src_flatten = proj_feats[enc_ind].flatten(2).permute(0, 2, 1) - if self.training or self.eval_spatial_size is None: - pos_embed = self.build_2d_sincos_position_embedding( - w, h, self.hidden_dim, self.pe_temperature - ).to(src_flatten.device) - else: - pos_embed = getattr(self, f"pos_embed{enc_ind}", None).to(src_flatten.device) - - memory: torch.Tensor = self.encoder[i](src_flatten, pos_embed=pos_embed) - proj_feats[enc_ind] = ( - memory.permute(0, 2, 1).reshape(-1, self.hidden_dim, h, w).contiguous() - ) - - # broadcasting and fusion - inner_outs = [proj_feats[-1]] - for idx in range(len(self.in_channels) - 1, 0, -1): - feat_heigh = inner_outs[0] - feat_low = proj_feats[idx - 1] - feat_heigh = self.lateral_convs[len(self.in_channels) - 1 - idx](feat_heigh) - inner_outs[0] = feat_heigh - upsample_feat = F.interpolate(feat_heigh, scale_factor=2.0, mode="nearest") - inner_out = self.fpn_blocks[len(self.in_channels) - 1 - idx]( - torch.concat([upsample_feat, feat_low], dim=1) - ) - inner_outs.insert(0, inner_out) - - outs = [inner_outs[0]] - for idx in range(len(self.in_channels) - 1): - feat_low = outs[-1] - feat_height = inner_outs[idx + 1] - downsample_feat = self.downsample_convs[idx](feat_low) - out = self.pan_blocks[idx](torch.concat([downsample_feat, feat_height], dim=1)) - outs.append(out) - - return outs diff --git a/vlm/detector/D-FINE.backup/src/zoo/dfine/matcher.py b/vlm/detector/D-FINE.backup/src/zoo/dfine/matcher.py deleted file mode 100644 index debe797..0000000 --- a/vlm/detector/D-FINE.backup/src/zoo/dfine/matcher.py +++ /dev/null @@ -1,160 +0,0 @@ -""" -Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved -Modules to compute the matching cost and solve the corresponding LSAP. - -Copyright (c) 2024 The D-FINE Authors All Rights Reserved. -""" - -from typing import Dict - -import numpy as np -import torch -import torch.nn as nn -import torch.nn.functional as F -from scipy.optimize import linear_sum_assignment - -from ...core import register -from .box_ops import box_cxcywh_to_xyxy, generalized_box_iou - - -@register() -class HungarianMatcher(nn.Module): - """This class computes an assignment between the targets and the predictions of the network - - For efficiency reasons, the targets don't include the no_object. Because of this, in general, - there are more predictions than targets. In this case, we do a 1-to-1 matching of the best predictions, - while the others are un-matched (and thus treated as non-objects). - """ - - __share__ = [ - "use_focal_loss", - ] - - def __init__(self, weight_dict, use_focal_loss=False, alpha=0.25, gamma=2.0): - """Creates the matcher - - Params: - cost_class: This is the relative weight of the classification error in the matching cost - cost_bbox: This is the relative weight of the L1 error of the bounding box coordinates in the matching cost - cost_giou: This is the relative weight of the giou loss of the bounding box in the matching cost - """ - super().__init__() - self.cost_class = weight_dict["cost_class"] - self.cost_bbox = weight_dict["cost_bbox"] - self.cost_giou = weight_dict["cost_giou"] - - self.use_focal_loss = use_focal_loss - self.alpha = alpha - self.gamma = gamma - - assert ( - self.cost_class != 0 or self.cost_bbox != 0 or self.cost_giou != 0 - ), "all costs cant be 0" - - @torch.no_grad() - def forward(self, outputs: Dict[str, torch.Tensor], targets, return_topk=False): - """Performs the matching - - Params: - outputs: This is a dict that contains at least these entries: - "pred_logits": Tensor of dim [batch_size, num_queries, num_classes] with the classification logits - "pred_boxes": Tensor of dim [batch_size, num_queries, 4] with the predicted box coordinates - - targets: This is a list of targets (len(targets) = batch_size), where each target is a dict containing: - "labels": Tensor of dim [num_target_boxes] (where num_target_boxes is the number of ground-truth - objects in the target) containing the class labels - "boxes": Tensor of dim [num_target_boxes, 4] containing the target box coordinates - - Returns: - A list of size batch_size, containing tuples of (index_i, index_j) where: - - index_i is the indices of the selected predictions (in order) - - index_j is the indices of the corresponding selected targets (in order) - For each batch element, it holds: - len(index_i) = len(index_j) = min(num_queries, num_target_boxes) - """ - bs, num_queries = outputs["pred_logits"].shape[:2] - - # We flatten to compute the cost matrices in a batch - if self.use_focal_loss: - out_prob = F.sigmoid(outputs["pred_logits"].flatten(0, 1)) - else: - out_prob = ( - outputs["pred_logits"].flatten(0, 1).softmax(-1) - ) # [batch_size * num_queries, num_classes] - - out_bbox = outputs["pred_boxes"].flatten(0, 1) # [batch_size * num_queries, 4] - - # Also concat the target labels and boxes - tgt_ids = torch.cat([v["labels"] for v in targets]) - tgt_bbox = torch.cat([v["boxes"] for v in targets]) - - # Compute the classification cost. Contrary to the loss, we don't use the NLL, - # but approximate it in 1 - proba[target class]. - # The 1 is a constant that doesn't change the matching, it can be ommitted. - if self.use_focal_loss: - out_prob = out_prob[:, tgt_ids] - neg_cost_class = ( - (1 - self.alpha) * (out_prob**self.gamma) * (-(1 - out_prob + 1e-8).log()) - ) - pos_cost_class = ( - self.alpha * ((1 - out_prob) ** self.gamma) * (-(out_prob + 1e-8).log()) - ) - cost_class = pos_cost_class - neg_cost_class - else: - cost_class = -out_prob[:, tgt_ids] - - # Compute the L1 cost between boxes - cost_bbox = torch.cdist(out_bbox, tgt_bbox, p=1) - - # Compute the giou cost betwen boxes - cost_giou = -generalized_box_iou(box_cxcywh_to_xyxy(out_bbox), box_cxcywh_to_xyxy(tgt_bbox)) - - # Final cost matrix 3 * self.cost_bbox + 2 * self.cost_class + self.cost_giou - C = self.cost_bbox * cost_bbox + self.cost_class * cost_class + self.cost_giou * cost_giou - C = C.view(bs, num_queries, -1).cpu() - - sizes = [len(v["boxes"]) for v in targets] - C = torch.nan_to_num(C, nan=1.0) - indices_pre = [linear_sum_assignment(c[i]) for i, c in enumerate(C.split(sizes, -1))] - indices = [ - (torch.as_tensor(i, dtype=torch.int64), torch.as_tensor(j, dtype=torch.int64)) - for i, j in indices_pre - ] - - # Compute topk indices - if return_topk: - return { - "indices_o2m": self.get_top_k_matches( - C, sizes=sizes, k=return_topk, initial_indices=indices_pre - ) - } - - return {"indices": indices} # , 'indices_o2m': C.min(-1)[1]} - - def get_top_k_matches(self, C, sizes, k=1, initial_indices=None): - indices_list = [] - # C_original = C.clone() - for i in range(k): - indices_k = ( - [linear_sum_assignment(c[i]) for i, c in enumerate(C.split(sizes, -1))] - if i > 0 - else initial_indices - ) - indices_list.append( - [ - (torch.as_tensor(i, dtype=torch.int64), torch.as_tensor(j, dtype=torch.int64)) - for i, j in indices_k - ] - ) - for c, idx_k in zip(C.split(sizes, -1), indices_k): - idx_k = np.stack(idx_k) - c[:, idx_k] = 1e6 - indices_list = [ - ( - torch.cat([indices_list[i][j][0] for i in range(k)], dim=0), - torch.cat([indices_list[i][j][1] for i in range(k)], dim=0), - ) - for j in range(len(sizes)) - ] - # C.copy_(C_original) - return indices_list diff --git a/vlm/detector/D-FINE.backup/src/zoo/dfine/postprocessor.py b/vlm/detector/D-FINE.backup/src/zoo/dfine/postprocessor.py deleted file mode 100644 index b1516b6..0000000 --- a/vlm/detector/D-FINE.backup/src/zoo/dfine/postprocessor.py +++ /dev/null @@ -1,93 +0,0 @@ -""" -Copied from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright(c) 2023 lyuwenyu. All Rights Reserved. -""" - -import torch -import torch.nn as nn -import torch.nn.functional as F -import torchvision - -from ...core import register - -__all__ = ["DFINEPostProcessor"] - - -def mod(a, b): - out = a - a // b * b - return out - - -@register() -class DFINEPostProcessor(nn.Module): - __share__ = ["num_classes", "use_focal_loss", "num_top_queries", "remap_mscoco_category"] - - def __init__( - self, num_classes=80, use_focal_loss=True, num_top_queries=300, remap_mscoco_category=False - ) -> None: - super().__init__() - self.use_focal_loss = use_focal_loss - self.num_top_queries = num_top_queries - self.num_classes = int(num_classes) - self.remap_mscoco_category = remap_mscoco_category - self.deploy_mode = False - - def extra_repr(self) -> str: - return f"use_focal_loss={self.use_focal_loss}, num_classes={self.num_classes}, num_top_queries={self.num_top_queries}" - - # def forward(self, outputs, orig_target_sizes): - def forward(self, outputs, orig_target_sizes: torch.Tensor): - logits, boxes = outputs["pred_logits"], outputs["pred_boxes"] - # orig_target_sizes = torch.stack([t["orig_size"] for t in targets], dim=0) - - bbox_pred = torchvision.ops.box_convert(boxes, in_fmt="cxcywh", out_fmt="xyxy") - bbox_pred *= orig_target_sizes.repeat(1, 2).unsqueeze(1) - - if self.use_focal_loss: - scores = F.sigmoid(logits) - scores, index = torch.topk(scores.flatten(1), self.num_top_queries, dim=-1) - # TODO for older tensorrt - # labels = index % self.num_classes - labels = mod(index, self.num_classes) - index = index // self.num_classes - boxes = bbox_pred.gather( - dim=1, index=index.unsqueeze(-1).repeat(1, 1, bbox_pred.shape[-1]) - ) - - else: - scores = F.softmax(logits)[:, :, :-1] - scores, labels = scores.max(dim=-1) - if scores.shape[1] > self.num_top_queries: - scores, index = torch.topk(scores, self.num_top_queries, dim=-1) - labels = torch.gather(labels, dim=1, index=index) - boxes = torch.gather( - boxes, dim=1, index=index.unsqueeze(-1).tile(1, 1, boxes.shape[-1]) - ) - - # TODO for onnx export - if self.deploy_mode: - return labels, boxes, scores - - # TODO - if self.remap_mscoco_category: - from ...data.dataset import mscoco_label2category - - labels = ( - torch.tensor([mscoco_label2category[int(x.item())] for x in labels.flatten()]) - .to(boxes.device) - .reshape(labels.shape) - ) - - results = [] - for lab, box, sco in zip(labels, boxes, scores): - result = dict(labels=lab, boxes=box, scores=sco) - results.append(result) - - return results - - def deploy( - self, - ): - self.eval() - self.deploy_mode = True - return self diff --git a/vlm/detector/D-FINE.backup/src/zoo/dfine/utils.py b/vlm/detector/D-FINE.backup/src/zoo/dfine/utils.py deleted file mode 100644 index dffdcb5..0000000 --- a/vlm/detector/D-FINE.backup/src/zoo/dfine/utils.py +++ /dev/null @@ -1,182 +0,0 @@ -""" -D-FINE: Redefine Regression Task of DETRs as Fine-grained Distribution Refinement -Copyright (c) 2024 The D-FINE Authors. All Rights Reserved. ---------------------------------------------------------------------------------- -Modified from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright (c) 2023 lyuwenyu. All Rights Reserved. -""" - -import math -from typing import List - -import torch -import torch.nn as nn -import torch.nn.functional as F - - -def inverse_sigmoid(x: torch.Tensor, eps: float = 1e-5) -> torch.Tensor: - x = x.clip(min=0.0, max=1.0) - return torch.log(x.clip(min=eps) / (1 - x).clip(min=eps)) - - -def bias_init_with_prob(prior_prob=0.01): - """initialize conv/fc bias value according to a given probability value.""" - bias_init = float(-math.log((1 - prior_prob) / prior_prob)) - return bias_init - - -def deformable_attention_core_func( - value, value_spatial_shapes, sampling_locations, attention_weights -): - """ - Args: - value (Tensor): [bs, value_length, n_head, c] - value_spatial_shapes (Tensor|List): [n_levels, 2] - value_level_start_index (Tensor|List): [n_levels] - sampling_locations (Tensor): [bs, query_length, n_head, n_levels, n_points, 2] - attention_weights (Tensor): [bs, query_length, n_head, n_levels, n_points] - - Returns: - output (Tensor): [bs, Length_{query}, C] - """ - bs, _, n_head, c = value.shape - _, Len_q, _, n_levels, n_points, _ = sampling_locations.shape - - split_shape = [h * w for h, w in value_spatial_shapes] - value_list = value.split(split_shape, dim=1) - sampling_grids = 2 * sampling_locations - 1 - sampling_value_list = [] - for level, (h, w) in enumerate(value_spatial_shapes): - # N_, H_*W_, M_, D_ -> N_, H_*W_, M_*D_ -> N_, M_*D_, H_*W_ -> N_*M_, D_, H_, W_ - value_l_ = value_list[level].flatten(2).permute(0, 2, 1).reshape(bs * n_head, c, h, w) - # N_, Lq_, M_, P_, 2 -> N_, M_, Lq_, P_, 2 -> N_*M_, Lq_, P_, 2 - sampling_grid_l_ = sampling_grids[:, :, :, level].permute(0, 2, 1, 3, 4).flatten(0, 1) - # N_*M_, D_, Lq_, P_ - sampling_value_l_ = F.grid_sample( - value_l_, sampling_grid_l_, mode="bilinear", padding_mode="zeros", align_corners=False - ) - sampling_value_list.append(sampling_value_l_) - # (N_, Lq_, M_, L_, P_) -> (N_, M_, Lq_, L_, P_) -> (N_*M_, 1, Lq_, L_*P_) - attention_weights = attention_weights.permute(0, 2, 1, 3, 4).reshape( - bs * n_head, 1, Len_q, n_levels * n_points - ) - output = ( - (torch.stack(sampling_value_list, dim=-2).flatten(-2) * attention_weights) - .sum(-1) - .reshape(bs, n_head * c, Len_q) - ) - - return output.permute(0, 2, 1) - - -def deformable_attention_core_func_v2( - value: torch.Tensor, - value_spatial_shapes, - sampling_locations: torch.Tensor, - attention_weights: torch.Tensor, - num_points_list: List[int], - method="default", -): - """ - Args: - value (Tensor): [bs, value_length, n_head, c] - value_spatial_shapes (Tensor|List): [n_levels, 2] - value_level_start_index (Tensor|List): [n_levels] - sampling_locations (Tensor): [bs, query_length, n_head, n_levels * n_points, 2] - attention_weights (Tensor): [bs, query_length, n_head, n_levels * n_points] - - Returns: - output (Tensor): [bs, Length_{query}, C] - """ - bs, n_head, c, _ = value[0].shape - _, Len_q, _, _, _ = sampling_locations.shape - - # sampling_offsets [8, 480, 8, 12, 2] - if method == "default": - sampling_grids = 2 * sampling_locations - 1 - - elif method == "discrete": - sampling_grids = sampling_locations - - sampling_grids = sampling_grids.permute(0, 2, 1, 3, 4).flatten(0, 1) - sampling_locations_list = sampling_grids.split(num_points_list, dim=-2) - - sampling_value_list = [] - for level, (h, w) in enumerate(value_spatial_shapes): - value_l = value[level].reshape(bs * n_head, c, h, w) - sampling_grid_l: torch.Tensor = sampling_locations_list[level] - - if method == "default": - sampling_value_l = F.grid_sample( - value_l, sampling_grid_l, mode="bilinear", padding_mode="zeros", align_corners=False - ) - - elif method == "discrete": - # n * m, seq, n, 2 - sampling_coord = ( - sampling_grid_l * torch.tensor([[w, h]], device=value_l.device) + 0.5 - ).to(torch.int64) - - # FIX ME? for rectangle input - sampling_coord = sampling_coord.clamp(0, h - 1) - sampling_coord = sampling_coord.reshape(bs * n_head, Len_q * num_points_list[level], 2) - - s_idx = ( - torch.arange(sampling_coord.shape[0], device=value_l.device) - .unsqueeze(-1) - .repeat(1, sampling_coord.shape[1]) - ) - sampling_value_l: torch.Tensor = value_l[ - s_idx, :, sampling_coord[..., 1], sampling_coord[..., 0] - ] # n l c - - sampling_value_l = sampling_value_l.permute(0, 2, 1).reshape( - bs * n_head, c, Len_q, num_points_list[level] - ) - - sampling_value_list.append(sampling_value_l) - - attn_weights = attention_weights.permute(0, 2, 1, 3).reshape( - bs * n_head, 1, Len_q, sum(num_points_list) - ) - weighted_sample_locs = torch.concat(sampling_value_list, dim=-1) * attn_weights - output = weighted_sample_locs.sum(-1).reshape(bs, n_head * c, Len_q) - - return output.permute(0, 2, 1) - - -def get_activation(act: str, inpace: bool = True): - """get activation""" - if act is None: - return nn.Identity() - - elif isinstance(act, nn.Module): - return act - - act = act.lower() - - if act == "silu" or act == "swish": - m = nn.SiLU() - - elif act == "relu": - m = nn.ReLU() - - elif act == "leaky_relu": - m = nn.LeakyReLU() - - elif act == "silu": - m = nn.SiLU() - - elif act == "gelu": - m = nn.GELU() - - elif act == "hardsigmoid": - m = nn.Hardsigmoid() - - else: - raise RuntimeError("") - - if hasattr(m, "inplace"): - m.inplace = inpace - - return m diff --git a/vlm/detector/D-FINE.backup/tools/benchmark/dataset.py b/vlm/detector/D-FINE.backup/tools/benchmark/dataset.py deleted file mode 100644 index 0eaeeef..0000000 --- a/vlm/detector/D-FINE.backup/tools/benchmark/dataset.py +++ /dev/null @@ -1,113 +0,0 @@ -""" -Copied from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright(c) 2023 lyuwenyu. All Rights Reserved. -""" - -import glob -import os - -import torch -import torch.utils.data as data -import torchvision -import torchvision.transforms as T -import torchvision.transforms.functional as F -from PIL import Image - -Image.MAX_IMAGE_PIXELS = None - - -class ToTensor(T.ToTensor): - def __init__(self) -> None: - super().__init__() - - def __call__(self, pic): - if isinstance(pic, torch.Tensor): - return pic - return super().__call__(pic) - - -class PadToSize(T.Pad): - def __init__(self, size, fill=0, padding_mode="constant"): - super().__init__(0, fill, padding_mode) - self.size = size - self.fill = fill - - def __call__(self, img): - """ - Args: - img (PIL Image or Tensor): Image to be padded. - - Returns: - PIL Image or Tensor: Padded image. - """ - w, h = F.get_image_size(img) - padding = (0, 0, self.size[0] - w, self.size[1] - h) - return F.pad(img, padding, self.fill, self.padding_mode) - - -class Dataset(data.Dataset): - def __init__(self, img_dir: str = "", preprocess: T.Compose = None, device="cuda:0") -> None: - super().__init__() - - self.device = device - self.size = 640 - - self.im_path_list = list(glob.glob(os.path.join(img_dir, "*.jpg"))) - - if preprocess is None: - self.preprocess = T.Compose( - [ - T.Resize(size=639, max_size=640), - PadToSize(size=(640, 640), fill=114), - ToTensor(), - T.ConvertImageDtype(torch.float), - ] - ) - else: - self.preprocess = preprocess - - def __len__( - self, - ): - return len(self.im_path_list) - - def __getitem__(self, index): - # im = Image.open(self.img_path_list[index]).convert('RGB') - im = torchvision.io.read_file(self.im_path_list[index]) - im = torchvision.io.decode_jpeg( - im, mode=torchvision.io.ImageReadMode.RGB, device=self.device - ) - _, h, w = im.shape # c,h,w - - im = self.preprocess(im) - - blob = { - "images": im, - "im_shape": torch.tensor([self.size, self.size]).to(im.device), - "scale_factor": torch.tensor([self.size / h, self.size / w]).to(im.device), - "orig_target_sizes": torch.tensor([w, h]).to(im.device), - } - - return blob - - @staticmethod - def post_process(): - pass - - @staticmethod - def collate_fn(): - pass - - -def draw_nms_result(blob, outputs, draw_score_threshold=0.25, name=""): - """show result - Keys: - 'num_dets', 'det_boxes', 'det_scores', 'det_classes' - """ - for i in range(blob["image"].shape[0]): - det_scores = outputs["det_scores"][i] - det_boxes = outputs["det_boxes"][i][det_scores > draw_score_threshold] - - im = (blob["image"][i] * 255).to(torch.uint8) - im = torchvision.utils.draw_bounding_boxes(im, boxes=det_boxes, width=2) - Image.fromarray(im.permute(1, 2, 0).cpu().numpy()).save(f"test_{name}_{i}.jpg") diff --git a/vlm/detector/D-FINE.backup/tools/benchmark/get_info.py b/vlm/detector/D-FINE.backup/tools/benchmark/get_info.py deleted file mode 100644 index bcef786..0000000 --- a/vlm/detector/D-FINE.backup/tools/benchmark/get_info.py +++ /dev/null @@ -1,60 +0,0 @@ -""" -Copyright (c) 2024 The D-FINE Authors. All Rights Reserved. -""" - -import os -import sys - -sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "../..")) - -import argparse - -import torch -import torch.nn as nn -from calflops import calculate_flops - -from src.core import YAMLConfig - - -def custom_repr(self): - return f"{{Tensor:{tuple(self.shape)}}} {original_repr(self)}" - - -original_repr = torch.Tensor.__repr__ -torch.Tensor.__repr__ = custom_repr - - -def main( - args, -): - """main""" - cfg = YAMLConfig(args.config, resume=None) - - class Model_for_flops(nn.Module): - def __init__( - self, - ) -> None: - super().__init__() - self.model = cfg.model.deploy() - - def forward(self, images): - outputs = self.model(images) - return outputs - - model = Model_for_flops().eval() - - flops, macs, _ = calculate_flops( - model=model, input_shape=(1, 3, 640, 640), output_as_string=True, output_precision=4 - ) - params = sum(p.numel() for p in model.parameters()) - print("Model FLOPs:%s MACs:%s Params:%s \n" % (flops, macs, params)) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "--config", "-c", default="configs/dfine/dfine_hgnetv2_l_coco.yml", type=str - ) - args = parser.parse_args() - - main(args) diff --git a/vlm/detector/D-FINE.backup/tools/benchmark/requirements.txt b/vlm/detector/D-FINE.backup/tools/benchmark/requirements.txt deleted file mode 100644 index 55a3c0f..0000000 --- a/vlm/detector/D-FINE.backup/tools/benchmark/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -onnxruntime -tensorrt -pycuda -calflops -tqdm -# onnx_graphsurgeon # for YOLOs diff --git a/vlm/detector/D-FINE.backup/tools/benchmark/trt_benchmark.py b/vlm/detector/D-FINE.backup/tools/benchmark/trt_benchmark.py deleted file mode 100644 index 5b8b3b5..0000000 --- a/vlm/detector/D-FINE.backup/tools/benchmark/trt_benchmark.py +++ /dev/null @@ -1,214 +0,0 @@ -""" -Copyright (c) 2024 The D-FINE Authors. All Rights Reserved. -""" - -import argparse -import glob -import os -import time -from collections import OrderedDict, namedtuple - -import numpy as np -import pycuda.driver as cuda -import tensorrt as trt -import torch -from dataset import Dataset -from tqdm import tqdm -from utils import TimeProfiler - - -def parse_args(): - parser = argparse.ArgumentParser(description="Argument Parser Example") - parser.add_argument( - "--infer_dir", - type=str, - default="/data/COCO2017/val2017", - help="Directory for images to perform inference on.", - ) - parser.add_argument("--engine_dir", type=str, help="Directory containing model engine files.") - parser.add_argument( - "--busy", action="store_true", help="Flag to indicate that other processes may be running." - ) - args = parser.parse_args() - return args - - -class TRTInference(object): - def __init__( - self, engine_path, device="cuda", backend="torch", max_batch_size=32, verbose=False - ): - self.engine_path = engine_path - self.device = device - self.backend = backend - self.max_batch_size = max_batch_size - - self.logger = trt.Logger(trt.Logger.VERBOSE) if verbose else trt.Logger(trt.Logger.INFO) - self.engine = self.load_engine(engine_path) - self.context = self.engine.create_execution_context() - self.bindings = self.get_bindings( - self.engine, self.context, self.max_batch_size, self.device - ) - self.bindings_addr = OrderedDict((n, v.ptr) for n, v in self.bindings.items()) - self.input_names = self.get_input_names() - self.output_names = self.get_output_names() - - if self.backend == "cuda": - self.stream = cuda.Stream() - self.time_profile = TimeProfiler() - self.time_profile_dataset = TimeProfiler() - - def init(self): - self.dynamic = False - - def load_engine(self, path): - trt.init_libnvinfer_plugins(self.logger, "") - with open(path, "rb") as f, trt.Runtime(self.logger) as runtime: - return runtime.deserialize_cuda_engine(f.read()) - - def get_input_names(self): - names = [] - for _, name in enumerate(self.engine): - if self.engine.get_tensor_mode(name) == trt.TensorIOMode.INPUT: - names.append(name) - return names - - def get_output_names(self): - names = [] - for _, name in enumerate(self.engine): - if self.engine.get_tensor_mode(name) == trt.TensorIOMode.OUTPUT: - names.append(name) - return names - - def get_bindings(self, engine, context, max_batch_size=32, device=None): - Binding = namedtuple("Binding", ("name", "dtype", "shape", "data", "ptr")) - bindings = OrderedDict() - for i, name in enumerate(engine): - shape = engine.get_tensor_shape(name) - dtype = trt.nptype(engine.get_tensor_dtype(name)) - - if shape[0] == -1: - dynamic = True - shape[0] = max_batch_size - if engine.get_tensor_mode(name) == trt.TensorIOMode.INPUT: - context.set_input_shape(name, shape) - - if self.backend == "cuda": - if engine.get_tensor_mode(name) == trt.TensorIOMode.INPUT: - data = np.random.randn(*shape).astype(dtype) - ptr = cuda.mem_alloc(data.nbytes) - bindings[name] = Binding(name, dtype, shape, data, ptr) - else: - data = cuda.pagelocked_empty(trt.volume(shape), dtype) - ptr = cuda.mem_alloc(data.nbytes) - bindings[name] = Binding(name, dtype, shape, data, ptr) - else: - data = torch.from_numpy(np.empty(shape, dtype=dtype)).to(device) - bindings[name] = Binding(name, dtype, shape, data, data.data_ptr()) - return bindings - - def run_torch(self, blob): - for n in self.input_names: - if self.bindings[n].shape != blob[n].shape: - self.context.set_input_shape(n, blob[n].shape) - self.bindings[n] = self.bindings[n]._replace(shape=blob[n].shape) - - self.bindings_addr.update({n: blob[n].data_ptr() for n in self.input_names}) - self.context.execute_v2(list(self.bindings_addr.values())) - outputs = {n: self.bindings[n].data for n in self.output_names} - return outputs - - def async_run_cuda(self, blob): - for n in self.input_names: - cuda.memcpy_htod_async(self.bindings_addr[n], blob[n], self.stream) - - bindings_addr = [int(v) for _, v in self.bindings_addr.items()] - self.context.execute_async_v2(bindings=bindings_addr, stream_handle=self.stream.handle) - - outputs = {} - for n in self.output_names: - cuda.memcpy_dtoh_async(self.bindings[n].data, self.bindings[n].ptr, self.stream) - outputs[n] = self.bindings[n].data - - self.stream.synchronize() - - return outputs - - def __call__(self, blob): - if self.backend == "torch": - return self.run_torch(blob) - elif self.backend == "cuda": - return self.async_run_cuda(blob) - - def synchronize(self): - if self.backend == "torch" and torch.cuda.is_available(): - torch.cuda.synchronize() - elif self.backend == "cuda": - self.stream.synchronize() - - def warmup(self, blob, n): - for _ in range(n): - _ = self(blob) - - def speed(self, blob, n, nonempty_process=False): - times = [] - self.time_profile_dataset.reset() - for i in tqdm(range(n), desc="Running Inference", unit="iteration"): - self.time_profile.reset() - with self.time_profile_dataset: - img = blob[i] - if img["images"] is not None: - img["image"] = img["input"] = img["images"].unsqueeze(0) - else: - img["images"] = img["input"] = img["image"].unsqueeze(0) - with self.time_profile: - _ = self(img) - times.append(self.time_profile.total) - - # end-to-end model only - times = sorted(times) - if len(times) > 100 and nonempty_process: - times = times[:100] - - avg_time = sum(times) / len(times) # Calculate the average of the remaining times - return avg_time - - -def main(): - FLAGS = parse_args() - dataset = Dataset(FLAGS.infer_dir) - im = torch.ones(1, 3, 640, 640).cuda() - blob = { - "image": im, - "images": im, - "input": im, - "im_shape": torch.tensor([640, 640]).to(im.device), - "scale_factor": torch.tensor([1, 1]).to(im.device), - "orig_target_sizes": torch.tensor([640, 640]).to(im.device), - } - - engine_files = glob.glob(os.path.join(FLAGS.engine_dir, "*.engine")) - results = [] - - for engine_file in engine_files: - print(f"Testing engine: {engine_file}") - model = TRTInference(engine_file, max_batch_size=1, verbose=False) - model.init() - model.warmup(blob, 1000) - t = [] - for _ in range(1): - t.append(model.speed(dataset, 1000, FLAGS.busy)) - avg_latency = 1000 * torch.tensor(t).mean() - results.append((engine_file, avg_latency)) - print(f"Engine: {engine_file}, Latency: {avg_latency:.2f} ms") - - del model - torch.cuda.empty_cache() - time.sleep(1) - - sorted_results = sorted(results, key=lambda x: x[1]) - for engine_file, latency in sorted_results: - print(f"Engine: {engine_file}, Latency: {latency:.2f} ms") - - -if __name__ == "__main__": - main() diff --git a/vlm/detector/D-FINE.backup/tools/benchmark/utils.py b/vlm/detector/D-FINE.backup/tools/benchmark/utils.py deleted file mode 100644 index aaa6c1e..0000000 --- a/vlm/detector/D-FINE.backup/tools/benchmark/utils.py +++ /dev/null @@ -1,95 +0,0 @@ -import contextlib -import time -from collections import OrderedDict - -import numpy as np -import onnx -import onnx_graphsurgeon -import torch -from PIL import Image - - -def to_binary_data(path, size=(640, 640), output_name="input_tensor.bin"): - """--loadInputs='image:input_tensor.bin'""" - im = Image.open(path).resize(size) - data = np.asarray(im, dtype=np.float32).transpose(2, 0, 1)[None] / 255.0 - data.tofile(output_name) - - -def yolo_insert_nms( - path, score_threshold=0.01, iou_threshold=0.7, max_output_boxes=300, simplify=False -): - """ - http://www.xavierdupre.fr/app/onnxcustom/helpsphinx/api/onnxops/onnx__EfficientNMS_TRT.html - https://huggingface.co/spaces/muttalib1326/Punjabi_Character_Detection/blob/3dd1e17054c64e5f6b2254278f96cfa2bf418cd4/utils/add_nms.py - """ - onnx_model = onnx.load(path) - - if simplify: - from onnxsim import simplify - - onnx_model, _ = simplify(onnx_model, overwrite_input_shapes={"image": [1, 3, 640, 640]}) - - graph = onnx_graphsurgeon.import_onnx(onnx_model) - graph.toposort() - graph.fold_constants() - graph.cleanup() - - topk = max_output_boxes - attrs = OrderedDict( - plugin_version="1", - background_class=-1, - max_output_boxes=topk, - score_threshold=score_threshold, - iou_threshold=iou_threshold, - score_activation=False, - box_coding=0, - ) - - outputs = [ - onnx_graphsurgeon.Variable("num_dets", np.int32, [-1, 1]), - onnx_graphsurgeon.Variable("det_boxes", np.float32, [-1, topk, 4]), - onnx_graphsurgeon.Variable("det_scores", np.float32, [-1, topk]), - onnx_graphsurgeon.Variable("det_classes", np.int32, [-1, topk]), - ] - - graph.layer( - op="EfficientNMS_TRT", - name="batched_nms", - inputs=[graph.outputs[0], graph.outputs[1]], - outputs=outputs, - attrs=attrs, - ) - - graph.outputs = outputs - graph.cleanup().toposort() - - onnx.save(onnx_graphsurgeon.export_onnx(graph), "yolo_w_nms.onnx") - - -class TimeProfiler(contextlib.ContextDecorator): - def __init__( - self, - ): - self.total = 0 - - def __enter__( - self, - ): - self.start = self.time() - return self - - def __exit__(self, type, value, traceback): - self.total += self.time() - self.start - - def reset( - self, - ): - self.total = 0 - - def time( - self, - ): - if torch.cuda.is_available(): - torch.cuda.synchronize() - return time.time() diff --git a/vlm/detector/D-FINE.backup/tools/dataset/remap_obj365.py b/vlm/detector/D-FINE.backup/tools/dataset/remap_obj365.py deleted file mode 100644 index 9102eea..0000000 --- a/vlm/detector/D-FINE.backup/tools/dataset/remap_obj365.py +++ /dev/null @@ -1,153 +0,0 @@ -""" -Copyright (c) 2024 The D-FINE Authors. All Rights Reserved. -""" - -import argparse -import json -import os - - -def update_image_paths(images, new_prefix): - print("Updating image paths with new prefix...") - for img in images: - split = img["file_name"].split("/")[1:] - img["file_name"] = os.path.join(new_prefix, *split) - print("Image paths updated.") - return images - - -def create_split_annotations(original_annotations, split_image_ids, new_prefix, output_file): - print(f"Creating split annotations for {output_file}...") - new_images = [img for img in original_annotations["images"] if img["id"] in split_image_ids] - print(f"Number of images selected: {len(new_images)}") - if new_prefix is not None: - new_images = update_image_paths(new_images, new_prefix) - - new_annotations = { - "images": new_images, - "annotations": [ - ann for ann in original_annotations["annotations"] if ann["image_id"] in split_image_ids - ], - "categories": original_annotations["categories"], - } - print(f'Number of annotations selected: {len(new_annotations["annotations"])}') - with open(output_file, "w") as f: - json.dump(new_annotations, f) - print(f"Annotations saved to {output_file}") - - -def parse_arguments(): - parser = argparse.ArgumentParser(description="Split and update dataset annotations.") - parser.add_argument( - "--base_dir", - type=str, - required=True, - help="Base directory of the dataset, e.g., /data/Objects365/data", - ) - parser.add_argument( - "--new_val_size", - type=int, - default=5000, - help="Number of images to include in the new validation set (default: 5000)", - ) - parser.add_argument( - "--output_suffix", - type=str, - default="new", - help="Suffix to add to new annotation files (default: new)", - ) - return parser.parse_args() - - -def main(): - args = parse_arguments() - base_dir = args.base_dir - new_val_size = args.new_val_size - output_suffix = args.output_suffix - - # Define paths based on the base directory - original_train_ann_file = os.path.join(base_dir, "train", "zhiyuan_objv2_train.json") - original_val_ann_file = os.path.join(base_dir, "val", "zhiyuan_objv2_val.json") - - new_val_ann_file = os.path.join(base_dir, "val", f"{output_suffix}_zhiyuan_objv2_val.json") - new_train_ann_file = os.path.join( - base_dir, "train", f"{output_suffix}_zhiyuan_objv2_train.json" - ) - - # Check if original annotation files exist - if not os.path.isfile(original_train_ann_file): - print(f"Error: Training annotation file not found at {original_train_ann_file}") - return - if not os.path.isfile(original_val_ann_file): - print(f"Error: Validation annotation file not found at {original_val_ann_file}") - return - - # Load the original training and validation annotations - print("Loading original training annotations...") - with open(original_train_ann_file, "r") as f: - train_annotations = json.load(f) - print("Training annotations loaded.") - - print("Loading original validation annotations...") - with open(original_val_ann_file, "r") as f: - val_annotations = json.load(f) - print("Validation annotations loaded.") - - # Extract image IDs from the original validation set - print("Extracting image IDs from the validation set...") - val_image_ids = [img["id"] for img in val_annotations["images"]] - print(f"Total validation images: {len(val_image_ids)}") - - # Split image IDs for the new training and validation sets - print( - f"Splitting validation images into new validation set of size {new_val_size} and training set..." - ) - new_val_image_ids = val_image_ids[:new_val_size] - new_train_image_ids = val_image_ids[new_val_size:] - print(f"New validation set size: {len(new_val_image_ids)}") - print(f"New training set size from validation images: {len(new_train_image_ids)}") - - # Create new validation annotation file - print("Creating new validation annotations...") - create_split_annotations(val_annotations, new_val_image_ids, None, new_val_ann_file) - print("New validation annotations created.") - - # Combine the remaining validation images and annotations with the original training data - print("Preparing new training images and annotations...") - new_train_images = [ - img for img in val_annotations["images"] if img["id"] in new_train_image_ids - ] - print(f"Number of images from validation to add to training: {len(new_train_images)}") - new_train_images = update_image_paths(new_train_images, "images_from_val") - new_train_annotations = [ - ann for ann in val_annotations["annotations"] if ann["image_id"] in new_train_image_ids - ] - print(f"Number of annotations from validation to add to training: {len(new_train_annotations)}") - - # Add the original training images and annotations - print("Adding original training images and annotations...") - new_train_images.extend(train_annotations["images"]) - new_train_annotations.extend(train_annotations["annotations"]) - print(f"Total training images: {len(new_train_images)}") - print(f"Total training annotations: {len(new_train_annotations)}") - - # Create a new training annotation dictionary - print("Creating new training annotations dictionary...") - new_train_annotations_dict = { - "images": new_train_images, - "annotations": new_train_annotations, - "categories": train_annotations["categories"], - } - print("New training annotations dictionary created.") - - # Save the new training annotations - print("Saving new training annotations...") - with open(new_train_ann_file, "w") as f: - json.dump(new_train_annotations_dict, f) - print(f"New training annotations saved to {new_train_ann_file}") - - print("Processing completed successfully.") - - -if __name__ == "__main__": - main() diff --git a/vlm/detector/D-FINE.backup/tools/dataset/resize_obj365.py b/vlm/detector/D-FINE.backup/tools/dataset/resize_obj365.py deleted file mode 100644 index 6ccaeac..0000000 --- a/vlm/detector/D-FINE.backup/tools/dataset/resize_obj365.py +++ /dev/null @@ -1,153 +0,0 @@ -""" -Copyright (c) 2024 The D-FINE Authors. All Rights Reserved. -""" - -import argparse -import json -import os -from concurrent.futures import ThreadPoolExecutor - -from PIL import Image - - -def resize_image_and_update_annotations(image_path, annotations, max_size=640): - print(f"Processing image: {image_path}") - try: - with Image.open(image_path) as img: - w, h = img.size - if max(w, h) <= max_size: - return annotations, w, h, False # No need to resize - - scale = max_size / max(w, h) - new_w = int(w * scale) - new_h = int(h * scale) - print(f"Resizing image to width={new_w}, height={new_h}") - - img = img.resize((new_w, new_h), Image.Resampling.LANCZOS) - new_image_path = image_path.replace(".jpg", "_resized.jpg") - img.save(new_image_path) - print(f"Resized image saved: {new_image_path}") - print(f"Original size: ({w}, {h}), New size: ({new_w}, {new_h})") - - # Update annotations - for ann in annotations: - ann["area"] = ann["area"] * (scale**2) - ann["bbox"] = [coord * scale for coord in ann["bbox"]] - if "orig_size" in ann: - ann["orig_size"] = (new_w, new_h) - if "size" in ann: - ann["size"] = (new_w, new_h) - - except Exception as e: - print(f"Error processing {image_path}: {e}") - return None - - return annotations, new_w, new_h, True - - -def resize_images_and_update_annotations(base_dir, subset, max_size=640, num_workers=4): - print(f"Starting to resize images and update annotations for subset: {subset}") - json_file = os.path.join(base_dir, subset, "new_zhiyuan_objv2_{}.json".format(subset)) - if not os.path.isfile(json_file): - print(f"Error: JSON file not found at {json_file}") - return - - print(f"Loading JSON file: {json_file}") - with open(json_file, "r") as f: - data = json.load(f) - print("JSON file loaded.") - - print("Preparing image annotations mapping...") - image_annotations = {img["id"]: [] for img in data["images"]} - for ann in data["annotations"]: - image_annotations[ann["image_id"]].append(ann) - print("Image annotations mapping prepared.") - - def process_image(image_info): - image_path = os.path.join(base_dir, subset, image_info["file_name"]) - results = resize_image_and_update_annotations( - image_path, image_annotations[image_info["id"]], max_size - ) - if results is None: - updated_annotations, new_w, new_h, resized = None, None, None, None - else: - updated_annotations, new_w, new_h, resized = results - return image_info, updated_annotations, new_w, new_h, resized - - print(f"Processing images with {num_workers} worker threads...") - with ThreadPoolExecutor(max_workers=num_workers) as executor: - results = list(executor.map(process_image, data["images"])) - print("Image processing completed.") - - new_images = [] - new_annotations = [] - - print("Updating image and annotation data...") - for image_info, updated_annotations, new_w, new_h, resized in results: - if updated_annotations is not None: - image_info["width"] = new_w - image_info["height"] = new_h - image_annotations[image_info["id"]] = updated_annotations - if resized: - image_info["file_name"] = image_info["file_name"].replace(".jpg", "_resized.jpg") - new_images.append(image_info) - new_annotations.extend(updated_annotations) - print(f"Total images processed: {len(new_images)}") - print(f"Total annotations updated: {len(new_annotations)}") - - new_data = { - "images": new_images, - "annotations": new_annotations, - "categories": data["categories"], - } - - new_json_file = json_file.replace(".json", "_resized.json") - print("Saving new training annotations...") - with open(new_json_file, "w") as f: - json.dump(new_data, f) - print(f"New JSON file saved to {new_json_file}") - - -def parse_arguments(): - parser = argparse.ArgumentParser( - description="Resize images and update dataset annotations for both train and val sets." - ) - parser.add_argument( - "--base_dir", - type=str, - required=True, - help="Base directory of the dataset, e.g., /data/Objects365/data", - ) - parser.add_argument( - "--max_size", - type=int, - default=640, - help="Maximum size for the longer side of the image (default: 640)", - ) - parser.add_argument( - "--num_workers", - type=int, - default=4, - help="Number of worker threads for parallel processing (default: 4)", - ) - args = parser.parse_args() - return args - - -def main(): - args = parse_arguments() - base_dir = args.base_dir - max_size = args.max_size - num_workers = args.num_workers - - subsets = ["train", "val"] - for subset in subsets: - print(f"Processing subset: {subset}") - resize_images_and_update_annotations( - base_dir=base_dir, subset=subset, max_size=max_size, num_workers=num_workers - ) - print("All subsets processed.") - - -if __name__ == "__main__": - main() diff --git a/vlm/detector/D-FINE.backup/tools/deployment/export_onnx.py b/vlm/detector/D-FINE.backup/tools/deployment/export_onnx.py deleted file mode 100644 index 58a12cd..0000000 --- a/vlm/detector/D-FINE.backup/tools/deployment/export_onnx.py +++ /dev/null @@ -1,128 +0,0 @@ -""" -D-FINE: Redefine Regression Task of DETRs as Fine-grained Distribution Refinement -Copyright (c) 2024 The D-FINE Authors. All Rights Reserved. ---------------------------------------------------------------------------------- -Modified from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright (c) 2023 lyuwenyu. All Rights Reserved. -""" - -import os -import sys - -sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "../..")) - -import torch -import torch.nn as nn - -from src.core import YAMLConfig - - -def main( - args, -): - """main""" - cfg = YAMLConfig(args.config, resume=args.resume) - - if "HGNetv2" in cfg.yaml_cfg: - cfg.yaml_cfg["HGNetv2"]["pretrained"] = False - - if args.resume: - checkpoint = torch.load(args.resume, map_location="cpu") - if "ema" in checkpoint: - state = checkpoint["ema"]["module"] - else: - state = checkpoint["model"] - - # NOTE load train mode state -> convert to deploy mode - cfg.model.load_state_dict(state) - - else: - # raise AttributeError('Only support resume to load model.state_dict by now.') - print("not load model.state_dict, use default init state dict...") - - class Model(nn.Module): - def __init__( - self, - ) -> None: - super().__init__() - self.model = cfg.model.deploy() - self.postprocessor = cfg.postprocessor.deploy() - - def forward(self, images, orig_target_sizes): - outputs = self.model(images) - outputs = self.postprocessor(outputs, orig_target_sizes) - return outputs - - model = Model() - - data = torch.rand(32, 3, 640, 640) - size = torch.tensor([[640, 640]]) - _ = model(data, size) - - dynamic_axes = { - "images": { - 0: "N", - }, - "orig_target_sizes": {0: "N"}, - } - - output_file = args.resume.replace(".pth", ".onnx") if args.resume else "model.onnx" - - torch.onnx.export( - model, - (data, size), - output_file, - input_names=["images", "orig_target_sizes"], - output_names=["labels", "boxes", "scores"], - dynamic_axes=dynamic_axes, - opset_version=16, - verbose=False, - do_constant_folding=True, - ) - - if args.check: - import onnx - - onnx_model = onnx.load(output_file) - onnx.checker.check_model(onnx_model) - print("Check export onnx model done...") - - if args.simplify: - import onnx - import onnxsim - - dynamic = True - # input_shapes = {'images': [1, 3, 640, 640], 'orig_target_sizes': [1, 2]} if dynamic else None - input_shapes = {"images": data.shape, "orig_target_sizes": size.shape} if dynamic else None - onnx_model_simplify, check = onnxsim.simplify(output_file, test_input_shapes=input_shapes) - onnx.save(onnx_model_simplify, output_file) - print(f"Simplify onnx model {check}...") - - -if __name__ == "__main__": - import argparse - - parser = argparse.ArgumentParser() - parser.add_argument( - "--config", - "-c", - default="configs/dfine/dfine_hgnetv2_l_coco.yml", - type=str, - ) - parser.add_argument( - "--resume", - "-r", - type=str, - ) - parser.add_argument( - "--check", - action="store_true", - default=True, - ) - parser.add_argument( - "--simplify", - action="store_true", - default=True, - ) - args = parser.parse_args() - main(args) diff --git a/vlm/detector/D-FINE.backup/tools/deployment/export_yolo_w_nms.py b/vlm/detector/D-FINE.backup/tools/deployment/export_yolo_w_nms.py deleted file mode 100644 index 206ae26..0000000 --- a/vlm/detector/D-FINE.backup/tools/deployment/export_yolo_w_nms.py +++ /dev/null @@ -1,75 +0,0 @@ -import numpy as np -import onnxruntime as ort -import torch -import torchvision -from utils import yolo_insert_nms - - -class YOLO11(torch.nn.Module): - def __init__(self, name) -> None: - super().__init__() - from ultralytics import YOLO - # Load a model - # build a new model from scratch - # model = YOLO(f'{name}.yaml') - - # load a pretrained model (recommended for training) - model = YOLO("yolo11n.pt") - self.model = model.model - - def forward(self, x): - """https://github.com/ultralytics/ultralytics/blob/main/ultralytics/nn/tasks.py#L216""" - pred: torch.Tensor = self.model(x)[0] # n 84 8400, - pred = pred.permute(0, 2, 1) - boxes, scores = pred.split([4, 80], dim=-1) - boxes = torchvision.ops.box_convert(boxes, in_fmt="cxcywh", out_fmt="xyxy") - - return boxes, scores - - -def export_onnx(name="yolov8n"): - """export onnx""" - m = YOLO11(name) - - x = torch.rand(1, 3, 640, 640) - dynamic_axes = {"image": {0: "-1"}} - torch.onnx.export( - m, - x, - f"{name}.onnx", - input_names=["image"], - output_names=["boxes", "scores"], - opset_version=13, - dynamic_axes=dynamic_axes, - ) - - data = np.random.rand(1, 3, 640, 640).astype(np.float32) - sess = ort.InferenceSession(f"{name}.onnx") - _ = sess.run(output_names=None, input_feed={"image": data}) - - import onnx - import onnxslim - - model_onnx = onnx.load(f"{name}.onnx") - model_onnx = onnxslim.slim(model_onnx) - onnx.save(model_onnx, f"{name}.onnx") - - -if __name__ == "__main__": - import argparse - - parser = argparse.ArgumentParser() - parser.add_argument("--name", type=str, default="yolo11n_tuned") - parser.add_argument("--score_threshold", type=float, default=0.01) - parser.add_argument("--iou_threshold", type=float, default=0.6) - parser.add_argument("--max_output_boxes", type=int, default=300) - args = parser.parse_args() - - export_onnx(name=args.name) - - yolo_insert_nms( - path=f"{args.name}.onnx", - score_threshold=args.score_threshold, - iou_threshold=args.iou_threshold, - max_output_boxes=args.max_output_boxes, - ) diff --git a/vlm/detector/D-FINE.backup/tools/inference/cppExample/onnx/CMakeLists.txt b/vlm/detector/D-FINE.backup/tools/inference/cppExample/onnx/CMakeLists.txt deleted file mode 100644 index 85f2ade..0000000 --- a/vlm/detector/D-FINE.backup/tools/inference/cppExample/onnx/CMakeLists.txt +++ /dev/null @@ -1,38 +0,0 @@ -cmake_minimum_required(VERSION 3.5.1) - -project( - onnxExample - LANGUAGES CXX - VERSION 1.0.0 -) - -set(CMAKE_CXX_STANDARD 17) - -# Compiler options -if (MSVC) - add_compile_options(/W4 /permissive-) - add_definitions(-DNOMINMAX) -else() - add_compile_options(-Wall -Wextra -Wpedantic) -endif() - -# Find ONNX Runtime package -find_package(onnxruntime REQUIRED) - -# Find OpenCV -find_package(OpenCV REQUIRED core imgcodecs imgproc) - -# Add executable -add_executable( - onnxExample - onnxExample.cpp -) - -# Link libraries -target_link_libraries( - onnxExample - onnxruntime::onnxruntime - opencv_core - opencv_imgcodecs - opencv_imgproc -) diff --git a/vlm/detector/D-FINE.backup/tools/inference/cppExample/onnx/onnxExample.cpp b/vlm/detector/D-FINE.backup/tools/inference/cppExample/onnx/onnxExample.cpp deleted file mode 100644 index 62c98db..0000000 --- a/vlm/detector/D-FINE.backup/tools/inference/cppExample/onnx/onnxExample.cpp +++ /dev/null @@ -1,331 +0,0 @@ -๏ปฟ -#include -#include -#include -#include -#include -#include - - -std::vector input_names = { "images", "orig_target_sizes" }; -std::vector output_names = { "labels", "boxes", "scores" }; - -/** - * @brief Draws bounding boxes, labels, and confidence scores on an image. - * - * This function takes an image, a list of labels, bounding boxes, and their corresponding confidence scores, - * and overlays the bounding boxes and labels on the image. The bounding boxes are adjusted to compensate - * for resizing and padding applied during preprocessing. - * - * @param image The input image (cv::Mat) on which to draw the bounding boxes and labels. - * @param labels A vector of integer labels corresponding to detected objects. - * @param boxes A vector of bounding boxes, where each box is represented as {x1, y1, x2, y2}. - * @param scores A vector of confidence scores corresponding to the bounding boxes. - * @param ratio The scaling factor used to resize the image during preprocessing. - * @param pad_w The horizontal padding applied to the image during preprocessing. - * @param pad_h The vertical padding applied to the image during preprocessing. - * @param thrh The confidence threshold; only boxes with scores above this value will be drawn (default is 0.4). - * @return A cv::Mat object containing the original image with bounding boxes, labels, and scores drawn on it. - */ -cv::Mat draw( - const cv::Mat& image, - const std::vector& labels, - const std::vector>& boxes, - const std::vector& scores, - float ratio, - int pad_w, - int pad_h, - float thrh = 0.4) -{ - // Clone the input image to preserve the original image - cv::Mat img = image.clone(); - - // Iterate over all detected objects - for (size_t i = 0; i < scores.size(); ++i) { - // Only process objects with confidence scores above the threshold - if (scores[i] > thrh) { - // Adjust bounding box coordinates to account for resizing and padding - float x1 = (boxes[i][0] - pad_w) / ratio; // Top-left x-coordinate - float y1 = (boxes[i][1] - pad_h) / ratio; // Top-left y-coordinate - float x2 = (boxes[i][2] - pad_w) / ratio; // Bottom-right x-coordinate - float y2 = (boxes[i][3] - pad_h) / ratio; // Bottom-right y-coordinate - - // Draw the bounding box on the image - cv::rectangle(img, cv::Point(x1, y1), cv::Point(x2, y2), cv::Scalar(0, 0, 255), 1); - - // Prepare the label text with class label and confidence score - std::string label_text = "Label: " + std::to_string(labels[i]) + - " Conf: " + std::to_string(scores[i]); - - // Draw the label text above the bounding box - cv::putText(img, label_text, cv::Point(x1, y1 - 10), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(255, 0, 0), 1); - } - } - - // Return the annotated image - return img; -} - - - -/** - * @brief Resizes an image while maintaining its aspect ratio and pads the resized image to a square of a specified size. - * - * This function scales the input image proportionally to fit within a square of the specified size while preserving - * the aspect ratio. It then pads the resized image with black pixels (value 0) to fill the remaining space, creating - * a square output image. - * - * @param image Input image (cv::Mat) to be resized and padded. - * @param size Target size of the square output image (both width and height will be equal to size). - * @param ratio Output parameter that will contain the scaling factor applied to the image. - * @param pad_w Output parameter that will contain the width of padding applied on the left and right sides. - * @param pad_h Output parameter that will contain the height of padding applied on the top and bottom sides. - * @return A cv::Mat object containing the resized and padded square image. - */ -cv::Mat resizeWithAspectRatio(const cv::Mat& image, int size, float& ratio, int& pad_w, int& pad_h) { - // Get the original width and height of the input image - int original_width = image.cols; - int original_height = image.rows; - - // Compute the scaling ratio to fit the image within the target size while maintaining aspect ratio - ratio = std::min(static_cast(size) / original_width, static_cast(size) / original_height); - int new_width = static_cast(original_width * ratio); // New width after scaling - int new_height = static_cast(original_height * ratio); // New height after scaling - - // Resize the image using the computed dimensions - cv::Mat resized_image; - cv::resize(image, resized_image, cv::Size(new_width, new_height)); - - // Calculate the padding required to center the resized image in the square output - pad_w = (size - new_width) / 2; // Horizontal padding (left and right) - pad_h = (size - new_height) / 2; // Vertical padding (top and bottom) - - // Create a square output image filled with black pixels (value 0) - cv::Mat padded_image(size, size, resized_image.type(), cv::Scalar(0, 0, 0)); - - // Copy the resized image into the center of the square output image - resized_image.copyTo(padded_image(cv::Rect(pad_w, pad_h, new_width, new_height))); - - // Return the resized and padded image - return padded_image; -} - -/** - * @brief Preprocess an input image, run inference using an ONNX model, and process the results. - * - * This function resizes the input image while maintaining its aspect ratio, prepares it for inference, - * runs the inference using the specified ONNX Runtime session, and processes the output to draw - * bounding boxes and labels on the original image. - * - * @param session The ONNX Runtime session used to perform inference. - * @param image The input image (OpenCV Mat) to process. - * @return cv::Mat The result image with bounding boxes and labels drawn. - */ -cv::Mat processImage(Ort::Session& session, const cv::Mat& image) { - float ratio; // Aspect ratio for resizing the image. - int pad_w, pad_h; // Padding added to maintain aspect ratio. - int target_size = 640; // Target size for resizing (typically square). - - // Step 1: Resize and pad the image to the target size while preserving the aspect ratio. - cv::Mat resized_image = resizeWithAspectRatio(image, target_size, ratio, pad_w, pad_h); - - // Step 2: Convert the resized image to RGB format as required by the model. - cv::cvtColor(resized_image, resized_image, cv::COLOR_BGR2RGB); - - // Step 3: Prepare the input tensor in NCHW format (channels-first). - std::vector input_dims = { 1, 3, target_size, target_size }; // Batch size = 1, Channels = 3, HxW = target_size. - std::vector input_tensor_values(input_dims[1] * input_dims[2] * input_dims[3]); - - // Populate the input tensor with normalized pixel values (range 0 to 1). - int index = 0; - for (int c = 0; c < 3; ++c) { // Loop through channels. - for (int i = 0; i < resized_image.rows; ++i) { // Loop through rows. - for (int j = 0; j < resized_image.cols; ++j) { // Loop through columns. - input_tensor_values[index++] = resized_image.at(i, j)[c] / 255.0f; // Normalize pixel value. - } - } - } - - // Step 4: Create ONNX Runtime input tensors. - Ort::MemoryInfo memory_info = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeDefault); - - // Tensor for the preprocessed image. - Ort::Value input_tensor_images = Ort::Value::CreateTensor( - memory_info, input_tensor_values.data(), input_tensor_values.size(), - input_dims.data(), input_dims.size() - ); - - // Tensor for the original target sizes (optional, used for postprocessing). - std::vector orig_size_dims = { 1, 2 }; - std::vector orig_size_values = { - static_cast(resized_image.rows), - static_cast(resized_image.cols) - }; - Ort::Value input_tensor_orig_target_sizes = Ort::Value::CreateTensor( - memory_info, orig_size_values.data(), orig_size_values.size(), - orig_size_dims.data(), orig_size_dims.size() - ); - - // Step 5: Run inference on the session. - auto outputs = session.Run( - Ort::RunOptions{ nullptr }, // Default run options. - input_names.data(), // Names of input nodes. - std::array{std::move(input_tensor_images), std::move(input_tensor_orig_target_sizes)}.data(), - input_names.size(), // Number of inputs. - output_names.data(), // Names of output nodes. - output_names.size() // Number of outputs. - ); - - // Step 6: Extract and process model outputs. - auto labels_ptr = outputs[0].GetTensorMutableData(); // Labels for detected objects. - auto boxes_ptr = outputs[1].GetTensorMutableData(); // Bounding boxes. - auto scores_ptr = outputs[2].GetTensorMutableData(); // Confidence scores. - - size_t num_boxes = outputs[2].GetTensorTypeAndShapeInfo().GetShape()[1]; // Number of detected boxes. - - // Convert raw output to structured data. - std::vector labels(labels_ptr, labels_ptr + num_boxes); - std::vector> boxes; - std::vector scores(scores_ptr, scores_ptr + num_boxes); - - auto boxes_shape = outputs[1].GetTensorTypeAndShapeInfo().GetShape(); - size_t num_coordinates = boxes_shape[2]; // Usually 4 coordinates: (x1, y1, x2, y2). - - // Populate the `boxes` vector. - for (size_t i = 0; i < num_boxes; ++i) { - boxes.push_back({ - boxes_ptr[i * num_coordinates + 0], // x1 - boxes_ptr[i * num_coordinates + 1], // y1 - boxes_ptr[i * num_coordinates + 2], // x2 - boxes_ptr[i * num_coordinates + 3] // y2 - }); - } - - // Step 7: Draw the results on the original image. - cv::Mat result_image = draw(image, labels, boxes, scores, ratio, pad_w, pad_h); - - // Return the annotated image. - return result_image; -} - -/** - * @brief Entry point of the application to perform object detection on an input source using a specified model. - * - * The program loads a pre-trained model, processes an input source (image, video, or webcam), and performs object - * detection using either a CPU or GPU for computation. The results are displayed or saved as appropriate. - * - * @param argc The number of command-line arguments passed to the program. - * @param argv The array of command-line arguments: - * - argv[0]: The name of the executable. - * - argv[1]: The path to the pre-trained model file. - * - argv[2]: The source of the input (image file, video file, or webcam index). - * - argv[3]: Flag to indicate whether to use GPU (1 for GPU, 0 for CPU). - * @return Exit status: - * - Returns 0 on success. - * - Returns -1 if incorrect arguments are provided. - */ -int main(int argc, char** argv) { - // Check if the required number of arguments is provided - if (argc < 4) { - // Display usage instructions if arguments are insufficient - std::cerr << "Usage: " << argv[0] - << " \n"; - return -1; - } - - // Parse arguments - std::string modelPath = argv[1]; - std::string source = argv[2]; - bool useGPU = std::stoi(argv[3]) != 0; - - // Initialize ONNX Runtime environment - Ort::Env env(ORT_LOGGING_LEVEL_ERROR, "ONNXExample"); - Ort::SessionOptions session_options; - session_options.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_EXTENDED); - - if (useGPU) { - OrtCUDAProviderOptions cudaOptions; - cudaOptions.device_id = 0; // Default to GPU 0 - session_options.AppendExecutionProvider_CUDA(cudaOptions); - std::cout << "Using GPU for inference.\n"; - } - else { - std::cout << "Using CPU for inference.\n"; - } - - // Load ONNX model - std::wstring widestr = std::wstring(modelPath.begin(), modelPath.end()); - const wchar_t* model_path = widestr.c_str(); - Ort::Session session(env, model_path, session_options); - - // Open source - cv::VideoCapture cap; - bool isVideo = false; - bool isWebcam = false; - bool isImage = false; - cv::Mat frame; - - if (source == "webcam") { - isWebcam = true; - cap.open(0); // Open webcam - } - else if (source.find(".mp4") != std::string::npos || - source.find(".avi") != std::string::npos || - source.find(".mkv") != std::string::npos) { - isVideo = true; - cap.open(source); // Open video file - } - else { - isImage = true; - frame = cv::imread(source); - if (frame.empty()) { - std::cerr << "Error: Could not read image file.\n"; - return -1; - } - } - - if ((isVideo || isWebcam) && !cap.isOpened()) { - std::cerr << "Error: Could not open video source.\n"; - return -1; - } - - // Process source - do { - if (isWebcam || isVideo) { - cap >> frame; - if (frame.empty()) { - if (isVideo) { - std::cout << "End of video reached.\n"; - } - break; - } - } - - // Process the frame/image with ONNX model - auto result_image = processImage(session, frame); - - cv::imshow("ONNX Result", result_image); - if (isImage) { - cv::waitKey(0); // Wait indefinitely for image - break; - } - else if (cv::waitKey(1) == 27) { // Exit on 'Esc' key for video/webcam - break; - } - - // FPS calculation for video/webcam - static int frame_count = 0; - static auto last_time = std::chrono::high_resolution_clock::now(); - frame_count++; - auto current_time = std::chrono::high_resolution_clock::now(); - std::chrono::duration elapsed = current_time - last_time; - if (elapsed.count() >= 1.0) { - std::cout << "FPS: " << frame_count / elapsed.count() << "\n"; - frame_count = 0; - last_time = current_time; - } - - } while (isWebcam || isVideo); - - return 0; -} diff --git a/vlm/detector/D-FINE.backup/tools/inference/cppExample/openvino/CMakeLists.txt b/vlm/detector/D-FINE.backup/tools/inference/cppExample/openvino/CMakeLists.txt deleted file mode 100644 index 9a51193..0000000 --- a/vlm/detector/D-FINE.backup/tools/inference/cppExample/openvino/CMakeLists.txt +++ /dev/null @@ -1,31 +0,0 @@ -cmake_minimum_required(VERSION 3.5.1) - -project( - openvinoExample - LANGUAGES CXX - VERSION 1.0.0 -) - -set(CMAKE_CXX_STANDARD 17) - -if (MSVC) - add_compile_options(-nologo) - add_definitions(-DNOMINMAX) -else() - add_compile_options(-Wall) -endif() - -find_package(OpenVINO CONFIG REQUIRED) -find_package(OpenCV CONFIG REQUIRED core dnn imgcodecs imgproc) - -add_executable( - openvinoExample - openvinoExample.cpp -) -target_link_libraries( - openvinoExample - openvino::runtime - opencv_core - opencv_dnn - opencv_imgcodecs -) diff --git a/vlm/detector/D-FINE.backup/tools/inference/cppExample/openvino/openvinoExample.cpp b/vlm/detector/D-FINE.backup/tools/inference/cppExample/openvino/openvinoExample.cpp deleted file mode 100644 index 9b5430f..0000000 --- a/vlm/detector/D-FINE.backup/tools/inference/cppExample/openvino/openvinoExample.cpp +++ /dev/null @@ -1,42 +0,0 @@ -#include -#include -#include - -int main(){ - try{ - ov::Core core; - ov::CompiledModel mCompiledModel = core.compile_model("DFINE.onnx","AUTO"); - cv::Mat imageMat = cv::imread("test.png"); - cv::cvtColor(imageMat, imageMat, cv::COLOR_BGR2RGB); - cv::Mat inferMat; - cv::dnn::blobFromImage(imageMat,inferMat,1.0 / 255.0); - std::vector im_shape = { (float)1/imageMat.rows, (float)1/imageMat.cols }; - auto ireq = mCompiledModel.create_infer_request(); - auto inputports = mCompiledModel.inputs(); - ov::Tensor input_tensor1(inputports[0].get_element_type(), { 1,3,640,640 }, inferMat.ptr()); - ireq.set_input_tensor(0,input_tensor1); - ov::Tensor input_tensor2(inputports[1].get_element_type(), { 1,2 }); - int64* input_tensor_data = input_tensor2.data(); - for (int i = 0; i < 2; i++) { - input_tensor_data[i] = 640; - } - ireq.set_input_tensor(1,input_tensor2); - ireq.infer(); - ov::Tensor labels_tensor = ireq.get_output_tensor(0); - ov::Tensor bboxs_tensor = ireq.get_output_tensor(1); - ov::Tensor scores_tensor = ireq.get_output_tensor(2); - float *bo = bboxs_tensor.data(); - - //example - float cx = bo[4] ; - float cy = bo[5] ; - float bx = bo[6] ; - float by = bo[7] ; - cv::rectangle(imageMat, cv::Rect(bo[0],bo[1], bo[2]-bo[0], bo[3]-bo[1]), cv::Scalar(0, 255, 0), 2); - cv::rectangle(imageMat, cv::Rect(cx,cy, bx-cx, by-cy), cv::Scalar(0, 255, 0), 2); - cv::imwrite("aimage.png",imageMat); - } - catch(const ov::Exception& e){ - std::cerr << e.what() << '\n'; - } -} diff --git a/vlm/detector/D-FINE.backup/tools/inference/cppExample/trt/CMakeLists.txt b/vlm/detector/D-FINE.backup/tools/inference/cppExample/trt/CMakeLists.txt deleted file mode 100644 index 452ef21..0000000 --- a/vlm/detector/D-FINE.backup/tools/inference/cppExample/trt/CMakeLists.txt +++ /dev/null @@ -1,35 +0,0 @@ -cmake_minimum_required(VERSION 3.20) -cmake_policy(SET CMP0146 OLD) # for disabling FindCUDA warning -project(DFineTRT) - -set(CMAKE_CXX_STANDARD 17) -set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(CMAKE_CXX_FLAGS "-O3") - -set(LIB_ROOT "$ENV{HOME}/Documents/libs") - -# opencv -find_package(OpenCV REQUIRED) -find_package(Iconv QUIET) -# cuda -find_package(CUDA REQUIRED) -# tensorrt -set(CUDA_RUNTIME_API_DIRS "/usr/local/cuda-12.9/targets/x86_64-linux/include") - -# dfine -add_executable(trtExample trtExample.cpp) -target_compile_options(trtExample PRIVATE - -Wall - -Wextra - -Wno-unused-function -) -target_link_libraries(trtExample - ${OpenCV_LIBS} - ${CUDA_LIBRARIES} - nvonnxparser - nvinfer -) -target_include_directories(trtExample PRIVATE - ${OpenCV_INCLUDE_DIRS} - ${CUDA_RUNTIME_API_DIRS} -) diff --git a/vlm/detector/D-FINE.backup/tools/inference/cppExample/trt/trtExample.cpp b/vlm/detector/D-FINE.backup/tools/inference/cppExample/trt/trtExample.cpp deleted file mode 100644 index 1ae871c..0000000 --- a/vlm/detector/D-FINE.backup/tools/inference/cppExample/trt/trtExample.cpp +++ /dev/null @@ -1,346 +0,0 @@ -/** - * NOTE: Convert model with --fp16 may lead to incorrect results - */ -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#define CUDA_CHECK(call) \ - do \ - { \ - const cudaError_t error_code = call; \ - if (error_code != cudaSuccess) \ - { \ - printf("CUDA_CHECK Error:\n"); \ - printf(" File: %s\n", __FILE__); \ - printf(" Line: %d\n", __LINE__); \ - printf(" Error code: %d\n", error_code); \ - printf(" Error text: %s\n", cudaGetErrorString(error_code)); \ - exit(1); \ - } \ - } while (0) - -struct Object -{ - cv::Rect_ rect; - int label; - float prob; -}; - -const std::vector labels = { - "Person", "Bicycle", "Car", "Motorcycle", "Airplane", "Bus", "Train", - "Truck", "Boat", "Traffic light", "Fire hydrant", "Stop sign", "Parking meter", - "Bench", "Bird", "Cat", "Dog", "Horse", "Sheep", "Cow", "Elephant", "Bear", - "Zebra", "Giraffe", "Backpack", "Umbrella", "Handbag", "Tie", "Suitcase", - "Frisbee", "Skis", "Snowboard", "Sports ball", "Kite", "Baseball bat", - "Baseball glove", "Skateboard", "Surfboard", "Tennis racket", "Bottle", - "Wine glass", "Cup", "Fork", "Knife", "Spoon", "Bowl", "Banana", "Apple", - "Sandwich", "Orange", "Broccoli", "Carrot", "Hot dog", "Pizza", "Donut", - "Cake", "Chair", "Couch", "Potted plant", "Bed", "Dining table", "Toilet", - "Tv", "Laptop", "Mouse", "Remote", "Keyboard", "Cell phone", "Microwave", - "Oven", "Toaster", "Sink", "Refrigerator", "Book", "Clock", "Vase", "Scissors", - "Teddy bear", "Hair drier", "Toothbrush" -}; - -class Logger : public nvinfer1::ILogger -{ -public: - void log(Severity severity, const char *msg) noexcept override - { - if (severity <= nvinfer1::ILogger::Severity::kWARNING) - { - std::cerr << "[TensorRT] "; - switch (severity) - { - case nvinfer1::ILogger::Severity::kINTERNAL_ERROR: std::cerr << "INTERNAL_ERROR: "; break; - case nvinfer1::ILogger::Severity::kERROR: std::cerr << "ERROR: "; break; - case nvinfer1::ILogger::Severity::kWARNING: std::cerr << "WARNING: "; break; - case nvinfer1::ILogger::Severity::kINFO: std::cerr << "INFO: "; break; - case nvinfer1::ILogger::Severity::kVERBOSE: std::cerr << "VERBOSE: "; break; - } - std::cerr << msg << "\n"; - } - } -}; - -static Logger logger; - -bool DrawObjects(cv::Mat &image, const std::vector &objects, - const std::vector &labels, bool isSilent) -{ - for (auto obj : objects) - { - if (obj.label >= static_cast(labels.size())) - return false; - - if (isSilent != true) - std::printf("%s = %.2f%% at (%.1f, %.1f) %.1f x %.1f\n", labels[obj.label].c_str(), obj.prob * 100.0f, - obj.rect.x, obj.rect.y, obj.rect.width, obj.rect.height); - - char text[256]; - snprintf(text, sizeof(text), "%s %.1f%%", labels[obj.label].c_str(), obj.prob * 100.0f); - - auto scalar = cv::Scalar(255, 255, 255); - cv::rectangle(image, obj.rect, scalar, 2); - - int baseLine = 5; - cv::Size label_size = cv::getTextSize(text, cv::FONT_HERSHEY_SIMPLEX, 0.75, 1, &baseLine); - - int x = obj.rect.x - 1; - int y = obj.rect.y - label_size.height - baseLine; - y = std::max(0, y); - if (x + label_size.width > image.cols) - x = image.cols - label_size.width; - - cv::rectangle(image, cv::Rect(cv::Point(x, y), cv::Size(label_size.width, label_size.height + baseLine)), - scalar, -1); - - cv::putText(image, text, cv::Point(x, y + label_size.height + baseLine / 2), - cv::FONT_HERSHEY_SIMPLEX, 0.75, cv::Scalar(0, 0, 0), 2); - } - - return true; -} - -size_t CountElement(const nvinfer1::Dims &dims) -{ - int64_t total = 1; - for (int32_t i = 0; i < dims.nbDims; ++i) - total *= dims.d[i]; - return static_cast(total); -} - -template -T Clamp(T val, T min, T max) -{ - return val > min ? (val < max ? val : max) : min; -} - -void GetLetterboxDimensions( - const int img_rows, const int img_cols, - const int target_size, - int &resize_rows, int &resize_cols, int &pad_rows, int &pad_cols, float &scale -) -{ - scale = static_cast(target_size) / std::max(img_rows, img_cols); - resize_rows = static_cast(std::round(img_rows * scale)); - resize_cols = static_cast(std::round(img_cols * scale)); - pad_rows = target_size - resize_rows; - pad_cols = target_size - resize_cols; -} - -int main(int argc, char *argv[]) -{ - // --- Settings --- - if (argc < 3) - { - std::printf("Usage: %s model image [conf] [target size]\n", argv[0]); - return 0; - } - const std::string model_path = std::string(argv[1]); - float conf_thres = 0.25f; - int target_size = 640; - if (argc >= 4 && std::stof(argv[3]) > 0.0f) - conf_thres = std::stof(argv[3]); - if (argc >= 5 && std::stoi(argv[4]) > 0 && - std::stoi(argv[4]) % 32 == 0 && std::stoi(argv[4]) > 32) - target_size = std::stoi(argv[4]); - - std::cout << "Model: " << model_path << "\n"; - std::cout << "Input: " << argv[2] << "\n"; - std::cout << "Conf: " << conf_thres << "\n"; - std::cout << "Target size: " << target_size << "\n"; - - // --- Init TRT --- - // load model data - std::ifstream engine_file(model_path, std::ios::binary); - if (!engine_file) - { - std::cerr << "Failed to open engine file\n"; - return -1; - } - engine_file.seekg(0, engine_file.end); - std::streamsize engine_size = engine_file.tellg(); - engine_file.seekg(0, engine_file.beg); - std::unique_ptr engine_data{std::make_unique(engine_size)}; - if (!engine_file.read(engine_data.get(), engine_size)) - { - std::cerr << "Failed to read engine file\n"; - return -1; - } - engine_file.close(); - - // create runtime, engine, context, and stream - auto runtime{nvinfer1::createInferRuntime(logger)}; - if (!runtime) - { - std::cerr << "Failed to create runtime\n"; - return -1; - } - auto engine{runtime->deserializeCudaEngine(engine_data.get(), engine_size)}; - if (!engine) - { - std::cerr << "Failed to deserialize engine\n"; - return -1; - } - auto context{engine->createExecutionContext()}; - if (!context) - { - std::cerr << "Failed to create contexts\n"; - return -1; - } - std::unique_ptr stream = std::make_unique(); - CUDA_CHECK(cudaStreamCreate(stream.get())); - - // get model info - std::vector> in_tensor_info, out_tensor_info; - for (int i = 0; i < engine->getNbIOTensors(); ++i) - { - const char *tensor_name = engine->getIOTensorName(i); - nvinfer1::TensorIOMode io_mode = engine->getTensorIOMode(tensor_name); - if (io_mode == nvinfer1::TensorIOMode::kINPUT) - in_tensor_info.push_back({i, std::string(tensor_name)}); - else if (io_mode == nvinfer1::TensorIOMode::kOUTPUT) - out_tensor_info.push_back({i, std::string(tensor_name)}); - } - - // create host memory - size_t max_in0_size_byte = CountElement(context->getTensorShape(in_tensor_info[0].second.c_str())) * sizeof(float); - size_t max_in1_size_byte = CountElement(context->getTensorShape(in_tensor_info[1].second.c_str())) * sizeof(int64_t); - size_t max_out0_size_byte = CountElement(context->getTensorShape(out_tensor_info[0].second.c_str())) * sizeof(int64_t); - size_t max_out1_size_byte = CountElement(context->getTensorShape(out_tensor_info[1].second.c_str())) * sizeof(float); - size_t max_out2_size_byte = CountElement(context->getTensorShape(out_tensor_info[2].second.c_str())) * sizeof(float); - std::vector> host_outs; - host_outs.resize(out_tensor_info.size()); - host_outs[0] = std::make_unique(max_out0_size_byte); - host_outs[1] = std::make_unique(max_out1_size_byte); - host_outs[2] = std::make_unique(max_out2_size_byte); - // create cuda memory - std::vector buffers{}; - buffers.resize(engine->getNbIOTensors()); - CUDA_CHECK(cudaMalloc(&buffers[in_tensor_info[0].first], max_in0_size_byte)); - CUDA_CHECK(cudaMalloc(&buffers[in_tensor_info[1].first], max_in1_size_byte)); - CUDA_CHECK(cudaMalloc(&buffers[out_tensor_info[0].first], max_out0_size_byte)); - CUDA_CHECK(cudaMalloc(&buffers[out_tensor_info[1].first], max_out1_size_byte)); - CUDA_CHECK(cudaMalloc(&buffers[out_tensor_info[2].first], max_out2_size_byte)); - - // set in/out tensor address - context->setInputTensorAddress(in_tensor_info[0].second.c_str(), buffers[in_tensor_info[0].first]); - context->setInputTensorAddress(in_tensor_info[1].second.c_str(), buffers[in_tensor_info[1].first]); - context->setOutputTensorAddress(out_tensor_info[0].second.c_str(), buffers[out_tensor_info[0].first]); - context->setOutputTensorAddress(out_tensor_info[1].second.c_str(), buffers[out_tensor_info[1].first]); - context->setOutputTensorAddress(out_tensor_info[2].second.c_str(), buffers[out_tensor_info[2].first]); - - // --- Detect --- - cv::Mat image = cv::imread(argv[2]); - if (image.empty()) - { - std::cout << "Failed to read image\n"; - return -1; - } - - // preprocessing - int img_rows = image.rows; - int img_cols = image.cols; - float scale; - int resize_rows, resize_cols, pad_rows, pad_cols; - GetLetterboxDimensions( - img_rows, img_cols, target_size, - resize_rows, resize_cols, pad_rows, pad_cols, scale - ); - cv::Mat letterbox, blob; - cv::resize(image, letterbox, cv::Size(resize_cols, resize_rows), 0, 0, cv::INTER_AREA); - cv::copyMakeBorder( - letterbox, letterbox, - pad_rows / 2, pad_rows - pad_rows / 2, - pad_cols / 2, pad_cols - pad_cols / 2, - cv::BORDER_CONSTANT, cv::Scalar(114.0, 114.0, 114.0) - ); - // no normalization - cv::dnn::blobFromImage(letterbox, blob, 1.0f / 255.0f, cv::Size(letterbox.cols, letterbox.rows), cv::Scalar(0, 0, 0), true, false, CV_32F); - - nvinfer1::Dims trt_in0_dims{}, trt_in1_dims{}; - trt_in0_dims.nbDims = 4; - trt_in0_dims.d[0] = 1; - trt_in0_dims.d[1] = 3; - trt_in0_dims.d[2] = letterbox.rows; - trt_in0_dims.d[3] = letterbox.cols; - context->setInputShape(in_tensor_info[0].second.c_str(), trt_in0_dims); - - std::vector orig_size{static_cast(letterbox.rows), static_cast(letterbox.cols)}; - trt_in1_dims.nbDims = 2; - trt_in1_dims.d[0] = 1; - trt_in1_dims.d[1] = 2; - context->setInputShape(in_tensor_info[1].second.c_str(), trt_in1_dims); - - // execute - CUDA_CHECK(cudaMemcpyAsync(buffers[0], blob.data, max_in0_size_byte, cudaMemcpyHostToDevice, *stream)); - CUDA_CHECK(cudaMemcpyAsync(buffers[1], orig_size.data(), max_in1_size_byte, cudaMemcpyHostToDevice, *stream)); - - context->enqueueV3(*stream); - - CUDA_CHECK(cudaMemcpyAsync(host_outs[0].get(), buffers[2], max_out0_size_byte, cudaMemcpyDeviceToHost, *stream)); - CUDA_CHECK(cudaMemcpyAsync(host_outs[1].get(), buffers[3], max_out1_size_byte, cudaMemcpyDeviceToHost, *stream)); - CUDA_CHECK(cudaMemcpyAsync(host_outs[2].get(), buffers[4], max_out2_size_byte, cudaMemcpyDeviceToHost, *stream)); - CUDA_CHECK(cudaStreamSynchronize(*stream)); - - const int64_t *labels_ptr = reinterpret_cast(host_outs[0].get()); - const float *boxes_ptr = reinterpret_cast(host_outs[1].get()); - const float *scores_ptr = reinterpret_cast(host_outs[2].get()); - - size_t num_box = 300; - size_t walk = 4; - float dw = pad_cols / 2, dh = pad_rows / 2; - std::vector objects; - for (size_t i = 0; i < num_box; ++i) - { - if (scores_ptr[i] < conf_thres) - continue; - - float x0 = boxes_ptr[i * walk]; - float y0 = boxes_ptr[i * walk + 1]; - float x1 = boxes_ptr[i * walk + 2]; - float y1 = boxes_ptr[i * walk + 3]; - - x0 = (x0 - dw) / scale; - y0 = (y0 - dh) / scale; - x1 = (x1 - dw) / scale; - y1 = (y1 - dh) / scale; - - x0 = Clamp(x0, 0.0f, static_cast(img_cols)); - y0 = Clamp(y0, 0.0f, static_cast(img_rows)); - x1 = Clamp(x1, x0, static_cast(img_cols)); - y1 = Clamp(y1, y0, static_cast(img_rows)); - - Object object; - object.rect.x = x0; - object.rect.y = y0; - object.rect.width = x1 - x0; - object.rect.height = y1 - y0; - object.prob = scores_ptr[i]; - object.label = static_cast(labels_ptr[i]); - objects.emplace_back(object); - } - - // save results - if (DrawObjects(image, objects, labels, false)) - cv::imwrite("./result.jpg", image); - else - std::cout << "Failed to draw objects\n"; - - // --- Release resources --- - for (const auto &buffer : buffers) - if (buffer) - CUDA_CHECK(cudaFree(buffer)); - if (stream && *stream) - CUDA_CHECK(cudaStreamDestroy(*stream)); - - return 0; -} diff --git a/vlm/detector/D-FINE.backup/tools/inference/onnx_inf.py b/vlm/detector/D-FINE.backup/tools/inference/onnx_inf.py deleted file mode 100644 index 3fb26c7..0000000 --- a/vlm/detector/D-FINE.backup/tools/inference/onnx_inf.py +++ /dev/null @@ -1,163 +0,0 @@ -""" -Copyright (c) 2024 The D-FINE Authors. All Rights Reserved. -""" - -import cv2 -import numpy as np -import onnxruntime as ort -import torch -import torchvision.transforms as T -from PIL import Image, ImageDraw - - -def resize_with_aspect_ratio(image, size, interpolation=Image.BILINEAR): - """Resizes an image while maintaining aspect ratio and pads it.""" - original_width, original_height = image.size - ratio = min(size / original_width, size / original_height) - new_width = int(original_width * ratio) - new_height = int(original_height * ratio) - image = image.resize((new_width, new_height), interpolation) - - # Create a new image with the desired size and paste the resized image onto it - new_image = Image.new("RGB", (size, size)) - new_image.paste(image, ((size - new_width) // 2, (size - new_height) // 2)) - return new_image, ratio, (size - new_width) // 2, (size - new_height) // 2 - - -def draw(images, labels, boxes, scores, ratios, paddings, thrh=0.4): - result_images = [] - for i, im in enumerate(images): - draw = ImageDraw.Draw(im) - scr = scores[i] - lab = labels[i][scr > thrh] - box = boxes[i][scr > thrh] - scr = scr[scr > thrh] - - ratio = ratios[i] - pad_w, pad_h = paddings[i] - - for lbl, bb in zip(lab, box): - # Adjust bounding boxes according to the resizing and padding - bb = [ - (bb[0] - pad_w) / ratio, - (bb[1] - pad_h) / ratio, - (bb[2] - pad_w) / ratio, - (bb[3] - pad_h) / ratio, - ] - draw.rectangle(bb, outline="red") - draw.text((bb[0], bb[1]), text=str(lbl), fill="blue") - - result_images.append(im) - return result_images - - -def process_image(sess, im_pil): - # Resize image while preserving aspect ratio - resized_im_pil, ratio, pad_w, pad_h = resize_with_aspect_ratio(im_pil, 640) - orig_size = torch.tensor([[resized_im_pil.size[1], resized_im_pil.size[0]]]) - - transforms = T.Compose( - [ - T.ToTensor(), - ] - ) - im_data = transforms(resized_im_pil).unsqueeze(0) - - output = sess.run( - output_names=None, - input_feed={"images": im_data.numpy(), "orig_target_sizes": orig_size.numpy()}, - ) - - labels, boxes, scores = output - - result_images = draw([im_pil], labels, boxes, scores, [ratio], [(pad_w, pad_h)]) - result_images[0].save("onnx_result.jpg") - print("Image processing complete. Result saved as 'result.jpg'.") - - -def process_video(sess, video_path): - cap = cv2.VideoCapture(video_path) - - # Get video properties - fps = cap.get(cv2.CAP_PROP_FPS) - orig_w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) - orig_h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) - - # Define the codec and create VideoWriter object - fourcc = cv2.VideoWriter_fourcc(*"mp4v") - out = cv2.VideoWriter("onnx_result.mp4", fourcc, fps, (orig_w, orig_h)) - - frame_count = 0 - print("Processing video frames...") - while cap.isOpened(): - ret, frame = cap.read() - if not ret: - break - - # Convert frame to PIL image - frame_pil = Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)) - - # Resize frame while preserving aspect ratio - resized_frame_pil, ratio, pad_w, pad_h = resize_with_aspect_ratio(frame_pil, 640) - orig_size = torch.tensor([[resized_frame_pil.size[1], resized_frame_pil.size[0]]]) - - transforms = T.Compose( - [ - T.ToTensor(), - ] - ) - im_data = transforms(resized_frame_pil).unsqueeze(0) - - output = sess.run( - output_names=None, - input_feed={"images": im_data.numpy(), "orig_target_sizes": orig_size.numpy()}, - ) - - labels, boxes, scores = output - - # Draw detections on the original frame - result_images = draw([frame_pil], labels, boxes, scores, [ratio], [(pad_w, pad_h)]) - frame_with_detections = result_images[0] - - # Convert back to OpenCV image - frame = cv2.cvtColor(np.array(frame_with_detections), cv2.COLOR_RGB2BGR) - - # Write the frame - out.write(frame) - frame_count += 1 - - if frame_count % 10 == 0: - print(f"Processed {frame_count} frames...") - - cap.release() - out.release() - print("Video processing complete. Result saved as 'result.mp4'.") - - -def main(args): - """Main function.""" - # Load the ONNX model - sess = ort.InferenceSession(args.onnx) - print(f"Using device: {ort.get_device()}") - - input_path = args.input - - try: - # Try to open the input as an image - im_pil = Image.open(input_path).convert("RGB") - process_image(sess, im_pil) - except IOError: - # Not an image, process as video - process_video(sess, input_path) - - -if __name__ == "__main__": - import argparse - - parser = argparse.ArgumentParser() - parser.add_argument("--onnx", type=str, required=True, help="Path to the ONNX model file.") - parser.add_argument( - "--input", type=str, required=True, help="Path to the input image or video file." - ) - args = parser.parse_args() - main(args) diff --git a/vlm/detector/D-FINE.backup/tools/inference/openvino_inf.py b/vlm/detector/D-FINE.backup/tools/inference/openvino_inf.py deleted file mode 100644 index 6d37e4b..0000000 --- a/vlm/detector/D-FINE.backup/tools/inference/openvino_inf.py +++ /dev/null @@ -1,101 +0,0 @@ -""" -Copyright (c) 2024 The D-FINE Authors. All Rights Reserved. -""" - -import cv2 -import numpy as np -import openvino -from openvino.runtime import Core - - -class OvInfer: - def __init__(self, model_path, device_name="AUTO"): - self.resized_image = None - self.ratio = None - self.resize_image = None - self.ori_image = None - self.device = device_name - self.model_path = model_path - self.core = Core() - self.available_device = self.core.available_devices - self.compile_model = self.core.compile_model(self.model_path, device_name) - self.target_size = [ - self.compile_model.inputs[0].get_partial_shape()[2].get_length(), - self.compile_model.inputs[0].get_partial_shape()[3].get_length(), - ] - self.query_num = self.compile_model.outputs[0].get_partial_shape()[1].get_length() - - def infer(self, inputs: dict): - infer_request = self.compile_model.create_infer_request() - for input_name, input_data in inputs.items(): - input_tensor = openvino.Tensor(input_data) - infer_request.set_tensor(input_name, input_tensor) - infer_request.infer() - outputs = { - "labels": infer_request.get_tensor("labels").data, - "boxes": infer_request.get_tensor("boxes").data, - "scores": infer_request.get_tensor("scores").data, - } - return outputs - - def process_image(self, ori_image, keep_ratio: bool): - self.ori_image = ori_image - h, w = ori_image.shape[:2] - if keep_ratio: - r = min(self.target_size[0] / h, self.target_size[1] / w) - self.ratio = r - new_w = int(w * r) - new_h = int(h * r) - temp_image = cv2.resize(ori_image, (new_w, new_h), interpolation=cv2.INTER_LINEAR) - resized_image = np.full( - (self.target_size[0], self.target_size[1], 3), 114, dtype=temp_image.dtype - ) - resized_image[:new_h, :new_w, :] = temp_image - self.resized_image = resized_image - else: - self.resized_image = cv2.resize( - ori_image, self.target_size, interpolation=cv2.INTER_LINEAR - ) - blob_image = cv2.dnn.blobFromImage(self.resized_image, 1.0 / 255.0) - orig_size = np.array([self.resized_image.shape[0], self.resized_image.shape[1]], dtype=np.int64).reshape( - 1, 2 - ) - - inputs = { - "images": blob_image, - "orig_target_sizes": orig_size, - } - return inputs - - def get_available_device(self): - return self.available_device - - def draw_and_save_image(self, infer_result, image_path, score_threshold=0.6): - draw_image = self.ori_image - scores = infer_result["scores"] - labels = infer_result["labels"] - boxes = infer_result["boxes"] - for i in range(self.query_num): - if scores[0, i] > score_threshold: - cx = boxes[0, i, 0] / self.ratio - cy = boxes[0, i, 1] / self.ratio - bx = boxes[0, i, 2] / self.ratio - by = boxes[0, i, 3] / self.ratio - cv2.rectangle( - draw_image, (int(cx), int(cy), int(bx - cx), int(by - cy)), (255, 0, 0), 1 - ) - cv2.imwrite(image_path, draw_image) - - -if __name__ == "__main__": - import argparse - - parser = argparse.ArgumentParser() - parser.add_argument("-image", "--image", type=str, required=True) - parser.add_argument("-ov_model", "--ov_model", type=str, required=True) - args = parser.parse_args() - img = cv2.imread(args.image) - mOvInfer = OvInfer(args.ov_model) - inputs = mOvInfer.process_image(img, True) - outputs = mOvInfer.infer(inputs) - mOvInfer.draw_and_save_image(outputs, "openvino_result.jpg") diff --git a/vlm/detector/D-FINE.backup/tools/inference/requirements.txt b/vlm/detector/D-FINE.backup/tools/inference/requirements.txt deleted file mode 100644 index 7dfc48a..0000000 --- a/vlm/detector/D-FINE.backup/tools/inference/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -onnxruntime -tensorrt -openvino diff --git a/vlm/detector/D-FINE.backup/tools/inference/torch_inf.py b/vlm/detector/D-FINE.backup/tools/inference/torch_inf.py deleted file mode 100644 index c5ba0ef..0000000 --- a/vlm/detector/D-FINE.backup/tools/inference/torch_inf.py +++ /dev/null @@ -1,166 +0,0 @@ -""" -Copyright (c) 2024 The D-FINE Authors. All Rights Reserved. -""" - -import os -import sys - -import cv2 # Added for video processing -import numpy as np -import torch -import torch.nn as nn -import torchvision.transforms as T -from PIL import Image, ImageDraw - -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../"))) -from src.core import YAMLConfig - - -def draw(images, labels, boxes, scores, thrh=0.4): - for i, im in enumerate(images): - draw = ImageDraw.Draw(im) - - scr = scores[i] - lab = labels[i][scr > thrh] - box = boxes[i][scr > thrh] - scrs = scr[scr > thrh] - - for j, b in enumerate(box): - draw.rectangle(list(b), outline="red") - draw.text( - (b[0], b[1]), - text=f"{lab[j].item()} {round(scrs[j].item(), 2)}", - fill="blue", - ) - - im.save("torch_results.jpg") - - -def process_image(model, device, file_path): - im_pil = Image.open(file_path).convert("RGB") - w, h = im_pil.size - orig_size = torch.tensor([[w, h]]).to(device) - - transforms = T.Compose( - [ - T.Resize((640, 640)), - T.ToTensor(), - ] - ) - im_data = transforms(im_pil).unsqueeze(0).to(device) - - output = model(im_data, orig_size) - labels, boxes, scores = output - - draw([im_pil], labels, boxes, scores) - - -def process_video(model, device, file_path): - cap = cv2.VideoCapture(file_path) - - # Get video properties - fps = cap.get(cv2.CAP_PROP_FPS) - orig_w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) - orig_h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) - - # Define the codec and create VideoWriter object - fourcc = cv2.VideoWriter_fourcc(*"mp4v") - out = cv2.VideoWriter("torch_results.mp4", fourcc, fps, (orig_w, orig_h)) - - transforms = T.Compose( - [ - T.Resize((640, 640)), - T.ToTensor(), - ] - ) - - frame_count = 0 - print("Processing video frames...") - while cap.isOpened(): - ret, frame = cap.read() - if not ret: - break - - # Convert frame to PIL image - frame_pil = Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)) - - w, h = frame_pil.size - orig_size = torch.tensor([[w, h]]).to(device) - - im_data = transforms(frame_pil).unsqueeze(0).to(device) - - output = model(im_data, orig_size) - labels, boxes, scores = output - - # Draw detections on the frame - draw([frame_pil], labels, boxes, scores) - - # Convert back to OpenCV image - frame = cv2.cvtColor(np.array(frame_pil), cv2.COLOR_RGB2BGR) - - # Write the frame - out.write(frame) - frame_count += 1 - - if frame_count % 10 == 0: - print(f"Processed {frame_count} frames...") - - cap.release() - out.release() - print("Video processing complete. Result saved as 'results_video.mp4'.") - - -def main(args): - """Main function""" - cfg = YAMLConfig(args.config, resume=args.resume) - - if "HGNetv2" in cfg.yaml_cfg: - cfg.yaml_cfg["HGNetv2"]["pretrained"] = False - - if args.resume: - checkpoint = torch.load(args.resume, map_location="cpu") - if "ema" in checkpoint: - state = checkpoint["ema"]["module"] - else: - state = checkpoint["model"] - else: - raise AttributeError("Only support resume to load model.state_dict by now.") - - # Load train mode state and convert to deploy mode - cfg.model.load_state_dict(state) - - class Model(nn.Module): - def __init__(self): - super().__init__() - self.model = cfg.model.deploy() - self.postprocessor = cfg.postprocessor.deploy() - - def forward(self, images, orig_target_sizes): - outputs = self.model(images) - outputs = self.postprocessor(outputs, orig_target_sizes) - return outputs - - device = args.device - model = Model().to(device) - - # Check if the input file is an image or a video - file_path = args.input - if os.path.splitext(file_path)[-1].lower() in [".jpg", ".jpeg", ".png", ".bmp"]: - # Process as image - process_image(model, device, file_path) - print("Image processing complete.") - else: - # Process as video - process_video(model, device, file_path) - - -if __name__ == "__main__": - import argparse - - parser = argparse.ArgumentParser() - parser.add_argument("-c", "--config", type=str, required=True) - parser.add_argument("-r", "--resume", type=str, required=True) - parser.add_argument("-i", "--input", type=str, required=True) - parser.add_argument("-d", "--device", type=str, default="cpu") - args = parser.parse_args() - main(args) diff --git a/vlm/detector/D-FINE.backup/tools/inference/trt_inf.py b/vlm/detector/D-FINE.backup/tools/inference/trt_inf.py deleted file mode 100644 index 731e0b5..0000000 --- a/vlm/detector/D-FINE.backup/tools/inference/trt_inf.py +++ /dev/null @@ -1,244 +0,0 @@ -""" -Copyright (c) 2024 The D-FINE Authors. All Rights Reserved. -""" - -import collections -import contextlib -import os -import time -from collections import OrderedDict - -import cv2 # Added for video processing -import numpy as np -import tensorrt as trt -import torch -import torchvision.transforms as T -from PIL import Image, ImageDraw - - -class TimeProfiler(contextlib.ContextDecorator): - def __init__(self): - self.total = 0 - - def __enter__(self): - self.start = self.time() - return self - - def __exit__(self, type, value, traceback): - self.total += self.time() - self.start - - def reset(self): - self.total = 0 - - def time(self): - if torch.cuda.is_available(): - torch.cuda.synchronize() - return time.time() - - -class TRTInference(object): - def __init__( - self, engine_path, device="cuda:0", backend="torch", max_batch_size=32, verbose=False - ): - self.engine_path = engine_path - self.device = device - self.backend = backend - self.max_batch_size = max_batch_size - - self.logger = trt.Logger(trt.Logger.VERBOSE) if verbose else trt.Logger(trt.Logger.INFO) - - self.engine = self.load_engine(engine_path) - self.context = self.engine.create_execution_context() - self.bindings = self.get_bindings( - self.engine, self.context, self.max_batch_size, self.device - ) - self.bindings_addr = OrderedDict((n, v.ptr) for n, v in self.bindings.items()) - self.input_names = self.get_input_names() - self.output_names = self.get_output_names() - self.time_profile = TimeProfiler() - - def load_engine(self, path): - trt.init_libnvinfer_plugins(self.logger, "") - with open(path, "rb") as f, trt.Runtime(self.logger) as runtime: - return runtime.deserialize_cuda_engine(f.read()) - - def get_input_names(self): - names = [] - for _, name in enumerate(self.engine): - if self.engine.get_tensor_mode(name) == trt.TensorIOMode.INPUT: - names.append(name) - return names - - def get_output_names(self): - names = [] - for _, name in enumerate(self.engine): - if self.engine.get_tensor_mode(name) == trt.TensorIOMode.OUTPUT: - names.append(name) - return names - - def get_bindings(self, engine, context, max_batch_size=32, device=None) -> OrderedDict: - Binding = collections.namedtuple("Binding", ("name", "dtype", "shape", "data", "ptr")) - bindings = OrderedDict() - - for i, name in enumerate(engine): - shape = engine.get_tensor_shape(name) - dtype = trt.nptype(engine.get_tensor_dtype(name)) - - if shape[0] == -1: - shape[0] = max_batch_size - if engine.get_tensor_mode(name) == trt.TensorIOMode.INPUT: - context.set_input_shape(name, shape) - - data = torch.from_numpy(np.empty(shape, dtype=dtype)).to(device) - bindings[name] = Binding(name, dtype, shape, data, data.data_ptr()) - - return bindings - - def run_torch(self, blob): - for n in self.input_names: - if blob[n].dtype is not self.bindings[n].data.dtype: - blob[n] = blob[n].to(dtype=self.bindings[n].data.dtype) - if self.bindings[n].shape != blob[n].shape: - self.context.set_input_shape(n, blob[n].shape) - self.bindings[n] = self.bindings[n]._replace(shape=blob[n].shape) - - assert self.bindings[n].data.dtype == blob[n].dtype, "{} dtype mismatch".format(n) - - self.bindings_addr.update({n: blob[n].data_ptr() for n in self.input_names}) - self.context.execute_v2(list(self.bindings_addr.values())) - outputs = {n: self.bindings[n].data for n in self.output_names} - - return outputs - - def __call__(self, blob): - if self.backend == "torch": - return self.run_torch(blob) - else: - raise NotImplementedError("Only 'torch' backend is implemented.") - - def synchronize(self): - if self.backend == "torch" and torch.cuda.is_available(): - torch.cuda.synchronize() - - -def draw(images, labels, boxes, scores, thrh=0.4): - for i, im in enumerate(images): - draw = ImageDraw.Draw(im) - scr = scores[i] - lab = labels[i][scr > thrh] - box = boxes[i][scr > thrh] - scrs = scr[scr > thrh] - - for j, b in enumerate(box): - draw.rectangle(list(b), outline="red") - draw.text( - (b[0], b[1]), - text=f"{lab[j].item()} {round(scrs[j].item(), 2)}", - fill="blue", - ) - - return images - - -def process_image(m, file_path, device): - im_pil = Image.open(file_path).convert("RGB") - w, h = im_pil.size - orig_size = torch.tensor([w, h])[None].to(device) - - transforms = T.Compose( - [ - T.Resize((640, 640)), - T.ToTensor(), - ] - ) - im_data = transforms(im_pil)[None] - - blob = { - "images": im_data.to(device), - "orig_target_sizes": orig_size.to(device), - } - - output = m(blob) - result_images = draw([im_pil], output["labels"], output["boxes"], output["scores"]) - result_images[0].save("trt_result.jpg") - print("Image processing complete. Result saved as 'result.jpg'.") - - -def process_video(m, file_path, device): - cap = cv2.VideoCapture(file_path) - - # Get video properties - fps = cap.get(cv2.CAP_PROP_FPS) - orig_w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) - orig_h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) - - # Define the codec and create VideoWriter object - fourcc = cv2.VideoWriter_fourcc(*"mp4v") - out = cv2.VideoWriter("trt_result.mp4", fourcc, fps, (orig_w, orig_h)) - - transforms = T.Compose( - [ - T.Resize((640, 640)), - T.ToTensor(), - ] - ) - - frame_count = 0 - print("Processing video frames...") - while cap.isOpened(): - ret, frame = cap.read() - if not ret: - break - - # Convert frame to PIL image - frame_pil = Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)) - - w, h = frame_pil.size - orig_size = torch.tensor([w, h])[None].to(device) - - im_data = transforms(frame_pil)[None] - - blob = { - "images": im_data.to(device), - "orig_target_sizes": orig_size.to(device), - } - - output = m(blob) - - # Draw detections on the frame - result_images = draw([frame_pil], output["labels"], output["boxes"], output["scores"]) - - # Convert back to OpenCV image - frame = cv2.cvtColor(np.array(result_images[0]), cv2.COLOR_RGB2BGR) - - # Write the frame - out.write(frame) - frame_count += 1 - - if frame_count % 10 == 0: - print(f"Processed {frame_count} frames...") - - cap.release() - out.release() - print("Video processing complete. Result saved as 'result_video.mp4'.") - - -if __name__ == "__main__": - import argparse - - parser = argparse.ArgumentParser() - parser.add_argument("-trt", "--trt", type=str, required=True) - parser.add_argument("-i", "--input", type=str, required=True) - parser.add_argument("-d", "--device", type=str, default="cuda:0") - - args = parser.parse_args() - - m = TRTInference(args.trt, device=args.device) - - file_path = args.input - if os.path.splitext(file_path)[-1].lower() in [".jpg", ".jpeg", ".png", ".bmp"]: - # Process as image - process_image(m, file_path, args.device) - else: - # Process as video - process_video(m, file_path, args.device) diff --git a/vlm/detector/D-FINE.backup/tools/visualization/fiftyone_vis.py b/vlm/detector/D-FINE.backup/tools/visualization/fiftyone_vis.py deleted file mode 100644 index cded6d9..0000000 --- a/vlm/detector/D-FINE.backup/tools/visualization/fiftyone_vis.py +++ /dev/null @@ -1,406 +0,0 @@ -""" -Copyright (c) 2024 The D-FINE Authors. All Rights Reserved. -""" - -import argparse -import os -import subprocess -import sys -import time - -import fiftyone as fo -import fiftyone.core.fields as fof -import fiftyone.core.labels as fol -import fiftyone.core.models as fom -import fiftyone.zoo as foz -import torch -import torchvision.transforms as transforms -import tqdm -from fiftyone import ViewField as F -from PIL import Image - -from src.core import YAMLConfig - -sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "../..")) - - -def kill_existing_mongod(): - try: - result = subprocess.run(["ps", "aux"], stdout=subprocess.PIPE) - processes = result.stdout.decode("utf-8").splitlines() - - for process in processes: - if "mongod" in process and "--dbpath" in process: - # find mongod PID - pid = int(process.split()[1]) - print(f"Killing existing mongod process with PID: {pid}") - # kill mongod session - os.kill(pid, 9) - except Exception as e: - print(f"Error occurred while killing mongod: {e}") - - -kill_existing_mongod() - - -label_map = { - 1: "person", - 2: "bicycle", - 3: "car", - 4: "motorbike", - 5: "aeroplane", - 6: "bus", - 7: "train", - 8: "truck", - 9: "boat", - 10: "trafficlight", - 11: "firehydrant", - 12: "streetsign", - 13: "stopsign", - 14: "parkingmeter", - 15: "bench", - 16: "bird", - 17: "cat", - 18: "dog", - 19: "horse", - 20: "sheep", - 21: "cow", - 22: "elephant", - 23: "bear", - 24: "zebra", - 25: "giraffe", - 26: "hat", - 27: "backpack", - 28: "umbrella", - 29: "shoe", - 30: "eyeglasses", - 31: "handbag", - 32: "tie", - 33: "suitcase", - 34: "frisbee", - 35: "skis", - 36: "snowboard", - 37: "sportsball", - 38: "kite", - 39: "baseballbat", - 40: "baseballglove", - 41: "skateboard", - 42: "surfboard", - 43: "tennisracket", - 44: "bottle", - 45: "plate", - 46: "wineglass", - 47: "cup", - 48: "fork", - 49: "knife", - 50: "spoon", - 51: "bowl", - 52: "banana", - 53: "apple", - 54: "sandwich", - 55: "orange", - 56: "broccoli", - 57: "carrot", - 58: "hotdog", - 59: "pizza", - 60: "donut", - 61: "cake", - 62: "chair", - 63: "sofa", - 64: "pottedplant", - 65: "bed", - 66: "mirror", - 67: "diningtable", - 68: "window", - 69: "desk", - 70: "toilet", - 71: "door", - 72: "tv", - 73: "laptop", - 74: "mouse", - 75: "remote", - 76: "keyboard", - 77: "cellphone", - 78: "microwave", - 79: "oven", - 80: "toaster", - 81: "sink", - 82: "refrigerator", - 83: "blender", - 84: "book", - 85: "clock", - 86: "vase", - 87: "scissors", - 88: "teddybear", - 89: "hairdrier", - 90: "toothbrush", - 91: "hairbrush", -} - - -class CustomModel(fom.Model): - def __init__(self, cfg): - super().__init__() - self.model = cfg.model.eval().cuda() - self.postprocessor = cfg.postprocessor.eval().cuda() - self.transform = transforms.Compose( - [ - transforms.ToTensor(), - transforms.Resize((640, 640)), # Resize to the size expected by your model - # transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) - ] - ) - - @property - def media_type(self): - return "image" - - @property - def has_logits(self): - return False - - @property - def has_embeddings(self): - return False - - @property - def ragged_batches(self): - return False - - @property - def transforms(self): - return None - - @property - def preprocess(self): - return True - - @preprocess.setter - def preprocess(self, value): - pass - - def _convert_predictions(self, predictions): - class_labels, bboxes, scores = ( - predictions[0]["labels"], - predictions[0]["boxes"], - predictions[0]["scores"], - ) - - detections = [] - for label, bbox, score in zip(class_labels, bboxes, scores): - detection = fol.Detection( - label=label_map[label.item()], - bounding_box=[ - bbox[0] / 640, # Normalized coordinates - bbox[1] / 640, - (bbox[2] - bbox[0]) / 640, - (bbox[3] - bbox[1]) / 640, - ], - confidence=score, - ) - detections.append(detection) - - return fol.Detections(detections=detections) - - def predict(self, image): - image = Image.fromarray(image).convert("RGB") - image_tensor = self.transform(image).unsqueeze(0).cuda() - outputs = self.model(image_tensor) - orig_target_sizes = torch.tensor([[640, 640]]).cuda() - predictions = self.postprocessor(outputs, orig_target_sizes) - return self._convert_predictions(predictions) - - def predict_all(self, images): - image_tensors = [] - for image in images: - image = Image.fromarray(image) - image_tensor = self.transform(image) - image_tensors.append(image_tensor) - image_tensors = torch.stack(image_tensors).cuda() - outputs = self.model(image_tensors) - orig_target_sizes = torch.tensor([[640, 640] for image in images]).cuda() - predictions = self.postprocessor(outputs, orig_target_sizes) - converted_predictions = [self._convert_predictions(pred) for pred in predictions] - - # Ensure the output is a list of lists of Detections - return converted_predictions - - -def filter_by_predictions5_confidence(predictions_view, confidence_threshold=0.3): - for j, sample in tqdm.tqdm(enumerate(predictions_view), total=len(predictions_view)): - has_modified = False - for i, detection in enumerate(sample["predictions0"].detections): - if "original_confidence" not in detection: - detection["original_confidence"] = detection["confidence"] - - if ( - detection["confidence"] <= confidence_threshold - and sample["predictions5"].detections[i]["confidence"] >= confidence_threshold - ) or ( - detection["confidence"] >= confidence_threshold - and sample["predictions5"].detections[i]["confidence"] <= confidence_threshold - ): - sample["predictions0"].detections[i]["confidence"] = sample[ - "predictions5" - ].detections[i]["confidence"] - has_modified = True - if has_modified: - sample.save() - - -def restore_confidence(predictions_view): - for j, sample in tqdm.tqdm(enumerate(predictions_view), total=len(predictions_view)): - for i, detection in enumerate(sample["predictions0"].detections): - if "original_confidence" in detection: - detection["confidence"] = detection["original_confidence"] - sample.save() - - -def fast_iou(bbox1, bbox2): - x1, y1, w1, h1 = bbox1 - x2, y2, w2, h2 = bbox2 - xA = max(x1, x2) - yA = max(y1, y2) - xB = min(x1 + w1, x2 + w2) - yB = min(y1 + h1, y2 + h2) - interArea = max(0, xB - xA) * max(0, yB - yA) - boxAArea = w1 * h1 - boxBArea = w2 * h2 - iou = interArea / float(boxAArea + boxBArea - interArea) - return iou - - -def assign_iou_diff(predictions_view): - for sample in predictions_view: - ious_0 = [ - detection.eval0_iou if "eval0_iou" in detection else None - for detection in sample["predictions0"].detections - ] - ious_5 = [ - detection.eval5_iou if "eval5_iou" in detection else None - for detection in sample["predictions5"].detections - ] - bbox_0 = [detection.bounding_box for detection in sample["predictions0"].detections] - bbox_5 = [detection.bounding_box for detection in sample["predictions5"].detections] - # iou_diffs = [abs(iou_5 - iou_0) if iou_0 is not None and iou_5 is not None else -1 for iou_0, iou_5 in zip(ious_0, ious_5)] - iou_inter = [fast_iou(b0, b5) for b0, b5 in zip(bbox_0, bbox_5)] - iou_diffs = [ - abs(iou_5 - iou_0) - if iou_0 is not None and iou_5 is not None and iou_inter > 0.5 - else -1 - for iou_0, iou_5, iou_inter in zip(ious_0, ious_5, iou_inter) - ] - - for detection, iou_diff in zip(sample["predictions0"].detections, iou_diffs): - detection["iou_diff"] = iou_diff - for detection, iou_diff in zip(sample["predictions5"].detections, iou_diffs): - detection["iou_diff"] = iou_diff - # for detection, iou_diff in zip(sample["predictions100"].detections, iou_diffs): - # detection["iou_diff"] = iou_diff - sample.save() - - -def main(args): - try: - if os.path.exists("saved_predictions_view") and os.path.exists("saved_filtered_view"): - print("Loading saved predictions and filtered views...") - dataset = foz.load_zoo_dataset( - "coco-2017", - split="validation", - dataset_name="evaluate-detections-tutorial", - dataset_dir="data/fiftyone", - ) - - dataset.persistent = True - session = fo.launch_app(dataset) - - predictions_view = fo.Dataset.from_dir( - dataset_dir="saved_predictions_view", dataset_type=fo.types.FiftyOneDataset - ).view() - filtered_view = fo.Dataset.from_dir( - dataset_dir="saved_filtered_view", dataset_type=fo.types.FiftyOneDataset - ).view() - else: - dataset = foz.load_zoo_dataset( - "coco-2017", - split="validation", - dataset_name="evaluate-detections-tutorial", - dataset_dir="data/fiftyone", - ) - - dataset.persistent = True - - session = fo.launch_app(dataset) - cfg = YAMLConfig(args.config, resume=args.resume) - if "HGNetv2" in cfg.yaml_cfg: - cfg.yaml_cfg["HGNetv2"]["pretrained"] = False - if args.resume: - checkpoint = torch.load(args.resume, map_location="cpu") - if "ema" in checkpoint: - state = checkpoint["ema"]["module"] - else: - state = checkpoint["model"] - else: - raise AttributeError("only support resume to load model.state_dict by now.") - - # NOTE load train mode state -> convert to deploy mode - cfg.model.load_state_dict(state) - predictions_view = dataset.take(100, seed=51) - - model = CustomModel(cfg) - L = model.model.decoder.decoder.eval_idx - # Apply models and save predictions in different label fields - for i in [L]: - model.model.decoder.decoder.eval_idx = i - label_field = "predictions{:d}".format(i) - predictions_view.apply_model(model, label_field=label_field) - - # filter_by_predictions5_confidence(predictions_view, confidence_threshold=0.3) - for i in [L]: - label_field = "predictions{:d}".format(i) - predictions_view = predictions_view.filter_labels( - label_field, F("confidence") > 0.5, only_matches=False - ) - eval_key = "eval{:d}".format(i) - _ = predictions_view.evaluate_detections( - label_field, - gt_field="ground_truth", - eval_key=eval_key, - compute_mAP=True, - ) - - # assign_iou_diff(predictions_view) - - # filtered_view = predictions_view.filter_labels("predictions0", F("iou_diff") > 0.05, only_matches=True) - # filtered_view = filtered_view.filter_labels("predictions5", F("iou_diff") > 0.05, only_matches=True) - # restore_confidence(filtered_view) - - predictions_view.export( - export_dir="saved_predictions_view", dataset_type=fo.types.FiftyOneDataset - ) - # filtered_view.export( - # export_dir="saved_filtered_view", - # dataset_type=fo.types.FiftyOneDataset - # ) - - # Display the filtered view - session.view = predictions_view - - # Keep the session open - while True: - time.sleep(1) - except Exception as e: - print(f"An error occurred: {e}") - finally: - print("Shutting down session") - if "session" in locals(): - session.close() - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--config", "-c", type=str) - parser.add_argument("--resume", "-r", type=str) - args = parser.parse_args() - - main(args) diff --git a/vlm/detector/D-FINE.backup/train.py b/vlm/detector/D-FINE.backup/train.py deleted file mode 100644 index d4aa451..0000000 --- a/vlm/detector/D-FINE.backup/train.py +++ /dev/null @@ -1,113 +0,0 @@ -""" -D-FINE: Redefine Regression Task of DETRs as Fine-grained Distribution Refinement -Copyright (c) 2024 The D-FINE Authors. All Rights Reserved. ---------------------------------------------------------------------------------- -Modified from RT-DETR (https://github.com/lyuwenyu/RT-DETR) -Copyright (c) 2023 lyuwenyu. All Rights Reserved. -""" - -import os -import sys -import torch - -sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) - -import argparse - -from src.core import YAMLConfig, yaml_utils -from src.misc import dist_utils -from src.solver import TASKS -from pprint import pprint - -debug = False - -if debug: - def custom_repr(self): - return f"{{Tensor:{tuple(self.shape)}}} {original_repr(self)}" - - original_repr = torch.Tensor.__repr__ - torch.Tensor.__repr__ = custom_repr - - -def safe_get_rank(): - if torch.distributed.is_available() and torch.distributed.is_initialized(): - return torch.distributed.get_rank() - else: - return 0 - - -def main(args) -> None: - """main""" - dist_utils.setup_distributed(args.print_rank, args.print_method, seed=args.seed) - - assert not all( - [args.tuning, args.resume] - ), "Only support from_scrach or resume or tuning at one time" - - update_dict = yaml_utils.parse_cli(args.update) - update_dict.update( - { - k: v - for k, v in args.__dict__.items() - if k - not in [ - "update", - ] - and v is not None - } - ) - - cfg = YAMLConfig(args.config, **update_dict) - - if args.resume or args.tuning: - if "HGNetv2" in cfg.yaml_cfg: - cfg.yaml_cfg["HGNetv2"]["pretrained"] = False - - if safe_get_rank() == 0: - print("cfg: ") - pprint(cfg.__dict__) - - solver = TASKS[cfg.yaml_cfg["task"]](cfg) - - if args.test_only: - solver.val() - else: - solver.fit() - - dist_utils.cleanup() - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - - # priority 0 - parser.add_argument("-c", "--config", type=str, required=True) - parser.add_argument("-r", "--resume", type=str, help="resume from checkpoint") - parser.add_argument("-t", "--tuning", type=str, help="tuning from checkpoint") - parser.add_argument( - "-d", - "--device", - type=str, - help="device", - ) - parser.add_argument("--seed", type=int, help="exp reproducibility") - parser.add_argument("--use-amp", action="store_true", help="auto mixed precision training") - parser.add_argument("--output-dir", type=str, help="output directoy") - parser.add_argument("--summary-dir", type=str, help="tensorboard summry") - parser.add_argument( - "--test-only", - action="store_true", - default=False, - ) - - # priority 1 - parser.add_argument("-u", "--update", nargs="+", help="update yaml config") - - # env - parser.add_argument("--print-method", type=str, default="builtin", help="print method") - parser.add_argument("--print-rank", type=int, default=0, help="print rank id") - - parser.add_argument("--local-rank", type=int, help="local rank id") - args = parser.parse_args() - - main(args) diff --git a/vlm/detector/__init__.py b/vlm/detector/__init__.py index b8242e3..bac1db1 100644 --- a/vlm/detector/__init__.py +++ b/vlm/detector/__init__.py @@ -1,5 +1,5 @@ """ -Object Detection Module for ApexNav. +Object detection module for InfoNav. Provides multiple detector backends: - YOLOv7: Fast real-time detection (original) diff --git a/vlm/detector/dfine.py b/vlm/detector/dfine.py index df31e8a..f8f1385 100644 --- a/vlm/detector/dfine.py +++ b/vlm/detector/dfine.py @@ -1,5 +1,5 @@ """ -D-FINE Object Detector Module for ApexNav. +D-FINE object detector module for InfoNav. Replaces YOLOv7 with the state-of-the-art D-FINE detector. D-FINE: Redefine Regression Task of DETRs as Fine-grained Distribution Refinement @@ -42,7 +42,7 @@ def _import_dfine(): class DFine(nn.Module): """ - D-FINE object detector wrapper compatible with ApexNav's detection interface. + D-FINE object detector wrapper compatible with InfoNav's detection interface. D-FINE is a powerful real-time object detector based on DETR architecture with fine-grained distribution refinement for improved accuracy. @@ -54,6 +54,7 @@ def __init__( checkpoint_path: str, device: Optional[str] = None, image_size: int = 640, + num_top_queries: int = 15, ): """ Initialize D-FINE detector. @@ -63,6 +64,8 @@ def __init__( checkpoint_path: Path to D-FINE checkpoint file device: Device to run inference on ('cuda' or 'cpu') image_size: Input image size for the model + num_top_queries: Number of highest-scoring detections retained by + the D-FINE postprocessor. InfoNav uses 15. """ super().__init__() @@ -72,6 +75,7 @@ def __init__( self.device = torch.device(device) self.image_size = image_size + self.num_top_queries = num_top_queries self._load_model(config_path, checkpoint_path) self.model.eval() @@ -91,6 +95,12 @@ def _load_model(self, config_path: str, checkpoint_path: str): try: cfg = _YAMLConfig(config_path) + # Keep the InfoNav inference setting outside the third-party + # submodule so a fresh clone reproduces the published behavior. + cfg.yaml_cfg["DFINEPostProcessor"]["num_top_queries"] = ( + self.num_top_queries + ) + # Handle special cases in config if "HGNetv2" in cfg.yaml_cfg: cfg.yaml_cfg["HGNetv2"]["pretrained"] = False @@ -266,6 +276,12 @@ class ObjectDetectorClient(DFineClient): default="pretrained_weights/dfine_x_coco.pth", help="Path to D-FINE checkpoint file" ) + parser.add_argument( + "--num-top-queries", + type=int, + default=15, + help="Number of D-FINE postprocessor queries retained (default: 15)", + ) args = parser.parse_args() print("Loading D-FINE model...") @@ -281,7 +297,11 @@ def process_payload(self, payload: dict) -> dict: conf_thres = payload.get("conf_thres", 0.4) return self.predict(image, conf_thres=conf_thres).to_json() - dfine = DFineServer(args.config, args.checkpoint) + dfine = DFineServer( + args.config, + args.checkpoint, + num_top_queries=args.num_top_queries, + ) print("D-FINE loaded!") print(f"Hosting on port {args.port}...") host_model(dfine, name="dfine", port=args.port) diff --git a/vlm/utils/get_itm_message.py b/vlm/utils/get_itm_message.py index 5222823..71c0ba2 100644 --- a/vlm/utils/get_itm_message.py +++ b/vlm/utils/get_itm_message.py @@ -1,20 +1,37 @@ import cv2 import numpy as np import os +import yaml +from pathlib import Path -# Check if parallel mode is enabled via environment variable -USE_PARALLEL_BLIP2 = os.environ.get('USE_PARALLEL_BLIP2', 'false').lower() == 'true' +# ============================================================ +# Load VLM Configuration +# ============================================================ +_config_path = Path(__file__).parent.parent.parent / "config" / "default.yaml" +if _config_path.exists(): + with open(_config_path, encoding='utf-8') as f: + _VLM_CONFIG = yaml.safe_load(f) +else: + _VLM_CONFIG = {} + +# Check if parallel mode is enabled (priority: env var > config file > default) +USE_PARALLEL_BLIP2 = os.environ.get('USE_PARALLEL_BLIP2', + str(_VLM_CONFIG.get('vlm', {}).get('blip2_itm', {}).get('parallel', {}).get('enabled', False))).lower() == 'true' + +# Get port configuration from config file (with fallback to hardcoded defaults) +_BLIP2_PORT = _VLM_CONFIG.get('vlm', {}).get('blip2_itm', {}).get('port', 12182) +_BLIP2_PARALLEL_PORTS = _VLM_CONFIG.get('vlm', {}).get('blip2_itm', {}).get('parallel', {}).get('ports', [12182, 12192]) if USE_PARALLEL_BLIP2: from vlm.itm.blip2itm_parallel import ParallelBLIP2ITMClient # Use 2 BLIP2 servers for parallel processing - itmclient = ParallelBLIP2ITMClient(ports=[12182, 12192]) - print("[ITM] Using ParallelBLIP2ITMClient with 2 servers (ports 12182, 12192)") + itmclient = ParallelBLIP2ITMClient(ports=_BLIP2_PARALLEL_PORTS) + print(f"[ITM] Using ParallelBLIP2ITMClient with {len(_BLIP2_PARALLEL_PORTS)} servers (ports {_BLIP2_PARALLEL_PORTS})") else: from vlm.itm.blip2itm import BLIP2ITMClient # Use single BLIP2 server - itmclient = BLIP2ITMClient(port=12182) - print("[ITM] Using single BLIP2ITMClient (port 12182)") + itmclient = BLIP2ITMClient(port=_BLIP2_PORT) + print(f"[ITM] Using single BLIP2ITMClient (port {_BLIP2_PORT})") def get_itm_message(rgb_image, label): txt = f"Is there a {label} in the image?" @@ -245,6 +262,68 @@ def get_ig_score_detailed(rgb_image): return result +def get_ig_score_dynamic(rgb_image, ig_prompts=None): + """ + Calculate Information Gain (IG) score with dynamic prompts. + + This function supports LLM-generated IG hypotheses for exploration potential + estimation. Falls back to default prompts if none provided. + + Args: + rgb_image: Current RGB observation (numpy array, HxWx3) + ig_prompts: List of IG prompts (optional). If None, uses default prompts. + + Returns: + dict: { + "ig_score": float, # Average IG score + "individual_scores": list, # Score for each prompt + "prompts": list[str], # The prompts used + "inference_time": float, # Time taken for inference + "is_dynamic": bool # Whether dynamic prompts were used + } + """ + import time + t_start = time.time() + + # Use provided prompts or fall back to default + if ig_prompts and len(ig_prompts) > 0: + prompts = ig_prompts + is_dynamic = True + else: + prompts = IG_CONNECTIVITY_PROMPTS + is_dynamic = False + + # Batch compute ITM scores for all prompts + cosine_scores = itmclient.cosine_batch(rgb_image, prompts) + + # Average the scores + ig_score = sum(cosine_scores) / len(cosine_scores) + + t_elapsed = time.time() - t_start + + # Log timing if available + try: + from basic_utils.logging import get_log_manager + logger = get_log_manager() + logger.log_vlm_detail( + f" [IG Score Dynamic] {ig_score:.3f} | Time: {t_elapsed:.3f}s | " + f"Prompts: {len(prompts)} ({'dynamic' if is_dynamic else 'default'}) | " + f"Scores: [{', '.join(f'{s:.3f}' for s in cosine_scores)}]" + ) + except ImportError: + pass + + result = { + "ig_score": ig_score, + "individual_scores": cosine_scores, + "prompts": prompts, + "inference_time": t_elapsed, + "is_dynamic": is_dynamic + } + + return result + + def get_multi_source_cosine_with_ig(rgb_image, label, semantic_hypotheses, ig_weights=None): """ ่ฎก็ฎ—ๅคšๆบ่ฏญไน‰ๅ‡่ฏด็š„cosineๅˆ†ๆ•ฐ + IGๅˆ†ๆ•ฐ @@ -328,4 +407,134 @@ def get_multi_source_cosine_with_ig(rgb_image, label, semantic_hypotheses, ig_we except ImportError: pass - return hypotheses_data, ig_data \ No newline at end of file + return hypotheses_data, ig_data + + +def get_multi_source_cosine_with_dynamic_ig(rgb_image, label, semantic_hypotheses, ig_prompts=None): + """ + ่ฎก็ฎ—ๅคšๆบ่ฏญไน‰ๅ‡่ฏด็š„cosineๅˆ†ๆ•ฐ + ๅŠจๆ€IGๅˆ†ๆ•ฐ + + ไธŽ get_multi_source_cosine_with_ig ็ฑปไผผ๏ผŒไฝ†ๆ”ฏๆŒLLMๅŠจๆ€็”Ÿๆˆ็š„IGๅ‡่ฎพ๏ผŒ + ่€Œไธๆ˜ฏๅ›บๅฎš็š„3ไธชconnectivity promptsใ€‚ + + HSVM (Hierarchical Semantic Value Map) ๅŒ…ๅซๅคšๅฑ‚็บงๅ‡่ฏด: + - room_type: ๆˆฟ้—ด็ฑปๅž‹ + - target_object: ็›ฎๆ ‡็‰ฉไฝ“ + - co_occurrence: ๅ…ฑ็Žฐ็‰ฉไฝ“ + - part_attribute: ็‰ฉไฝ“้ƒจไปถ + + IGๅ‡่ฎพๆ˜ฏ็›ฎๆ ‡ๆ— ๅ…ณ็š„่ฟž้€šๆ€งๆ่ฟฐ๏ผˆcorridors, doorways, passages็ญ‰๏ผ‰ใ€‚ + + Args: + rgb_image: ๅฝ“ๅ‰่ง‚ๆต‹็š„RGBๅ›พๅƒ + label: ็›ฎๆ ‡็‰ฉไฝ“ๅ็งฐ (็”จไบŽๆ—ฅๅฟ—่ฎฐๅฝ•) + semantic_hypotheses: LLM็”Ÿๆˆ็š„HSVMๅ‡่ฏดๅˆ—่กจ + ๆ ผๅผ: [{"id": 1, "type": "room_type", "prompt": "bedroom", + "confidence": 0.8, "navigation_value": 0.9, "weight": 0.72}, ...] + ig_prompts: LLM็”Ÿๆˆ็š„IGๅ‡่ฏดpromptsๅˆ—่กจ (ๅฏ้€‰) + ๆ ผๅผ: ["corridor leading to bedrooms", "open doorway", ...] + ๅฆ‚ๆžœไธบNoneๆˆ–็ฉบ๏ผŒไฝฟ็”จ้ป˜่ฎค็š„IG_CONNECTIVITY_PROMPTS + + Returns: + tuple: (hypotheses_data, ig_data) + - hypotheses_data: List[dict] - ่ฏญไน‰ๅ‡่ฏด็ป“ๆžœ + - ig_data: dict - IGๅˆ†ๆ•ฐ็ป“ๆžœ (ๅŠจๆ€ๆˆ–้ป˜่ฎคprompts) + """ + import time + t_start_total = time.time() + + # ========== 1. Build HSVM semantic prompts ========== + prompts_list = [] + metadata_list = [] + + for hyp in semantic_hypotheses: + hyp_description = hyp.get("prompt") or hyp.get("description", "") + hyp_type = hyp.get("type") or hyp.get("hypothesis_type", "unknown") + hyp_confidence = hyp.get("confidence", 0.5) + hyp_nav_value = hyp.get("navigation_value", 0.5) + hyp_weight = hyp.get("weight") or (hyp_confidence * hyp_nav_value) + + hyp_prompt = f"Seems like there is a {hyp_description} ahead?" + prompts_list.append(hyp_prompt) + metadata_list.append({ + "id": hyp.get("id", -1), + "type": hyp_type, + "prompt": hyp_prompt, + "confidence": hyp_confidence, + "navigation_value": hyp_nav_value, + "weight": hyp_weight + }) + + # A missing/corrupt hypothesis file must not silently turn the semantic + # map into IG-only exploration. Keep one explicit target-object channel. + if not any(item["type"] == "target_object" for item in metadata_list): + target_prompt = f"Seems like there is a {label} ahead?" + prompts_list.insert(0, target_prompt) + metadata_list.insert(0, { + "id": 0, + "type": "target_object", + "prompt": target_prompt, + "confidence": 1.0, + "navigation_value": 1.0, + "weight": 1.0 + }) + + # ========== 2. Select dynamic/default IG prompts ========== + if ig_prompts and len(ig_prompts) > 0: + active_ig_prompts = ig_prompts + is_dynamic = True + else: + active_ig_prompts = IG_CONNECTIVITY_PROMPTS + is_dynamic = False + + # ========== 3. One BLIP-2 request for every value-map channel ========== + # The same RGB embedding supports all HSVM layers and the IG map. A second + # BLIP-2 replica is therefore only a throughput optimization, not a + # functional requirement for multiple value maps. + t0 = time.time() + combined_prompts = prompts_list + list(active_ig_prompts) + combined_scores = itmclient.cosine_batch(rgb_image, combined_prompts) + t_combined_inference = time.time() - t0 + + semantic_count = len(prompts_list) + semantic_scores = combined_scores[:semantic_count] + ig_scores = combined_scores[semantic_count:] + + hypotheses_data = [] + for metadata, score in zip(metadata_list, semantic_scores): + metadata["score"] = score + hypotheses_data.append(metadata) + + ig_score_avg = sum(ig_scores) / len(ig_scores) if ig_scores else 0.0 + + # Build ig_data + ig_data = { + "ig_score": ig_score_avg, + "individual_scores": ig_scores, + "prompts": active_ig_prompts, + "is_dynamic": is_dynamic + } + + # For backward compatibility, include fixed keys if using default prompts + if not is_dynamic and len(ig_scores) >= 3: + ig_data["corridor_score"] = ig_scores[0] + ig_data["doorway_score"] = ig_scores[1] + ig_data["passage_score"] = ig_scores[2] + + t_total = time.time() - t_start_total + + # Log timing + try: + from basic_utils.logging import get_log_manager + logger = get_log_manager() + logger.log_vlm_detail( + f" [VLM] Total: {t_total:.3f}s | " + f"Combined BLIP2: {t_combined_inference:.3f}s " + f"({len(prompts_list)} HSVM + {len(active_ig_prompts)} IG prompts) | " + f"IG: ({len(active_ig_prompts)} prompts, " + f"{'dynamic' if is_dynamic else 'default'}, score: {ig_score_avg:.3f})" + ) + except ImportError: + pass + + return hypotheses_data, ig_data diff --git a/vlm/utils/get_object_utils.py b/vlm/utils/get_object_utils.py index d7ac8b8..6ebad73 100644 --- a/vlm/utils/get_object_utils.py +++ b/vlm/utils/get_object_utils.py @@ -1,5 +1,7 @@ import cv2 import numpy as np +import yaml +from pathlib import Path from vlm.coco_classes import COCO_CLASSES from vlm.detector.dfine import DFineClient from vlm.segmentor.sam import MobileSAMClient @@ -8,6 +10,26 @@ from vlm.utils.get_itm_message import get_itm_message from vlm.detector.yolov7 import YOLOv7Client +# ============================================================ +# Load VLM Configuration +# ============================================================ +# Try to load from InfoNav/config/default.yaml +_config_path = Path(__file__).parent.parent.parent / "config" / "default.yaml" +if _config_path.exists(): + with open(_config_path, encoding='utf-8') as f: + _VLM_CONFIG = yaml.safe_load(f) + print(f"[VLM Config] Loaded from {_config_path}") +else: + _VLM_CONFIG = {} + print(f"[VLM Config] File not found at {_config_path}, using defaults") + +# Extract VLM server ports (with fallback to hardcoded defaults) +_BLIP2_PORT = _VLM_CONFIG.get('vlm', {}).get('blip2_itm', {}).get('port', 12182) +_DINO_PORT = _VLM_CONFIG.get('vlm', {}).get('grounding_dino', {}).get('port', 12181) +_SAM_PORT = _VLM_CONFIG.get('vlm', {}).get('sam', {}).get('port', 12183) +_YOLOV7_PORT = _VLM_CONFIG.get('vlm', {}).get('yolov7', {}).get('port', 12184) +_DFINE_PORT = _VLM_CONFIG.get('vlm', {}).get('dfine', {}).get('port', 12185) + # ============== Detection Mode Configuration ============== # Mode 1: YOLOv7 + GroundingDINO (ApexNav original, better false positive reduction) # Mode 2: D-FINE + GroundingDINO (current InfoNav, for comparison experiments) @@ -16,13 +38,13 @@ # Port configuration matches start_vlm_servers.sh # COCO object detectors (mutually exclusive based on mode) -yolov7_detector = YOLOv7Client(port=12184) # Mode 1: YOLOv7 -dfine_detector = DFineClient(port=12185) # Mode 2: D-FINE +yolov7_detector = YOLOv7Client(port=_YOLOV7_PORT) # Mode 1: YOLOv7 +dfine_detector = DFineClient(port=_DFINE_PORT) # Mode 2: D-FINE # Shared components (used in both modes) -blip2_itm = BLIP2ITMClient(port=12182) -sam_segmentor = MobileSAMClient(port=12183) -dino_detector = GroundingDINOClient(port=12181) # Used for non-COCO objects in both modes +blip2_itm = BLIP2ITMClient(port=_BLIP2_PORT) +sam_segmentor = MobileSAMClient(port=_SAM_PORT) +dino_detector = GroundingDINOClient(port=_DINO_PORT) # Used for non-COCO objects in both modes def get_segmentation(segmented_img, idx, detections, img, label, score, color): diff --git a/vlm/utils/http_vlm_client.py b/vlm/utils/http_vlm_client.py index d6dce02..188caf3 100644 --- a/vlm/utils/http_vlm_client.py +++ b/vlm/utils/http_vlm_client.py @@ -3,8 +3,6 @@ Connects to local VLM service at http://localhost:20004 Supports vision-language tasks with OpenAI format. - -Author: Zager-Zhang """ import requests diff --git a/vlm_environment_analyzer.py b/vlm_environment_analyzer.py index 3250bd0..cf6d97a 100644 --- a/vlm_environment_analyzer.py +++ b/vlm_environment_analyzer.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """ ไฝฟ็”จ่ฏดๆ˜Ž๏ผš2026-2-1 -#python vlm_environment_analyzer.py single --env hm3dv2 --task 0 --api-type dashscope --model qwen3-vl-flash --env-base-dir /home/yangz/Nav/InfoNav/env +# python vlm_environment_analyzer.py single --env hm3dv2 --task 0 --api-type dashscope --model qwen3-vl-flash --env-base-dir ./env VLM ็Žฏๅขƒๆ„Ÿ็Ÿฅๅˆ†ๆžๅทฅๅ…ท (Unified Version) =====================================