feat(openevaluator): implement open evaluator for generation level tasks

#4

Author: little1d

.gitignore

@@ -16,4 +16,5 @@
 
 # Local data & playground
 data/
+data_test/
 playground/

spectrumlab/benchmark/base.py

@@ -2,6 +2,7 @@
 from pathlib import Path
 from typing import List, Dict, Union
 import json
+import os
 
 
 class BaseGroup(ABC):
@@ -60,14 +61,26 @@ def _load_from_remote(self, local_level_path: Path):
         # TODO
         self.datasets = {}
 
-    def _fix_image_path(self, image_path: str) -> str:
-        if not image_path or not image_path.strip():
+    def _fix_image_path(self, image_path):
+        if isinstance(image_path, list):
+            return [self._fix_image_path(p) for p in image_path]
+        if not image_path or not str(image_path).strip():
             return image_path
-        if image_path.startswith("./data/"):
-            relative_part = image_path[7:]
+        # 支持 ./data/ 和 data/ 开头
+        s = str(image_path)
+        if s.startswith("./data/"):
+            relative_part = s[7:]
             corrected_path = self.data_root / relative_part
             return str(corrected_path)
-        return image_path
+        if s.startswith("data/"):
+            corrected_path = self.data_root / s[5:]
+            return str(corrected_path)
+        # 如果已经是绝对路径，直接返回
+        if os.path.isabs(s):
+            return s
+        # 其它相对路径，拼到 data_root 下
+        corrected_path = self.data_root / s
+        return str(corrected_path)
 
     def _load_json(self, file_path: Path) -> List[Dict]:
         try:
@@ -79,7 +92,19 @@ def _load_json(self, file_path: Path) -> List[Dict]:
                             if item["image_path"]:
                                 original_path = item["image_path"]
                                 item["image_path"] = self._fix_image_path(original_path)
-
+                        # 修正 answer 字段（如果是图片路径或图片路径 list）
+                        if isinstance(item, dict) and "answer" in item:
+                            answer = item["answer"]
+                            # 只修正字符串类型且像图片路径的 answer
+                            if isinstance(answer, str) and answer.lower().endswith(
+                                (".png", ".jpg", ".jpeg", ".bmp", ".gif", ".webp")
+                            ):
+                                item["answer"] = self._fix_image_path(answer)
+                            # 如果 answer 是 list（极少见），也递归修正
+                            if isinstance(answer, list):
+                                item["answer"] = [
+                                    self._fix_image_path(a) for a in answer
+                                ]
                     return data
                 else:
                     print(f"Warning: Expected list in {file_path}, got {type(data)}")

spectrumlab/evaluator/base.py

@@ -107,61 +107,61 @@ def evaluate_many(
     ) -> Dict:
         """
         Evaluate a single model on data_items with parallel processing.
-        
+
         Args:
             data_items: List of data items to evaluate
             model: Model instance to evaluate
             max_out_len: Maximum output length for model generation
             batch_size: Batch size for processing (if None, will be auto-calculated)
             save_path: Base path to save results
             n_jobs: Number of parallel jobs (-1 for all available cores)
-            
+
         Returns:
             Dictionary containing evaluation results
         """
         import multiprocessing as mp
         from concurrent.futures import ThreadPoolExecutor, as_completed
         import math
-        
+
         if not data_items:
             print("❌ No data items provided")
             return {"error": "No data items provided"}
-            
+
         # Set number of jobs
         if n_jobs == -1:
             n_jobs = mp.cpu_count()
-            
+
         # Calculate batch size if not provided
         if batch_size is None:
             batch_size = max(1, math.ceil(len(data_items) / n_jobs))
-            
+
         print(f"🔄 Starting parallel evaluation on {len(data_items)} items...")
         print(f"📝 Model: {type(model).__name__}")
         print(f"⚡ Using {n_jobs} parallel workers with batch size {batch_size}")
-        
+
         # Split data into batches
         batches = [
-            data_items[i:i + batch_size] 
+            data_items[i : i + batch_size]
             for i in range(0, len(data_items), batch_size)
         ]
-        
+
         print(f"📦 Split into {len(batches)} batches")
-        
+
         # Build prompts for all items
         print("📝 Building prompts...")
         all_prompts = [self._build_prompt(item) for item in data_items]
-        
+
         # Split prompts into batches
         prompt_batches = [
-            all_prompts[i:i + batch_size] 
+            all_prompts[i : i + batch_size]
             for i in range(0, len(all_prompts), batch_size)
         ]
-        
+
         def process_batch(batch_data):
             """Process a batch of prompts and return responses."""
             batch_prompts, batch_indices = batch_data
             batch_responses = []
-            
+
             for i, prompt in enumerate(batch_prompts):
                 try:
                     response = model.generate(prompt, max_out_len)
@@ -171,41 +171,41 @@ def process_batch(batch_data):
                     original_index = batch_indices[i]
                     print(f"\n⚠️  Error on item {original_index + 1}: {e}")
                     batch_responses.append(f"Error: {str(e)}")
-                    
+
             return batch_indices, batch_responses
-        
+
         # Prepare batch data with indices
         batch_data_list = []
         for i, prompt_batch in enumerate(prompt_batches):
             start_idx = i * batch_size
             end_idx = min(start_idx + batch_size, len(data_items))
             batch_indices = list(range(start_idx, end_idx))
             batch_data_list.append((prompt_batch, batch_indices))
-        
+
         # Execute parallel processing
         all_responses = [None] * len(data_items)
-        
+
         with ThreadPoolExecutor(max_workers=n_jobs) as executor:
             # Submit all batch tasks
             future_to_batch = {
-                executor.submit(process_batch, batch_data): batch_data[1][0] 
+                executor.submit(process_batch, batch_data): batch_data[1][0]
                 for batch_data in batch_data_list
             }
-            
+
             # Collect results as they complete
             for future in tqdm(
-                as_completed(future_to_batch), 
+                as_completed(future_to_batch),
                 total=len(future_to_batch),
                 desc="Processing batches",
-                unit="batch"
+                unit="batch",
             ):
                 try:
                     batch_indices, batch_responses = future.result()
                     for idx, response in zip(batch_indices, batch_responses):
                         all_responses[idx] = response
                 except Exception as e:
                     print(f"❌ Error processing batch: {e}")
-        
+
         # Process responses and calculate results
         print("🔍 Processing responses...")
         processed_items = []
@@ -225,27 +225,27 @@ def process_batch(batch_data):
             item_copy["pass"] = is_correct
 
             processed_items.append(item_copy)
-        
+
         # Save results
         saved_files = self._save_results(processed_items, save_path)
         print(f"💾 Results saved to: {saved_files}")
-        
+
         # Calculate metrics
         print("📊 Calculating metrics...")
         metrics = self._calculate_metrics(processed_items)
-        
+
         # Print results
         self._print_results(metrics)
-        
+
         return {
             "metrics": metrics,
             "saved_files": saved_files,
             "total_items": len(data_items),
             "parallel_info": {
                 "n_jobs": n_jobs,
                 "batch_size": batch_size,
-                "n_batches": len(batches)
-            }
+                "n_batches": len(batches),
+            },
         }
 
     def _save_results(self, results_data: List[Dict], save_path: str) -> List[str]:
@@ -302,7 +302,10 @@ def _calculate_metrics(self, processed_items: List[Dict]) -> Dict:
             sub_category = item.get("sub_category", "Unknown")
 
             # Check if prediction exists
-            if not prediction or prediction.strip() == "":
+            # 兼容 prediction 可能为 float（如 OpenEvaluator），也可能为 str（如 ChoiceEvaluator）
+            if prediction is None or (
+                isinstance(prediction, str) and prediction.strip() == ""
+            ):
                 no_prediction += 1
 
             # Use the pre-calculated "pass" field

spectrumlab/evaluator/choice_evaluator.py

@@ -26,9 +26,10 @@ def _build_prompt(self, item: Dict) -> str:
             "Available options:",
             options_block,
             "",
-            "Please think step by step and provide your reasoning.",
-            "After your analysis, indicate your final choice by putting it in \\box{}.",
-            "For example: \\box{Option A}",
+            "Please analyze the question and options carefully. Your answer must be exactly one of the provided options, and must be copied verbatim from the options above.",
+            "Return your answer using the format \\answer{...}, where the content inside the braces is exactly the text of your chosen option (not the option letter or number, and do not use \\box{} or any other wrapper).",
+            "For example, if you choose the option '~1700 cm⁻¹', you should return: \\answer{~1700 cm⁻¹}",
+            "Do not return just a value like '~1700 cm' or any partial/incomplete answer. The answer must match one of the options exactly.",
             "",
             "Your response:",
         ]
@@ -39,6 +40,9 @@ def _build_prompt(self, item: Dict) -> str:
         image_paths = normalize_image_paths(image_paths_field)
 
         if image_paths:
+            assert all(
+                isinstance(p, str) for p in image_paths
+            ), f"image_paths should be List[str], got {image_paths}"
             # Prepare image data
             image_data = prepare_images_for_prompt(image_paths)
 
@@ -50,24 +54,13 @@ def _build_prompt(self, item: Dict) -> str:
         return text_content
 
     def _extract_prediction(self, response: str, item: Dict) -> str:
-        """Extract prediction from model response using \\box{} pattern."""
+        """只提取 \\answer{...} 内的内容"""
         if not response:
             return ""
-
-        choices = item.get("choices", [])
-
-        # Look for \\box{} pattern
-        box_pattern = r"\\box\{([^}]+)\}"
-        matches = re.findall(box_pattern, response)
-
+        answer_pattern = r"\\answer\{([^}]+)\}"
+        matches = re.findall(answer_pattern, response)
         if matches:
-            extracted = matches[-1].strip()
-            # Try to match with actual choices
-            for choice in choices:
-                if choice.lower() == extracted.lower():
-                    return choice
-            return extracted
-
+            return matches[-1].strip()
         return ""
 
     def _calculate_accuracy(self, answer: str, prediction: str, item: Dict) -> bool: