feat: integrate html2text for improved HTML content extraction in evaluators and dripper extractor

Author: darkrush

webmainbench/evaluator/main_html_evaluator.py

@@ -24,6 +24,9 @@ def __init__(self, metric_config: Dict[str, Any] = None):
         """
         self.metric_calculator = MainHTMLMetricCalculator(metric_config)
         self.metric_config = metric_config or {}
+        self.html2text = html2text.HTML2Text(bodywidth=0)
+        self.html2text.ignore_links = True
+        self.html2text.ignore_images = True
 
 
     def evaluate(self, 
@@ -149,7 +152,8 @@ def _evaluate_sample(self, sample: DataSample, extractor: BaseExtractor) -> Dict
             return sample_result
 
         main_html = extract_main_html(sample.html)
-        convert_gt_main_content = html2text.html2text(main_html, sample.url, bodywidth=0)
+        self.html2text.baseurl = sample.url
+        convert_gt_main_content = self.html2text.handle(main_html)
         sample_result['groundtruth_content'] = sample.groundtruth_content
         sample_result['gt_main_html'] = main_html
         sample_result['convert_gt_main_content'] = convert_gt_main_content

webmainbench/extractors/dripper_extractor.py

@@ -28,12 +28,15 @@ class DripperExtractor(BaseExtractor):
     def __init__(self, name: str, config: Optional[Dict[str, Any]] = None):
         # 先初始化inference_config，再调用父类初始化（因为父类会调用_setup()）
         self.dripper = Dripper(config)
+        self.html2text = html2text.HTML2Text(bodywidth=0)
+        self.html2text.ignore_links = True
+        self.html2text.ignore_images = True
 
         # 现在可以安全地调用父类初始化（会调用_setup()）
         super().__init__(name, config)
 
     def _setup(self) -> None:
-        #self.dripper.get_llm()
+        self.dripper.get_llm()
         self.dripper.get_tokenizer()
 
     def _extract_content(self, html: str, url: str = None) -> ExtractionResult:
@@ -54,7 +57,8 @@ def _extract_content(self, html: str, url: str = None) -> ExtractionResult:
             dripper_output : DripperOutput = self.dripper.process([dripper_input])[0]
 
             main_html = dripper_output.main_html
-            main_content = html2text.html2text(dripper_output.main_html, url, bodywidth=0)
+            self.html2text.baseurl = url
+            main_content = self.html2text.handle(main_html)
 
 
             extraction_time = time.time() - start_time