feat: replace html2text with HTML2TextWrapper for improved HTML to text conversion in evaluators and extractors

Author: darkrush

webmainbench/evaluator/main_html_evaluator.py

@@ -4,13 +4,12 @@
 
 from typing import Dict, Any, List, Optional, Union
 from datetime import datetime
-import html2text
 
 from ..data import BenchmarkDataset, DataSample
 from ..extractors import BaseExtractor, ExtractorFactory
 from ..metrics import  MainHTMLMetricCalculator
 from .evaluator import EvaluationResult, Evaluator
-from ..utils import extract_main_html
+from ..utils import extract_main_html, HTML2TextWrapper
 
 class MainHTMLEvaluator(Evaluator):
     """Main html evaluator for web content extraction benchmarks."""
@@ -24,9 +23,7 @@ def __init__(self, metric_config: Dict[str, Any] = None):
         """
         self.metric_calculator = MainHTMLMetricCalculator(metric_config)
         self.metric_config = metric_config or {}
-        self.html2text = html2text.HTML2Text(bodywidth=0)
-        self.html2text.ignore_links = True
-        self.html2text.ignore_images = True
+        self.html2text = HTML2TextWrapper()
 
 
     def evaluate(self, 
@@ -152,8 +149,7 @@ def _evaluate_sample(self, sample: DataSample, extractor: BaseExtractor) -> Dict
             return sample_result
 
         main_html = extract_main_html(sample.html)
-        self.html2text.baseurl = sample.url
-        convert_gt_main_content = self.html2text.handle(main_html)
+        convert_gt_main_content = self.html2text(main_html, sample.url)
         sample_result['groundtruth_content'] = sample.groundtruth_content
         sample_result['gt_main_html'] = main_html
         sample_result['convert_gt_main_content'] = convert_gt_main_content

webmainbench/extractors/dripper_extractor.py

@@ -7,13 +7,12 @@
 import time
 from typing import Dict, Any, Optional, List
 
-import html2text
 from dripper.api import Dripper
 from dripper.base import DripperInput, DripperOutput
 from .base import BaseExtractor, ExtractionResult
 from .factory import extractor
 
-from ..utils import extract_main_html
+from ..utils import HTML2TextWrapper
 
 
 
@@ -28,9 +27,7 @@ class DripperExtractor(BaseExtractor):
     def __init__(self, name: str, config: Optional[Dict[str, Any]] = None):
         # 先初始化inference_config，再调用父类初始化（因为父类会调用_setup()）
         self.dripper = Dripper(config)
-        self.html2text = html2text.HTML2Text(bodywidth=0)
-        self.html2text.ignore_links = True
-        self.html2text.ignore_images = True
+        self.html2text = HTML2TextWrapper()
 
         # 现在可以安全地调用父类初始化（会调用_setup()）
         super().__init__(name, config)
@@ -57,10 +54,8 @@ def _extract_content(self, html: str, url: str = None) -> ExtractionResult:
             dripper_output : DripperOutput = self.dripper.process([dripper_input])[0]
 
             main_html = dripper_output.main_html
-            self.html2text.baseurl = url
-            main_content = self.html2text.handle(main_html)
+            main_content = self.html2text(main_html, url)
 
-            
             extraction_time = time.time() - start_time
 
             # 创建结果对象

webmainbench/utils/__init__.py

@@ -3,11 +3,12 @@
 """
 
 from .helpers import setup_logging, validate_config, format_results
-from .main_html import extract_main_html
+from .main_html import extract_main_html, HTML2TextWrapper
 
 __all__ = [
     "setup_logging",
     "validate_config", 
     "format_results",
     "extract_main_html",
+    "HTML2TextWrapper",
 ] 

webmainbench/utils/main_html.py

@@ -1,11 +1,25 @@
 
 from lxml import html
-import html2text
+
 
 
 SELECT_ATTR = 'cc-select'
 
 
+class HTML2TextWrapper:
+    def __init__(self):
+        import html2text
+        self.converter = html2text.HTML2Text(bodywidth=0)
+        self.converter.ignore_links = True
+        self.converter.ignore_images = True
+    
+    def __call__(self, html_str: str, url: str = '') -> str:
+        self.converter.baseurl = url
+        text = self.converter.handle(html_str)
+        self.converter.baseurl = ''
+        return text
+
+
 def html_to_element(html_str: str) -> html.HtmlElement:
     parser = html.HTMLParser(
         collect_ids=False,