fix - table分割支持更多场景

- code删除单反引号匹配

Author: pekopoke

tests/test_table_extraction.py

@@ -114,6 +114,51 @@ def test_complex_markdown_table(self):
         # 验证文本中表格被移除
         # self.assertNotIn('| 姓名 | 年龄 | 职业 | 薪资 |', result['text'])
 
+    def test_corner_case_markdown_table(self):
+        """测试复杂Markdown表格"""
+        text = """| |
+|---|---|
+| Scientific classification | |
+| Domain: | Eukaryota |
+| Kingdom: | Animalia |
+| Phylum: | Arthropoda |
+| Class: | Insecta |
+| Order: | Lepidoptera |
+| Family: | Autostichidae |
+| Genus: | Pantacordis
+|
+| Species: | P. scotinella |
+| Binomial name | |
+Pantacordis scotinella | |
+| Synonyms | |
+*Borkhausenia scotinella*Rebel, 1916*Pantacordis scotinellum*
+|"""
+
+        result = self.metric._extract_from_markdown(text)
+        expected_table = """| |
+|---|---|
+| Scientific classification | |
+| Domain: | Eukaryota |
+| Kingdom: | Animalia |
+| Phylum: | Arthropoda |
+| Class: | Insecta |
+| Order: | Lepidoptera |
+| Family: | Autostichidae |
+| Genus: | Pantacordis
+|
+| Species: | P. scotinella |
+| Binomial name | |
+Pantacordis scotinella | |
+| Synonyms | |
+*Borkhausenia scotinella*Rebel, 1916*Pantacordis scotinellum*
+|"""
+
+        # 验证提取结果与预期一致
+        self.assertEqual(result['table'], expected_table)
+
+        # 验证文本中表格被移除
+        # self.assertNotIn('| 姓名 | 年龄 | 职业 | 薪资 |', result['text'])
+
 
 
     def test_table_with_alignment(self):

webmainbench/extractors/magic_html_extractor.py

@@ -9,6 +9,7 @@
 from magic_html import GeneralExtractor
 import re
 import html2text
+from ..utils import HTML2TextWrapper
 
 
 @extractor("magic-html")
@@ -32,7 +33,10 @@ def _extract_content(self, html: str, url: str = None) -> ExtractionResult:
 
             # 从输出中提取所需信息
             extracted_html = data.get('html', '')
-            markdown = html2text.html2text(extracted_html)
+            # 使用内部HTML2Text方法生成markdown
+            h = HTML2TextWrapper()
+            markdown = h(extracted_html)
+            # markdown = html2text.html2text(extracted_html)
             title = data.get('title', '')
             # 简单地将提取的 HTML 作为内容
             content = markdown

webmainbench/metrics/base.py

@@ -284,7 +284,7 @@ def _extract_from_markdown(text: str) -> Dict[str, str]:
 
         def is_md_table_line(line):
             """判断是否可能是 Markdown 表格行"""
-            if line.count("|") < 3:  # 至少三个竖线
+            if line.count("|") < 1:  # 至少三个竖线
                 return False
             return True
 
@@ -294,13 +294,17 @@ def is_md_separator_line(line):
             # 检查是否所有部分都是分隔符格式
             for p in parts:
                 if p and not re.match(r"^:?\-{3,}:?$", p):
+
+                # if p and not re.match(r"^:?\-+(:?)$", p):  # 调整正则为允许1个及以上短横线
                     return False
             return True
 
         def save_table():
             """保存当前表格并清空缓存"""
             nonlocal table_lines
-            # 只有当表格行数大于等于2，且第二行是分隔行时才保存
+            # # 原逻辑要求至少2行且第2行是分隔行，改为只要有2行及以上且包含至少1个分隔行即可
+            # if len(table_lines) >= 2 and any(is_md_separator_line(line) for line in table_lines):
+            #     只有当表格行数大于等于2，且第二行是分隔行时才保存
             if len(table_lines) >= 2 and is_md_separator_line(table_lines[1]):
                 md_table = '\n'.join(table_lines)
                 extracted_segments.append(md_table)