Merge pull request #40 from pekopoke/dev

Dev：优化表格分割、删除code行内分割、teds性能提升

Author: e06084

examples/multi_extractor_compare.py

@@ -8,7 +8,7 @@ def all_extractor_comparison():
     print("\n=== 多抽取器对比演示 ===\n")
 
     # 创建数据集
-    dataset_path = Path("data/sample_dataset.jsonl")
+    dataset_path = Path("/home/lulindong/Pycharm_projects/cc/1827_split_jsonl/1-200.jsonl")
     dataset = DataLoader.load_jsonl(dataset_path)
 
     # 创建webkit抽取器

requirements.txt

@@ -11,4 +11,5 @@ trafilatura
 https://github.com/opendatalab/magic-html/releases/download/magic_html-0.1.5-released/magic_html-0.1.5-py3-none-any.whl
 streamlit
 markdown
-jieba
+jieba
+apted

results/leaderboard.csv

@@ -1,5 +1,5 @@
-extractor,dataset,total_samples,success_rate,overall,code_edit,formula_edit,table_TEDS,table_edit,text_edit
-llm-webkit,sample_dataset,4,1.0,0.262,0.5,0.0,0.0,0.0,0.81
-magic-html,sample_dataset,4,1.0,0.174,0.1007,0.0,0.0,0.0,0.7693
-resiliparse,sample_dataset,4,1.0,0.1667,0.0,0.0,0.0,0.0,0.8333
-trafilatura,sample_dataset,4,1.0,0.1325,0.1007,0.0,0.0,0.0,0.5618
+extractor,dataset,total_samples,success_rate,overall,code_edit,formula_edit,table_TEDS,table_edit,text_edit
+llm-webkit,sample_dataset,4,1.0,0.262,0.5,0.0,0.0,0.0,0.81
+magic-html,sample_dataset,4,1.0,0.174,0.1007,0.0,0.0,0.0,0.7693
+resiliparse,sample_dataset,4,1.0,0.1667,0.0,0.0,0.0,0.0,0.8333
+trafilatura,sample_dataset,4,1.0,0.1325,0.1007,0.0,0.0,0.0,0.5618

tests/test_code_extraction.py

@@ -38,13 +38,13 @@ def test_empty_text(self):
         self.assertEqual(result['code'], '')
         self.assertEqual(result['text'], '')
 
-    def test_inline_code(self):
-        """测试行内代码"""
-        text = "这是一个`行内代码`的例子"
-        result = BaseMetric._extract_from_markdown(text)
-        print(result)
-        self.assertEqual(result['code'], '行内代码')
-        self.assertEqual(result['text'], text)
+    # def test_inline_code(self):
+    #     """测试行内代码"""
+    #     text = "这是一个`行内代码`的例子"
+    #     result = BaseMetric._extract_from_markdown(text)
+    #     print(result)
+    #     self.assertEqual(result['code'], '行内代码')
+    #     self.assertEqual(result['text'], text)
 
     def test_code_block(self):
         """测试代码块"""
@@ -63,7 +63,6 @@ def test_code_block(self):
 
         # 验证提取的代码
         expected_code = ("""
-"aaaabbbb"
 >>> mystr = "abcdefghijkl"
 >>> mystr[-4:]
 'ijkl'
@@ -79,19 +78,19 @@ def test_code_block(self):
         self.assertEqual(result['text'], text)
         self.assertEqual(result['formula'], '')
 
-    def test_code_with_leading_trailing_spaces(self):
-        """测试代码前后有空格的情况"""
-        text = "前面 `  code  ` 后面"
-        result = BaseMetric._extract_from_markdown(text)
-        self.assertEqual(result['code'], 'code')  # 应该去除空格
-        self.assertEqual(result['text'], text)
-
-    def test_multiline_inline_code(self):
-        """测试多行行内代码（不应该匹配）"""
-        text = "`第一行\n第二行`"
-        result = BaseMetric._extract_from_markdown(text)
-        self.assertEqual(result['code'], '')  # 不应该匹配多行行内代码
-        self.assertEqual(result['text'], text)  # 原样保留
+    # def test_code_with_leading_trailing_spaces(self):
+    #     """测试代码前后有空格的情况"""
+    #     text = "前面 `  code  ` 后面"
+    #     result = BaseMetric._extract_from_markdown(text)
+    #     self.assertEqual(result['code'], 'code')  # 应该去除空格
+    #     self.assertEqual(result['text'], text)
+
+    # def test_multiline_inline_code(self):
+    #     """测试多行行内代码（不应该匹配）"""
+    #     text = "`第一行\n第二行`"
+    #     result = BaseMetric._extract_from_markdown(text)
+    #     self.assertEqual(result['code'], '')  # 不应该匹配多行行内代码
+    #     self.assertEqual(result['text'], text)  # 原样保留
 
 if __name__ == '__main__':
     unittest.main()