add text format segmentation

Author: pingzhili

EduNLP/SIF/parser/parser.py

@@ -76,7 +76,7 @@ def _is_formula_legal(self, formula_str):
 
         """
         legal_tags = ['FormFigureID', 'FormFigureBase64', 'FigureID', 'FigureBase64',
-                      'SIFBlank', 'SIFChoice', 'SIFTag', 'SIFSep', 'SIFUnderline']
+                      'SIFBlank', 'SIFChoice', 'SIFTag', 'SIFSep', 'SIFUnderline', 'textf']
         for tag in legal_tags:
             if tag in formula_str:
                 return True

EduNLP/SIF/segment/__init__.py

@@ -2,4 +2,4 @@
 # 2021/5/18 @ tongshiwei
 
 from .segment import (SegmentList, TextSegment, FigureFormulaSegment, LatexFormulaSegment, FigureSegment,
-                      QuesMarkSegment, Figure, TagSegment, SepSegment, seg)
+                      QuesMarkSegment, Figure, TagSegment, SepSegment, seg, TextFSegment)

EduNLP/SIF/segment/segment.py

@@ -75,6 +75,10 @@ class SepSegment(str):
     pass
 
 
+class TextFSegment(str):
+    pass
+
+
 class SegmentList(object):
     def __init__(self, item, figures: dict = None):
         self._segments = []
@@ -104,6 +108,9 @@ def __init__(self, item, figures: dict = None):
                 self.append(TagSegment(segment[1:-1]))
             elif re.match(r"\$\\SIFSep\$", segment):
                 self.append(SepSegment(segment[1:-1]))
+            elif re.match(r"\$\\textf\{[^,]+?,b?d?i?t?u?w?}\$", segment):
+                seg_capture = re.match(r"\$\\textf\{([^,]+?),b?d?i?t?u?w?}\$", segment)
+                self.append(TextFSegment(seg_capture.group(1)))
             else:
                 self.append(LatexFormulaSegment(segment[1:-1]))
         self._seg_idx = None
@@ -115,8 +122,12 @@ def __len__(self):
         return len(self._segments)
 
     def append(self, segment) -> None:
-        if isinstance(segment, TextSegment):
-            self._text_segments.append(len(self))
+        if isinstance(segment, TextSegment) or isinstance(segment, TextFSegment):
+            if len(self._text_segments) != 0 and self._text_segments[-1] == len(self) - 1:
+                self._segments[-1] = self._segments[-1] + segment
+            else:
+                self._text_segments.append(len(self))
+                self._segments.append(segment)
         elif isinstance(segment, (LatexFormulaSegment, FigureFormulaSegment)):
             self._formula_segments.append(len(self))
         elif isinstance(segment, FigureSegment):
@@ -129,7 +140,10 @@ def append(self, segment) -> None:
             self._sep_segments.append(len(self))
         else:
             raise TypeError("Unknown Segment Type: %s" % type(segment))
-        self._segments.append(segment)
+        if isinstance(segment, TextFSegment) or isinstance(segment, TextSegment):
+            pass
+        else:
+            self._segments.append(segment)
 
     @property
     def segments(self):

tests/test_sif/test_segement.py

@@ -19,6 +19,10 @@ def test_segment(figure0, figure1, figure0_base64, figure1_base64):
         r"如图所示，则$\FormFigureBase64{%s}$的面积是$\SIFBlank$。$\FigureBase64{%s}$" % (figure0_base64, figure1_base64),
         figures=True
     )
-
     with pytest.raises(TypeError):
         s.append("123")
+    seg_test_text = seg(
+        r"如图所示，有三组$\textf{机器人,bu}$在踢$\textf{足球,b}$",
+        figures=True
+    )
+    assert seg_test_text.text_segments == ['如图所示，有三组机器人在踢足球']