Merge pull request #4 from clamsproject/normalization

Added naive name normalization functionality

Author: keighrim

README.md

@@ -23,6 +23,8 @@ annotations generated by an OCR app such as [doctr-wrapper](https://github.com/c
 or [Tesseract](https://github.com/clamsproject/app-tesseractocr-wrapper).
 
 The app outputs a [`TextDocument`]('https://mmif.clams.ai/vocabulary/TextDocument/v1/') annotation corresponding to each
-of the input annotations, containing the original `text` split into `name-as-written` and `attributes` fields in an escaped
-JSON string. Each annotation also contains identifying information for the new annotation, source annotation, and source [`VideoDocument`]('https://mmif.clams.ai/vocabulary/VideoDocument/v1/').
+of the input annotations, containing the original `text` split into `name-as-written` and `attributes` fields
+in an escaped JSON string; optionally, the output may also include a `name-normalized` field.
+Each annotation also contains identifying information for the new annotation,
+source annotation, and source [`VideoDocument`]('https://mmif.clams.ai/vocabulary/VideoDocument/v1/').
 For more details, see the `output` section of the app metadata.

app.py

@@ -12,6 +12,7 @@
 
 import interpreter
 
+
 class HeuristicChyronInterpreter(ClamsApp):
 
     def __init__(self):
@@ -23,27 +24,27 @@ def _appmetadata(self):
         pass
 
     def _annotate(self, mmif: Mmif, **parameters) -> Mmif:
-
         self.mmif = mmif if isinstance(mmif, Mmif) else Mmif(mmif)
 
         new_view = self.mmif.new_view()
         self.sign_view(new_view, parameters)
         new_view.new_contain(DocumentTypes.TextDocument)
 
         for doc in self.mmif.get_documents_by_type(DocumentTypes.TextDocument):
-            self._run_interpreter(doc, new_view)
+            self._run_interpreter(doc, new_view, not parameters.get('note4mode'))
 
         return self.mmif
 
-    def _run_interpreter(self, doc, new_view):
+    def _run_interpreter(self, doc, new_view, do_normalize):
         """
         Run the chyron interpreter over the document and add annotations to the view.
         """
-        text = doc.text_value
-        content = interpreter.split_text(text)
+        content = interpreter.split_text(doc.text_value, do_normalize)
         mmif_vids = self.mmif.get_documents_by_type(DocumentTypes.VideoDocument)
         vid_id = mmif_vids[0].long_id
-        out_doc = new_view.new_textdocument(text=content, document=vid_id, origin=doc.long_id, provenance='derived', mime='application/json')
+        out_doc = new_view.new_textdocument(text=content, document=vid_id, origin=doc.long_id, provenance='derived',
+                                            mime='application/json')
+
 
 def get_app():
     """

interpreter.py

@@ -4,22 +4,57 @@
 Reformats chyron text output from an OCR app as a name and list of attributes
 presented as an escaped json string.
 """
-
-__VERSION__ = 'v0.1'
-
 import json
 
-def split_text(text: str) -> str:
+
+def split_text(text: str, normalize=False) -> str:
     """
     Splits input string on newline and creates a dictionary with 'name-as-written'
-    and 'attributes' keys, with string content as values.
+    and 'attributes' keys, with string content as values. If normalization parameter
+    is True, also adds a 'name-normalized' item to the dictionary.
     Returns escaped json string of dictionary.
     """
     content = {}
-    lines = text.split("\n")
+    lines = list(filter(lambda x: x is not None and len(x) > 0, text.split("\n")))
     if lines:
         content["name-as-written"] = lines[0]
-        content["attributes"] = [x for x in lines[1:] if x != '']
+        if normalize:
+            content["name-normalized"] = normalize_text(lines[0])
+            last_line_processed = 0
+        else:
+            if len(lines) > 1:
+                content["name-normalized"] = lines[1]
+                last_line_processed = 1
+            else:
+                content["name-normalized"] = ''
+                last_line_processed = 0
+        content["attributes"] = lines[(last_line_processed+1):]
     content = json.dumps(content)
 
-    return content
+    return content
+
+
+def normalize_text(text: str) -> str:
+    """
+    Converts input string to a 'normalized' name form.
+    Returns text in a 'Lastname, Firstname' format.
+    """
+    # Depending on the quality of the OCR, this may not represent the original chyron as well as desired.
+
+    if text.find('.') != -1: # Will only strip 1 abbreviated title, not multiple or full-word titles.
+        text = text[text.find('.') + 1:]
+    if text.find('(') != -1:
+        text = text[:text.find('(')]
+    if text.find(', ') != -1:
+        text = text[:text.find(', ')]
+    parts = text.split()
+    normal_parts = []
+    for word in parts:
+        word = ''.join([word[0].upper(), word[1:].lower()])
+        normal_parts.append(word)
+    if len(normal_parts) == 1:
+        normal_name = normal_parts[0]
+    else:
+        normal_name = ' '.join(normal_parts[1:]) + ', ' + normal_parts[0]  # to handle names with more than two words
+
+    return normal_name

metadata.py

@@ -18,6 +18,7 @@ def appmetadata() -> AppMetadata:
     - https://sdk.clams.ai/appmetadata.html metadata specification.
     - https://sdk.clams.ai/autodoc/clams.appmetadata.html python API
     
+    
     :return: AppMetadata object holding all necessary information.
     """
     metadata = AppMetadata(
@@ -32,12 +33,18 @@ def appmetadata() -> AppMetadata:
     # I/O Spec
     in_doc = metadata.add_input(DocumentTypes.TextDocument)
     in_doc.add_description('Text content transcribed from video input by docTR/Tesseract/LLAVA.')
-    out_doc = metadata.add_output(DocumentTypes.TextDocument, **{'document': '*', 'origin': '*'})
+    out_doc = metadata.add_output(DocumentTypes.TextDocument, 
+                                  document='*', origin='*', provenance='derived', mime='application/json')
     out_doc.add_description('Reformatted chyron text. `document` property stores the ID of the original source '
                             '`VideoDocument`. `origin` property stores the ID of the original OCR `TextDocument` '
-                            'annotation. ')
+                            'annotation. Reformatted text is escaped JSON string with three fields: '
+                            '`name-as-written`, `name-normalized`, and `attributes`. ')
 
-    # No runtime parameters besides universals.
+    metadata.add_parameter(name='note4mode', default=False, type='boolean',
+                           description='Boolean to set the app to run in "note-4" mode and to take the second line '
+                                       '(if available) from the input text to be the `name-normalized` value. The '
+                                       'default is false, which means the app will try to generate normalization from'
+                                       '`name-as-written` (from the first line) value. ')
 
     return metadata