Merge commit 'c5bd5808563456204441275c32b2d6a513519a61'

Author: keighrim

.gitignore

@@ -74,3 +74,6 @@ tags
 # static archival files
 static/tmp*
 
+# VSCode
+.devcontainer
+devcontainer.json

Containerfile

@@ -0,0 +1,11 @@
+FROM ghcr.io/clamsproject/clams-python-opencv4:1.0.0
+
+WORKDIR ./app
+
+COPY ./requirements.txt .
+
+RUN pip install -r requirements.txt
+
+COPY ./ ./
+
+CMD ["python", "app.py"]

Dockerfile

@@ -1,19 +1,28 @@
 # The MMIF Visualization Server
 
-This application creates an HTML server that visualizes annotation components in a [MMIF](https://mmif.clams.ai) file. Supported annotations are:
+This application creates an HTML server that visualizes annotation components in a [MMIF](https://mmif.clams.ai) file. It contains the following visualizations for any valid MMIF:
 
-- Video or Audio file player with HTML5.
-- [WebVTT](https://www.w3.org/TR/webvtt1/) for showing alignments.
+- Video or Audio file player with HTML5 (assuming file refers to video and/or audio document).
 - Pretty-printed MMIF contents.
-- Javascript for bounding boxes.
-- Named entity annotations with [displaCy.](https://explosion.ai/demos/displacy-ent)
+- Interactive, searchable MMIF tree view with [JSTree](https://www.jstree.com/).
+- Embedded [Universal Viewer](https://universalviewer.io/) (assuming file refers to video and/or image document).
+
+
+The application also includes tailored visualizations depending on the annotations present in the input MMIF:
+| Visualization | Supported CLAMS apps |
+|---|---|
+| [WebVTT](https://www.w3.org/TR/webvtt1/) for showing alignments of video captions. | [Whisper](https://github.com/clamsproject/app-whisper-wrapper), [Kaldi](https://github.com/clamsproject/app-aapb-pua-kaldi-wrapper) |
+| Javascript bounding boxes for image and OCR annotations. | [Tesseract](https://github.com/clamsproject/app-tesseractocr-wrapper), [EAST](https://github.com/clamsproject/app-east-textdetection) |
+| Named entity annotations with [displaCy.](https://explosion.ai/demos/displacy-ent) | [SPACY](https://github.com/clamsproject/app-spacy-wrapper) |                                                                        |
+
+
 
 Requirements:
 
 - A command line interface.
 - Git (to get the code).
-- [Docker](https://www.docker.com/)  (if you run the visualizer using Docker).
-- Python 3.6 or later (if you want to run the server without Docker).
+- [Docker](https://www.docker.com/) or [Podman](https://podman.io/) (if you run the visualizer in a container).
+- Python 3.6 or later (if you want to run the server containerless).
 
 To get this code if you don't already have it:
 
@@ -23,12 +32,12 @@ $ git clone https://github.com/clamsproject/mmif-visualizer
 
 
 
-## Running the server in a Docker container
+## Running the server in a container
 
-Download or clone this repository and build an image using the `Dockerfile` (you may use another name for the -t parameter, for this example we use `clams-mmif-visualizer` throughout).
+Download or clone this repository and build an image using the `Dockerfile` (you may use another name for the -t parameter, for this example we use `clams-mmif-visualizer` throughout). **NOTE**: if using podman, just substitute `docker` for `podman` in the following commands.
 
 ```bash
-$ docker build -t clams-mmif-visualizer .
+$ docker build . -f Containerfile -t clams-mmif-visualizer
 ```
 
 In these notes we assume that the data are in a local directory named `/Users/Shared/archive` with sub directories `audio`, `image`, `text` and `video` (those subdirectories are standard in CLAMS, but the parent directory could be any directory depending on your local set up). We can now run a Docker container with
@@ -56,7 +65,7 @@ With this, the mounted directory `/data` in the container is accessable from ins
 
 
 
-## Running the server without Docker
+## Running the server without Docker/Podman
 
 First install the python dependencies listed in `requirements.txt`:
 

README.md

@@ -19,13 +19,11 @@ def index():
 def ocrpage():
     data = request.form
     try:
-        # print(html.unescape(data['frames_pages']))
         frames_pages = eval(html.unescape(data['frames_pages']))
         page_number = int(data['page_number'])
 
         return (render_ocr(data['vid_path'], frames_pages, page_number))
     except Exception as e:
-        print(html.unescape(data['frames_pages']))
         return f'<p class="error">Unexpected error of type {type(e)}: {e}</h1>'
         pass
 
@@ -35,8 +33,11 @@ def upload():
     # unavailable because no secret key was set). This was solved in the
     # __main__ block by setting a key.
     if request.method == 'POST':
-        # check if the post request has the file part
-        if 'file' not in request.files:
+        # Check if request is coming from elasticsearch
+        if 'data' in request.form:
+            return render_mmif(request.form['data'])
+        # Otherwise, check if the post request has the file part
+        elif 'file' not in request.files:
             flash('WARNING: post request has no file part')
             return redirect(request.url)
         file = request.files['file']

app.py

@@ -30,19 +30,19 @@ def entity_dict(mmif, view, document_id, app_root):
     displacy_dict['ents'] = []
     for ann in view['annotations']:
         if ann.at_type == Uri.NE:
-            displacy_dict['ents'].append(entity(ann))
+            displacy_dict['ents'].append(entity(view, ann))
     return displacy_dict
 
 
 def get_text_documents(mmif):
     """Return a dictionary indexed on document identifiers (with the view identifier
     if needed) with text documents as the values."""
-    tds = [d for d in mmif.documents if str(d.at_type).endswith('TextDocument')]
+    tds = [d for d in mmif.documents if "TextDocument" in str(d.at_type)]
     tds = {td.id:td for td in tds}
     for view in mmif.views:
         # TODO: add check for TextDocument in metadata.contains (saves time)
         for annotation in view.annotations:
-            if str(annotation.at_type).endswith('TextDocument'):
+            if "TextDocument" in str(annotation.at_type):
                 tds["%s:%s" % (view.id, annotation.id)] = annotation
     return tds
 
@@ -74,7 +74,7 @@ def mmif_to_dict(mmif: Mmif):
     # to a TextDocument in the views or a set of TextDocuments in the views.
     transcript_location = None
     for document in mmif.documents:
-        if document.at_type.endswith('TextDocument'):
+        if "TextDocument" in document.at_type:
             transcript_location = document.location
     transcript_location = transcript_location
     displacy_dict = {}
@@ -85,13 +85,19 @@ def mmif_to_dict(mmif: Mmif):
         displacy_dict['ents'] = []
         for ann in ne_view['annotations']:
             if ann.at_type == Uri.NE:
-                displacy_dict['ents'].append(entity(ann))
+                displacy_dict['ents'].append(entity(ne_view, ann))
     return displacy_dict
 
 
-def entity(annotation: Annotation):
-    return {'start': annotation.properties['start'],
-            'end': annotation.properties['end'],
+def entity(view: View, annotation: Annotation):
+    if "targets" in annotation.properties:
+        start = min([view.annotations[target].properties["start"] for target in annotation.properties["targets"]])
+        end = max([view.annotations[target].properties["end"] for target in annotation.properties["targets"]])
+    else:
+        start = annotation.properties['start']
+        end = annotation.properties['end']
+    return {'start': start,
+            'end': end,
             'label': annotation.properties['category']}
 
 

displacy/__init__.py

@@ -6,8 +6,8 @@
 from flask import render_template
 
 
-def add_bounding_box(anno, frames):
-    frame_num = anno.properties["frame"]
+def add_bounding_box(anno, frames, fps):
+    frame_num = anno.properties.get("frame") or anno.properties.get("timePoint")
     box_id = anno.properties["id"]
     boxType = anno.properties["boxType"]
     coordinates = anno.properties["coordinates"]
@@ -21,17 +21,18 @@ def add_bounding_box(anno, frames):
         frames[frame_num]["bb_ids"].append(box_id)
     else:
         frames[frame_num] = {"boxes": [box], "text": [], "bb_ids": [box_id], "timestamp": None, "secs": None, "repeat": False}
+    if fps:
+        secs = int(frame_num/fps)
+        frames[frame_num]["timestamp"] = str(datetime.timedelta(seconds=secs))
+        frames[frame_num]["secs"] = secs
+
     return frames
 
 
-def align_annotations(frames_list, alignments, text_docs, fps):
+def align_annotations(frames_list, alignments, text_docs):
     """Link alignments with frames"""
     prev_frame = None
     for frame_num, frame in frames_list:
-        if fps:
-            secs = int(frame_num/fps)
-            frame["timestamp"] = str(datetime.timedelta(seconds=secs))
-            frame["secs"] = secs
         for box_id in frame["bb_ids"]:
             text_id = alignments[box_id]
             frame["text"].append(text_docs[text_id])
@@ -96,16 +97,10 @@ def round_boxes(boxes):
     return rounded_boxes
 
 def get_ocr_views(mmif):
-    """Return OCR views, which have TextDocument and Alignment annotations, but no
-    other annotations."""
+    """Return OCR views, which have TextDocument, BoundingBox, and Alignment annotations"""
     views = []
-    # TODO: not sure why we use the full URL
-    needed_types = set([
-        "http://mmif.clams.ai/0.4.0/vocabulary/TextDocument",
-        "http://mmif.clams.ai/0.4.0/vocabulary/BoundingBox",
-        "http://mmif.clams.ai/0.4.0/vocabulary/Alignment" ])
+    ocr_apps = ["east-textdetection", "tesseract"]
     for view in mmif.views:
-        annotation_types = view.metadata.contains.keys()
-        if needed_types.issubset(annotation_types) and len(annotation_types) == 3:
+        if any([view.metadata.app.find(ocr_app) for ocr_app in ocr_apps]):
             views.append(view)
     return views

ocr.py

@@ -1,4 +1,2 @@
 spacy==2.3.2
-lapps==0.0.2
-clams-python==0.5.0
-opencv-python==4.4.0.44
+clams-python==1.0.0

requirements.txt

@@ -75,7 +75,7 @@
             <li data-jstree='{"type":"view"}'>{{view.metadata.app}} ({{view.id}})
                 <ul>
                     {% for annotation in view.annotations %}
-                        <li data-jstree='{"type": "{{"annotation-highlighted" if cluster.highlighted and is_aligned else "annotation"}}"}'>{{annotation.at_type}}
+                        <li data-jstree='{"type": "{{"annotation-highlighted" if cluster.highlighted and view.id in aligned_views else "annotation"}}"}'>{{annotation.at_type}}
                             <ul>
                                 <li data-jstree='{"type":"properties"}'>{{annotation.properties}}</li>
                             </ul>

templates/interactive.html

@@ -13,10 +13,12 @@
                     <h4>
                         frame: {{frame_num}}<br>
                         timestamp: <a class="timestamp" onclick="SetCurTime('{{secs}}')">{{frame["timestamp"]}}</a><br>
-                        text detected:<br>
-                        {% for text in frame["text"] %}
-                        &emsp;{{text}}<br>
-                        {% endfor %}
+                        {% if frame["text"] %}
+                            text detected:<br>
+                            {% for text in frame["text"] %}
+                                &emsp;{{text}}<br>
+                            {% endfor %}
+                        {% endif %}
                     </h4>
                 </div>
             </div>