Added option to double click on a word to jump to that point in media

Author: Pietro Passarelli

package-lock.json

@@ -57,15 +57,17 @@
     "@fortawesome/free-solid-svg-icons": "^5.12.1",
     "@fortawesome/react-fontawesome": "^0.1.5",
     "@storybook/addon-storysource": "^5.3.18",
+    "difflib": "^0.2.4",
     "docx": "4.7.1",
+    "number-to-words": "^1.2.4",
     "prop-types": "^15.7.2",
     "react-bootstrap": "^1.0.1",
     "sbd": "^1.0.18",
     "slate": "^0.59.0",
     "slate-history": "^0.59.0",
     "slate-react": "^0.59.0",
     "smpte-timecode": "^1.2.3",
-    "stt-align-node": "^1.1.7"
+    "stt-align-node": "^1.1.8"
   },
   "husky": {
     "hooks": {

package.json

@@ -42,6 +42,8 @@ import updateTimestamps from '../util/export-adapters/slate-to-dpe/update-timest
 import exportAdapter from '../util/export-adapters';
 import isEmpty from '../util/is-empty';
 import generatePreviousTimingsUpToCurrent from '../util/dpe-to-slate/generate-previous-timings-up-to-current';
+import getSelectionNodes from '../util/get-selection-nodes';
+
 const PLAYBACK_RATE_VALUES = [0.2, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2, 2.25, 2.5, 3, 3.5];
 const SEEK_BACK_SEC = 15;
 const PAUSE_WHILTE_TYPING_TIMEOUT_MILLISECONDS = 1500;
@@ -276,6 +278,10 @@ export default function SlateTranscriptEditor(props) {
               style={{ cursor: 'pointer' }}
               className={'timecode text-muted unselectable'}
               onClick={handleTimedTextClick}
+              // onClick={(e) => {
+              //   e.preventDefault();
+              // }}
+              onDoubleClick={handleTimedTextClick}
               title={props.element.startTimecode}
               data-start={props.element.start}
             >
@@ -321,11 +327,19 @@ export default function SlateTranscriptEditor(props) {
         mediaRef.current.play();
       }
     } else if (e.target.dataset.slateString) {
+      console.log('e.target.dataset.slateString');
       if (e.target.parentNode.dataset.start) {
-        const start = e.target.parentNode.dataset.start;
-        if (mediaRef && mediaRef.current && start) {
-          mediaRef.current.currentTime = parseFloat(start);
+        const { startSec } = getSelectionNodes(editor, editor.selection);
+        console.log('startSec', startSec);
+        if (mediaRef && mediaRef.current && startSec) {
+          mediaRef.current.currentTime = parseFloat(startSec);
           mediaRef.current.play();
+        } else {
+          // const start = e.target.parentNode.dataset.start;
+          // if (mediaRef && mediaRef.current && start) {
+          //   mediaRef.current.currentTime = parseFloat(start);
+          //   mediaRef.current.play();
+          // }
         }
       }
     }

src/components/index.js

@@ -22,6 +22,8 @@ import { shortTimecode } from '../timecode-converter';
  * @param {Number} time - float, time in seconds
  */
 
+import getWordsForParagraph from '../get-words-for-paragraph';
+
 export const generatePreviousTimings = (time) => {
   // https://stackoverflow.com/questions/3746725/how-to-create-an-array-containing-1-n
   if (time) {
@@ -96,7 +98,7 @@ const convertDpeToSlate = (transcript) => {
       {
         text: generateText(paragraph, words),
         // Adding list of words in slateJs paragraphs
-        words,
+        words: getWordsForParagraph(paragraph, words),
       },
     ],
   }));

src/util/dpe-to-slate/index.js

@@ -3,8 +3,6 @@
  * https://github.com/pietrop/react-transcript-editor/blob/master/packages/components/timed-text-editor/UpdateTimestamps/index.js
  * similar to updateTimestamps
  */
-// import { alignSTT } from 'stt-align-node';
-// import slateToText from '../txt';
 import countWords from '../../count-words';
 import updateTimestampsHelper from './update-timestamps/update-timestamps-helper';
 /**

src/util/export-adapters/slate-to-dpe/index.js

@@ -7,6 +7,7 @@ import { shortTimecode } from '../../../timecode-converter';
 import { generatePreviousTimingsUpToCurrentOne } from '../../../dpe-to-slate';
 import countWords from '../../../count-words';
 import updateTimestampsHelper from './update-timestamps-helper';
+import getWordsForParagraph from '../../../get-words-for-paragraph';
 /**
  * Transposes the timecodes from stt json list of words onto
  * slateJs value paragraphs
@@ -27,6 +28,8 @@ export const createSlateContentFromSlateJsParagraphs = (currentContent, newEntit
     const blockEntites = newEntities.slice(totalWords, totalWords + wordsInBlock);
     let speaker = block.speaker;
     const start = parseFloat(blockEntites[0].start);
+    const end = parseFloat(blockEntites[blockEntites.length - 1].end);
+    const currentParagraph = { start, end };
     if (!speaker) {
       speaker = 'U_UKN';
     }
@@ -36,7 +39,13 @@ export const createSlateContentFromSlateJsParagraphs = (currentContent, newEntit
       start,
       previousTimings: generatePreviousTimingsUpToCurrentOne(blockEntites, start),
       startTimecode: shortTimecode(start),
-      children: [{ text }],
+      children: [
+        {
+          text,
+          words: blockEntites,
+          // getWordsForParagraph(currentParagraph, newEntities)
+        },
+      ],
     };
 
     updatedBlockArray.push(updatedBlock);
@@ -53,6 +62,8 @@ export const createSlateContentFromSlateJsParagraphs = (currentContent, newEntit
  */
 const updateTimestamps = (currentContent, words) => {
   const alignedWords = updateTimestampsHelper(currentContent, words);
+  // TODO: there seem to be some words without text attribute, so doing a quick fix clean up
+
   const updatedContent = createSlateContentFromSlateJsParagraphs(currentContent, alignedWords);
   return updatedContent;
 };

src/util/export-adapters/slate-to-dpe/update-timestamps/index.js

@@ -4,17 +4,30 @@
  *
  */
 import { alignSTT } from 'stt-align-node';
+// import alignSTT from '../../../stt-align-node';
 import slateToText from '../../txt';
+// Yo
 /**
  * Update timestamps usign stt-align module
  * @param {*} currentContent - slate js value
  * @param {*} words - list of stt words
  * @return slateJS value
  */
 export const updateTimestampsHelper = (currentContent, words) => {
-  const currentText = slateToText({ value: currentContent, speakers: false, timecodes: false, atlasFormat: false });
-  const alignedWords = alignSTT(words, currentText);
-  return alignedWords;
+  let tmpWords = words;
+  const currentText = slateToText({
+    value: currentContent,
+    speakers: false,
+    timecodes: false,
+    atlasFormat: false,
+  });
+  const alignedWords = alignSTT(tmpWords, currentText);
+  const alignedWordsCleanedUp = alignedWords.filter((word) => {
+    if (word.text) {
+      return word;
+    }
+  });
+  return alignedWordsCleanedUp;
 };
 
 export default updateTimestampsHelper;

src/util/export-adapters/slate-to-dpe/update-timestamps/update-timestamps-helper.js

@@ -0,0 +1,61 @@
+const getSelectionNodes = (editor, selection) => {
+  console.log('selection', selection);
+  try {
+    const orderedSelection = [selection.anchor, selection.focus].sort((a, b) => {
+      return a.path[0] - b.path[0];
+    });
+    const selectionStart = orderedSelection[0];
+    const selectionEnd = orderedSelection[1];
+    let counterAnchor = 0;
+    let goalAnchor = selectionStart.offset;
+    let targetWordIndexAnchor = null;
+    let selectedLeafWordsAnchor = editor.children[selectionStart.path[0]].children[0].words;
+    // let pathValue = selectionStart.path;
+    // console.log('editor.children[1]', editor.children[1]);
+    // let selectedLeafWordsAnchor2 = editor.children[selectionStart.path].children[0].words;
+    // console.log('selectedLeafWordsAnchor2', selectedLeafWordsAnchor2);
+    selectedLeafWordsAnchor.forEach((word, wordIndex) => {
+      const wordLength = (word.text + ' ').length;
+
+      counterAnchor = counterAnchor + wordLength;
+      if (counterAnchor <= goalAnchor) {
+        targetWordIndexAnchor = wordIndex;
+      }
+    });
+
+    const startWord = selectedLeafWordsAnchor[targetWordIndexAnchor + 1];
+    console.log('startWord', startWord);
+    console.log('selection', selection);
+
+    let counter = 0;
+    let goal = selectionEnd.offset;
+    console.log('goal', goal);
+    let targetWordIndex = null;
+    let selectedLeafWords = editor.children[selectionEnd.path[0]].children[0].words;
+    selectedLeafWords.forEach((word, wordIndex) => {
+      const wordLength = (word.text + ' ').length;
+
+      counter = counter + wordLength;
+      if (counter <= goal) {
+        targetWordIndex = wordIndex;
+      }
+    });
+
+    const endWord = selectedLeafWords[targetWordIndex + 1];
+    // console.log('endWord', endWord);
+    // console.log('selection', selection);
+    console.log('startWord', startWord);
+    // const startSec = startWord.start / 1000;
+    // const endSec = endWord.end / 1000;
+    // console.log('startSec', startSec);
+    // const selObj = window.getSelection();
+    // console.log(selObj.toString());
+    // setSearch(selObj.toString())
+    // createUserClip(startSec, endSec);
+    return { startSec: startWord.start, endSec: endWord.end };
+  } catch (error) {
+    console.error('error creating clip', error);
+  }
+};
+
+export default getSelectionNodes;

src/util/get-selection-nodes/index.js

@@ -0,0 +1,14 @@
+/**
+ *
+ * @param {*} currentParagraph a dpe paragraph object, with start, and end attribute eg in seconds
+ * @param {*} words a list of word objects with start and end attributes
+ * @returns a lsit of words obejcts that are included in the given paragraphs
+ */
+const getWordsForParagraph = (currentParagraph, words) => {
+  const { start, end } = currentParagraph;
+  return words.filter((word) => {
+    return word.start >= start && word.end <= end;
+  });
+};
+
+export default getWordsForParagraph;