Merge pull request #44 from coreofscience/bibtex-support

jdalzatec · web-flow · commit ad720c8836ac · 2020-10-19T13:04:49.000-05:00
Scopus support
diff --git a/src/components/tree/Reference.tsx b/src/components/tree/Reference.tsx
@@ -34,7 +34,6 @@ const Reference: FC<Props> = ({
   simple = true,
 }: Props) => (
   <div className="reference" id={label}>
-    {!!title && simple && <div className="title">{title}</div>}
     {!!authors && (
       <Fragment>
         <span className="authors">
@@ -50,7 +49,7 @@ const Reference: FC<Props> = ({
         {". "}
       </Fragment>
     )}
-    {!!title && !simple && (
+    {!!title && (
       <Fragment>
         <span className="title">{title}</span>
         {". "}
diff --git a/src/components/tree/Tree.tsx b/src/components/tree/Tree.tsx
@@ -4,10 +4,10 @@ import sortBy from "lodash.sortby";
 import StarImgage from "../vectors/StarImage";
 
 import Reference from "./Reference";
+import { mostCommon } from "../../utils/arrays";
 import { Article } from "../../utils/customTypes";
 
 import "./Tree.css";
-import { mostCommon } from "../../utils/isiUtils";
 
 interface Props {
   data: { [section: string]: Article[] };
diff --git a/src/components/upload/FileCard.tsx b/src/components/upload/FileCard.tsx
@@ -3,7 +3,7 @@ import React, { FC } from "react";
 import CancelFile from "../vectors/CancelFile";
 import MoveFirstIcon from "../vectors/MoveFirstIcon";
 
-import { round } from "../../utils/mathUtils";
+import { round } from "../../utils/math";
 import { MAX_SIZE } from "../../utils/computeQuantities";
 
 import "./FileCard.css";
diff --git a/src/components/upload/FileDropper.tsx b/src/components/upload/FileDropper.tsx
@@ -2,7 +2,8 @@ import React, { FC, useCallback } from "react";
 import { useDropzone } from "react-dropzone";
 import "./FileDropper.css";
 
-import { looksLikeIsi } from "../../utils/isiUtils";
+import { looksLikeIsi } from "../../utils/isi";
+import { looksLikeScopus } from "../../utils/scopus";
 import useUpload from "../../hooks/useUpload";
 import useError from "../../hooks/useError";
 
@@ -27,7 +28,7 @@ const FileDropper: FC<Props> = () => {
           .map((file) => file.text().then((text) => ({ text, file })))
       ).then((data) => {
         data.forEach(({ text, file }) => {
-          if (looksLikeIsi(text)) {
+          if (looksLikeIsi(text) || looksLikeScopus(text)) {
             upload(Object(file).name, file);
           } else {
             error(Object(file).name, file, FileErrorMap.isi);
diff --git a/src/components/upload/Home.tsx b/src/components/upload/Home.tsx
@@ -11,7 +11,7 @@ import useFiles from "../../hooks/useFiles";
 import FirebaseContext from "../../context/FirebaseContext";
 
 import computeQuantities, { MAX_SIZE } from "../../utils/computeQuantities";
-import { round } from "../../utils/mathUtils";
+import { round } from "../../utils/math";
 
 import "./Home.css";
 
diff --git a/src/components/upload/UploadIndicator.tsx b/src/components/upload/UploadIndicator.tsx
@@ -1,11 +1,12 @@
 import React, { FC, useContext, useEffect, useState } from "react";
-import "./UploadIndicator.css";
 
 import FileCard from "./FileCard";
 import FileContext from "../../context/FileContext";
 import useFiles from "../../hooks/useFiles";
 import { MAX_SIZE } from "../../utils/computeQuantities";
 
+import "./UploadIndicator.css";
+
 interface Props {}
 
 const UploadIndicator: FC<Props> = () => {
diff --git a/src/hooks/useUpload.ts b/src/hooks/useUpload.ts
@@ -1,39 +1,24 @@
-import md5 from "md5";
 import { useCallback, useContext } from "react";
 
 import FirebaseContext from "../context/FirebaseContext";
 import FileContext from "../context/FileContext";
-import {
-  countArticles,
-  countReferences,
-  mostCommonKeywords,
-} from "../utils/isiUtils";
+import metadata from "../utils/metadata";
 
 const useUpload = () => {
   const { add, track } = useContext(FileContext);
   const firebase = useContext(FirebaseContext);
 
   const upload = useCallback(
     (name: string, blob: Blob) => {
-      blob.text().then((text) => {
+      metadata(name, blob).then((meta) => {
         if (firebase === null) return;
-        const hash = md5(text);
-        const metadata = {
-          name,
-          blob,
-          hash,
-          keywords: mostCommonKeywords(text, 3),
-          articles: countArticles(text),
-          citations: countReferences(text),
-          valid: true,
-        };
-        add(metadata);
-        const ref = firebase.storage().ref(`isi-files/${hash}`);
+        add(meta);
+        const ref = firebase.storage().ref(`isi-files/${meta.hash}`);
 
         ref
           .getDownloadURL()
           .then(() => {
-            track(hash, 100);
+            track(meta.hash, 100);
           })
           .catch(() => {
             const task = ref.put(blob);
@@ -42,11 +27,11 @@ const useUpload = () => {
               (snapshot) => {
                 const percent =
                   (snapshot.bytesTransferred / snapshot.totalBytes) * 100;
-                track(hash, percent);
+                track(meta.hash, percent);
               },
               (error) => console.log,
               () => {
-                track(hash, 100);
+                track(meta.hash, 100);
               }
             );
           });
diff --git a/src/utils/arrays.ts b/src/utils/arrays.ts
@@ -0,0 +1,12 @@
+const mostCommon = (keywordsList: string[], max: number): string[] => {
+  let count: { [keyword: string]: number } = {};
+  for (let keyword of keywordsList) {
+    count[keyword] = (count[keyword] ? count[keyword] : 0) + 1;
+  }
+  const sortCount = Object.entries(count).sort((first, second) =>
+    first[1] < second[1] ? 1 : -1
+  );
+  return sortCount.slice(0, max).map((item) => item[0]);
+};
+
+export { mostCommon };
diff --git a/src/utils/customTypes.tsx b/src/utils/customTypes.tsx
@@ -34,5 +34,4 @@ export interface Article {
   issue?: string | null;
   page?: string | null;
   doi?: string | null;
-  simple?: boolean;
 }
diff --git a/src/utils/isi.ts b/src/utils/isi.ts
@@ -13,7 +13,7 @@ const looksLikeIsi = (content: string): boolean => {
   return true;
 };
 
-const getKeywordsList = (text: string) => {
+const keywords = (text: string): string[] => {
   const identifier = "ID ";
   const keywordsLines = text
     .split("\n")
@@ -30,28 +30,12 @@ const getKeywordsList = (text: string) => {
     .flat();
 };
 
-function mostCommon(keywordsList: string[], max: number) {
-  let count: { [keyword: string]: number } = {};
-  for (let keyword of keywordsList) {
-    count[keyword] = (count[keyword] ? count[keyword] : 0) + 1;
-  }
-  const sortCount = Object.entries(count).sort((first, second) =>
-    first[1] < second[1] ? 1 : -1
-  );
-  return sortCount.slice(0, max).map((item) => item[0]);
-}
-
-const mostCommonKeywords = (text: string, max: number = 3) => {
-  const keywordsList = getKeywordsList(text);
-  return mostCommon(keywordsList, max);
-};
-
-const countArticles = (text: string) => {
+const countArticles = (text: string): number => {
   const identifier = "PT ";
   return text.split("\n").filter((line) => line.startsWith(identifier)).length;
 };
 
-const countReferences = (text: string) => {
+const countReferences = (text: string): number => {
   const identifier = "NR ";
   return text
     .split("\n")
@@ -60,11 +44,4 @@ const countReferences = (text: string) => {
     .reduce((n, m) => n + m, 0);
 };
 
-export {
-  ISI_PATTERN,
-  looksLikeIsi,
-  mostCommonKeywords,
-  mostCommon,
-  countArticles,
-  countReferences,
-};
+export { ISI_PATTERN, looksLikeIsi, keywords, countArticles, countReferences };
diff --git a/src/utils/math.ts b/src/utils/math.ts
diff --git a/src/utils/metadata.ts b/src/utils/metadata.ts
@@ -0,0 +1,43 @@
+import { FileMetadata } from "./customTypes";
+import { mostCommon } from "./arrays";
+import * as isi from "./isi";
+import * as scopus from "./scopus";
+import md5 from "md5";
+
+const metadata = async (name: string, blob: Blob): Promise<FileMetadata> => {
+  const content = await blob.text();
+  const hash = md5(content);
+
+  if (isi.looksLikeIsi(content)) {
+    return {
+      name,
+      blob,
+      hash,
+      keywords: mostCommon(isi.keywords(content), 3),
+      articles: isi.countArticles(content),
+      citations: isi.countReferences(content),
+      valid: true,
+    };
+  }
+
+  if (scopus.looksLikeScopus(content)) {
+    return {
+      name,
+      blob,
+      hash,
+      keywords: mostCommon(scopus.keywords(content), 3),
+      articles: scopus.countArticles(content),
+      citations: scopus.countReferences(content),
+      valid: true,
+    };
+  }
+
+  return {
+    name,
+    hash,
+    blob,
+    valid: false,
+  };
+};
+
+export default metadata;
diff --git a/src/utils/scopus.ts b/src/utils/scopus.ts
@@ -0,0 +1,75 @@
+const RIS_PATTERN = /^(?<key>[A-Z0-9]{2})[ ]{2}-[ ](?<value>.*)$/;
+
+const looksLikeScopus = (content: string): boolean => {
+  let currentKey = null;
+  for (const line of content.split("\n")) {
+    if (!line) {
+      continue;
+    }
+    const match = line.match(RIS_PATTERN);
+    if (!match && currentKey !== "N1") {
+      return false;
+    }
+    if (match && match.groups && match.groups.key) {
+      currentKey = match.groups.key;
+    }
+  }
+  return true;
+};
+
+const keywords = (content: string): string[] =>
+  content
+    .split("\n")
+    .filter((line) => !!line)
+    .map((line) => line.match(RIS_PATTERN))
+    .map(
+      (match) =>
+        !!match &&
+        match.groups &&
+        match.groups.key &&
+        match.groups.value &&
+        match.groups.key === "KW" &&
+        match.groups.value
+    )
+    .filter((keyword: string | false | undefined) => !!keyword) as string[];
+
+const countArticles = (content: string): number =>
+  content
+    .split("\n")
+    .filter((line) => !!line)
+    .map((line) => line.match(RIS_PATTERN))
+    .filter(
+      (match) =>
+        !!match && match.groups && match.groups.key && match.groups.key === "ER"
+    ).length;
+
+const countReferences = (content: string): number =>
+  content
+    .split("\n")
+    .filter((line) => !!line)
+    .map((line) => line.match(RIS_PATTERN))
+    .reduce(
+      ({ counting, count }, match) => {
+        if (counting && !match) {
+          return { counting, count: ++count };
+        }
+        if (!match || !match.groups) {
+          return { counting, count };
+        }
+        if (counting && match.groups.key) {
+          return { counting: false, count };
+        }
+        if (!counting && match.groups.key && match.groups.key === "N1") {
+          if (
+            match.groups.value &&
+            match.groups.value.startsWith("References:")
+          ) {
+            return { counting: true, count: ++count };
+          }
+        }
+        return { counting, count };
+      },
+      { counting: false, count: 0 }
+    ).count;
+
+export { looksLikeScopus, keywords, countArticles, countReferences };

Original file line number	Diff line number	Diff line change
`@@ -34,5 +34,4 @@ export interface Article {`
`34`	`34`	`issue?: string \| null;`
`35`	`35`	`page?: string \| null;`
`36`	`36`	`doi?: string \| null;`
`37`		`- simple?: boolean;`
`38`	`37`	`}`