Merge pull request #134 from ezpaarse-project/development

Development
ezpaarse-project · Oct 20, 2023 · 2080ac3 · 2080ac3
2 parents 7d94697 + bb9e946
commit 2080ac3
Show file tree

Hide file tree

Showing 15 changed files with 86 additions and 31 deletions.
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
    "name": "ezunpaywall",
-   "version": "1.2.2",
+   "version": "1.3.2",
    "description": "",
    "author": "felixleo22",
    "license": "CeCILL",

diff --git a/src/apikey/package.json b/src/apikey/package.json
@@ -1,6 +1,6 @@
 {
   "name": "ezunpaywall-apikey",
-  "version": "1.3.0",
+  "version": "1.3.2",
   "description": "",
   "main": "app.js",
   "author": "felixleo22",

diff --git a/src/enrich/lib/csv.js b/src/enrich/lib/csv.js
@@ -239,8 +239,7 @@ async function enrichInFile(data, enrichConfig, state) {
     response = await requestGraphql(data, args, index, state.apikey);
   } catch (err) {
     logger.error(`[graphql] Cannot request graphql service at ${config.get('graphql.host')}/graphql`, JSON.stringify(err?.response?.data?.errors));
-    await fail(state);
-    return;
+    throw err;
   }
 
   // enrichment
@@ -336,7 +335,14 @@ async function processEnrichCSV(id, index, args, state, prefix, separator) {
           data = [];
           await parser.pause();
 
-          await enrichInFile(copyData, enrichConfig, state);
+          try {
+            await enrichInFile(copyData, enrichConfig, state);
+          } catch (err) {
+            logger.error(`[job csv] Cannot enrich in file [${enrichedFile}]`, err);
+            await fail(state);
+            return;
+          }
+
           await parser.resume();
         }
       },
@@ -349,9 +355,14 @@ async function processEnrichCSV(id, index, args, state, prefix, separator) {
       enrichConfig.headers = await enrichHeaderCSV(headers, args);
       await writeHeaderCSV(headers, separator, enrichedFile);
     }
-    await enrichInFile(data, enrichConfig, state);
+    try {
+      await enrichInFile(data, enrichConfig, state);
+    } catch (err) {
+      logger.error(`[job csv] Cannot enrich in file [${enrichedFile}]`, err);
+      await fail(state);
+      return;
+    }
   }
-
   logger.info(`[job csv] ${state.enrichedLines}/${state.linesRead} enriched lines`);
 }
 

diff --git a/src/enrich/lib/json.js b/src/enrich/lib/json.js
@@ -140,8 +140,7 @@ async function enrichInFile(data, enrichConfig, state) {
     response = await requestGraphql(data, args, index, state.apikey);
   } catch (err) {
     logger.error(`[graphql] Cannot request graphql service at ${config.get('graphql.host')}/graphql`, JSON.stringify(err?.response?.data?.errors));
-    await fail(state);
-    return;
+    throw err;
   }
   // enrichment
   const enrichedData = enrichArray(data, response);
@@ -221,14 +220,26 @@ async function processEnrichJSON(id, index, args, prefix, state) {
     }
 
     if (data.length === 1000) {
-      await enrichInFile(data, enrichConfig, state);
+      try {
+        await enrichInFile(data, enrichConfig, state);
+      } catch (err) {
+        logger.error(`[job jsonl] Cannot enrich in file [${enrichedFile}]`, err);
+        await fail(state);
+        return;
+      }
       data = [];
     }
   }
 
   // last insertion
   if (data.length !== 0) {
-    await enrichInFile(data, enrichConfig, state);
+    try {
+      await enrichInFile(data, enrichConfig, state);
+    } catch (err) {
+      logger.error(`[job jsonl] Cannot enrich in file [${enrichedFile}]`, err);
+      await fail(state);
+      return;
+    }
   }
   logger.info(`[job jsonl] ${state.enrichedLines}/${state.linesRead} enriched lines`);
 }

diff --git a/src/enrich/package.json b/src/enrich/package.json
@@ -1,6 +1,6 @@
 {
   "name": "ezunpaywall-enrich",
-  "version": "1.3.0",
+  "version": "1.3.2",
   "description": "",
   "main": "app.js",
   "author": "felixleo22",

diff --git a/src/fakeUnpaywall/package.json b/src/fakeUnpaywall/package.json
@@ -1,6 +1,6 @@
 {
   "name": "fakeunpaywall",
-  "version": "1.3.0",
+  "version": "1.3.2",
   "description": "",
   "main": "app.js",
   "author": "felixleo22",

diff --git a/src/frontend/package.json b/src/frontend/package.json
@@ -1,6 +1,6 @@
 {
   "name": "client",
-  "version": "1.3.0",
+  "version": "1.3.2",
   "private": true,
   "scripts": {
     "build": "nuxt build",

diff --git a/src/graphql/app.js b/src/graphql/app.js
@@ -9,6 +9,7 @@ const { expressMiddleware } = require('@apollo/server/express4');
 
 const { pingRedis, startConnectionRedis } = require('./lib/services/redis');
 
+const getNumberOfDOI = require('./lib/middlewares/args');
 const auth = require('./lib/middlewares/auth');
 
 const logger = require('./lib/logger');
@@ -55,7 +56,11 @@ const server = new ApolloServer({
   await server.start();
 
   app.use('/graphql', cors(), json(), auth, expressMiddleware(server, {
-    context: async ({ req }) => req,
+    context: async ({ req }) => {
+      req.countDOI = getNumberOfDOI(req);
+
+      return req;
+    },
   }));
 
   app.listen(3000, async () => {

diff --git a/src/graphql/lib/middlewares/args.js b/src/graphql/lib/middlewares/args.js
@@ -0,0 +1,32 @@
+function getNumberOfDOI(req) {
+  const patternBetweenBracket = /.*?(\[.*?\]).*?$/i;
+  // BODY
+  // {
+  //  query: 'query ($dois: [ID!]!) {GetByDOI(dois: $dois) {doi, is_oa}}',
+  //  variables: { dois: [ '10.1186/s40510-015-0109-6' ] }
+  // }
+  if (req?.body?.variables?.dois) {
+    return req.body.variables.dois.length;
+  }
+  // BODY
+  // query: '{GetByDOI(dois:["10.1186/s40510-015-0109-6","Coin Coin"]){doi, is_oa}}'
+  if (req?.body?.query) {
+    const match = patternBetweenBracket.exec(req?.body?.query);
+    if (match?.length >= 1) {
+      const listOfDOI = JSON.parse(match[1]);
+      return listOfDOI.length;
+    }
+  }
+  // query: '{ GetByDOI(dois:["10.1186/s40510-015-0109-6"]) { doi, is_oa } }
+  if (req?.query?.query) {
+    const match = patternBetweenBracket.exec(req.query.query);
+    if (match?.length >= 1) {
+      const listOfDOI = JSON.parse(match[1]);
+      return listOfDOI.length;
+    }
+  }
+
+  return 0;
+}
+
+module.exports = getNumberOfDOI;
diff --git a/src/graphql/lib/resolvers/getByDOI.js b/src/graphql/lib/resolvers/getByDOI.js
@@ -39,8 +39,8 @@ function flatten(obj) {
   return flattened;
 }
 
-async function GetByDOI(parent, args, req, info) {
-  const apikey = req.get('X-API-KEY');
+async function GetByDOI(parent, args, contextValue, info) {
+  const apikey = contextValue.get('X-API-KEY');
 
   if (!apikey) {
     throw Error('Not authorized');
@@ -80,9 +80,9 @@ async function GetByDOI(parent, args, req, info) {
     }
   }
 
-  let index = req?.get('index');
+  let index = contextValue?.get('index');
 
-  const { attributes } = req;
+  const { attributes } = contextValue;
 
   if (!index) {
     index = config.get('elasticsearch.indexAlias');
@@ -101,8 +101,6 @@ async function GetByDOI(parent, args, req, info) {
 
   const dois = [];
 
-  req.countDOI = args?.dois?.length;
-
   // Normalize request
   args.dois.forEach((doi) => {
     dois.push(doi.toLowerCase());

diff --git a/src/graphql/lib/resolvers/unpaywall.js b/src/graphql/lib/resolvers/unpaywall.js
@@ -39,8 +39,8 @@ function flatten(obj) {
   return flattened;
 }
 
-async function unpaywall(parent, args, req, info) {
-  const apikey = req.get('X-API-KEY');
+async function unpaywall(parent, args, contextValue, info) {
+  const apikey = contextValue.get('X-API-KEY');
 
   if (!apikey) {
     throw Error('Not authorized');
@@ -80,9 +80,9 @@ async function unpaywall(parent, args, req, info) {
     }
   }
 
-  let index = req?.get('index');
+  let index = contextValue?.get('index');
 
-  const { attributes } = req;
+  const { attributes } = contextValue;
 
   if (!index) {
     index = config.get('elasticsearch.indexAlias');
@@ -101,8 +101,6 @@ async function unpaywall(parent, args, req, info) {
 
   const dois = [];
 
-  req.countDOI = args?.dois?.length;
-
   // Normalize request
   args.dois.forEach((doi) => {
     dois.push(doi.toLowerCase());

diff --git a/src/graphql/package.json b/src/graphql/package.json
@@ -1,6 +1,6 @@
 {
   "name": "ezunpaywall-graphql-api",
-  "version": "1.3.0",
+  "version": "1.3.2",
   "description": "",
   "main": "app.js",
   "author": "felixleo22",

diff --git a/src/health/package.json b/src/health/package.json
@@ -1,6 +1,6 @@
 {
   "name": "ezunpaywall-health",
-  "version": "1.3.0",
+  "version": "1.3.2",
   "description": "",
   "main": "app.js",
   "author": "felixleo22",

diff --git a/src/mail/package.json b/src/mail/package.json
@@ -1,6 +1,6 @@
 {
   "name": "mail",
-  "version": "1.3.0",
+  "version": "1.3.2",
   "description": "",
   "main": "app.js",
   "scripts": {

diff --git a/src/update/package.json b/src/update/package.json
@@ -1,6 +1,6 @@
 {
   "name": "ezunpaywall-update",
-  "version": "1.3.0",
+  "version": "1.3.2",
   "description": "",
   "main": "app.js",
   "author": "felixleo22",