release(2.1.0): search_after support added (#134)

bharathkontham · web-flow · commit 5dd48e267ce7 · 2020-05-03T20:11:45.000+05:30
* task(Readme): readme file update

* fix(eslint): all lint issues, remove unused files

* fix(eslint): constructor, prototype issues for ESConnector class

* fix(lint): lint issues fixed in all.js and buildOrder.js

* feature(Client): all client APIs updated

* fix(replaceOrCreate): response fix

* task(README, examples): update README and example server

* Release: Version 2.0.0

* task(filter): added _source filter support for fields

* release(2.0.1): version update

* task(search_after): add elasticsearch search after support in filter

* task(search-after): updated readme

* release(2.1.0): search_after support added
diff --git a/README.md b/README.md
@@ -14,7 +14,9 @@ Elasticsearch(versions 6.x and 7.x) datasource connector for [Loopback 3.x](http
   - [Recommended properties](#recommended)
   - [Optional properties](#optional)
   - [Sample for copy paste](#sample)
-- [How to achieve Instant search](#how-to-achieve-instant-search)
+
+- [Elasticsearch SearchAfter Support](#elasticsearch-searchafter-support)
+- [Example](#about-the-example-app)
 - [Troubleshooting](#troubleshooting)
 - [Contributing](#contributing)
 - [Frequently Asked Questions](#faqs)
@@ -154,6 +156,60 @@ npm install loopback-connector-esv6 --save --save-exact
 
 2.You can peek at `/examples/server/datasources.json` for more hints.
 
+## Elasticsearch SearchAfter Support
+
+- ```search_after``` feature of elasticsearch is supported in loopback filter.
+- For this, you need to create a property in model called ```_search_after``` with loopback type ```["any"]```. This field cannot be updated using in connector.
+- Elasticsearch ```sort``` value will return in this field.
+- You need pass ```_search_after``` value in ```searchafter``` key of loopback filter.
+- Example filter query for ```find```.
+
+```json
+{
+  "where": {
+    "username": "hello"
+  },
+  "order": "created DESC",
+  "searchafter": [
+    1580902552905
+  ],
+  "limit": 4
+}
+```
+
+- Example result.
+
+```json
+[
+  {
+    "id": "1bb2dd63-c7b9-588e-a942-15ca4f891a80",
+    "username": "test",
+    "_search_after": [
+      1580902552905
+    ],
+    "created": "2020-02-05T11:35:52.905Z"
+  },
+  {
+    "id": "fd5ea4df-f159-5816-9104-22147f2a740f",
+    "username": "test3",
+    "_search_after": [
+      1580902552901
+    ],
+    "created": "2020-02-05T11:35:52.901Z"
+  },
+  {
+    "id": "047c0adb-13ea-5f80-a772-3d2a4691d47a",
+    "username": "test4",
+    "_search_after": [
+      1580902552897
+    ],
+    "created": "2020-02-05T11:35:52.897Z"
+  }
+]
+```
+
+- This is useful for pagination. To go to previous page, change sorting order.
+
 ## About the example app
 
 1. The `examples` directory contains a loopback app which uses this connector.
diff --git a/examples/common/models/user-model.json b/examples/common/models/user-model.json
@@ -2,7 +2,11 @@
   "name": "UserModel",
   "base": "User",
   "idInjection": true,
-  "properties": {},
+  "properties": {
+    "_search_after": {
+      "type": ["any"]
+    }
+  },
   "validations": [],
   "relations": {
     "globalConfigModels": {
diff --git a/lib/buildFilter.js b/lib/buildFilter.js
@@ -44,6 +44,11 @@ function buildFilter(modelName, idName, criteria = {}, size = null, offset = nul
         };
       }
     }
+    if (criteria.searchafter && Array.isArray(criteria.searchafter)
+    && criteria.searchafter.length) {
+      filter.body.search_after = criteria.searchafter;
+      filter.from = undefined;
+    }
     if (criteria.order) {
       log('ESConnector.prototype.buildFilter', 'will delegate sorting to buildOrder()');
       filter.body.sort = self.buildOrder(modelName, idName, criteria.order);
diff --git a/lib/create.js b/lib/create.js
@@ -1,5 +1,7 @@
 const _ = require('lodash');
 const log = require('debug')('loopback:connector:elasticsearch');
+// CONSTANTS
+const SEARCHAFTERKEY = '_search_after';
 
 function create(model, data, done) {
   const self = this;
@@ -26,6 +28,9 @@ function create(model, data, done) {
     method = 'index'; // if there is no/empty id field, we must use the index method to create it (API 5.0)
   }
   document.body.docType = model;
+  if (document.body[SEARCHAFTERKEY]) {
+    document.body[SEARCHAFTERKEY] = undefined;
+  }
   self.db[method](
     document
   ).then(
diff --git a/lib/esConnector.js b/lib/esConnector.js
@@ -35,6 +35,9 @@ const { updateAll } = require('./updateAll');
 const { updateAttributes } = require('./updateAttributes');
 const { updateOrCreate } = require('./updateOrCreate');
 
+// CONSTANTS
+const SEARCHAFTERKEY = '_search_after';
+
 /**
  * Connector constructor
  * @param {object} datasource settings
@@ -212,7 +215,7 @@ ESConnector.prototype.getValueFromProperty = function (property, value) {
  * @param {Object} data from DB
  * @returns {object} modeled document
  */
-ESConnector.prototype.matchDataToModel = function (modelName, data, esId, idName) {
+ESConnector.prototype.matchDataToModel = function (modelName, data, esId, idName, sort) {
   /*
   log('ESConnector.prototype.matchDataToModel', 'modelName',
     modelName, 'data', JSON.stringify(data,null,0));
@@ -239,6 +242,7 @@ ESConnector.prototype.matchDataToModel = function (modelName, data, esId, idName
         );
       }
     });
+    document[SEARCHAFTERKEY] = sort;
     log('ESConnector.prototype.matchDataToModel', 'document', JSON.stringify(document, null, 0));
     return document;
   } catch (err) {
@@ -258,7 +262,7 @@ ESConnector.prototype.dataSourceToModel = function (modelName, data, idName) {
 
   // return data._source; // TODO: super-simplify?
   // eslint-disable-next-line no-underscore-dangle
-  return this.matchDataToModel(modelName, data._source, data._id, idName);
+  return this.matchDataToModel(modelName, data._source, data._id, idName, data.sort || []);
 };
 
 /**
diff --git a/lib/find.js b/lib/find.js
@@ -8,12 +8,12 @@ function find(modelName, id, done) {
   if (id === undefined || id === null) {
     throw new Error('id not set!');
   }
-
+  const idName = self.idName(modelName);
   const defaults = self.addDefaults(modelName, 'find');
   self.db.get(_.defaults({
     id: self.getDocumentId(id)
   }, defaults)).then(({ body }) => {
-    done(null, self.dataSourceToModel(modelName, body));
+    done(null, self.dataSourceToModel(modelName, body, idName));
   }).catch((error) => {
     log('ESConnector.prototype.find', error.message);
     done(error);
diff --git a/lib/replaceById.js b/lib/replaceById.js
@@ -1,5 +1,7 @@
 const _ = require('lodash');
 const log = require('debug')('loopback:connector:elasticsearch');
+// CONSTANTS
+const SEARCHAFTERKEY = '_search_after';
 
 function replaceById(modelName, id, data, options, callback) {
   const self = this;
@@ -21,6 +23,9 @@ function replaceById(modelName, id, data, options, callback) {
   if (Object.prototype.hasOwnProperty.call(modelProperties, idName)) {
     document.body[idName] = id;
   }
+  if (document.body[SEARCHAFTERKEY]) {
+    document.body[SEARCHAFTERKEY] = undefined;
+  }
   log('ESConnector.prototype.replaceById', 'document', document);
   self.db.index(
     document
diff --git a/lib/replaceOrCreate.js b/lib/replaceOrCreate.js
@@ -1,5 +1,7 @@
 const _ = require('lodash');
 const log = require('debug')('loopback:connector:elasticsearch');
+// CONSTANTS
+const SEARCHAFTERKEY = '_search_after';
 
 function replaceOrCreate(modelName, data, callback) {
   const self = this;
@@ -16,6 +18,9 @@ function replaceOrCreate(modelName, data, callback) {
   document.body = {};
   _.assign(document.body, data);
   document.body.docType = modelName;
+  if (document.body[SEARCHAFTERKEY]) {
+    document.body[SEARCHAFTERKEY] = undefined;
+  }
   log('ESConnector.prototype.replaceOrCreate', 'document', document);
   self.db.index(
     document
diff --git a/lib/save.js b/lib/save.js
@@ -1,5 +1,7 @@
 const _ = require('lodash');
 const log = require('debug')('loopback:connector:elasticsearch');
+// CONSTANTS
+const SEARCHAFTERKEY = '_search_after';
 
 // eslint-disable-next-line consistent-return
 function save(model, data, done) {
@@ -16,6 +18,9 @@ function save(model, data, done) {
     return done('Document id not setted!', null);
   }
   data.docType = model;
+  if (data[SEARCHAFTERKEY]) {
+    data[SEARCHAFTERKEY] = undefined;
+  }
   self.db.update(_.defaults({
     id,
     body: {
diff --git a/lib/updateAll.js b/lib/updateAll.js
@@ -1,5 +1,7 @@
 const _ = require('lodash');
 const log = require('debug')('loopback:connector:elasticsearch');
+// CONSTANTS
+const SEARCHAFTERKEY = '_search_after';
 
 function updateAll(model, where, data, options, cb) {
   const self = this;
@@ -20,7 +22,7 @@ function updateAll(model, where, data, options, cb) {
     params: {}
   };
   _.forEach(data, (value, key) => {
-    if (key !== '_id' || key !== idName) {
+    if (key !== '_id' && key !== idName && key !== SEARCHAFTERKEY) {
       // default language for inline scripts is painless if ES 5, so this needs the extra params.
       reqBody.script.inline += `ctx._source.${key}=params.${key};`;
       reqBody.script.params[key] = value;
diff --git a/lib/updateAttributes.js b/lib/updateAttributes.js
@@ -1,5 +1,7 @@
 const _ = require('lodash');
 const log = require('debug')('loopback:connector:elasticsearch');
+// CONSTANTS
+const SEARCHAFTERKEY = '_search_after';
 
 function updateAttributes(modelName, id, data, callback) {
   const self = this;
@@ -22,7 +24,7 @@ function updateAttributes(modelName, id, data, callback) {
     params: {}
   };
   _.forEach(data, (value, key) => {
-    if (key !== '_id' || key !== idName) {
+    if (key !== '_id' && key !== idName && key !== SEARCHAFTERKEY) {
       // default language for inline scripts is painless if ES 5, so this needs the extra params.
       reqBody.script.inline += `ctx._source.${key}=params.${key};`;
       reqBody.script.params[key] = value;
diff --git a/lib/updateOrCreate.js b/lib/updateOrCreate.js
@@ -1,5 +1,7 @@
 const _ = require('lodash');
 const log = require('debug')('loopback:connector:elasticsearch');
+// CONSTANTS
+const SEARCHAFTERKEY = '_search_after';
 
 function updateOrCreate(modelName, data, callback) {
   const self = this;
@@ -13,6 +15,9 @@ function updateOrCreate(modelName, data, callback) {
 
   const defaults = self.addDefaults(modelName, 'updateOrCreate');
   data.docType = modelName;
+  if (data[SEARCHAFTERKEY]) {
+    data[SEARCHAFTERKEY] = undefined;
+  }
   self.db.update(_.defaults({
     id,
     body: {
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "loopback-connector-esv6",
-  "version": "2.0.1",
+  "version": "2.1.0",
   "description": "LoopBack Connector for Elasticsearch 6.x and 7.x",
   "main": "index.js",
   "scripts": {

Original file line number	Diff line number	Diff line change
`@@ -44,6 +44,11 @@ function buildFilter(modelName, idName, criteria = {}, size = null, offset = nul`
`44`	`44`	`};`
`45`	`45`	`}`
`46`	`46`	`}`
	`47`	`+ if (criteria.searchafter && Array.isArray(criteria.searchafter)`
	`48`	`+ && criteria.searchafter.length) {`
	`49`	`+ filter.body.search_after = criteria.searchafter;`
	`50`	`+ filter.from = undefined;`
	`51`	`+ }`
`47`	`52`	`if (criteria.order) {`
`48`	`53`	`log('ESConnector.prototype.buildFilter', 'will delegate sorting to buildOrder()');`
`49`	`54`	`filter.body.sort = self.buildOrder(modelName, idName, criteria.order);`
Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "loopback-connector-esv6",`
`3`		`- "version": "2.0.1",`
	`3`	`+ "version": "2.1.0",`
`4`	`4`	`"description": "LoopBack Connector for Elasticsearch 6.x and 7.x",`
`5`	`5`	`"main": "index.js",`
`6`	`6`	`"scripts": {`