Skip to content

Commit 810b7bf

Browse files
holashchandholashchand
committed
Added pagination for search and list apis including both of elastic and native search
1 parent a23865b commit 810b7bf

File tree

16 files changed

+201
-96
lines changed

16 files changed

+201
-96
lines changed

‎docker-compose-v1.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ version: '2.4'
22

33
services:
44
es:
5-
image: docker.elastic.co/elasticsearch/elasticsearch:7.17.13
5+
image: docker.elastic.co/elasticsearch/elasticsearch:6.8.23
66
volumes:
77
- ./${ES_DIR-es-data}:/usr/share/elasticsearch/data/*
88
environment:

‎docker-compose.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ version: '2.4'
22

33
services:
44
es:
5-
image: docker.elastic.co/elasticsearch/elasticsearch:7.17.13
5+
image: docker.elastic.co/elasticsearch/elasticsearch:6.8.23
66
volumes:
77
- ./${ES_DIR-es-data}:/usr/share/elasticsearch/data/*
88
environment:

‎java/apitest/src/test/java/e2e/registry/fusionauth-registry.feature

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,4 +57,4 @@ Feature: Registry api tests
5757
And header Authorization = student_token
5858
When method get
5959
Then status 200
60-
And response[0].osid.length > 0
60+
And response.data[0].osid.length > 0

‎java/apitest/src/test/java/e2e/registry/registry.feature

Lines changed: 17 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -151,8 +151,8 @@ Feature: Registry api tests
151151
And header Authorization = student_token
152152
When method get
153153
Then status 200
154-
And response[0].osid.length > 0
155-
* def studentOsid = response[0].osid
154+
And response.data[0].osid.length > 0
155+
* def studentOsid = response.data[0].osid
156156
# update student info
157157
Given url registryUrl
158158
And path 'api/v1/Student/' + studentOsid
@@ -168,7 +168,7 @@ Feature: Registry api tests
168168
And header Authorization = student_token
169169
When method get
170170
Then status 200
171-
And response[0].name == "xyz"
171+
And response.data[0].name == "xyz"
172172
# get student
173173
Given url registryUrl
174174
And path 'api/v1/Student/search'
@@ -177,8 +177,8 @@ Feature: Registry api tests
177177
Then status 200
178178
* print response
179179
And response.length == 1
180-
And match response[0].contact == '#notpresent'
181-
And match response[0].favoriteSubject == '#notpresent'
180+
And match response.data[0].contact == '#notpresent'
181+
And match response.data[0].favoriteSubject == '#notpresent'
182182
# delete student info
183183
Given url registryUrl
184184
And path 'api/v1/Student/' + studentOsid
@@ -191,7 +191,8 @@ Feature: Registry api tests
191191
And path 'api/v1/Student'
192192
And header Authorization = student_token
193193
When method get
194-
Then status 404
194+
Then status 200
195+
And response.totalCount == 0
195196

196197
@env=async
197198
Scenario: Create a teacher schema and create teacher entity asynchronously
@@ -240,17 +241,17 @@ Feature: Registry api tests
240241
When method post
241242
Then status 200
242243
* print response
243-
And response.length == 1
244-
And response[0].contact == '#notpresent'
244+
And response.totalCount == 1
245+
And response.data[0].contact == '#notpresent'
245246
# get teacher info
246247
Given url registryUrl
247248
And path 'api/v1/Teacher/search'
248249
And request {"filters":{ "name": { "endsWith": "abc" }}}
249250
When method post
250251
Then status 200
251252
* print response
252-
And response.length == 1
253-
And response[0].contact == '#notpresent'
253+
And response.totalCount == 1
254+
And response.data[0].contact == '#notpresent'
254255

255256
@envnot=fusionauth
256257
Scenario: Create Board and invite institutes
@@ -316,7 +317,7 @@ Feature: Registry api tests
316317
And header Authorization = board_token
317318
When method get
318319
Then status 200
319-
And response[0].osid.length > 0
320+
And response.data[0].osid.length > 0
320321

321322
# invite institute with token
322323
Given url registryUrl
@@ -347,9 +348,9 @@ Feature: Registry api tests
347348
And header Authorization = institute_token
348349
When method get
349350
Then status 200
350-
And response[0].osid.length > 0
351-
* def instituteOsid = response[0].osid
352-
* def address = response[0].address
351+
And response.data[0].osid.length > 0
352+
* def instituteOsid = response.data[0].osid
353+
* def address = response.data[0].address
353354

354355
# update property of institute
355356
Given url registryUrl
@@ -366,8 +367,8 @@ Feature: Registry api tests
366367
And header Authorization = institute_token
367368
When method get
368369
Then status 200
369-
And assert response[0].address[0].phoneNo.length == 1
370-
And assert response[0].address[0].phoneNo[0] == "444"
370+
And assert response.data[0].address[0].phoneNo.length == 1
371+
And assert response.data[0].address[0].phoneNo[0] == "444"
371372

372373
@envnot=fusionauth
373374
Scenario: write a api test, to test the schema not found error

‎java/elastic-search/src/main/java/dev/sunbirdrc/elastic/ElasticServiceImpl.java

Lines changed: 17 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import com.fasterxml.jackson.databind.ObjectMapper;
55
import com.fasterxml.jackson.databind.node.ArrayNode;
66
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
7+
import com.fasterxml.jackson.databind.node.ObjectNode;
78
import dev.sunbirdrc.pojos.ComponentHealthInfo;
89
import dev.sunbirdrc.pojos.Filter;
910
import dev.sunbirdrc.pojos.FilterOperators;
@@ -49,8 +50,7 @@
4950
import org.springframework.retry.annotation.Backoff;
5051
import org.springframework.retry.annotation.Retryable;
5152

52-
import static dev.sunbirdrc.registry.middleware.util.Constants.CONNECTION_FAILURE;
53-
import static dev.sunbirdrc.registry.middleware.util.Constants.SUNBIRD_ELASTIC_SERVICE_NAME;
53+
import static dev.sunbirdrc.registry.middleware.util.Constants.*;
5454

5555
public class ElasticServiceImpl implements IElasticService {
5656
private static Map<String, RestHighLevelClient> esClient = new HashMap<String, RestHighLevelClient>();
@@ -283,21 +283,25 @@ public JsonNode search(String index, SearchQuery searchQuery) throws IOException
283283
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder()
284284
.query(query)
285285
.size(searchQuery.getLimit())
286-
.from(searchQuery.getOffset());
286+
.from(searchQuery.getOffset())
287+
.trackTotalHits(true);
287288
SearchRequest searchRequest = new SearchRequest(index).source(sourceBuilder);
288-
ArrayNode resultArray = JsonNodeFactory.instance.arrayNode();
289+
ObjectNode resultNode = JsonNodeFactory.instance.objectNode();
290+
ArrayNode dataArray = JsonNodeFactory.instance.arrayNode();
289291
ObjectMapper mapper = new ObjectMapper();
290-
SearchResponse searchResponse = getClient(index).search(searchRequest, RequestOptions.DEFAULT);
291-
for (SearchHit hit : searchResponse.getHits()) {
292-
JsonNode node = mapper.readValue(hit.getSourceAsString(), JsonNode.class);
293-
// TODO: Add draft mode condition
294-
if(node.get("_status") == null || node.get("_status").asBoolean()) {
295-
resultArray.add(node);
296-
}
292+
SearchResponse searchResponse = getClient(index).search(searchRequest, RequestOptions.DEFAULT);
293+
for (SearchHit hit : searchResponse.getHits()) {
294+
JsonNode node = mapper.readValue(hit.getSourceAsString(), JsonNode.class);
295+
// TODO: Add draft mode condition
296+
if(node.get(STATUS_KEYWORD) == null || node.get(STATUS_KEYWORD).asBoolean()) {
297+
dataArray.add(node);
297298
}
298-
logger.debug("Total search records found " + resultArray.size());
299+
}
300+
resultNode.set(ENTITY_LIST, dataArray);
301+
resultNode.put(TOTAL_COUNT, searchResponse.getHits().getTotalHits());
302+
logger.debug("Total search records found " + dataArray.size());
299303

300-
return resultArray;
304+
return resultNode;
301305

302306
}
303307

‎java/middleware-commons/src/main/java/dev/sunbirdrc/registry/middleware/util/Constants.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,10 @@ public class Constants {
4343
public static final String END_DATE="endDate";
4444
public static final String LIMIT="limit";
4545
public static final String OFFSET="offset";
46+
public static final String NEXT_PAGE="nextPage";
47+
public static final String PREV_PAGE="prevPage";
48+
public static final String TOTAL_COUNT="totalCount";
49+
public static final String ENTITY_LIST="data";
4650

4751
// JSON LD specific
4852
public static final String CONTEXT_KEYWORD = "@context";

‎java/middleware-commons/src/main/java/dev/sunbirdrc/registry/middleware/util/JSONUtil.java

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
import java.io.IOException;
2121
import java.lang.reflect.Type;
22+
import java.nio.charset.StandardCharsets;
2223
import java.util.*;
2324
import java.util.regex.Matcher;
2425
import java.util.regex.Pattern;
@@ -30,7 +31,8 @@
3031
import org.apache.commons.lang3.exception.ExceptionUtils;
3132
import org.slf4j.Logger;
3233
import org.slf4j.LoggerFactory;
33-
import org.springframework.core.NestedExceptionUtils;
34+
35+
import static dev.sunbirdrc.registry.middleware.util.Constants.*;
3436

3537
public class JSONUtil {
3638

@@ -545,4 +547,28 @@ public static JsonNode extractPropertyDataFromEntity(JsonNode entityNode, Map<St
545547
}
546548
return result;
547549
}
550+
551+
public static ObjectNode getSearchPageUrls(JsonNode inputNode, long defaultLimit, long defaultOffset, long totalCount, String url) throws IOException {
552+
ObjectNode result = JsonNodeFactory.instance.objectNode();
553+
JsonNode searchNode = objectMapper.readTree(inputNode.toString());
554+
long limit = searchNode.get(LIMIT) == null ? defaultLimit : searchNode.get(LIMIT).asLong(defaultLimit);
555+
long offset = searchNode.get(OFFSET) == null ? defaultOffset : searchNode.get(OFFSET).asLong(defaultOffset);
556+
((ObjectNode) searchNode).set(OFFSET, JsonNodeFactory.instance.numberNode(offset - limit));
557+
String prevPageToken = Base64.getEncoder().encodeToString(searchNode.toString().getBytes(StandardCharsets.UTF_8));
558+
((ObjectNode) searchNode).set(OFFSET, JsonNodeFactory.instance.numberNode(offset + limit));
559+
String nextPageToken = Base64.getEncoder().encodeToString(searchNode.toString().getBytes(StandardCharsets.UTF_8));
560+
if(offset - limit >=0) result.put(PREV_PAGE, url + "?search=" + prevPageToken);
561+
if(offset + limit <= totalCount) result.put(NEXT_PAGE, url + "?search=" + nextPageToken);
562+
return result;
563+
}
564+
565+
public static ObjectNode parseSearchToken(String endcodedValue) {
566+
try {
567+
byte[] decoded = Base64.getDecoder().decode(endcodedValue);
568+
return (ObjectNode) objectMapper.readTree(decoded);
569+
} catch (Exception ignored) {
570+
logger.warn("Unable to parse next page token");
571+
}
572+
return null;
573+
}
548574
}

‎java/registry/src/main/java/dev/sunbirdrc/registry/config/SchemaLoader.java

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,7 @@
2121
import java.util.ArrayList;
2222

2323
import static dev.sunbirdrc.registry.Constants.Schema;
24-
import static dev.sunbirdrc.registry.middleware.util.Constants.ENTITY_TYPE;
25-
import static dev.sunbirdrc.registry.middleware.util.Constants.FILTERS;
24+
import static dev.sunbirdrc.registry.middleware.util.Constants.*;
2625

2726
@Component
2827
public class SchemaLoader implements ApplicationListener<ContextRefreshedEvent> {
@@ -46,14 +45,14 @@ private void loadSchemasFromDB() {
4645
objectNode.set(FILTERS, JsonNodeFactory.instance.objectNode());
4746
try {
4847
JsonNode searchResults = searchService.search(objectNode, "");
49-
for (JsonNode schemaNode : searchResults.get(Schema)) {
48+
for (JsonNode schemaNode : searchResults.get(Schema).get(ENTITY_LIST)) {
5049
try {
5150
schemaService.addSchema(JsonNodeFactory.instance.objectNode().set(Schema, schemaNode));
5251
} catch (Exception e) {
5352
logger.error("Failed loading schema to definition manager: {}", ExceptionUtils.getStackTrace(e));
5453
}
5554
}
56-
logger.error("Loaded {} schema from DB", searchResults.get(Schema).size());
55+
logger.error("Loaded {} schema from DB", searchResults.get(Schema).get(TOTAL_COUNT));
5756
} catch (IOException e) {
5857
logger.error("Exception occurred while loading schema from db: {}", ExceptionUtils.getStackTrace(e));
5958
} catch (Exception e) {

‎java/registry/src/main/java/dev/sunbirdrc/registry/controller/RegistryController.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ public ResponseEntity<Response> searchEntity(@RequestHeader HttpHeaders header)
5656

5757
try {
5858
watch.start("RegistryController.searchEntity");
59-
JsonNode result = registryHelper.searchEntity(payload, null);
59+
JsonNode result = registryHelper.searchEntity(payload, null, false);
6060

6161
response.setResult(result);
6262
responseParams.setStatus(Response.Status.SUCCESSFUL);

‎java/registry/src/main/java/dev/sunbirdrc/registry/controller/RegistryEntityController.java

Lines changed: 31 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -48,12 +48,14 @@
4848
import org.springframework.web.bind.annotation.*;
4949

5050
import javax.servlet.http.HttpServletRequest;
51+
import javax.ws.rs.BadRequestException;
5152
import java.lang.reflect.InvocationTargetException;
5253
import java.util.*;
5354

5455
import static dev.sunbirdrc.registry.Constants.*;
5556
import static dev.sunbirdrc.registry.helper.RegistryHelper.ServiceNotEnabledResponse;
5657
import static dev.sunbirdrc.registry.middleware.util.Constants.ENTITY_TYPE;
58+
import static dev.sunbirdrc.registry.middleware.util.Constants.TOTAL_COUNT;
5759

5860
@RestController
5961
public class RegistryEntityController extends AbstractController {
@@ -83,6 +85,10 @@ public class RegistryEntityController extends AbstractController {
8385
boolean securityEnabled;
8486
@Value("${certificate.enableExternalTemplates:false}")
8587
boolean externalTemplatesEnabled;
88+
@Value("${search.offset:0}")
89+
private int searchOffset;
90+
@Value("${search.limit:2000}")
91+
private int searchLimit;
8692

8793
@RequestMapping(value = "/api/v1/{entityName}/invite", method = RequestMethod.POST)
8894
public ResponseEntity<Object> invite(
@@ -191,8 +197,11 @@ public ResponseEntity<Object> deleteEntity(
191197
}
192198
}
193199

194-
@RequestMapping(value = "/api/v1/{entityName}/search", method = RequestMethod.POST)
195-
public ResponseEntity<Object> searchEntity(@PathVariable String entityName, @RequestHeader HttpHeaders header, @RequestBody ObjectNode searchNode) {
200+
@RequestMapping(value = "/api/v1/{entityName}/search", method = {RequestMethod.POST, RequestMethod.GET})
201+
public ResponseEntity<Object> searchEntity(@PathVariable String entityName,
202+
HttpServletRequest request,
203+
@RequestHeader HttpHeaders header, @RequestBody(required = false) ObjectNode searchNode,
204+
@RequestParam(value = "search", required = false) String searchQueryString) {
196205

197206
ResponseParams responseParams = new ResponseParams();
198207
Response response = new Response(Response.API_ID.SEARCH, "OK", responseParams);
@@ -201,12 +210,21 @@ public ResponseEntity<Object> searchEntity(@PathVariable String entityName, @Req
201210
watch.start("RegistryController.searchEntity");
202211
ArrayNode entity = JsonNodeFactory.instance.arrayNode();
203212
entity.add(entityName);
213+
if(searchNode == null && (searchQueryString == null || searchQueryString.isEmpty())) {
214+
throw new BadRequestException("Search Request body not found");
215+
}
216+
if(searchNode == null) {
217+
searchNode = JsonNodeFactory.instance.objectNode();
218+
registryHelper.addSearchTokenToQuery(searchQueryString, searchNode);
219+
}
204220
searchNode.set(ENTITY_TYPE, entity);
205221
checkEntityNameInDefinitionManager(entityName);
206222
if (definitionsManager.getDefinition(entityName).getOsSchemaConfiguration().getEnableSearch()) {
207-
JsonNode result = registryHelper.searchEntity(searchNode, null);
223+
JsonNode result = registryHelper.searchEntity(searchNode, null, false).get(entityName);
224+
ObjectNode pageUrls = JSONUtil.getSearchPageUrls(searchNode, searchLimit, searchOffset, result.get(TOTAL_COUNT).asLong(), request.getRequestURL().toString());
225+
((ObjectNode) result).setAll(pageUrls);
208226
watch.stop("RegistryController.searchEntity");
209-
return new ResponseEntity<>(result.get(entityName), HttpStatus.OK);
227+
return new ResponseEntity<>(result, HttpStatus.OK);
210228
} else {
211229
watch.stop("RegistryController.searchEntity");
212230
logger.error("Searching on entity {} not allowed", entityName);
@@ -669,17 +687,21 @@ private JsonNode getEntityJsonNode(@PathVariable String entityName, @PathVariabl
669687

670688
@RequestMapping(value = "/api/v1/{entityName}", method = RequestMethod.GET)
671689
public ResponseEntity<Object> getEntityByToken(@PathVariable String entityName, HttpServletRequest request,
672-
@RequestHeader(required = false) String viewTemplateId) throws RecordNotFoundException {
690+
@RequestHeader(required = false) String viewTemplateId,
691+
@RequestParam(value = "search", required = false) String searchToken) throws RecordNotFoundException {
673692
ResponseParams responseParams = new ResponseParams();
674693
Response response = new Response(Response.API_ID.GET, "OK", responseParams);
675694
try {
676695
checkEntityNameInDefinitionManager(entityName);
677696
String userId = registryHelper.getUserId(entityName);
678697
if (!Strings.isEmpty(userId)) {
679-
JsonNode responseFromDb = registryHelper.searchEntitiesByUserId(entityName, userId, viewTemplateId);
680-
JsonNode entities = responseFromDb.get(entityName);
681-
if (!entities.isEmpty()) {
682-
return new ResponseEntity<>(entities, HttpStatus.OK);
698+
JsonNode searchQuery = registryHelper.searchQueryByUserId(entityName, userId, searchToken, viewTemplateId);
699+
JsonNode responseFromDb = registryHelper.searchEntity(searchQuery, userId, true);
700+
JsonNode results = responseFromDb.get(entityName);
701+
if (!results.isEmpty()) {
702+
ObjectNode pageUrls = JSONUtil.getSearchPageUrls(searchQuery, searchLimit, searchOffset, results.get(TOTAL_COUNT).asLong(), request.getRequestURL().toString());
703+
((ObjectNode) results).setAll(pageUrls);
704+
return new ResponseEntity<>(results, HttpStatus.OK);
683705
} else {
684706
responseParams.setErrmsg("No record found");
685707
responseParams.setStatus(Response.Status.UNSUCCESSFUL);

‎java/registry/src/main/java/dev/sunbirdrc/registry/dao/SearchDaoImpl.java

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@
2222
import org.slf4j.Logger;
2323
import org.slf4j.LoggerFactory;
2424

25+
import static dev.sunbirdrc.registry.middleware.util.Constants.ENTITY_LIST;
26+
import static dev.sunbirdrc.registry.middleware.util.Constants.TOTAL_COUNT;
2527
import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.has;
2628
import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.hasNot;
2729

@@ -45,10 +47,15 @@ public JsonNode search(Graph graphFromStore, SearchQuery searchQuery, boolean ex
4547
GraphTraversal<Vertex, Vertex> parentTraversal = resultGraphTraversal.asAdmin();
4648

4749
resultGraphTraversal = getFilteredResultTraversal(resultGraphTraversal, filterList)
48-
.or(hasNot(Constants.STATUS_KEYWORD), has(Constants.STATUS_KEYWORD, Constants.STATUS_ACTIVE))
49-
.range(offset, offset + searchQuery.getLimit()).limit(searchQuery.getLimit());
50+
.or(hasNot(Constants.STATUS_KEYWORD), has(Constants.STATUS_KEYWORD, Constants.STATUS_ACTIVE));
51+
GraphTraversal<Vertex, Vertex> resultGraphTraversalCount = resultGraphTraversal.asAdmin().clone();
52+
resultGraphTraversal.range(offset, offset + searchQuery.getLimit());
5053
JsonNode result = getResult(graphFromStore, resultGraphTraversal, parentTraversal, expandInternal);
51-
resultNode.set(entity, result);
54+
ObjectNode response = JsonNodeFactory.instance.objectNode();
55+
response.set(ENTITY_LIST, result);
56+
long count = resultGraphTraversalCount.count().next();
57+
response.set(TOTAL_COUNT, JsonNodeFactory.instance.numberNode(count));
58+
resultNode.set(entity, response);
5259
}
5360

5461
return resultNode;

0 commit comments

Comments
 (0)