Bug Fix: Return correct count values for RC/DRC and PC/DPC (#2405)

Fixes #2296

Author: amarsan

pom.xml

@@ -769,6 +769,12 @@
       <artifactId>json-smart</artifactId>
       <version>2.4.9</version>
     </dependency>
+    <dependency>
+      <groupId>net.minidev</groupId>
+      <artifactId>asm</artifactId>
+      <version>1.0.2</version>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-lang3</artifactId>

src/main/java/org/ohdsi/webapi/cdmresults/service/CDMCacheService.java

@@ -145,9 +145,9 @@ private void cacheRecordsById(Source source, List<Integer> ids) {
       PreparedStatementRenderer psr = getPartialPreparedStatementRenderer(source, idsSlice);
       Set<Integer> cachedIds = loadCache(source, psr, mapper, jdbcTemplate);
       // in this batch, need to save any concepts that were not found when loading cache
-      idsSlice.removeAll(cachedIds);
-      if (!idsSlice.isEmpty()) { // store zeros in cache
-        List<CDMCacheEntity> zeroConcepts = idsSlice.stream().map(id -> {
+      List<Integer> notFoundIds = new ArrayList<Integer>(CollectionUtils.subtract(idsSlice, cachedIds));
+      if (!notFoundIds.isEmpty()) { // store zeros in cache
+        List<CDMCacheEntity> zeroConcepts = notFoundIds.stream().map(id -> {
           CDMCacheEntity ce = new CDMCacheEntity();
           ce.setConceptId(id);
           ce.setRecordCount(0L);

src/main/resources/ddl/results/achilles_result_concept_count.sql

@@ -0,0 +1,8 @@
+IF OBJECT_ID('@results_schema.achilles_result_concept_count', 'U') IS NULL
+CREATE TABLE @results_schema.achilles_result_concept_count (
+  concept_id                int,
+  record_count              bigint,
+  descendant_record_count   bigint,
+  person_count              bigint,
+  descendant_person_count   bigint
+);

src/test/java/org/ohdsi/webapi/test/CDMResultsServiceIT.java

@@ -0,0 +1,81 @@
+package org.ohdsi.webapi.test;
+
+import com.odysseusinc.arachne.commons.types.DBMSType;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.ohdsi.circe.helper.ResourceHelper;
+import org.ohdsi.sql.SqlRender;
+import org.ohdsi.sql.SqlTranslate;
+import org.ohdsi.webapi.source.SourceRepository;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.http.ResponseEntity;
+
+public class CDMResultsServiceIT extends WebApiIT {
+    private static final String CDM_RESULTS_FILE_PATH = "/database/cdm_results.sql";
+
+    @Value("${cdmResultsService.endpoint.conceptRecordCount}")
+    private String conceptRecordCountEndpoint;
+
+    @Autowired
+    private SourceRepository sourceRepository;
+
+    @Before
+    public void init() throws Exception {
+        truncateTable(String.format("%s.%s", "public", "source"));
+        resetSequence(String.format("%s.%s", "public", "source_sequence"));
+        sourceRepository.saveAndFlush(getCdmSource());
+        prepareCdmSchema();
+        prepareResultSchema();
+        addCDMResults();
+    }
+
+    private void addCDMResults() {
+        String resultSql = SqlRender.renderSql(ResourceHelper.GetResourceAsString(
+                CDM_RESULTS_FILE_PATH),
+                new String[] { "results_schema" }, new String[] { RESULT_SCHEMA_NAME });
+        String sql = SqlTranslate.translateSql(resultSql, DBMSType.POSTGRESQL.getOhdsiDB());
+        jdbcTemplate.execute(sql);
+    }
+
+    @Test
+    public void requestConceptRecordCounts_firstTime_returnsResults() {
+
+        // Arrange
+        List<Integer> conceptIds = Arrays.asList(1);
+        Map<String, String> queryParameters = new HashMap<String, String>();
+        queryParameters.put("sourceName", SOURCE_KEY);
+
+        List<LinkedHashMap<String, List<Integer>>> list = new ArrayList<>();
+        @SuppressWarnings("unchecked")
+        Class<List<LinkedHashMap<String, List<Integer>>>> returnClass = (Class<List<LinkedHashMap<String, List<Integer>>>>)list.getClass(); 
+
+        // Act
+        final ResponseEntity<List<LinkedHashMap<String, List<Integer>>>> entity = getRestTemplate().postForEntity(this.conceptRecordCountEndpoint, conceptIds, 
+                returnClass, queryParameters );
+
+        // Assertion
+        assertOK(entity);
+        List<LinkedHashMap<String, List<Integer>>> results = entity.getBody();
+        assertEquals(1, results.size());
+        LinkedHashMap<String, List<Integer>> resultHashMap = results.get(0);
+        assertEquals(1, resultHashMap.size());
+        assertTrue(resultHashMap.containsKey("1"));
+        List<Integer> counts = resultHashMap.get("1");
+        assertEquals(100, counts.get(0).intValue());
+        assertEquals(101, counts.get(1).intValue());
+        assertEquals(102, counts.get(2).intValue());
+        assertEquals(103, counts.get(3).intValue());
+    }
+}

src/test/java/org/ohdsi/webapi/test/WebApiIT.java

@@ -64,7 +64,8 @@ public abstract class WebApiIT {
             "/ddl/results/pathway_analysis_codes.sql",
             "/ddl/results/pathway_analysis_events.sql",
             "/ddl/results/pathway_analysis_paths.sql",
-            "/ddl/results/pathway_analysis_stats.sql"
+            "/ddl/results/pathway_analysis_stats.sql",
+            "/ddl/results/achilles_result_concept_count.sql"
     );
 
 

src/test/resources/application-test.properties

@@ -1,6 +1,7 @@
 baseUri=http://localhost:${local.server.port}${server.context-path}
 security.db.datasource.url=http://localhost:${datasource.url}/arachne_portal_enterprise
 vocabularyservice.endpoint=${baseUri}/vocabulary
+cdmResultsService.endpoint=${baseUri}/cdmresults
 #GET vocabularies
 vocabularyservice.endpoint.vocabularies=${vocabularyservice.endpoint}/vocabularies
 #GET domains
@@ -10,6 +11,9 @@ vocabularyservice.endpoint.concept=${vocabularyservice.endpoint}/concept/1
 #GET cohortdefinitions
 cohortdefinitionservice.endpoint.cohortdefinitions=${baseUri}/cohortdefinition
 
+#POST cdmResults
+cdmResultsService.endpoint.conceptRecordCount=${cdmResultsService.endpoint}/{sourceName}/conceptRecordCount
+
 #Example application service
 exampleservice.endpoint=${baseUri}/example
 

src/test/resources/database/cdm_results.sql

@@ -0,0 +1,2 @@
+insert into @results_schema.achilles_result_concept_count (concept_id, record_count, descendant_record_count, person_count, descendant_person_count)
+values (1,100,101,102,103);