[ALS-8191] Link facet and concept filtering a bit better

hms-dbmi · Feb 5, 2025 · 3bc87d8 · 3bc87d8
1 parent 9c9e654
commit 3bc87d8
Show file tree

Hide file tree

Showing 6 changed files with 39 additions and 19 deletions.
diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptFilterQueryGenerator.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptFilterQueryGenerator.java
@@ -3,6 +3,7 @@
 import edu.harvard.dbmi.avillach.dictionary.facet.Facet;
 import edu.harvard.dbmi.avillach.dictionary.filter.Filter;
 import edu.harvard.dbmi.avillach.dictionary.filter.QueryParamPair;
+import edu.harvard.dbmi.avillach.dictionary.util.QueryUtility;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.data.domain.Pageable;
@@ -88,23 +89,21 @@ public QueryParamPair generateFilterQuery(Filter filter, Pageable pageable) {
     }
 
     private String createValuelessNodeFilter(String search, List<String> consents) {
-        String rankQuery = "0 as rank";
+        String rankQuery = "0";
         String rankWhere = "";
         if (StringUtils.hasLength(search)) {
             // we rank search results via two factors:
             // 1. (more important) does the raw search term appear in the concept's values?
             // 2. (less important) does psql think the tokenized search term matches something in the searchable_fields
-            rankQuery = "(CAST(LOWER(categorical_values.VALUE) LIKE '%' || LOWER(:search) || '%' as integer) * 10) + "
-                + "ts_rank(searchable_fields, (phraseto_tsquery(:search)::text || ':*')::tsquery) as rank";
-            rankWhere = "(LOWER(categorical_values.VALUE) LIKE '%' || LOWER(:search) || '%' OR "
-                + "concept_node.searchable_fields @@ (phraseto_tsquery(:search)::text || ':*')::tsquery) AND";
+            rankQuery = QueryUtility.SEARCH_QUERY;
+            rankWhere = QueryUtility.SEARCH_WHERE + " AND ";
         }
         String consentWhere = CollectionUtils.isEmpty(consents) ? "" : CONSENT_QUERY;
         // concept nodes that have no values and no min/max should not get returned by search
         return """
             SELECT
                 concept_node.concept_node_id,
-                %s
+                (%s) as rank
             FROM
                 concept_node
                 LEFT JOIN dataset ON concept_node.dataset_id = dataset.dataset_id
@@ -135,8 +134,8 @@ private List<String> createFacetFilter(Filter filter, MapSqlParameterSource para
             String rankQuery = "0";
             String rankWhere = "";
             if (StringUtils.hasLength(filter.search())) {
-                rankQuery = "ts_rank(searchable_fields, (phraseto_tsquery(:search)::text || ':*')::tsquery)";
-                rankWhere = "concept_node.searchable_fields @@ (phraseto_tsquery(:search)::text || ':*')::tsquery AND";
+                rankQuery = QueryUtility.SEARCH_QUERY;
+                rankWhere = QueryUtility.SEARCH_WHERE + " AND ";
             }
             return """
                 (
@@ -147,6 +146,7 @@ private List<String> createFacetFilter(Filter filter, MapSqlParameterSource para
                         LEFT JOIN facet__concept_node ON facet__concept_node.facet_id = facet.facet_id
                         JOIN facet_category ON facet_category.facet_category_id = facet.facet_category_id
                         JOIN concept_node ON concept_node.concept_node_id = facet__concept_node.concept_node_id
+                        LEFT JOIN concept_node_meta AS categorical_values ON concept_node.concept_node_id = categorical_values.concept_node_id AND categorical_values.KEY = 'values'
                         LEFT JOIN dataset ON concept_node.dataset_id = dataset.dataset_id
                     WHERE
                         %s
@@ -155,7 +155,8 @@ facet.name IN (:facets_for_category_%s ) AND facet_category.name = :category_%s
                     GROUP BY
                         facet__concept_node.concept_node_id
                 )
-                """.formatted(rankQuery, rankWhere, consentWhere, facetsForCategory.getKey(), facetsForCategory.getKey());
+                """
+                .formatted(rankQuery, rankWhere, consentWhere, facetsForCategory.getKey(), facetsForCategory.getKey());
         }).toList();
     }
 

diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetQueryGenerator.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetQueryGenerator.java
@@ -1,6 +1,7 @@
 package edu.harvard.dbmi.avillach.dictionary.facet;
 
 import edu.harvard.dbmi.avillach.dictionary.filter.Filter;
+import edu.harvard.dbmi.avillach.dictionary.util.QueryUtility;
 import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
 import org.springframework.stereotype.Component;
 import org.springframework.util.CollectionUtils;
@@ -102,15 +103,15 @@ private String createMultiCategorySQLWithSearch(
                         LEFT JOIN concept_node_meta AS categorical_values ON concept_node.concept_node_id = categorical_values.concept_node_id AND categorical_values.KEY = 'values'
                     WHERE
                         (fc.name, facet.name) IN (:facets_in_cat_%s)
-                        AND concept_node.searchable_fields @@ (phraseto_tsquery(:search)::text || ':*')::tsquery
+                        AND %s
                         AND (
                             continuous_min.value <> '' OR
                             continuous_max.value <> '' OR
                             categorical_values.value <> ''
                         )
                 )
                 """
-                .formatted(categoryKeys.get(category), categoryKeys.get(category));
+                .formatted(categoryKeys.get(category), categoryKeys.get(category), QueryUtility.SEARCH_WHERE);
         }).collect(Collectors.joining(",\n"));
         /*
          * Categories with no selected facets contribute no concepts, so ignore them for now. Now, for each category with selected facets,
@@ -298,7 +299,7 @@ facet.facet_id, count(*) as facet_count
                 WHERE
                     %s
                     fc.name = :facet_category_name
-                    AND concept_node.searchable_fields @@ (phraseto_tsquery(:search)::text || ':*')::tsquery
+                    AND %s
                     AND (
                         continuous_min.value <> '' OR
                         continuous_max.value <> '' OR
@@ -325,7 +326,7 @@ WITH matching_concepts AS (
                     WHERE
                         fc.name = :facet_category_name
                         AND facet.name IN (:facets)
-                        AND concept_node.searchable_fields @@ (phraseto_tsquery(:search)::text || ':*')::tsquery
+                        AND %s
                         AND (
                             continuous_min.value <> '' OR
                             continuous_max.value <> '' OR
@@ -350,7 +351,7 @@ facet.facet_id, count(*) as facet_count
                     facet_count DESC
             )
             """
-            .formatted(consentWhere, consentWhere);
+            .formatted(consentWhere, QueryUtility.SEARCH_WHERE, QueryUtility.SEARCH_WHERE, consentWhere);
     }
 
     private String createSingleCategorySQLNoSearch(List<Facet> facets, String consentWhere, MapSqlParameterSource params) {
@@ -447,7 +448,7 @@ facet.facet_id, count(*) as facet_count
                 LEFT JOIN concept_node_meta AS categorical_values ON concept_node.concept_node_id = categorical_values.concept_node_id AND categorical_values.KEY = 'values'
             WHERE
                 %s
-                concept_node.searchable_fields @@ (phraseto_tsquery(:search)::text || ':*')::tsquery
+                %s
                 AND (
                     continuous_min.value <> '' OR
                     continuous_max.value <> '' OR
@@ -458,7 +459,7 @@ facet.facet_id, count(*) as facet_count
             ORDER BY
                 facet_count DESC
             """
-            .formatted(consentWhere);
+            .formatted(consentWhere, QueryUtility.SEARCH_WHERE);
 
     }
 

diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/util/QueryUtility.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/util/QueryUtility.java
@@ -16,4 +16,9 @@ AND concept_node_meta.KEY IN (:disallowed_meta_keys)
                 concept_node.concept_node_id
         )
         """;
+
+    public static final String SEARCH_QUERY =
+        "CAST(LOWER(categorical_values.VALUE) LIKE '%' || LOWER(:search) || '%' as integer) * 10 + ts_rank(searchable_fields, (phraseto_tsquery(:search)::text || ':*')::tsquery)";
+    public static final String SEARCH_WHERE =
+        "(LOWER(categorical_values.VALUE) LIKE '%' || LOWER(:search) || '%' OR concept_node.searchable_fields @@ (phraseto_tsquery(:search)::text || ':*')::tsquery)";
 }
diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetQueryGeneratorTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetQueryGeneratorTest.java
@@ -61,8 +61,8 @@ void shouldCountFacetsWithNoSearchAndNoSelectedFacetsAndNoConsents() {
 
         List<IdCountPair> actual = template.query(query, params, new IdCountPairMapper());
         List<IdCountPair> expected = List.of(
-            new IdCountPair(22, 13), new IdCountPair(31, 3), new IdCountPair(27, 3), new IdCountPair(26, 3), new IdCountPair(28, 3),
-            new IdCountPair(23, 2), new IdCountPair(25, 2), new IdCountPair(21, 1), new IdCountPair(20, 1)
+            new IdCountPair(22, 13), new IdCountPair(26, 3), new IdCountPair(31, 3), new IdCountPair(27, 3), new IdCountPair(28, 3),
+            new IdCountPair(23, 2), new IdCountPair(21, 2), new IdCountPair(25, 2), new IdCountPair(20, 1)
         );
 
         Assertions.assertEquals(expected, actual);
@@ -225,4 +225,16 @@ void shouldCountFacetsNoSearchAndTwoSelectedFacetsInDifferentCatsAndConsents() {
 
         Assertions.assertEquals(expected, actual);
     }
+
+    @Test
+    void shouldCountFacetsSearchMatchesValueNotSearchString() {
+        Filter filter = new Filter(List.of(), "gremlin", List.of());
+        MapSqlParameterSource params = new MapSqlParameterSource();
+        String query = subject.createFacetSQLAndPopulateParams(filter, params);
+
+        List<IdCountPair> actual = template.query(query, params, new IdCountPairMapper());
+        List<IdCountPair> expected = List.of(new IdCountPair(21, 1));
+
+        Assertions.assertEquals(expected, actual);
+    }
 }
diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetRepositoryTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetRepositoryTest.java
@@ -71,8 +71,8 @@ void shouldGetAllFacets() {
             new FacetCategory(
                 "nsrr_harmonized", "Common Data Element Collection", "",
                 List.of(
+                    new Facet("PhenX", "PhenX", null, null, 2, List.of(), "nsrr_harmonized", null),
                     new Facet("LOINC", "LOINC", null, null, 1, List.of(), "nsrr_harmonized", null),
-                    new Facet("PhenX", "PhenX", null, null, 1, List.of(), "nsrr_harmonized", null),
                     new Facet(
                         "gad_7", "Generalized Anxiety Disorder Assessment (GAD-7)", null, null, 0, List.of(), "nsrr_harmonized", null
                     ),

diff --git a/src/test/resources/seed.sql b/src/test/resources/seed.sql
@@ -739,6 +739,7 @@ COPY public.facet__concept_node (facet__concept_node_id, facet_id, concept_node_
 91	18	261
 92	20	229
 93	21	229
+94	21	271
 \.
-Original file line number
+Diff line change
@@ Expand Up @@
 	18	261
 	20	229
 	21	229
+	21	271
     \.
@@ Expand Down @@