From 516ea48ab9056486a6d2cd1d65ee5fd8e382ba22 Mon Sep 17 00:00:00 2001 From: Luke Sikina Date: Thu, 30 Jan 2025 14:23:52 -0500 Subject: [PATCH] [ALS-8191] Link facet and concept filtering a bit better --- .../concept/ConceptFilterQueryGenerator.java | 19 ++++++++++--------- .../dictionary/facet/FacetQueryGenerator.java | 15 ++++++++------- .../dictionary/util/QueryUtility.java | 5 +++++ .../facet/FacetQueryGeneratorTest.java | 16 ++++++++++++++-- .../dictionary/facet/FacetRepositoryTest.java | 2 +- src/test/resources/seed.sql | 1 + 6 files changed, 39 insertions(+), 19 deletions(-) diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptFilterQueryGenerator.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptFilterQueryGenerator.java index bf5722c..9c4a38e 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptFilterQueryGenerator.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptFilterQueryGenerator.java @@ -3,6 +3,7 @@ import edu.harvard.dbmi.avillach.dictionary.facet.Facet; import edu.harvard.dbmi.avillach.dictionary.filter.Filter; import edu.harvard.dbmi.avillach.dictionary.filter.QueryParamPair; +import edu.harvard.dbmi.avillach.dictionary.util.QueryUtility; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.data.domain.Pageable; @@ -88,23 +89,21 @@ public QueryParamPair generateFilterQuery(Filter filter, Pageable pageable) { } private String createValuelessNodeFilter(String search, List consents) { - String rankQuery = "0 as rank"; + String rankQuery = "0"; String rankWhere = ""; if (StringUtils.hasLength(search)) { // we rank search results via two factors: // 1. (more important) does the raw search term appear in the concept's values? // 2. (less important) does psql think the tokenized search term matches something in the searchable_fields - rankQuery = "(CAST(LOWER(categorical_values.VALUE) LIKE '%' || LOWER(:search) || '%' as integer) * 10) + " - + "ts_rank(searchable_fields, (phraseto_tsquery(:search)::text || ':*')::tsquery) as rank"; - rankWhere = "(LOWER(categorical_values.VALUE) LIKE '%' || LOWER(:search) || '%' OR " - + "concept_node.searchable_fields @@ (phraseto_tsquery(:search)::text || ':*')::tsquery) AND"; + rankQuery = QueryUtility.SEARCH_QUERY; + rankWhere = QueryUtility.SEARCH_WHERE + " AND "; } String consentWhere = CollectionUtils.isEmpty(consents) ? "" : CONSENT_QUERY; // concept nodes that have no values and no min/max should not get returned by search return """ SELECT concept_node.concept_node_id, - %s + (%s) as rank FROM concept_node LEFT JOIN dataset ON concept_node.dataset_id = dataset.dataset_id @@ -135,8 +134,8 @@ private List createFacetFilter(Filter filter, MapSqlParameterSource para String rankQuery = "0"; String rankWhere = ""; if (StringUtils.hasLength(filter.search())) { - rankQuery = "ts_rank(searchable_fields, (phraseto_tsquery(:search)::text || ':*')::tsquery)"; - rankWhere = "concept_node.searchable_fields @@ (phraseto_tsquery(:search)::text || ':*')::tsquery AND"; + rankQuery = QueryUtility.SEARCH_QUERY; + rankWhere = QueryUtility.SEARCH_WHERE + " AND "; } return """ ( @@ -147,6 +146,7 @@ private List createFacetFilter(Filter filter, MapSqlParameterSource para LEFT JOIN facet__concept_node ON facet__concept_node.facet_id = facet.facet_id JOIN facet_category ON facet_category.facet_category_id = facet.facet_category_id JOIN concept_node ON concept_node.concept_node_id = facet__concept_node.concept_node_id + LEFT JOIN concept_node_meta AS categorical_values ON concept_node.concept_node_id = categorical_values.concept_node_id AND categorical_values.KEY = 'values' LEFT JOIN dataset ON concept_node.dataset_id = dataset.dataset_id WHERE %s @@ -155,7 +155,8 @@ facet.name IN (:facets_for_category_%s ) AND facet_category.name = :category_%s GROUP BY facet__concept_node.concept_node_id ) - """.formatted(rankQuery, rankWhere, consentWhere, facetsForCategory.getKey(), facetsForCategory.getKey()); + """ + .formatted(rankQuery, rankWhere, consentWhere, facetsForCategory.getKey(), facetsForCategory.getKey()); }).toList(); } diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetQueryGenerator.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetQueryGenerator.java index c56fbd1..9e79259 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetQueryGenerator.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetQueryGenerator.java @@ -1,6 +1,7 @@ package edu.harvard.dbmi.avillach.dictionary.facet; import edu.harvard.dbmi.avillach.dictionary.filter.Filter; +import edu.harvard.dbmi.avillach.dictionary.util.QueryUtility; import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; @@ -102,7 +103,7 @@ private String createMultiCategorySQLWithSearch( LEFT JOIN concept_node_meta AS categorical_values ON concept_node.concept_node_id = categorical_values.concept_node_id AND categorical_values.KEY = 'values' WHERE (fc.name, facet.name) IN (:facets_in_cat_%s) - AND concept_node.searchable_fields @@ (phraseto_tsquery(:search)::text || ':*')::tsquery + AND %s AND ( continuous_min.value <> '' OR continuous_max.value <> '' OR @@ -110,7 +111,7 @@ private String createMultiCategorySQLWithSearch( ) ) """ - .formatted(categoryKeys.get(category), categoryKeys.get(category)); + .formatted(categoryKeys.get(category), categoryKeys.get(category), QueryUtility.SEARCH_WHERE); }).collect(Collectors.joining(",\n")); /* * Categories with no selected facets contribute no concepts, so ignore them for now. Now, for each category with selected facets, @@ -298,7 +299,7 @@ facet.facet_id, count(*) as facet_count WHERE %s fc.name = :facet_category_name - AND concept_node.searchable_fields @@ (phraseto_tsquery(:search)::text || ':*')::tsquery + AND %s AND ( continuous_min.value <> '' OR continuous_max.value <> '' OR @@ -325,7 +326,7 @@ WITH matching_concepts AS ( WHERE fc.name = :facet_category_name AND facet.name IN (:facets) - AND concept_node.searchable_fields @@ (phraseto_tsquery(:search)::text || ':*')::tsquery + AND %s AND ( continuous_min.value <> '' OR continuous_max.value <> '' OR @@ -350,7 +351,7 @@ facet.facet_id, count(*) as facet_count facet_count DESC ) """ - .formatted(consentWhere, consentWhere); + .formatted(consentWhere, QueryUtility.SEARCH_WHERE, QueryUtility.SEARCH_WHERE, consentWhere); } private String createSingleCategorySQLNoSearch(List facets, String consentWhere, MapSqlParameterSource params) { @@ -447,7 +448,7 @@ facet.facet_id, count(*) as facet_count LEFT JOIN concept_node_meta AS categorical_values ON concept_node.concept_node_id = categorical_values.concept_node_id AND categorical_values.KEY = 'values' WHERE %s - concept_node.searchable_fields @@ (phraseto_tsquery(:search)::text || ':*')::tsquery + %s AND ( continuous_min.value <> '' OR continuous_max.value <> '' OR @@ -458,7 +459,7 @@ facet.facet_id, count(*) as facet_count ORDER BY facet_count DESC """ - .formatted(consentWhere); + .formatted(consentWhere, QueryUtility.SEARCH_WHERE); } diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/util/QueryUtility.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/util/QueryUtility.java index 69eedc4..9fcf5ff 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/util/QueryUtility.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/util/QueryUtility.java @@ -16,4 +16,9 @@ AND concept_node_meta.KEY IN (:disallowed_meta_keys) concept_node.concept_node_id ) """; + + public static final String SEARCH_QUERY = + "CAST(LOWER(categorical_values.VALUE) LIKE '%' || LOWER(:search) || '%' as integer) * 10 + ts_rank(searchable_fields, (phraseto_tsquery(:search)::text || ':*')::tsquery)"; + public static final String SEARCH_WHERE = + "(LOWER(categorical_values.VALUE) LIKE '%' || LOWER(:search) || '%' OR concept_node.searchable_fields @@ (phraseto_tsquery(:search)::text || ':*')::tsquery)"; } diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetQueryGeneratorTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetQueryGeneratorTest.java index c9d8b59..12ff05c 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetQueryGeneratorTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetQueryGeneratorTest.java @@ -61,8 +61,8 @@ void shouldCountFacetsWithNoSearchAndNoSelectedFacetsAndNoConsents() { List actual = template.query(query, params, new IdCountPairMapper()); List expected = List.of( - new IdCountPair(22, 13), new IdCountPair(31, 3), new IdCountPair(27, 3), new IdCountPair(26, 3), new IdCountPair(28, 3), - new IdCountPair(23, 2), new IdCountPair(25, 2), new IdCountPair(21, 1), new IdCountPair(20, 1) + new IdCountPair(22, 13), new IdCountPair(26, 3), new IdCountPair(31, 3), new IdCountPair(27, 3), new IdCountPair(28, 3), + new IdCountPair(23, 2), new IdCountPair(21, 2), new IdCountPair(25, 2), new IdCountPair(20, 1) ); Assertions.assertEquals(expected, actual); @@ -225,4 +225,16 @@ void shouldCountFacetsNoSearchAndTwoSelectedFacetsInDifferentCatsAndConsents() { Assertions.assertEquals(expected, actual); } + + @Test + void shouldCountFacetsSearchMatchesValueNotSearchString() { + Filter filter = new Filter(List.of(), "gremlin", List.of()); + MapSqlParameterSource params = new MapSqlParameterSource(); + String query = subject.createFacetSQLAndPopulateParams(filter, params); + + List actual = template.query(query, params, new IdCountPairMapper()); + List expected = List.of(new IdCountPair(21, 1)); + + Assertions.assertEquals(expected, actual); + } } diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetRepositoryTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetRepositoryTest.java index 93a2b00..b9d998f 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetRepositoryTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetRepositoryTest.java @@ -71,8 +71,8 @@ void shouldGetAllFacets() { new FacetCategory( "nsrr_harmonized", "Common Data Element Collection", "", List.of( + new Facet("PhenX", "PhenX", null, null, 2, List.of(), "nsrr_harmonized", null), new Facet("LOINC", "LOINC", null, null, 1, List.of(), "nsrr_harmonized", null), - new Facet("PhenX", "PhenX", null, null, 1, List.of(), "nsrr_harmonized", null), new Facet( "gad_7", "Generalized Anxiety Disorder Assessment (GAD-7)", null, null, 0, List.of(), "nsrr_harmonized", null ), diff --git a/src/test/resources/seed.sql b/src/test/resources/seed.sql index c397a2e..296ad84 100644 --- a/src/test/resources/seed.sql +++ b/src/test/resources/seed.sql @@ -739,6 +739,7 @@ COPY public.facet__concept_node (facet__concept_node_id, facet_id, concept_node_ 91 18 261 92 20 229 93 21 229 +94 21 271 \.