Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[ALS-8191] Link facet and concept filtering a bit better #66

Merged
merged 1 commit into from
Jan 30, 2025
Merged

[ALS-8191] Link facet and concept filtering a bit better #66

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 10 additions & 9 deletions src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptFilterQueryGenerator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import edu.harvard.dbmi.avillach.dictionary.facet.Facet;
import edu.harvard.dbmi.avillach.dictionary.filter.Filter;
import edu.harvard.dbmi.avillach.dictionary.filter.QueryParamPair;
import edu.harvard.dbmi.avillach.dictionary.util.QueryUtility;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.domain.Pageable;
Expand Down Expand Up @@ -88,23 +89,21 @@ public QueryParamPair generateFilterQuery(Filter filter, Pageable pageable) {
}

private String createValuelessNodeFilter(String search, List<String> consents) {
String rankQuery = "0 as rank";
String rankQuery = "0";
String rankWhere = "";
if (StringUtils.hasLength(search)) {
// we rank search results via two factors:
// 1. (more important) does the raw search term appear in the concept's values?
// 2. (less important) does psql think the tokenized search term matches something in the searchable_fields
rankQuery = "(CAST(LOWER(categorical_values.VALUE) LIKE '%' || LOWER(:search) || '%' as integer) * 10) + "
+ "ts_rank(searchable_fields, (phraseto_tsquery(:search)::text || ':*')::tsquery) as rank";
rankWhere = "(LOWER(categorical_values.VALUE) LIKE '%' || LOWER(:search) || '%' OR "
+ "concept_node.searchable_fields @@ (phraseto_tsquery(:search)::text || ':*')::tsquery) AND";
rankQuery = QueryUtility.SEARCH_QUERY;
rankWhere = QueryUtility.SEARCH_WHERE + " AND ";
}
String consentWhere = CollectionUtils.isEmpty(consents) ? "" : CONSENT_QUERY;
// concept nodes that have no values and no min/max should not get returned by search
return """
SELECT
concept_node.concept_node_id,
%s
(%s) as rank
FROM
concept_node
LEFT JOIN dataset ON concept_node.dataset_id = dataset.dataset_id
Expand Down Expand Up @@ -135,8 +134,8 @@ private List<String> createFacetFilter(Filter filter, MapSqlParameterSource para
String rankQuery = "0";
String rankWhere = "";
if (StringUtils.hasLength(filter.search())) {
rankQuery = "ts_rank(searchable_fields, (phraseto_tsquery(:search)::text || ':*')::tsquery)";
rankWhere = "concept_node.searchable_fields @@ (phraseto_tsquery(:search)::text || ':*')::tsquery AND";
rankQuery = QueryUtility.SEARCH_QUERY;
rankWhere = QueryUtility.SEARCH_WHERE + " AND ";
}
return """
(
Expand All @@ -147,6 +146,7 @@ private List<String> createFacetFilter(Filter filter, MapSqlParameterSource para
LEFT JOIN facet__concept_node ON facet__concept_node.facet_id = facet.facet_id
JOIN facet_category ON facet_category.facet_category_id = facet.facet_category_id
JOIN concept_node ON concept_node.concept_node_id = facet__concept_node.concept_node_id
LEFT JOIN concept_node_meta AS categorical_values ON concept_node.concept_node_id = categorical_values.concept_node_id AND categorical_values.KEY = 'values'
LEFT JOIN dataset ON concept_node.dataset_id = dataset.dataset_id
WHERE
%s
Expand All @@ -155,7 +155,8 @@ facet.name IN (:facets_for_category_%s ) AND facet_category.name = :category_%s
GROUP BY
facet__concept_node.concept_node_id
)
""".formatted(rankQuery, rankWhere, consentWhere, facetsForCategory.getKey(), facetsForCategory.getKey());
"""
.formatted(rankQuery, rankWhere, consentWhere, facetsForCategory.getKey(), facetsForCategory.getKey());
}).toList();
}

Expand Down
15 changes: 8 additions & 7 deletions src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetQueryGenerator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package edu.harvard.dbmi.avillach.dictionary.facet;

import edu.harvard.dbmi.avillach.dictionary.filter.Filter;
import edu.harvard.dbmi.avillach.dictionary.util.QueryUtility;
import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
Expand Down Expand Up @@ -102,15 +103,15 @@ private String createMultiCategorySQLWithSearch(
LEFT JOIN concept_node_meta AS categorical_values ON concept_node.concept_node_id = categorical_values.concept_node_id AND categorical_values.KEY = 'values'
WHERE
(fc.name, facet.name) IN (:facets_in_cat_%s)
AND concept_node.searchable_fields @@ (phraseto_tsquery(:search)::text || ':*')::tsquery
AND %s
AND (
continuous_min.value <> '' OR
continuous_max.value <> '' OR
categorical_values.value <> ''
)
)
"""
.formatted(categoryKeys.get(category), categoryKeys.get(category));
.formatted(categoryKeys.get(category), categoryKeys.get(category), QueryUtility.SEARCH_WHERE);
}).collect(Collectors.joining(",\n"));
/*
* Categories with no selected facets contribute no concepts, so ignore them for now. Now, for each category with selected facets,
Expand Down Expand Up @@ -298,7 +299,7 @@ facet.facet_id, count(*) as facet_count
WHERE
%s
fc.name = :facet_category_name
AND concept_node.searchable_fields @@ (phraseto_tsquery(:search)::text || ':*')::tsquery
AND %s
AND (
continuous_min.value <> '' OR
continuous_max.value <> '' OR
Expand All @@ -325,7 +326,7 @@ WITH matching_concepts AS (
WHERE
fc.name = :facet_category_name
AND facet.name IN (:facets)
AND concept_node.searchable_fields @@ (phraseto_tsquery(:search)::text || ':*')::tsquery
AND %s
AND (
continuous_min.value <> '' OR
continuous_max.value <> '' OR
Expand All @@ -350,7 +351,7 @@ facet.facet_id, count(*) as facet_count
facet_count DESC
)
"""
.formatted(consentWhere, consentWhere);
.formatted(consentWhere, QueryUtility.SEARCH_WHERE, QueryUtility.SEARCH_WHERE, consentWhere);
}

private String createSingleCategorySQLNoSearch(List<Facet> facets, String consentWhere, MapSqlParameterSource params) {
Expand Down Expand Up @@ -447,7 +448,7 @@ facet.facet_id, count(*) as facet_count
LEFT JOIN concept_node_meta AS categorical_values ON concept_node.concept_node_id = categorical_values.concept_node_id AND categorical_values.KEY = 'values'
WHERE
%s
concept_node.searchable_fields @@ (phraseto_tsquery(:search)::text || ':*')::tsquery
%s
AND (
continuous_min.value <> '' OR
continuous_max.value <> '' OR
Expand All @@ -458,7 +459,7 @@ facet.facet_id, count(*) as facet_count
ORDER BY
facet_count DESC
"""
.formatted(consentWhere);
.formatted(consentWhere, QueryUtility.SEARCH_WHERE);

}

Expand Down
5 changes: 5 additions & 0 deletions src/main/java/edu/harvard/dbmi/avillach/dictionary/util/QueryUtility.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,4 +16,9 @@ AND concept_node_meta.KEY IN (:disallowed_meta_keys)
concept_node.concept_node_id
)
""";

public static final String SEARCH_QUERY =
"CAST(LOWER(categorical_values.VALUE) LIKE '%' || LOWER(:search) || '%' as integer) * 10 + ts_rank(searchable_fields, (phraseto_tsquery(:search)::text || ':*')::tsquery)";
public static final String SEARCH_WHERE =
"(LOWER(categorical_values.VALUE) LIKE '%' || LOWER(:search) || '%' OR concept_node.searchable_fields @@ (phraseto_tsquery(:search)::text || ':*')::tsquery)";
}
16 changes: 14 additions & 2 deletions src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetQueryGeneratorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,8 @@ void shouldCountFacetsWithNoSearchAndNoSelectedFacetsAndNoConsents() {

List<IdCountPair> actual = template.query(query, params, new IdCountPairMapper());
List<IdCountPair> expected = List.of(
new IdCountPair(22, 13), new IdCountPair(31, 3), new IdCountPair(27, 3), new IdCountPair(26, 3), new IdCountPair(28, 3),
new IdCountPair(23, 2), new IdCountPair(25, 2), new IdCountPair(21, 1), new IdCountPair(20, 1)
new IdCountPair(22, 13), new IdCountPair(26, 3), new IdCountPair(31, 3), new IdCountPair(27, 3), new IdCountPair(28, 3),
new IdCountPair(23, 2), new IdCountPair(21, 2), new IdCountPair(25, 2), new IdCountPair(20, 1)
);

Assertions.assertEquals(expected, actual);
Expand Down Expand Up @@ -225,4 +225,16 @@ void shouldCountFacetsNoSearchAndTwoSelectedFacetsInDifferentCatsAndConsents() {

Assertions.assertEquals(expected, actual);
}

@Test
void shouldCountFacetsSearchMatchesValueNotSearchString() {
Filter filter = new Filter(List.of(), "gremlin", List.of());
MapSqlParameterSource params = new MapSqlParameterSource();
String query = subject.createFacetSQLAndPopulateParams(filter, params);

List<IdCountPair> actual = template.query(query, params, new IdCountPairMapper());
List<IdCountPair> expected = List.of(new IdCountPair(21, 1));

Assertions.assertEquals(expected, actual);
}
}
2 changes: 1 addition & 1 deletion src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetRepositoryTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,8 @@ void shouldGetAllFacets() {
new FacetCategory(
"nsrr_harmonized", "Common Data Element Collection", "",
List.of(
new Facet("PhenX", "PhenX", null, null, 2, List.of(), "nsrr_harmonized", null),
new Facet("LOINC", "LOINC", null, null, 1, List.of(), "nsrr_harmonized", null),
new Facet("PhenX", "PhenX", null, null, 1, List.of(), "nsrr_harmonized", null),
new Facet(
"gad_7", "Generalized Anxiety Disorder Assessment (GAD-7)", null, null, 0, List.of(), "nsrr_harmonized", null
),
Expand Down
1 change: 1 addition & 0 deletions src/test/resources/seed.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -739,6 +739,7 @@ COPY public.facet__concept_node (facet__concept_node_id, facet_id, concept_node_
91 18 261
92 20 229
93 21 229
94 21 271
\.


Expand Down