From fa754cb90b3fb149c2bfbcf6a413fa090e519a58 Mon Sep 17 00:00:00 2001 From: Guillaume Poirier-Morency Date: Fri, 3 May 2024 10:02:18 -0700 Subject: [PATCH 01/61] Update versions for hotfix --- gemma-cli/pom.xml | 2 +- gemma-core/pom.xml | 2 +- gemma-groovy-support/pom.xml | 2 +- gemma-rest/pom.xml | 2 +- gemma-web/pom.xml | 2 +- pom.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/gemma-cli/pom.xml b/gemma-cli/pom.xml index d3f9f11206..83f5f23341 100644 --- a/gemma-cli/pom.xml +++ b/gemma-cli/pom.xml @@ -3,7 +3,7 @@ gemma gemma - 1.31.5 + 1.31.6-SNAPSHOT 4.0.0 gemma-cli diff --git a/gemma-core/pom.xml b/gemma-core/pom.xml index 7772bc83b5..6323dc48f0 100644 --- a/gemma-core/pom.xml +++ b/gemma-core/pom.xml @@ -3,7 +3,7 @@ gemma gemma - 1.31.5 + 1.31.6-SNAPSHOT 4.0.0 gemma-core diff --git a/gemma-groovy-support/pom.xml b/gemma-groovy-support/pom.xml index 693adebfca..34ffe5d10a 100644 --- a/gemma-groovy-support/pom.xml +++ b/gemma-groovy-support/pom.xml @@ -6,7 +6,7 @@ gemma gemma - 1.31.5 + 1.31.6-SNAPSHOT gemma-groovy-support diff --git a/gemma-rest/pom.xml b/gemma-rest/pom.xml index 775af3c9b8..154c417244 100644 --- a/gemma-rest/pom.xml +++ b/gemma-rest/pom.xml @@ -5,7 +5,7 @@ gemma gemma - 1.31.5 + 1.31.6-SNAPSHOT 4.0.0 diff --git a/gemma-web/pom.xml b/gemma-web/pom.xml index fb580fe55f..b1c181e211 100644 --- a/gemma-web/pom.xml +++ b/gemma-web/pom.xml @@ -3,7 +3,7 @@ gemma gemma - 1.31.5 + 1.31.6-SNAPSHOT 4.0.0 gemma-web diff --git a/pom.xml b/pom.xml index 0ee6c97550..9d7baa68c6 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ Gemma gemma gemma - 1.31.5 + 1.31.6-SNAPSHOT 2005 The Gemma Project for meta-analysis of genomics data https://gemma.msl.ubc.ca From 55757e848911f022928ed509fbd15e68f8ccb4af Mon Sep 17 00:00:00 2001 From: Guillaume Poirier-Morency Date: Fri, 3 May 2024 10:02:35 -0700 Subject: [PATCH 02/61] When a full-text ontology query fails, reattempt it escaped --- .../gemma/core/search/lucene/LuceneQueryUtils.java | 9 ++++++++- .../core/search/source/OntologySearchSource.java | 12 +++++++++++- .../core/search/source/OntologySearchSourceTest.java | 12 ++++++++++++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/gemma-core/src/main/java/ubic/gemma/core/search/lucene/LuceneQueryUtils.java b/gemma-core/src/main/java/ubic/gemma/core/search/lucene/LuceneQueryUtils.java index 139e3afea7..d1b5035198 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/search/lucene/LuceneQueryUtils.java +++ b/gemma-core/src/main/java/ubic/gemma/core/search/lucene/LuceneQueryUtils.java @@ -49,7 +49,7 @@ public static Query parseSafely( String query, QueryParser queryParser ) throws try { return queryParser.parse( query ); } catch ( ParseException e ) { - String strippedQuery = LUCENE_RESERVED_CHARS.matcher( query ).replaceAll( "\\\\$0" ); + String strippedQuery = escape( query ); log.debug( String.format( "Failed to parse '%s': %s.", query, ExceptionUtils.getRootCauseMessage( e ) ), e ); try { return queryParser.parse( strippedQuery ); @@ -63,6 +63,13 @@ public static Query parseSafely( String query, QueryParser queryParser ) throws } } + /** + * Escape all reserved Lucene characters in the given query. + */ + public static String escape( String query ) { + return LUCENE_RESERVED_CHARS.matcher( query ).replaceAll( "\\\\$0" ); + } + /** * Extract terms, regardless of their logical organization. *

diff --git a/gemma-core/src/main/java/ubic/gemma/core/search/source/OntologySearchSource.java b/gemma-core/src/main/java/ubic/gemma/core/search/source/OntologySearchSource.java index d07a85abc7..aa2fb98d65 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/search/source/OntologySearchSource.java +++ b/gemma-core/src/main/java/ubic/gemma/core/search/source/OntologySearchSource.java @@ -3,6 +3,7 @@ import lombok.EqualsAndHashCode; import lombok.Value; import lombok.extern.apachecommons.CommonsLog; +import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.commons.lang3.time.StopWatch; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -11,6 +12,7 @@ import ubic.basecode.ontology.search.OntologySearchResult; import ubic.gemma.core.ontology.OntologyService; import ubic.gemma.core.search.*; +import ubic.gemma.core.search.lucene.LuceneParseSearchException; import ubic.gemma.core.search.lucene.LuceneQueryUtils; import ubic.gemma.model.common.Identifiable; import ubic.gemma.model.common.description.Characteristic; @@ -194,7 +196,15 @@ private SearchResultSet doSearchExpressionExperiment( Sear ontologyResults.add( resource ); } else { // Search ontology classes matches to the full-text query - matchingTerms = ontologyService.findTerms( settings.getQuery(), 5000, Math.max( timeoutMs - watch.getTime(), 0L ), TimeUnit.MILLISECONDS ); + try { + matchingTerms = ontologyService.findTerms( settings.getQuery(), 5000, + Math.max( timeoutMs - watch.getTime(), 0L ), TimeUnit.MILLISECONDS ); + } catch ( LuceneParseSearchException e ) { + log.debug( String.format( "Failed to parse '%s': %s.", settings.getQuery(), ExceptionUtils.getRootCauseMessage( e ) ), e ); + // reattempt it without escaped + matchingTerms = ontologyService.findTerms( LuceneQueryUtils.escape( settings.getQuery() ), 5000, + Math.max( timeoutMs - watch.getTime(), 0L ), TimeUnit.MILLISECONDS ); + } matchingTerms.stream() // ignore bnodes .filter( t -> t.getResult().getUri() != null ) diff --git a/gemma-core/src/test/java/ubic/gemma/core/search/source/OntologySearchSourceTest.java b/gemma-core/src/test/java/ubic/gemma/core/search/source/OntologySearchSourceTest.java index ceb748e6f1..a25127e79c 100644 --- a/gemma-core/src/test/java/ubic/gemma/core/search/source/OntologySearchSourceTest.java +++ b/gemma-core/src/test/java/ubic/gemma/core/search/source/OntologySearchSourceTest.java @@ -15,6 +15,7 @@ import ubic.gemma.core.search.SearchException; import ubic.gemma.core.search.SearchResult; import ubic.gemma.core.search.SearchSource; +import ubic.gemma.core.search.lucene.LuceneParseSearchException; import ubic.gemma.model.common.search.SearchSettings; import ubic.gemma.model.expression.experiment.ExpressionExperiment; import ubic.gemma.persistence.service.common.description.CharacteristicService; @@ -149,6 +150,17 @@ public void testSearchExpressionExperimentWithBooleanQuery() throws SearchExcept verifyNoMoreInteractions( ontologyService ); } + @Test + public void setInvalidSearchSyntax() throws SearchException { + when( ontologyService.findTerms( eq( "1-[(2S)-butan-2-yl]-N-[(4,6-dimethyl-2-oxo-1H-pyridin-3-yl)methyl]-3-methyl-6-[6-(1-piperazinyl)-3-pyridinyl]-4-indolecarboxamide" ), anyInt(), anyLong(), any() ) ) + .thenThrow( LuceneParseSearchException.class ); + ontologySearchSource.searchExpressionExperiment( SearchSettings.expressionExperimentSearch( "1-[(2S)-butan-2-yl]-N-[(4,6-dimethyl-2-oxo-1H-pyridin-3-yl)methyl]-3-methyl-6-[6-(1-piperazinyl)-3-pyridinyl]-4-indolecarboxamide" ) ); + verify( ontologyService ).findTerms( eq( "1-[(2S)-butan-2-yl]-N-[(4,6-dimethyl-2-oxo-1H-pyridin-3-yl)methyl]-3-methyl-6-[6-(1-piperazinyl)-3-pyridinyl]-4-indolecarboxamide" ), eq( 5000 ), longThat( l -> l <= 30000L ), eq( TimeUnit.MILLISECONDS ) ); + // fallback to an escaped query + verify( ontologyService ).findTerms( eq( "1\\-\\[\\(2S\\)\\-butan\\-2\\-yl\\]\\-N\\-\\[\\(4,6\\-dimethyl\\-2\\-oxo\\-1H\\-pyridin\\-3\\-yl\\)methyl\\]\\-3\\-methyl\\-6\\-\\[6\\-\\(1\\-piperazinyl\\)\\-3\\-pyridinyl\\]\\-4\\-indolecarboxamide" ), eq( 5000 ), longThat( l -> l <= 30000L ), eq( TimeUnit.MILLISECONDS ) ); + verifyNoMoreInteractions( ontologyService ); + } + @Test public void testGetLabelFromTermUri() { assertEquals( "GO:0004016", getLabelFromTermUri( URI.create( "http://purl.obolibrary.org/obo/GO_0004016" ) ) ); From 3b8312ac8f72522c5ee83ffa8a91deb88a538cda Mon Sep 17 00:00:00 2001 From: Paul Pavlidis Date: Thu, 2 May 2024 16:39:52 -0700 Subject: [PATCH 03/61] fix a nasty bug that caused too many RNA-seq experiments to not be batchable My fault. We should now detect valid batching structure more often rather than concluding there are singleton batches. --- .../BatchInfoPopulationHelperServiceImpl.java | 64 +++++++++---------- .../RNASeqBatchInfoPopulationTest.java | 13 +++- .../GSE83115.fastq-headers-table.txt | 56 ++++++++++++++++ .../webapp/scripts/api/userHelpMessages.js | 2 +- 4 files changed, 101 insertions(+), 34 deletions(-) create mode 100644 gemma-core/src/test/resources/data/analysis/preprocess/batcheffects/fastqheaders/GSE83115.fastq-headers-table.txt diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchInfoPopulationHelperServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchInfoPopulationHelperServiceImpl.java index a2ed01beb9..b4b39637fa 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchInfoPopulationHelperServiceImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchInfoPopulationHelperServiceImpl.java @@ -45,7 +45,7 @@ public class BatchInfoPopulationHelperServiceImpl implements BatchInfoPopulation /** * For RNA-seq, the minimum number of samples per batch, if possible */ - private static final double MINIMUM_SAMPLES_PER_RNASEQ_BATCH = 2.0; + private static final int MINIMUM_SAMPLES_PER_RNASEQ_BATCH = 2; /** * For microarrays (that come with scan timestamps): How many hours do we allow to pass between samples, before we @@ -372,13 +372,13 @@ private Map> convertHeadersToBatches( ExpressionExper } // DEBUG CODE - // log.info( "--------------------------" ); - // for ( String b : result.keySet() ) { - // log.info( "Batch: " + b ); - // for ( String batchmember : result.get( b ) ) { - // log.info( " " + batchmember ); - // } - // } +// log.info( "--------------------------" ); +// for ( String b : result.keySet() ) { +// log.info( "Batch: " + b ); +// for ( String batchmember : result.get( b ) ) { +// log.info( " " + batchmember ); +// } +// } /* * Finalize @@ -436,6 +436,14 @@ private Map> batch( Map> batch( Map> updatedBatchInfos = dropResolution( batchInfos ); + } else if ( anyTooSmallBatches ) { - if ( updatedBatchInfos.size() == batchInfos.size() ) { - // we've reached the bottom - return updatedBatchInfos; - } + // too few samples for at least one batch. Try to reduce resolution and recount. + log.info( "Too few samples for at least one batch. Reducing resolution." ); + Map> updatedBatchInfos = dropResolution( batchInfos ); - batch( updatedBatchInfos, numSamples ); // recurse - } + if ( updatedBatchInfos.size() == batchInfos.size() ) { + // we've reached the bottom + return updatedBatchInfos; } + return batch( updatedBatchInfos, numSamples ); // start over with lower resolution } // reasonable number of samples per batch -- proceed. return batchInfos; - } /* @@ -479,23 +478,21 @@ private Map> dropResolution( Map> result = new HashMap<>(); for ( FastqHeaderData fhd : batchInfos.keySet() ) { - // if ( !fhd.hadUseableHeader() ) { - // // cannot drop resolution. - // result.put( fhd, batchInfos.get( fhd ) ); - // continue; - // } - FastqHeaderData updated = fhd.dropResolution(); if ( updated.equals( fhd ) ) { // we can reduce resolution no more return batchInfos; } - log.info( "Adding: " + updated ); if ( !result.containsKey( updated ) ) { + log.info( "Adding: " + updated ); result.put( updated, new HashSet() ); } + + // reassociate the samples with the new batch info result.get( updated ).addAll( batchInfos.get( fhd ) ); + // make sure the old one is gone. + result.remove( fhd ); } return result; @@ -671,10 +668,13 @@ public String getUnusableHeader() { private FastqHeaderData dropResolution() { // note that 'device' is the GPL if the header wasn't usable if ( this.lane != null ) { + log.debug( "Dropping lane" ); return new FastqHeaderData( this.device, this.run, this.flowCell, null ); } else if ( this.flowCell != null ) { + log.debug( "Dropping flowCell" ); return new FastqHeaderData( this.device, this.run, null, null ); } else if ( this.run != null ) { + log.debug( "Dropping device / GPL" ); return new FastqHeaderData( this.device, null, null, null ); } else if ( this.unusableHeader != null ) { // fallback diff --git a/gemma-core/src/test/java/ubic/gemma/core/analysis/preprocess/batcheffects/RNASeqBatchInfoPopulationTest.java b/gemma-core/src/test/java/ubic/gemma/core/analysis/preprocess/batcheffects/RNASeqBatchInfoPopulationTest.java index f0382cdf36..14786491a3 100644 --- a/gemma-core/src/test/java/ubic/gemma/core/analysis/preprocess/batcheffects/RNASeqBatchInfoPopulationTest.java +++ b/gemma-core/src/test/java/ubic/gemma/core/analysis/preprocess/batcheffects/RNASeqBatchInfoPopulationTest.java @@ -409,7 +409,7 @@ public void testBatchP() throws Exception { s.convertHeadersToBatches( h.values() ); } - @Test(expected = SingletonBatchesException.class) + @Test public void testBatchQ() throws Exception { //GSE173137 - BatchInfoPopulationHelperServiceImpl s = new BatchInfoPopulationHelperServiceImpl(); @@ -418,6 +418,17 @@ public void testBatchQ() throws Exception { s.convertHeadersToBatches( h.values() ); } + @Test + public void testBatchR() throws Exception { + // GSE83115 - singleton batches if we use lane, dropping to device gives two batches. + BatchInfoPopulationHelperServiceImpl s = new BatchInfoPopulationHelperServiceImpl(); + BatchInfoPopulationServiceImpl bs = new BatchInfoPopulationServiceImpl(); + Map h = bs.readFastqHeaders( "GSE83115" ); + + Map> batches = s.convertHeadersToBatches( h.values() ); + assertEquals( 2, batches.size() ); + } + @After public void teardown() { if ( ee != null ) diff --git a/gemma-core/src/test/resources/data/analysis/preprocess/batcheffects/fastqheaders/GSE83115.fastq-headers-table.txt b/gemma-core/src/test/resources/data/analysis/preprocess/batcheffects/fastqheaders/GSE83115.fastq-headers-table.txt new file mode 100644 index 0000000000..206c671ee1 --- /dev/null +++ b/gemma-core/src/test/resources/data/analysis/preprocess/batcheffects/fastqheaders/GSE83115.fastq-headers-table.txt @@ -0,0 +1,56 @@ +GSM2193201 SRR3647485_1 GPL15456 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830403 @SRR3647485.1 V0112_0150:1:1101:1206:1939 length=101 +GSM2193201 SRR3647485_2 GPL15456 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830403 @SRR3647485.1 V0112_0150:1:1101:1206:1939 length=101 +GSM2193213 SRR3647497_1 GPL15456 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830415 @SRR3647497.1 V0112_0150:7:1101:1159:1974 length=101 +GSM2193213 SRR3647497_2 GPL15456 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830415 @SRR3647497.1 V0112_0150:7:1101:1159:1974 length=101 +GSM2193195 SRR3647479_1 GPL11154 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830397 @SRR3647479.1 FCD03U8ACXX:5:1101:1075:2242 length=90 +GSM2193195 SRR3647479_2 GPL11154 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830397 @SRR3647479.1 FCD03U8ACXX:5:1101:1075:2242 length=90 +GSM2193198 SRR3647482_1 GPL11154 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830400 @SRR3647482.1 FCD03U8ACXX:4:1101:1284:2233 length=90 +GSM2193198 SRR3647482_2 GPL11154 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830400 @SRR3647482.1 FCD03U8ACXX:4:1101:1284:2233 length=90 +GSM2193196 SRR3647480_1 GPL11154 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830398 @SRR3647480.1 FCD03U8ACXX:5:1101:1215:2220 length=90 +GSM2193196 SRR3647480_2 GPL11154 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830398 @SRR3647480.1 FCD03U8ACXX:5:1101:1215:2220 length=90 +GSM2193194 SRR3647478_1 GPL11154 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830396 @SRR3647478.1 FCD03U8ACXX:5:1101:1113:2220 length=90 +GSM2193194 SRR3647478_2 GPL11154 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830396 @SRR3647478.1 FCD03U8ACXX:5:1101:1113:2220 length=90 +GSM2193210 SRR3647494_1 GPL15456 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830412 @SRR3647494.1 V0112_0150:6:1101:1035:1954 length=101 +GSM2193210 SRR3647494_2 GPL15456 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830412 @SRR3647494.1 V0112_0150:6:1101:1035:1954 length=101 +GSM2193209 SRR3647493_1 GPL15456 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830411 @SRR3647493.1 V0112_0150:7:1101:1203:1968 length=101 +GSM2193209 SRR3647493_2 GPL15456 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830411 @SRR3647493.1 V0112_0150:7:1101:1203:1968 length=101 +GSM2193206 SRR3647490_1 GPL15456 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830408 @SRR3647490.1 V0112_0150:4:1101:1061:1980 length=101 +GSM2193206 SRR3647490_2 GPL15456 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830408 @SRR3647490.1 V0112_0150:4:1101:1061:1980 length=101 +GSM2193199 SRR3647483_1 GPL11154 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830401 @SRR3647483.1 FCD03U8ACXX:6:1101:1099:2235 length=90 +GSM2193199 SRR3647483_2 GPL11154 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830401 @SRR3647483.1 FCD03U8ACXX:6:1101:1099:2235 length=90 +GSM2193207 SRR3647491_1 GPL15456 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830409 @SRR3647491.1 V0112_0150:3:1101:1246:1953 length=101 +GSM2193207 SRR3647491_2 GPL15456 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830409 @SRR3647491.1 V0112_0150:3:1101:1246:1953 length=101 +GSM2193197 SRR3647481_1 GPL11154 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830399 @SRR3647481.1 FCD03U8ACXX:4:1101:1192:2226 length=90 +GSM2193197 SRR3647481_2 GPL11154 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830399 @SRR3647481.1 FCD03U8ACXX:4:1101:1192:2226 length=90 +GSM2193211 SRR3647495_1 GPL15456 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830413 @SRR3647495.1 V0112_0150:6:1101:1180:1973 length=101 +GSM2193211 SRR3647495_2 GPL15456 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830413 @SRR3647495.1 V0112_0150:6:1101:1180:1973 length=101 +GSM2193192 SRR3647476_1 GPL11154 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830394 @SRR3647476.1 FCD03U8ACXX:3:1101:1456:2237 length=90 +GSM2193192 SRR3647476_2 GPL11154 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830394 @SRR3647476.1 FCD03U8ACXX:3:1101:1456:2237 length=90 +GSM2193200 SRR3647484_1 GPL15456 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830402 @SRR3647484.1 V0112_0150:1:1101:1181:1938 length=101 +GSM2193200 SRR3647484_2 GPL15456 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830402 @SRR3647484.1 V0112_0150:1:1101:1181:1938 length=101 +GSM2193186 SRR3647470_1 GPL11154 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830388 @SRR3647470.1 FCD03U8ACXX:1:1101:1076:2217 length=90 +GSM2193186 SRR3647470_2 GPL11154 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830388 @SRR3647470.1 FCD03U8ACXX:1:1101:1076:2217 length=90 +GSM2193204 SRR3647488_1 GPL15456 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830406 @SRR3647488.1 V0112_0150:4:1101:1088:1974 length=101 +GSM2193204 SRR3647488_2 GPL15456 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830406 @SRR3647488.1 V0112_0150:4:1101:1088:1974 length=101 +GSM2193205 SRR3647489_1 GPL15456 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830407 @SRR3647489.1 V0112_0150:4:1101:1028:1968 length=101 +GSM2193205 SRR3647489_2 GPL15456 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830407 @SRR3647489.1 V0112_0150:4:1101:1028:1968 length=101 +GSM2193212 SRR3647496_1 GPL15456 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830414 @SRR3647496.1 V0112_0150:6:1101:1135:1968 length=101 +GSM2193212 SRR3647496_2 GPL15456 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830414 @SRR3647496.1 V0112_0150:6:1101:1135:1968 length=101 +GSM2193208 SRR3647492_1 GPL15456 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830410 @SRR3647492.1 V0112_0150:7:1101:1120:1995 length=101 +GSM2193208 SRR3647492_2 GPL15456 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830410 @SRR3647492.1 V0112_0150:7:1101:1120:1995 length=101 +GSM2193188 SRR3647472_1 GPL11154 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830390 @SRR3647472.1 FCD03U8ACXX:1:1101:1236:2220 length=90 +GSM2193188 SRR3647472_2 GPL11154 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830390 @SRR3647472.1 FCD03U8ACXX:1:1101:1236:2220 length=90 +GSM2193189 SRR3647473_1 GPL11154 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830391 @SRR3647473.1 FCD03U8ACXX:2:1101:1226:2245 length=90 +GSM2193189 SRR3647473_2 GPL11154 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830391 @SRR3647473.1 FCD03U8ACXX:2:1101:1226:2245 length=90 +GSM2193203 SRR3647487_1 GPL15456 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830405 @SRR3647487.1 V0112_0150:3:1101:1151:1953 length=101 +GSM2193203 SRR3647487_2 GPL15456 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830405 @SRR3647487.1 V0112_0150:3:1101:1151:1953 length=101 +GSM2193187 SRR3647471_1 GPL11154 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830389 @SRR3647471.1 FCD03U8ACXX:1:1101:1097:2225 length=90 +GSM2193187 SRR3647471_2 GPL11154 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830389 @SRR3647471.1 FCD03U8ACXX:1:1101:1097:2225 length=90 +GSM2193202 SRR3647486_1 GPL15456 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830404 @SRR3647486.1 V0112_0150:2:1101:1045:1954 length=101 +GSM2193202 SRR3647486_2 GPL15456 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830404 @SRR3647486.1 V0112_0150:2:1101:1045:1954 length=101 +GSM2193190 SRR3647474_1 GPL11154 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830392 @SRR3647474.1 FCD03U8ACXX:3:1101:1267:2240 length=90 +GSM2193190 SRR3647474_2 GPL11154 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830392 @SRR3647474.1 FCD03U8ACXX:3:1101:1267:2240 length=90 +GSM2193193 SRR3647477_1 GPL11154 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830395 @SRR3647477.1 FCD03U8ACXX:2:1101:1993:2248 length=90 +GSM2193193 SRR3647477_2 GPL11154 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830395 @SRR3647477.1 FCD03U8ACXX:2:1101:1993:2248 length=90 +GSM2193191 SRR3647475_1 GPL11154 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830393 @SRR3647475.1 FCD03U8ACXX:3:1101:1722:2238 length=90 +GSM2193191 SRR3647475_2 GPL11154 https://www.ncbi.nlm.nih.gov/sra?term=SRX1830393 @SRR3647475.1 FCD03U8ACXX:3:1101:1722:2238 length=90 diff --git a/gemma-web/src/main/webapp/scripts/api/userHelpMessages.js b/gemma-web/src/main/webapp/scripts/api/userHelpMessages.js index 491d0ec004..32eec7a558 100755 --- a/gemma-web/src/main/webapp/scripts/api/userHelpMessages.js +++ b/gemma-web/src/main/webapp/scripts/api/userHelpMessages.js @@ -445,7 +445,7 @@ Gemma.HelpText.WidgetDefaults = { dataReprocessed: "Reprocessed from raw data by Gemma.", dataExternal: "Data are from external source.", noBatchInfo: "Information on sample batching was not available", - noBatchesSingletons: "Gemma was unable to form a reasonable sample batching structure", + noBatchesSingletons: "Gemma was unable to form a reasonable batching structure due to batch(es) with one sample", noBatchesBadHeaders: "Information on sample batching was not extractable from the available data", statusUnsuitableForDEA: "Data or experimental design not suitable for differential expression analysis." }, From 0183642ad2a45feb9270b3e9461d6275ed9e9b82 Mon Sep 17 00:00:00 2001 From: Paul Pavlidis Date: Thu, 2 May 2024 16:47:35 -0700 Subject: [PATCH 04/61] quiet --- .../batcheffects/BatchInfoPopulationHelperServiceImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchInfoPopulationHelperServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchInfoPopulationHelperServiceImpl.java index b4b39637fa..44104053c2 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchInfoPopulationHelperServiceImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchInfoPopulationHelperServiceImpl.java @@ -485,7 +485,7 @@ private Map> dropResolution( Map() ); } From 531642cf9ce6e5ec45913f0fff4537076c975f8c Mon Sep 17 00:00:00 2001 From: Paul Pavlidis Date: Fri, 3 May 2024 12:58:47 -0700 Subject: [PATCH 05/61] attempt to make tests a bit more consistent --- .../core/util/test/BaseSpringContextTest.java | 3 +++ .../util/test/PersistentDummyObjectHelper.java | 15 ++++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/gemma-core/src/test/java/ubic/gemma/core/util/test/BaseSpringContextTest.java b/gemma-core/src/test/java/ubic/gemma/core/util/test/BaseSpringContextTest.java index 6e1c577e32..395b0ae86d 100644 --- a/gemma-core/src/test/java/ubic/gemma/core/util/test/BaseSpringContextTest.java +++ b/gemma-core/src/test/java/ubic/gemma/core/util/test/BaseSpringContextTest.java @@ -395,10 +395,12 @@ protected ExpressionExperiment getTestPersistentCompleteExpressionExperiment( bo * @return EE */ protected ExpressionExperiment getTestPersistentCompleteExpressionExperimentWithSequences() { + testHelper.resetSeed(); return testHelper.getTestExpressionExperimentWithAllDependencies( true ); } protected ExpressionExperiment getNewTestPersistentCompleteExpressionExperiment() { + testHelper.resetSeed(); return testHelper.getTestExpressionExperimentWithAllDependencies( false ); } @@ -408,6 +410,7 @@ protected ExpressionExperiment getNewTestPersistentCompleteExpressionExperiment( */ protected ExpressionExperiment getTestPersistentCompleteExpressionExperimentWithSequences( ExpressionExperiment prototype ) { + testHelper.resetSeed(); return testHelper.getTestExpressionExperimentWithAllDependencies( prototype ); } diff --git a/gemma-core/src/test/java/ubic/gemma/core/util/test/PersistentDummyObjectHelper.java b/gemma-core/src/test/java/ubic/gemma/core/util/test/PersistentDummyObjectHelper.java index 9782c616a1..a7fe9a107f 100644 --- a/gemma-core/src/test/java/ubic/gemma/core/util/test/PersistentDummyObjectHelper.java +++ b/gemma-core/src/test/java/ubic/gemma/core/util/test/PersistentDummyObjectHelper.java @@ -106,6 +106,15 @@ public class PersistentDummyObjectHelper { @Autowired private ArrayDesignService adService; + // setting seed globally does not guarantee reproducibliity always as methods could access + // different parts of the sequence if called in different orders, so callers should reset it using resetSeed() + private static Random randomizer = new Random( 12345 ); + + // Tests can call this to ensure reproducibility + public void resetSeed() { + randomizer = new Random( 12345 ); + } + @Autowired private DifferentialExpressionAnalysisService differentialExpressionAnalysisService; @@ -629,7 +638,7 @@ public Set getTestPersistentBioSequence2GeneProducts( B Collection b2gCol = new HashSet<>(); BlatAssociation b2g = BlatAssociation.Factory.newInstance(); - b2g.setScore( new Random().nextDouble() ); + b2g.setScore( this.randomizer.nextDouble() ); b2g.setBioSequence( bioSequence ); b2g.setGeneProduct( this.getTestPersistentGeneProduct( this.getTestPersistentGene() ) ); b2g.setBlatResult( this.getTestPersistentBlatResult( bioSequence, null ) ); @@ -974,10 +983,10 @@ private byte[] getDoubleData() { double[] data = new double[PersistentDummyObjectHelper.NUM_BIOMATERIALS]; double bump = 0.0; for ( int j = 0; j < data.length; j++ ) { - data[j] = new Random().nextDouble() + bump; + data[j] = this.randomizer.nextDouble() + bump; if ( j % 3 == 0 ) { // add some correlation structure to the data. - bump += 0.5; + bump += 0.3; // adjusted to possibly avoid outlier samples from being generated by accident. } } ByteArrayConverter bconverter = new ByteArrayConverter(); From 5e7eae8b1fea56d0e3858a265ab2ef0d8f77503e Mon Sep 17 00:00:00 2001 From: Paul Pavlidis Date: Fri, 3 May 2024 14:21:21 -0700 Subject: [PATCH 06/61] force GO to be initialized --- .../gemma/core/apps/NCBIGene2GOAssociationLoaderCLI.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/NCBIGene2GOAssociationLoaderCLI.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/NCBIGene2GOAssociationLoaderCLI.java index 62fa71004f..eb8d996e63 100755 --- a/gemma-cli/src/main/java/ubic/gemma/core/apps/NCBIGene2GOAssociationLoaderCLI.java +++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/NCBIGene2GOAssociationLoaderCLI.java @@ -26,6 +26,8 @@ import ubic.gemma.core.loader.association.NCBIGene2GOAssociationLoader; import ubic.gemma.core.loader.association.NCBIGene2GOAssociationParser; import ubic.gemma.core.loader.util.fetcher.HttpFetcher; +import ubic.gemma.core.ontology.OntologyUtils; +import ubic.gemma.core.ontology.providers.GeneOntologyService; import ubic.gemma.core.util.AbstractAuthenticatedCLI; import ubic.gemma.core.util.AbstractCLI; import ubic.gemma.model.common.description.ExternalDatabase; @@ -61,6 +63,9 @@ public class NCBIGene2GOAssociationLoaderCLI extends AbstractAuthenticatedCLI { @Autowired private ExternalDatabaseService externalDatabaseService; + @Autowired + private GeneOntologyService goService; + private String filePath = null; @Override @@ -79,7 +84,7 @@ protected void buildOptions( Options options ) { @Override protected void doWork() throws Exception { NCBIGene2GOAssociationLoader gene2GOAssLoader = new NCBIGene2GOAssociationLoader(); - + OntologyUtils.ensureInitialized( goService ); gene2GOAssLoader.setPersisterHelper( persisterHelper ); Collection taxa = taxonService.loadAll(); From 19cca201a76a7f39df86da72d54b2b3a4b0a2b9b Mon Sep 17 00:00:00 2001 From: Guillaume Poirier-Morency Date: Fri, 3 May 2024 13:42:26 -0700 Subject: [PATCH 07/61] Fix various javadocs and minor warnings --- .../preprocess/batcheffects/ComBat.java | 26 +++++++---------- ...nExperimentBatchCorrectionServiceImpl.java | 16 +++++----- .../expression/geo/service/GeoService.java | 4 +-- .../ontology/OntologyIndividualSimple.java | 6 ++-- .../gemma/core/ontology/OntologyService.java | 2 -- .../gemma/core/ontology/OntologyUtils.java | 5 ---- .../arrayDesign/ArrayDesignService.java | 4 ++- .../ExpressionExperimentService.java | 11 ++----- .../ExpressionExperimentServiceImpl.java | 8 ++--- .../gemma/rest/AnnotationsWebService.java | 2 +- .../java/ubic/gemma/rest/TaxaWebService.java | 2 +- .../expression/experiment/DEDVController.java | 2 +- .../ExperimentalDesignController.java | 29 +++++++------------ 13 files changed, 44 insertions(+), 73 deletions(-) diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/ComBat.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/ComBat.java index 97cbb6ed2c..f395613fa2 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/ComBat.java +++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/ComBat.java @@ -42,7 +42,6 @@ import ubic.basecode.math.distribution.Histogram; import ubic.basecode.math.linearmodels.DesignMatrix; import ubic.basecode.math.linearmodels.LeastSquaresFit; -import ubic.gemma.model.expression.experiment.FactorValueBasicValueObject; import java.awt.*; import java.io.File; @@ -64,7 +63,7 @@ *

*/ @SuppressWarnings({ "unused", "WeakerAccess" }) // Possible external use -public class ComBat { +class ComBat { private static final String BATCH_COLUMN_NAME = "batch"; private static final Log log = LogFactory.getLog( ComBat.class ); @@ -79,7 +78,7 @@ public class ComBat { private LinkedHashMap> batches; private Map> originalLocationsInMatrix; - private Algebra solver; + private final Algebra solver; private DoubleMatrix2D varpooled; private DoubleMatrix2D standMean; private DoubleMatrix2D gammaHat = null; @@ -106,8 +105,6 @@ public class ComBat { /** * Constructor that can be used just for testing correctability (data is not provided) - FIXME refactor so it's not a constructor. - * @param sampleInfo - * @throws ComBatException */ public ComBat( ObjectMatrix sampleInfo ) throws ComBatException { this.sampleInfo = sampleInfo; @@ -426,15 +423,12 @@ private void runNonParametric( final DoubleMatrix2D sdata, DoubleMatrix2D gammas final int firstBatch = i * batchesPerThread; final int lastBatch = i == ( numThreads - 1 ) ? batches.size() : firstBatch + batchesPerThread; - futures[i] = service.submit( new Runnable() { - @Override - public void run() { - for ( int k = firstBatch; k < lastBatch; k++ ) { - String batchId = batchIds[k]; - DoubleMatrix2D batchData = ComBat.this.getBatchData( sdata, batchId ); - DoubleMatrix1D[] batchResults = ComBat.this.nonParametricFit( batchData, gammaHat.viewRow( k ), deltaHat.viewRow( k ) ); - results.put( batchId, batchResults ); - } + futures[i] = service.submit( () -> { + for ( int k = firstBatch; k < lastBatch; k++ ) { + String batchId = batchIds[k]; + DoubleMatrix2D batchData = ComBat.this.getBatchData( sdata, batchId ); + DoubleMatrix1D[] batchResults = ComBat.this.nonParametricFit( batchData, gammaHat.viewRow( k ), deltaHat.viewRow( k ) ); + results.put( batchId, batchResults ); } } ); } @@ -657,8 +651,8 @@ private void initPartA() { C sampleName = sampleInfo.getRowName( i ); String batchId = ( String ) sampleInfo.get( i, batchColumnIndex ); if ( !batches.containsKey( batchId ) ) { - batches.put( batchId, new ArrayList() ); - originalLocationsInMatrix.put( batchId, new LinkedHashMap() ); + batches.put( batchId, new ArrayList<>() ); + originalLocationsInMatrix.put( batchId, new LinkedHashMap<>() ); } batches.get( batchId ).add( sampleName ); diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/ExpressionExperimentBatchCorrectionServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/ExpressionExperimentBatchCorrectionServiceImpl.java index ecbd85d2c4..ab64d055a8 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/ExpressionExperimentBatchCorrectionServiceImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/ExpressionExperimentBatchCorrectionServiceImpl.java @@ -24,7 +24,6 @@ import ubic.basecode.dataStructure.matrix.ObjectMatrix; import ubic.basecode.dataStructure.matrix.ObjectMatrixImpl; import ubic.basecode.math.MatrixStats; -import ubic.basecode.math.linearmodels.DesignMatrix; import ubic.basecode.util.FileTools; import ubic.gemma.core.analysis.expression.diff.LinearModelAnalyzer; import ubic.gemma.core.analysis.util.ExperimentalDesignUtils; @@ -46,6 +45,8 @@ import java.util.*; import java.util.stream.Collectors; +import static java.util.Objects.requireNonNull; + /** * Methods for correcting batch effects. * @@ -128,7 +129,7 @@ public boolean checkCorrectability( ExpressionExperiment ee ) { ObjectMatrix design = this.getDesign( ee ); ObjectMatrix designU = this.convertFactorValuesToStrings( design ); try { - new ComBat( designU ); // without data, just to check + new ComBat<>( designU ); // without data, just to check } catch ( ComBatException c ) { // probably because it's not full rank. log.info( c.getMessage() ); return false; @@ -140,7 +141,8 @@ public boolean checkCorrectability( ExpressionExperiment ee ) { @Override public ExpressionDataDoubleMatrix comBat( ExpressionDataDoubleMatrix originalDataMatrix ) { - ExpressionExperiment ee = originalDataMatrix.getExpressionExperiment(); + ExpressionExperiment ee = requireNonNull( originalDataMatrix.getExpressionExperiment(), + "The data matrix must have an associated experiment to perform ComBat." ); ee = expressionExperimentService.thawLite( ee ); @@ -163,8 +165,6 @@ public ExpressionDataDoubleMatrix comBat( ExpressionDataDoubleMatrix originalDat /** * Remove outlier samples from the data matrix, based on outliers that were flagged in the experiment (not just candidate outliers) - * @param originalDataMatrix - * @param ee * @return the original matrix, or if outliers were present, a new matrix with the outliers removed */ public static ExpressionDataDoubleMatrix removeOutliers( ExpressionDataDoubleMatrix originalDataMatrix, ExpressionExperiment ee ) { @@ -323,10 +323,9 @@ private ObjectMatrix getDesign( Express /** * Extract sample information, format into something ComBat can use. - * + *

* Certain factors are removed at this stage, notably "DE_Exclude/Include" factors. * - * @param ee * @param mat only used to get sample ordering? * @return design matrix */ @@ -336,7 +335,7 @@ private ObjectMatrix getDesign( Express Collection experimentalFactors = ee.getExperimentalDesign().getExperimentalFactors(); /* remove experimental factors that are for DE_Exclude */ - List retainedFactors = experimentalFactors.stream().filter( ef -> retainForBatchCorrection( ef ) ).collect( Collectors.toList() ); + List retainedFactors = experimentalFactors.stream().filter( this::retainForBatchCorrection ).collect( Collectors.toList() ); List orderedSamples = new ArrayList<>(); if ( mat == null ) { @@ -376,6 +375,7 @@ private QuantitationType makeNewQuantitationType( QuantitationType oldQt ) { newQt.setIsBackground( oldQt.getIsBackground() ); newQt.setIsBackgroundSubtracted( oldQt.getIsBackgroundSubtracted() ); newQt.setGeneralType( oldQt.getGeneralType() ); + //noinspection deprecation newQt.setIsMaskedPreferred( oldQt.getIsMaskedPreferred() ); newQt.setIsPreferred( oldQt.getIsPreferred() ); newQt.setIsRatio( oldQt.getIsRatio() ); diff --git a/gemma-core/src/main/java/ubic/gemma/core/loader/expression/geo/service/GeoService.java b/gemma-core/src/main/java/ubic/gemma/core/loader/expression/geo/service/GeoService.java index 489aa2b2f9..8e893fc50b 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/loader/expression/geo/service/GeoService.java +++ b/gemma-core/src/main/java/ubic/gemma/core/loader/expression/geo/service/GeoService.java @@ -16,7 +16,6 @@ import ubic.gemma.core.loader.expression.geo.GeoDomainObjectGenerator; import ubic.gemma.model.expression.arrayDesign.ArrayDesign; -import ubic.gemma.model.expression.experiment.ExpressionExperiment; import java.util.Collection; @@ -62,7 +61,6 @@ Collection fetchAndLoad( String geoAccession, boolean loadPlatformOnly, boole /** * Refetch and reprocess the GEO series, updating select information. Currently only implemented for experiments (GSEs) - * @param geoAccession */ void updateFromGEO( String geoAccession ); @@ -83,5 +81,5 @@ Collection fetchAndLoad( String geoAccession, boolean loadPlatformOnly, boole * @param softFile the full path to the SOFT file. The file name has to be [accession].soft.gz * @return a single experiment. */ - Collection loadFromSoftFile( String accession, String softFile, boolean loadPlatformOnly, boolean doSampleMatching, boolean splitByPlatform ); + Collection loadFromSoftFile( String accession, String softFile, boolean loadPlatformOnly, boolean doSampleMatching, boolean splitByPlatform ); } \ No newline at end of file diff --git a/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyIndividualSimple.java b/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyIndividualSimple.java index 1b60150d41..4d01720ebd 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyIndividualSimple.java +++ b/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyIndividualSimple.java @@ -10,9 +10,9 @@ public class OntologyIndividualSimple extends AbstractOntologyResourceSimple imp private final OntologyTermSimple instanceOf; /** - * - * @param uri - * @param label + * Create a new simple ontology individual. + * @param uri a URI for the term, of null for a free-text term + * @param label a label for the term * @param instanceOf the term this individual is an instance of which must be simple since this class has to be * {@link java.io.Serializable}. */ diff --git a/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyService.java b/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyService.java index c0ac4b80ae..bd0523dab6 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyService.java +++ b/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyService.java @@ -56,8 +56,6 @@ default Collection findExperimentsCharacteristicTags( * * @param searchQuery search query * @param useNeuroCartaOntology use neurocarta ontology - * @param timeout - * @param timeUnit * @return characteristic vos */ @Deprecated diff --git a/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyUtils.java b/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyUtils.java index f0bcf59228..4417bbce7b 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyUtils.java +++ b/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyUtils.java @@ -1,12 +1,7 @@ package ubic.gemma.core.ontology; import lombok.extern.apachecommons.CommonsLog; -import ubic.basecode.ontology.model.OntologyTerm; import ubic.basecode.ontology.providers.OntologyService; -import ubic.gemma.model.common.description.Characteristic; - -import java.util.Collection; -import java.util.HashSet; /** * Utilities for working with ontologies. diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/arrayDesign/ArrayDesignService.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/arrayDesign/ArrayDesignService.java index bd4c575a5b..eb5598ff5b 100644 --- a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/arrayDesign/ArrayDesignService.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/arrayDesign/ArrayDesignService.java @@ -319,6 +319,7 @@ public interface ArrayDesignService extends CuratableService addFactorValues( ExpressionExperiment ee, Map fvs ); + void addFactorValues( ExpressionExperiment ee, Map fvs ); /** * Used when we want to add data for a quantitation type. Does not remove any existing vectors. @@ -352,7 +348,6 @@ class CharacteristicWithUsageStatisticsAndOntologyTerm { * Calculate the usage frequency of platforms by the datasets matching the provided filters. * * @param filters a set of filters to be applied as per {@link #load(Filters, Sort, int, int)} - * @param extraIds * @param maxResults the maximum of results, or unlimited if less than 1 */ Map getArrayDesignUsedOrOriginalPlatformUsageFrequency( @Nullable Filters filters, @Nullable Set extraIds, int maxResults ); @@ -451,7 +446,7 @@ class CharacteristicWithUsageStatisticsAndOntologyTerm { Map getLastProcessedDataUpdate( Collection ids ); /** - * @return a count of expression experiments, grouped by Taxon + * @return counts of expression experiments grouped by taxon */ Map getPerTaxonCount(); @@ -489,7 +484,7 @@ class CharacteristicWithUsageStatisticsAndOntologyTerm { boolean hasProcessedExpressionData( ExpressionExperiment ee ); /** - * @return count of an expressionExperiment's design element data vectors, grouped by quantitation type + * @return counts design element data vectors grouped by quantitation type */ @Secured({ "IS_AUTHENTICATED_ANONYMOUSLY", "ACL_SECURABLE_READ" }) Map getQuantitationTypeCount( ExpressionExperiment ee ); diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentServiceImpl.java index 2a54368260..2392717699 100755 --- a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentServiceImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentServiceImpl.java @@ -180,10 +180,9 @@ public FactorValue addFactorValue( ExpressionExperiment ee, FactorValue fv ) { @Override @Transactional - public Map addFactorValues( ExpressionExperiment ee, Map fvs ) { + public void addFactorValues( ExpressionExperiment ee, Map fvs ) { ExpressionExperiment experiment = requireNonNull( expressionExperimentDao.load( ee.getId() ) ); Collection efs = experiment.getExperimentalDesign().getExperimentalFactors(); - Map result = new HashMap<>(); int count = 0; for ( BioMaterial bm : fvs.keySet() ) { FactorValue fv = fvs.get( bm ); @@ -197,7 +196,6 @@ public Map addFactorValues( ExpressionExperiment ee, M } } bm.getFactorValues().add( fv ); - result.put( bm, fv ); ++count; if ( count % 50 == 0 ) { log.info( "Processed: " + count + " biomaterials for new factor values" ); @@ -205,9 +203,7 @@ public Map addFactorValues( ExpressionExperiment ee, M } log.info( "Processed: " + count + " biomaterials for new factor values, updating ..." ); // expressionExperimentDao.update( experiment ); - bioMaterialService.update( result.keySet() ); - - return result; + bioMaterialService.update( fvs.keySet() ); } @Override diff --git a/gemma-rest/src/main/java/ubic/gemma/rest/AnnotationsWebService.java b/gemma-rest/src/main/java/ubic/gemma/rest/AnnotationsWebService.java index 52791a1540..1a1471575b 100644 --- a/gemma-rest/src/main/java/ubic/gemma/rest/AnnotationsWebService.java +++ b/gemma-rest/src/main/java/ubic/gemma/rest/AnnotationsWebService.java @@ -110,7 +110,7 @@ public AnnotationsWebService( OntologyService ontologyService, SearchService sea * * @param query the search query. Either plain text, or an ontology term URI * @return response data object with a collection of found terms, each wrapped in a CharacteristicValueObject. - * @see OntologyService#findTermsInexact(String, Taxon) for better description of the search process. + * @see OntologyService#findTermsInexact(String, int, Taxon) for better description of the search process. * @see CharacteristicValueObject for the output object structure. */ @GET diff --git a/gemma-rest/src/main/java/ubic/gemma/rest/TaxaWebService.java b/gemma-rest/src/main/java/ubic/gemma/rest/TaxaWebService.java index c300f68a51..8afc5362fe 100644 --- a/gemma-rest/src/main/java/ubic/gemma/rest/TaxaWebService.java +++ b/gemma-rest/src/main/java/ubic/gemma/rest/TaxaWebService.java @@ -222,7 +222,7 @@ public ResponseDataObject> getGeneLocationsInT } /** - * Retrieves datasets for the given taxon. Filtering allowed exactly like in {@link DatasetsWebService#getDatasets(String, FilterArg, OffsetArg, LimitArg, SortArg)}. + * Retrieves datasets for the given taxon. Filtering allowed exactly like in {@link DatasetsWebService#getDatasets(QueryArg, FilterArg, OffsetArg, LimitArg, SortArg)}. * * @param taxonArg can either be Taxon ID, Taxon NCBI ID, or one of its string identifiers: * scientific name, common name. It is recommended to use the ID for efficiency. diff --git a/gemma-web/src/main/java/ubic/gemma/web/controller/expression/experiment/DEDVController.java b/gemma-web/src/main/java/ubic/gemma/web/controller/expression/experiment/DEDVController.java index 2d0b57bf86..50814c4470 100644 --- a/gemma-web/src/main/java/ubic/gemma/web/controller/expression/experiment/DEDVController.java +++ b/gemma-web/src/main/java/ubic/gemma/web/controller/expression/experiment/DEDVController.java @@ -406,7 +406,7 @@ public VisualizationValueObject[] getDEDVForDiffExVisualizationByExperiment( Lon * * @param resultSetId The resultset we're specifically interested. Note that this is what is used to choose the * vectors, since it could be a subset of an experiment. - * @param givenThreshold + * @param givenThreshold If non-null, a P-value threshold for retrieving associated vectors * @param primaryFactorID If non-null, the factor to use for sorting the samples before other factors are considered * @return collection of visualization value objects */ diff --git a/gemma-web/src/main/java/ubic/gemma/web/controller/expression/experiment/ExperimentalDesignController.java b/gemma-web/src/main/java/ubic/gemma/web/controller/expression/experiment/ExperimentalDesignController.java index e71d45f05d..549ccb05f2 100644 --- a/gemma-web/src/main/java/ubic/gemma/web/controller/expression/experiment/ExperimentalDesignController.java +++ b/gemma-web/src/main/java/ubic/gemma/web/controller/expression/experiment/ExperimentalDesignController.java @@ -21,7 +21,6 @@ import gemma.gsec.SecurityService; import org.apache.commons.lang.time.StopWatch; import org.apache.commons.lang3.StringUtils; -import org.jetbrains.annotations.NotNull; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.security.access.AccessDeniedException; import org.springframework.stereotype.Controller; @@ -74,6 +73,7 @@ */ @Controller @RequestMapping("/experimentalDesign") +@SuppressWarnings("unused") public class ExperimentalDesignController extends BaseController { @Autowired @@ -126,11 +126,11 @@ public void createDesignFromFile( Long eeid, String filePath ) { } /** + * Create an experimental factor. * - * - * @param e experimentalDesign to add the factor to + * @param e experimentalDesign to add the factor to * @param efvo non-null if we are pre-populating the factor values based on an existing set of BioMaterialCharacteristic, - * see https://github.com/PavlidisLab/Gemma/issues/987 + * see #987 */ public void createExperimentalFactor( EntityDelegator e, ExperimentalFactorValueWebUIObject efvo ) { if ( e == null || e.getId() == null ) return; @@ -149,7 +149,7 @@ public void createExperimentalFactor( EntityDelegator e, Exp */ // experimentalFactorService.create( ef ); - if ( ed.getExperimentalFactors() == null ) ed.setExperimentalFactors( new HashSet() ); + if ( ed.getExperimentalFactors() == null ) ed.setExperimentalFactors( new HashSet<>() ); ed.getExperimentalFactors().add( ef ); experimentalDesignService.update( ed ); @@ -196,7 +196,7 @@ public void createExperimentalFactor( EntityDelegator e, Exp Double.parseDouble( cvo.getValue() ); } catch ( NumberFormatException err ) { // try to handle missing data reasonably. - if ( cvo.getValue().toUpperCase().equals( "NA" ) || cvo.getValue().toUpperCase().equals( "N/A" ) ) { + if ( cvo.getValue().equalsIgnoreCase( "NA" ) || cvo.getValue().equalsIgnoreCase( "N/A" ) ) { cvo.setValue( null ); } else { // clean up after ourselves. @@ -364,12 +364,12 @@ public Collection getBioMaterials( EntityDelegator getBioMaterialValueObjects( ExpressionExperiment ee ) { + */ + private Collection getBioMaterialValueObjects( ExpressionExperiment ee ) { ee = expressionExperimentService.thawLite( ee ); Collection result = new HashSet<>(); for ( BioAssay assay : ee.getBioAssays() ) { @@ -386,14 +386,13 @@ public Collection getBioMaterials( EntityDelegator getBioMaterialCharacteristicCategories( Long experimentalDesignID ) { ExpressionExperiment ee = experimentalDesignService.getExpressionExperiment( experimentalDesignService.loadOrFail( experimentalDesignID ) ); Collection bmvos = getBioMaterialValueObjects( ee ); - if ( bmvos == null || bmvos.isEmpty() ) { + if ( bmvos.isEmpty() ) { return Collections.emptyList(); } @@ -413,9 +412,6 @@ public Collection getBioMaterialCharacteristicCategor * */ private void filterCharacteristics( Collection result ) { - - int c = result.size(); - Collection toremove = new HashSet<>(); // build map of categories to bmos. No category: can't use. @@ -438,7 +434,7 @@ private void filterCharacteristics( Collection result ) } if ( !map.containsKey( category ) ) { - map.put( category, new HashSet() ); + map.put( category, new HashSet<>() ); } if ( map.get( category ).contains( bmo ) ) { @@ -469,7 +465,6 @@ private void filterCharacteristics( Collection result ) } Collection vals = new HashSet<>(); - boolean keeper = false; bms: for ( BioMaterialValueObject bm : map.get( category ) ) { // log.info( "inspecting " + bm ); @@ -637,7 +632,7 @@ public void updateBioMaterials( BioMaterialValueObject[] bmvos ) { Collection biomaterials = bioMaterialService.updateBioMaterials( Arrays.asList( bmvos ) ); - log.info( String.format( "Updating biomaterials took %.2f seconds", (double)w.getTime() / 1000.0 ) ); + log.info( String.format( "Updating biomaterials took %.2f seconds", ( double ) w.getTime() / 1000.0 ) ); if ( biomaterials.isEmpty() ) return; @@ -846,7 +841,6 @@ public void updateFactorValueCharacteristics( FactorValueValueObject[] fvvos ) { public void markFactorValuesAsNeedsAttention( Long[] fvvos, String note ) { Set fvs = new HashSet<>( fvvos.length ); - int i = 0; for ( Long fvo : fvvos ) { FactorValue fv = factorValueService.loadOrFail( fvo, EntityNotFoundException::new, String.format( "No FactorValue with ID %d", fvo ) ); if ( fv.getNeedsAttention() ) { @@ -866,7 +860,6 @@ public void markFactorValuesAsNeedsAttention( Long[] fvvos, String note ) { public void clearFactorValuesNeedsAttention( Long[] fvvos, String note ) { Set fvs = new HashSet<>( fvvos.length ); - int i = 0; for ( Long fvo : fvvos ) { FactorValue fv = factorValueService.loadOrFail( fvo, EntityNotFoundException::new, String.format( "No FactorValue with ID %d", fvo ) ); if ( !fv.getNeedsAttention() ) { From ab6991989bc8a34abf6fa834fe9971f4d5e06208 Mon Sep 17 00:00:00 2001 From: Guillaume Poirier-Morency Date: Fri, 3 May 2024 14:31:29 -0700 Subject: [PATCH 08/61] Delete Gene2GO associations by batch --- .../association/Gene2GOAssociationDaoImpl.java | 9 +++++---- .../ubic/gemma/persistence/util/QueryUtils.java | 13 +++++++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/association/Gene2GOAssociationDaoImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/association/Gene2GOAssociationDaoImpl.java index 53134f666a..6653346d6e 100644 --- a/gemma-core/src/main/java/ubic/gemma/persistence/service/association/Gene2GOAssociationDaoImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/association/Gene2GOAssociationDaoImpl.java @@ -20,6 +20,7 @@ import org.apache.commons.lang3.time.StopWatch; import org.hibernate.Criteria; +import org.hibernate.Query; import org.hibernate.SessionFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Repository; @@ -31,6 +32,7 @@ import ubic.gemma.persistence.util.BusinessKey; import ubic.gemma.persistence.util.EntityUtils; import ubic.gemma.persistence.util.HibernateUtils; +import ubic.gemma.persistence.util.QueryUtils; import javax.annotation.Nullable; import java.util.*; @@ -168,10 +170,9 @@ public int removeAll() { .executeUpdate(); int removedCharacteristics; if ( !cIds.isEmpty() ) { - removedCharacteristics = getSessionFactory().getCurrentSession() - .createQuery( "delete from Characteristic where id in :cIds" ) - .setParameterList( "cIds", optimizeParameterList( cIds ) ) - .executeUpdate(); + Query query = getSessionFactory().getCurrentSession() + .createQuery( "delete from Characteristic where id in :cIds" ); + removedCharacteristics = QueryUtils.executeUpdateByBatch( query, "cIds", cIds, 2048 ); } else { removedCharacteristics = 0; } diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/util/QueryUtils.java b/gemma-core/src/main/java/ubic/gemma/persistence/util/QueryUtils.java index 8c4e527d5c..a2185785c3 100644 --- a/gemma-core/src/main/java/ubic/gemma/persistence/util/QueryUtils.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/util/QueryUtils.java @@ -151,6 +151,19 @@ public static , T> Stream streamByBatch( Query query, .flatMap( List::stream ); } + /** + * Execute an update query by a fixed batch size. + * @see Query#executeUpdate() + * @return the sum of all performed update executions + */ + public static > int executeUpdateByBatch( Query query, String batchParam, Collection list, int batchSize ) { + int updated = 0; + for ( List batch : batchParameterList( list, batchSize ) ) { + updated += query.setParameterList( batchParam, batch ).executeUpdate(); + } + return updated; + } + public static String escapeLike( String s ) { return s.replaceAll( "[%_\\\\]", "\\\\$0" ); } From 108e3e861556d25e56c44da476d401230cfeec25 Mon Sep 17 00:00:00 2001 From: Guillaume Poirier-Morency Date: Fri, 15 Mar 2024 16:50:59 -0700 Subject: [PATCH 09/61] Only update EE2C and EE2AD entries that were modified since last update Add a since parameter to only update entries from datasets modified after the given date. Rewrite the secure job to be more flexible so that it can also be used for direct Quartz job implementations. --- .../ubic/gemma/core/apps/UpdateEE2CCli.java | 2 +- .../service/TableMaintenanceUtil.java | 9 ++- .../service/TableMaintenanceUtilImpl.java | 47 +++++++++--- .../search/SearchServiceIntegrationTest.java | 2 +- .../CharacteristicServiceTest.java | 2 +- ...ssionExperimentServiceIntegrationTest.java | 4 +- .../TableMaintenanceUtilIntegrationTest.java | 21 ++--- .../CharacteristicDaoImplTest.java | 6 +- .../gemma/web/scheduler/Ee2AdUpdateJob.java | 22 ++++++ .../gemma/web/scheduler/Ee2cUpdateJob.java | 28 +++++++ .../gemma/web/scheduler/SecureInvoker.java | 58 ++++++++++++++ ...ureMethodInvokingJobDetailFactoryBean.java | 64 +++------------- .../web/scheduler/SecureQuartzJobBean.java | 32 ++++++++ .../gemma/applicationContext-schedule.xml | 76 +++++++++++-------- .../web/scheduler/SchedulerSecurityTest.java | 60 +++++++++++++-- 15 files changed, 313 insertions(+), 120 deletions(-) create mode 100644 gemma-web/src/main/java/ubic/gemma/web/scheduler/Ee2AdUpdateJob.java create mode 100644 gemma-web/src/main/java/ubic/gemma/web/scheduler/Ee2cUpdateJob.java create mode 100644 gemma-web/src/main/java/ubic/gemma/web/scheduler/SecureInvoker.java create mode 100644 gemma-web/src/main/java/ubic/gemma/web/scheduler/SecureQuartzJobBean.java diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/UpdateEE2CCli.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/UpdateEE2CCli.java index 9a457755c3..e292529b47 100644 --- a/gemma-cli/src/main/java/ubic/gemma/core/apps/UpdateEE2CCli.java +++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/UpdateEE2CCli.java @@ -30,7 +30,7 @@ protected void processOptions( CommandLine commandLine ) throws ParseException { @Override protected void doWork() throws Exception { - tableMaintenanceUtil.updateExpressionExperiment2CharacteristicEntries( truncate ); + tableMaintenanceUtil.updateExpressionExperiment2CharacteristicEntries( null, truncate ); } @Nullable diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/TableMaintenanceUtil.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/TableMaintenanceUtil.java index c0b944df2d..70161fb58b 100644 --- a/gemma-core/src/main/java/ubic/gemma/persistence/service/TableMaintenanceUtil.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/TableMaintenanceUtil.java @@ -20,6 +20,9 @@ import org.springframework.security.access.annotation.Secured; +import javax.annotation.Nullable; +import java.util.Date; + /** * @author paul */ @@ -68,7 +71,7 @@ public interface TableMaintenanceUtil { * @return the number of records that were created or updated */ @Secured({ "GROUP_AGENT" }) - int updateExpressionExperiment2CharacteristicEntries( boolean truncate ); + int updateExpressionExperiment2CharacteristicEntries( @Nullable Date sinceLastUpdate, boolean truncate ); /** * Update a specific level of the {@code EXPRESSION_EXPERIMENT2CHARACTERISTIC} table. @@ -77,14 +80,14 @@ public interface TableMaintenanceUtil { * @return the number of records that were created or updated */ @Secured({ "GROUP_AGENT" }) - int updateExpressionExperiment2CharacteristicEntries( Class level, boolean truncate ); + int updateExpressionExperiment2CharacteristicEntries( Class level, @Nullable Date sinceLastUpdate, boolean truncate ); /** * Update the {@code EXPRESSION_EXPERIMENT2_ARRAY_DESIGN} table. * @return the number of records that were created or updated */ @Secured({ "GROUP_AGENT" }) - int updateExpressionExperiment2ArrayDesignEntries(); + int updateExpressionExperiment2ArrayDesignEntries( @Nullable Date sinceLastUpdate ); // for tests only, to keep from getting emails. @Secured({ "GROUP_ADMIN" }) diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/TableMaintenanceUtilImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/TableMaintenanceUtilImpl.java index c722704922..1fdceb01b6 100644 --- a/gemma-core/src/main/java/ubic/gemma/persistence/service/TableMaintenanceUtilImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/TableMaintenanceUtilImpl.java @@ -28,6 +28,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.springframework.util.Assert; import ubic.gemma.model.common.Auditable; import ubic.gemma.model.common.auditAndSecurity.AuditEvent; import ubic.gemma.model.common.auditAndSecurity.eventType.ArrayDesignGeneMappingEvent; @@ -91,41 +92,52 @@ public class TableMaintenanceUtilImpl implements TableMaintenanceUtil { private static final String EE2C_EE_QUERY = "select MIN(C.ID), C.NAME, C.DESCRIPTION, C.CATEGORY, C.CATEGORY_URI, C.`VALUE`, C.VALUE_URI, C.ORIGINAL_VALUE, C.EVIDENCE_CODE, I.ID, (" + SELECT_ANONYMOUS_MASK + "), cast(? as char(255)) " + "from INVESTIGATION I " + + "join CURATION_DETAILS CD on I.CURATION_DETAILS_FK = CD.ID " + "join CHARACTERISTIC C on I.ID = C.INVESTIGATION_FK " + "where I.class = 'ExpressionExperiment' " + + "and COALESCE(CD.LAST_UPDATED, 0) >= COALESCE(:since, 0) " + "group by I.ID, COALESCE(C.CATEGORY_URI, C.CATEGORY), COALESCE(C.VALUE_URI, C.`VALUE`)"; private static final String EE2C_BM_QUERY = "select MIN(C.ID), C.NAME, C.DESCRIPTION, C.CATEGORY, C.CATEGORY_URI, C.`VALUE`, C.VALUE_URI, C.ORIGINAL_VALUE, C.EVIDENCE_CODE, I.ID, (" + SELECT_ANONYMOUS_MASK + "), cast(? as char(255)) " + "from INVESTIGATION I " + + "join CURATION_DETAILS CD on I.CURATION_DETAILS_FK = CD.ID " + "join BIO_ASSAY BA on I.ID = BA.EXPRESSION_EXPERIMENT_FK " + "join BIO_MATERIAL BM on BA.SAMPLE_USED_FK = BM.ID " + "join CHARACTERISTIC C on BM.ID = C.BIO_MATERIAL_FK " + "where I.class = 'ExpressionExperiment' " + + "and COALESCE(CD.LAST_UPDATED, 0) >= COALESCE(:since, 0) " + "group by I.ID, COALESCE(C.CATEGORY_URI, C.CATEGORY), COALESCE(C.VALUE_URI, C.`VALUE`)"; private static final String EE2C_ED_QUERY = "select MIN(C.ID), C.NAME, C.DESCRIPTION, C.CATEGORY, C.CATEGORY_URI, C.`VALUE`, C.VALUE_URI, C.ORIGINAL_VALUE, C.EVIDENCE_CODE, I.ID, (" + SELECT_ANONYMOUS_MASK + "), cast(? as char(255)) " + "from INVESTIGATION I " + + "join CURATION_DETAILS CD on I.CURATION_DETAILS_FK = CD.ID " + "join EXPERIMENTAL_DESIGN on I.EXPERIMENTAL_DESIGN_FK = EXPERIMENTAL_DESIGN.ID " + "join EXPERIMENTAL_FACTOR EF on EXPERIMENTAL_DESIGN.ID = EF.EXPERIMENTAL_DESIGN_FK " + "join FACTOR_VALUE FV on FV.EXPERIMENTAL_FACTOR_FK = EF.ID " + "join CHARACTERISTIC C on FV.ID = C.FACTOR_VALUE_FK " + + "where I.class = 'ExpressionExperiment' " // remove C.class = 'Statement' once the old-style characteristics are removed (see https://github.com/PavlidisLab/Gemma/issues/929 for details) - + "where I.class = 'ExpressionExperiment' and C.class = 'Statement' " + + "and C.class = 'Statement' " + + "and COALESCE(CD.LAST_UPDATED, 0) >= COALESCE(:since, 0) " + "group by I.ID, COALESCE(C.CATEGORY_URI, C.CATEGORY), COALESCE(C.VALUE_URI, C.`VALUE`)"; private static final String EE2AD_QUERY = "insert into EXPRESSION_EXPERIMENT2ARRAY_DESIGN (EXPRESSION_EXPERIMENT_FK, ARRAY_DESIGN_FK, IS_ORIGINAL_PLATFORM, ACL_IS_AUTHENTICATED_ANONYMOUSLY_MASK) " + "select I.ID, AD.ID, FALSE, (" + SELECT_ANONYMOUS_MASK + ") from INVESTIGATION I " + + "join CURATION_DETAILS CD on I.CURATION_DETAILS_FK = CD.ID " + "join BIO_ASSAY BA on I.ID = BA.EXPRESSION_EXPERIMENT_FK " + "join ARRAY_DESIGN AD on BA.ARRAY_DESIGN_USED_FK = AD.ID " + "where I.class = 'ExpressionExperiment' " + + "and COALESCE(CD.LAST_UPDATED, 0) >= COALESCE(:since, 0) " + "group by I.ID, AD.ID " + "union " + "select I.ID, AD.ID, TRUE, (" + SELECT_ANONYMOUS_MASK + ") from INVESTIGATION I " + + "join CURATION_DETAILS CD on I.CURATION_DETAILS_FK = CD.ID " + "join BIO_ASSAY BA on I.ID = BA.EXPRESSION_EXPERIMENT_FK " + "join ARRAY_DESIGN AD on BA.ORIGINAL_PLATFORM_FK = AD.ID " + "where I.class = 'ExpressionExperiment' " + + "and COALESCE(CD.LAST_UPDATED, 0) >= COALESCE(:since, 0) " + "group by I.ID, AD.ID " + "on duplicate key update ACL_IS_AUTHENTICATED_ANONYMOUSLY_MASK = VALUES(ACL_IS_AUTHENTICATED_ANONYMOUSLY_MASK)"; @@ -218,8 +230,10 @@ public void updateGene2CsEntries() { @Override @Transactional @Timed - public int updateExpressionExperiment2CharacteristicEntries( boolean truncate ) { - log.info( "Updating the EXPRESSION_EXPERIMENT2CHARACTERISTIC table..." ); + public int updateExpressionExperiment2CharacteristicEntries( @Nullable Date sinceLastUpdate, boolean truncate ) { + Assert.isTrue( !( sinceLastUpdate != null && truncate ), "Cannot perform a partial update with sinceLastUpdate with truncate." ); + log.info( String.format( "Updating the EXPRESSION_EXPERIMENT2CHARACTERISTIC table%s...", + sinceLastUpdate != null ? " since " + sinceLastUpdate : "" ) ); if ( truncate ) { log.info( "Truncating EXPRESSION_EXPERIMENT2CHARACTERISTIC..." ); sessionFactory.getCurrentSession() @@ -239,15 +253,19 @@ public int updateExpressionExperiment2CharacteristicEntries( boolean truncate ) .setParameter( 0, ExpressionExperiment.class ) .setParameter( 1, BioMaterial.class ) .setParameter( 2, ExperimentalDesign.class ) + .setParameter( "since", sinceLastUpdate ) .executeUpdate(); - log.info( String.format( "Done updating the EXPRESSION_EXPERIMENT2CHARACTERISTIC table; %d entries were updated.", updated ) ); + log.info( String.format( "Done updating the EXPRESSION_EXPERIMENT2CHARACTERISTIC table; %d entries were updated%s.", + updated, + sinceLastUpdate != null ? " since " + sinceLastUpdate : "" ) ); return updated; } @Override @Timed @Transactional - public int updateExpressionExperiment2CharacteristicEntries( Class level, boolean truncate ) { + public int updateExpressionExperiment2CharacteristicEntries( Class level, @Nullable Date sinceLastUpdate, boolean truncate ) { + Assert.isTrue( !( sinceLastUpdate != null && truncate ), "Cannot perform a partial update with sinceLastUpdate with truncate." ); String query; if ( level.equals( ExpressionExperiment.class ) ) { query = EE2C_EE_QUERY; @@ -258,7 +276,9 @@ public int updateExpressionExperiment2CharacteristicEntries( Class level, boo } else { throw new IllegalArgumentException( "Level must be one of ExpressionExperiment.class, BioMaterial.class or ExperimentalDesign.class." ); } - log.info( "Updating the EXPRESSION_EXPERIMENT2CHARACTERISTIC table at " + level.getSimpleName() + " level..." ); + log.info( String.format( "Updating the EXPRESSION_EXPERIMENT2CHARACTERISTIC table at %s level%s...", + level.getSimpleName(), + sinceLastUpdate != null ? " since " + sinceLastUpdate : "" ) ); if ( truncate ) { log.info( "Truncating EXPRESSION_EXPERIMENT2CHARACTERISTIC at " + level.getSimpleName() + " level..." ); sessionFactory.getCurrentSession() @@ -273,20 +293,25 @@ public int updateExpressionExperiment2CharacteristicEntries( Class level, boo + "on duplicate key update NAME = VALUES(NAME), DESCRIPTION = VALUES(DESCRIPTION), CATEGORY = VALUES(CATEGORY), CATEGORY_URI = VALUES(CATEGORY_URI), `VALUE` = VALUES(`VALUE`), VALUE_URI = VALUES(VALUE_URI), ORIGINAL_VALUE = VALUES(ORIGINAL_VALUE), EVIDENCE_CODE = VALUES(EVIDENCE_CODE), ACL_IS_AUTHENTICATED_ANONYMOUSLY_MASK = VALUES(ACL_IS_AUTHENTICATED_ANONYMOUSLY_MASK), LEVEL = VALUES(LEVEL)" ) .addSynchronizedQuerySpace( EE2C_QUERY_SPACE ) .setParameter( 0, level ) + .setParameter( "since", sinceLastUpdate ) .executeUpdate(); - log.info( String.format( "Done updating the EXPRESSION_EXPERIMENT2CHARACTERISTIC table at %s level; %d entries were updated.", - level.getSimpleName(), updated ) ); + log.info( String.format( "Done updating the EXPRESSION_EXPERIMENT2CHARACTERISTIC table at %s level; %d entries were updated%s.", + level.getSimpleName(), updated, + sinceLastUpdate != null ? " since " + sinceLastUpdate : "" ) ); return updated; } @Override @Transactional - public int updateExpressionExperiment2ArrayDesignEntries() { - log.info( "Updating the EXPRESSION_EXPERIMENT2ARRAY_DESIGN table..." ); + public int updateExpressionExperiment2ArrayDesignEntries( @Nullable Date sinceLastUpdate ) { + log.info( String.format( "Updating the EXPRESSION_EXPERIMENT2ARRAY_DESIGN table%s...", + sinceLastUpdate != null ? " since " + sinceLastUpdate : "" ) ); int updated = sessionFactory.getCurrentSession().createSQLQuery( EE2AD_QUERY ) .addSynchronizedQuerySpace( EE2AD_QUERY_SPACE ) + .setParameter( "since", sinceLastUpdate ) .executeUpdate(); - log.info( String.format( "Done updating the EXPRESSION_EXPERIMENT2ARRAY_DESIGN table; %d entries were updated.", updated ) ); + log.info( String.format( "Done updating the EXPRESSION_EXPERIMENT2ARRAY_DESIGN table; %d entries were updated%s.", + updated, sinceLastUpdate != null ? " since " + sinceLastUpdate : "" ) ); return updated; } diff --git a/gemma-core/src/test/java/ubic/gemma/core/search/SearchServiceIntegrationTest.java b/gemma-core/src/test/java/ubic/gemma/core/search/SearchServiceIntegrationTest.java index a460267bb7..fd5b7313ce 100644 --- a/gemma-core/src/test/java/ubic/gemma/core/search/SearchServiceIntegrationTest.java +++ b/gemma-core/src/test/java/ubic/gemma/core/search/SearchServiceIntegrationTest.java @@ -130,7 +130,7 @@ public void setUp() throws Exception { gene.setNcbiGeneId( new Integer( geneNcbiId ) ); geneService.update( gene ); - tableMaintenanceUtil.updateExpressionExperiment2CharacteristicEntries( false ); + tableMaintenanceUtil.updateExpressionExperiment2CharacteristicEntries( null, false ); } @After diff --git a/gemma-core/src/test/java/ubic/gemma/model/common/description/CharacteristicServiceTest.java b/gemma-core/src/test/java/ubic/gemma/model/common/description/CharacteristicServiceTest.java index 66f8ae450a..973867085a 100644 --- a/gemma-core/src/test/java/ubic/gemma/model/common/description/CharacteristicServiceTest.java +++ b/gemma-core/src/test/java/ubic/gemma/model/common/description/CharacteristicServiceTest.java @@ -91,7 +91,7 @@ public void setUp() throws Exception { fv.setCharacteristics( this.getTestPersistentStatements( 1 ) ); fvService.update( fv ); - tableMaintenanceUtil.updateExpressionExperiment2CharacteristicEntries( false ); + tableMaintenanceUtil.updateExpressionExperiment2CharacteristicEntries( null, false ); } @Test diff --git a/gemma-core/src/test/java/ubic/gemma/model/expression/experiment/ExpressionExperimentServiceIntegrationTest.java b/gemma-core/src/test/java/ubic/gemma/model/expression/experiment/ExpressionExperimentServiceIntegrationTest.java index 4995edf017..1cd8cb7701 100644 --- a/gemma-core/src/test/java/ubic/gemma/model/expression/experiment/ExpressionExperimentServiceIntegrationTest.java +++ b/gemma-core/src/test/java/ubic/gemma/model/expression/experiment/ExpressionExperimentServiceIntegrationTest.java @@ -430,7 +430,7 @@ public void testCacheInvalidationWhenACharacteristicIsDeleted() throws TimeoutEx assertThat( c2.getNumberOfExpressionExperiments() ).isEqualTo( 1L ); }; - tableMaintenanceUtil.updateExpressionExperiment2CharacteristicEntries( false ); + tableMaintenanceUtil.updateExpressionExperiment2CharacteristicEntries( null, false ); assertThat( expressionExperimentService.getAnnotationsUsageFrequency( null, null, null, null, null, 0, null, 0 ) ) .noneSatisfy( consumer ); @@ -444,7 +444,7 @@ public void testCacheInvalidationWhenACharacteristicIsDeleted() throws TimeoutEx .noneSatisfy( consumer ); // update the pivot table - tableMaintenanceUtil.updateExpressionExperiment2CharacteristicEntries( false ); + tableMaintenanceUtil.updateExpressionExperiment2CharacteristicEntries( null, false ); assertThat( expressionExperimentService.getAnnotationsUsageFrequency( null, null, null, null, null, 0, null, 0 ) ) .satisfiesOnlyOnce( consumer ); diff --git a/gemma-core/src/test/java/ubic/gemma/persistence/service/TableMaintenanceUtilIntegrationTest.java b/gemma-core/src/test/java/ubic/gemma/persistence/service/TableMaintenanceUtilIntegrationTest.java index d81d6884bb..211da6bab0 100644 --- a/gemma-core/src/test/java/ubic/gemma/persistence/service/TableMaintenanceUtilIntegrationTest.java +++ b/gemma-core/src/test/java/ubic/gemma/persistence/service/TableMaintenanceUtilIntegrationTest.java @@ -17,6 +17,7 @@ import java.io.File; import java.nio.file.Path; +import java.util.Date; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -60,24 +61,26 @@ public void testWhenUserIsAnonymous() { @Test @WithMockUser(authorities = "GROUP_AGENT") public void testUpdateExpressionExperiment2CharacteristicEntries() { - tableMaintenanceUtil.updateExpressionExperiment2CharacteristicEntries( false ); - tableMaintenanceUtil.updateExpressionExperiment2CharacteristicEntries( ExpressionExperiment.class, false ); - tableMaintenanceUtil.updateExpressionExperiment2CharacteristicEntries( BioMaterial.class, false ); - tableMaintenanceUtil.updateExpressionExperiment2CharacteristicEntries( ExperimentalDesign.class, false ); - assertThatThrownBy( () -> { - tableMaintenanceUtil.updateExpressionExperiment2CharacteristicEntries( FactorValue.class, false ); - } ).isInstanceOf( IllegalArgumentException.class ); + tableMaintenanceUtil.updateExpressionExperiment2CharacteristicEntries( null, false ); + tableMaintenanceUtil.updateExpressionExperiment2CharacteristicEntries( ExpressionExperiment.class, null, false ); + tableMaintenanceUtil.updateExpressionExperiment2CharacteristicEntries( BioMaterial.class, null, false ); + tableMaintenanceUtil.updateExpressionExperiment2CharacteristicEntries( ExperimentalDesign.class, null, false ); + assertThatThrownBy( () -> tableMaintenanceUtil.updateExpressionExperiment2CharacteristicEntries( FactorValue.class, null, false ) ) + .isInstanceOf( IllegalArgumentException.class ); + tableMaintenanceUtil.updateExpressionExperiment2CharacteristicEntries( new Date(), false ); + assertThatThrownBy( () -> tableMaintenanceUtil.updateExpressionExperiment2CharacteristicEntries( new Date(), true ) ) + .isInstanceOf( IllegalArgumentException.class ); } @Test(expected = AccessDeniedException.class) public void testUpdateEE2CAsUser() { this.runAsAnonymous(); - tableMaintenanceUtil.updateExpressionExperiment2CharacteristicEntries( false ); + tableMaintenanceUtil.updateExpressionExperiment2CharacteristicEntries( null, false ); } @Test @WithMockUser(authorities = "GROUP_AGENT") public void testUpdateExpressionExperiment2ArrayDesignEntries() { - tableMaintenanceUtil.updateExpressionExperiment2ArrayDesignEntries(); + tableMaintenanceUtil.updateExpressionExperiment2ArrayDesignEntries( null ); } } diff --git a/gemma-core/src/test/java/ubic/gemma/persistence/service/common/description/CharacteristicDaoImplTest.java b/gemma-core/src/test/java/ubic/gemma/persistence/service/common/description/CharacteristicDaoImplTest.java index 460a6ba3f4..04a224ccce 100644 --- a/gemma-core/src/test/java/ubic/gemma/persistence/service/common/description/CharacteristicDaoImplTest.java +++ b/gemma-core/src/test/java/ubic/gemma/persistence/service/common/description/CharacteristicDaoImplTest.java @@ -174,7 +174,7 @@ public void testFindExperimentsByUris() { acl.insertAce( 0, BasePermission.READ, new AclPrincipalSid( "bob" ), false ); aclService.updateAcl( acl ); - int updated = tableMaintenanceUtil.updateExpressionExperiment2CharacteristicEntries( false ); + int updated = tableMaintenanceUtil.updateExpressionExperiment2CharacteristicEntries( null, false ); assertThat( updated ).isEqualTo( 1 ); sessionFactory.getCurrentSession().flush(); // ranking by level uses the order by field() which is not supported @@ -201,7 +201,7 @@ public void testFindExperimentsByUrisAsAnonymous() { aclService.updateAcl( acl ); sessionFactory.getCurrentSession().flush(); - int updated = tableMaintenanceUtil.updateExpressionExperiment2CharacteristicEntries( false ); + int updated = tableMaintenanceUtil.updateExpressionExperiment2CharacteristicEntries( null, false ); assertThat( updated ).isEqualTo( 1 ); sessionFactory.getCurrentSession().flush(); @@ -233,7 +233,7 @@ public void testFindExperimentsByUrisAsAdmin() { sessionFactory.getCurrentSession().persist( ee ); sessionFactory.getCurrentSession().flush(); aclService.createAcl( new AclObjectIdentity( ExpressionExperiment.class, ee.getId() ) ); - int updated = tableMaintenanceUtil.updateExpressionExperiment2CharacteristicEntries( false ); + int updated = tableMaintenanceUtil.updateExpressionExperiment2CharacteristicEntries( null, false ); assertThat( updated ).isEqualTo( 1 ); sessionFactory.getCurrentSession().flush(); // ranking by level uses the order by field() which is not supported diff --git a/gemma-web/src/main/java/ubic/gemma/web/scheduler/Ee2AdUpdateJob.java b/gemma-web/src/main/java/ubic/gemma/web/scheduler/Ee2AdUpdateJob.java new file mode 100644 index 0000000000..cc6ed1f5f7 --- /dev/null +++ b/gemma-web/src/main/java/ubic/gemma/web/scheduler/Ee2AdUpdateJob.java @@ -0,0 +1,22 @@ +package ubic.gemma.web.scheduler; + +import lombok.Setter; +import org.quartz.JobExecutionContext; +import org.quartz.StatefulJob; +import org.springframework.util.Assert; +import ubic.gemma.persistence.service.TableMaintenanceUtil; + +/** + * @author poirigui + */ +@Setter +public class Ee2AdUpdateJob extends SecureQuartzJobBean implements StatefulJob { + + private TableMaintenanceUtil tableMaintenanceUtil; + + @Override + protected void executeAsAgent( JobExecutionContext context ) { + Assert.notNull( tableMaintenanceUtil, "The tableMaintenanceUtil bean was not set." ); + tableMaintenanceUtil.updateExpressionExperiment2ArrayDesignEntries( context.getPreviousFireTime() ); + } +} diff --git a/gemma-web/src/main/java/ubic/gemma/web/scheduler/Ee2cUpdateJob.java b/gemma-web/src/main/java/ubic/gemma/web/scheduler/Ee2cUpdateJob.java new file mode 100644 index 0000000000..7f444de293 --- /dev/null +++ b/gemma-web/src/main/java/ubic/gemma/web/scheduler/Ee2cUpdateJob.java @@ -0,0 +1,28 @@ +package ubic.gemma.web.scheduler; + +import lombok.Setter; +import org.quartz.JobExecutionContext; +import org.quartz.StatefulJob; +import org.springframework.util.Assert; +import ubic.gemma.persistence.service.TableMaintenanceUtil; + +/** + * @author poirigui + */ +@Setter +public class Ee2cUpdateJob extends SecureQuartzJobBean implements StatefulJob { + + private TableMaintenanceUtil tableMaintenanceUtil; + + private Class level = null; + + @Override + public void executeAsAgent( JobExecutionContext context ) { + Assert.notNull( tableMaintenanceUtil, "The tableMaintenanceUtil bean was not set." ); + if ( level == null ) { + tableMaintenanceUtil.updateExpressionExperiment2CharacteristicEntries( context.getPreviousFireTime(), false ); + } else { + tableMaintenanceUtil.updateExpressionExperiment2CharacteristicEntries( level, context.getPreviousFireTime(), false ); + } + } +} diff --git a/gemma-web/src/main/java/ubic/gemma/web/scheduler/SecureInvoker.java b/gemma-web/src/main/java/ubic/gemma/web/scheduler/SecureInvoker.java new file mode 100644 index 0000000000..268ae8233c --- /dev/null +++ b/gemma-web/src/main/java/ubic/gemma/web/scheduler/SecureInvoker.java @@ -0,0 +1,58 @@ +package ubic.gemma.web.scheduler; + +import gemma.gsec.authentication.ManualAuthenticationService; +import lombok.Setter; +import lombok.extern.apachecommons.CommonsLog; +import org.springframework.security.core.AuthenticationException; +import org.springframework.security.core.context.SecurityContext; +import org.springframework.security.core.context.SecurityContextHolder; + +import java.util.concurrent.Callable; + +/** + * Invoke a callable as an authenticated user with given credentials. + * @author poirigui + */ +@Setter +@CommonsLog +public class SecureInvoker { + + private final ManualAuthenticationService manualAuthenticationService; + + private String userName; + private String password; + + /** + * Fallback to an anonymous authentication if the authentication fails. + */ + private boolean fallbackToAnonymous = false; + + public SecureInvoker( ManualAuthenticationService manualAuthenticationService ) { + this.manualAuthenticationService = manualAuthenticationService; + } + + public T invoke( Callable callable ) throws Exception { + SecurityContext previousSecurityContext = SecurityContextHolder.getContext(); + try { + try { + SecurityContext securityContext = SecurityContextHolder.createEmptyContext(); + securityContext.setAuthentication( manualAuthenticationService.attemptAuthentication( userName, password ) ); + SecurityContextHolder.setContext( securityContext ); + } catch ( AuthenticationException e ) { + if ( fallbackToAnonymous ) { + log.error( "Failed to authenticate schedule job, jobs probably won't work, but trying anonymous" ); + SecurityContext securityContext = SecurityContextHolder.createEmptyContext(); + SecurityContextHolder.setContext( securityContext ); + // gsec will call SecurityContextHolder.getContext().setAuthentication() + manualAuthenticationService.authenticateAnonymously(); + } else { + throw e; + } + } + assert SecurityContextHolder.getContext().getAuthentication() != null; + return callable.call(); + } finally { + SecurityContextHolder.setContext( previousSecurityContext ); + } + } +} diff --git a/gemma-web/src/main/java/ubic/gemma/web/scheduler/SecureMethodInvokingJobDetailFactoryBean.java b/gemma-web/src/main/java/ubic/gemma/web/scheduler/SecureMethodInvokingJobDetailFactoryBean.java index fc7b3073f4..de67a873c5 100644 --- a/gemma-web/src/main/java/ubic/gemma/web/scheduler/SecureMethodInvokingJobDetailFactoryBean.java +++ b/gemma-web/src/main/java/ubic/gemma/web/scheduler/SecureMethodInvokingJobDetailFactoryBean.java @@ -18,17 +18,9 @@ */ package ubic.gemma.web.scheduler; -import gemma.gsec.authentication.ManualAuthenticationService; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.springframework.beans.factory.annotation.Autowired; +import lombok.Setter; import org.springframework.scheduling.quartz.MethodInvokingJobDetailFactoryBean; -import org.springframework.security.core.AuthenticationException; -import org.springframework.security.core.context.SecurityContext; -import org.springframework.security.core.context.SecurityContextHolder; -import ubic.gemma.core.security.audit.AuditAdvice; -import ubic.gemma.core.security.authentication.UserManager; -import ubic.gemma.persistence.util.Settings; +import org.springframework.util.Assert; import java.lang.reflect.InvocationTargetException; @@ -37,56 +29,22 @@ * thread where Quartz is being run is authenticated as GROUP_AGENT. * * @author paul + * @see SecureQuartzJobBean */ -@SuppressWarnings({ "unused", "WeakerAccess" }) // Possible external use +@Setter public class SecureMethodInvokingJobDetailFactoryBean extends MethodInvokingJobDetailFactoryBean { - private static final Log log = LogFactory.getLog( AuditAdvice.class.getName() ); - @Autowired - ManualAuthenticationService manualAuthenticationService; - @Autowired - UserManager userManager; + private SecureInvoker secureInvoker; @Override public Object invoke() throws InvocationTargetException, IllegalAccessException { - - String serverUserName = Settings.getString( "gemma.agent.userName" ); - String serverPassword = Settings.getString( "gemma.agent.password" ); - Object result; - SecurityContext previousSecurityContext = SecurityContextHolder.getContext(); - + Assert.notNull( secureInvoker, "groupAgentInvoker is not set." ); try { - try { - assert manualAuthenticationService != null; - SecurityContext securityContext = SecurityContextHolder.createEmptyContext(); - securityContext.setAuthentication( manualAuthenticationService.attemptAuthentication( serverUserName, serverPassword ) ); - SecurityContextHolder.setContext( securityContext ); - } catch ( AuthenticationException e ) { - SecureMethodInvokingJobDetailFactoryBean.log - .error( "Failed to authenticate schedule job, jobs probably won't work, but trying anonymous" ); - SecurityContext securityContext = SecurityContextHolder.createEmptyContext(); - SecurityContextHolder.setContext( securityContext ); - // gsec will call SecurityContextHolder.getContext().setAuthentication() - manualAuthenticationService.authenticateAnonymously(); - } - assert SecurityContextHolder.getContext().getAuthentication() != null; - return super.invoke(); - } finally { - SecurityContextHolder.setContext( previousSecurityContext ); + return secureInvoker.invoke( super::invoke ); + } catch ( InvocationTargetException | IllegalAccessException | RuntimeException e ) { + throw e; + } catch ( Exception e ) { + throw new InvocationTargetException( e ); } } - - /** - * @param manualAuthenticationService the manualAuthenticationService to set - */ - public void setManualAuthenticationService( ManualAuthenticationService manualAuthenticationService ) { - this.manualAuthenticationService = manualAuthenticationService; - } - - /** - * @param userManager the userManager to set - */ - public void setUserManager( UserManager userManager ) { - this.userManager = userManager; - } } diff --git a/gemma-web/src/main/java/ubic/gemma/web/scheduler/SecureQuartzJobBean.java b/gemma-web/src/main/java/ubic/gemma/web/scheduler/SecureQuartzJobBean.java new file mode 100644 index 0000000000..02287a1480 --- /dev/null +++ b/gemma-web/src/main/java/ubic/gemma/web/scheduler/SecureQuartzJobBean.java @@ -0,0 +1,32 @@ +package ubic.gemma.web.scheduler; + +import lombok.Setter; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; +import org.springframework.scheduling.quartz.QuartzJobBean; +import org.springframework.util.Assert; + +/** + * + */ +@Setter +public abstract class SecureQuartzJobBean extends QuartzJobBean { + + private SecureInvoker secureInvoker; + + @Override + protected final void executeInternal( JobExecutionContext context ) throws JobExecutionException { + Assert.notNull( secureInvoker, "The secureInvoker bean is not set." ); + try { + secureInvoker.invoke( () -> { + executeAsAgent( context ); + return null; + } ); + } catch ( Exception e ) { + throw new JobExecutionException( e ); + } + } + + protected abstract void executeAsAgent( JobExecutionContext context ) throws JobExecutionException; +} + diff --git a/gemma-web/src/main/resources/ubic/gemma/applicationContext-schedule.xml b/gemma-web/src/main/resources/ubic/gemma/applicationContext-schedule.xml index 7949b18295..5e4ac1fb79 100644 --- a/gemma-web/src/main/resources/ubic/gemma/applicationContext-schedule.xml +++ b/gemma-web/src/main/resources/ubic/gemma/applicationContext-schedule.xml @@ -35,6 +35,14 @@ + + + + + + + + @@ -58,6 +67,7 @@ + @@ -70,6 +80,7 @@ + @@ -82,6 +93,7 @@ + @@ -94,6 +106,7 @@ + @@ -101,16 +114,15 @@ - - - - - - ubic.gemma.model.expression.experiment.ExpressionExperiment - false - + + + + + + + + - @@ -118,16 +130,15 @@ - - - - - - ubic.gemma.model.expression.biomaterial.BioMaterial - false - + + + + + + + + - @@ -136,16 +147,15 @@ - - - - - - ubic.gemma.model.expression.experiment.ExperimentalDesign - false - + + + + + + + + - @@ -154,10 +164,14 @@ - - - - + + + + + + + + diff --git a/gemma-web/src/test/java/ubic/gemma/web/scheduler/SchedulerSecurityTest.java b/gemma-web/src/test/java/ubic/gemma/web/scheduler/SchedulerSecurityTest.java index 8d3fb8ef19..8c0f1e66bf 100644 --- a/gemma-web/src/test/java/ubic/gemma/web/scheduler/SchedulerSecurityTest.java +++ b/gemma-web/src/test/java/ubic/gemma/web/scheduler/SchedulerSecurityTest.java @@ -18,16 +18,27 @@ */ package ubic.gemma.web.scheduler; -import gemma.gsec.authentication.ManualAuthenticationService; import org.apache.commons.lang3.RandomStringUtils; import org.junit.Test; +import org.quartz.JobDataMap; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.security.core.authority.SimpleGrantedAuthority; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.stereotype.Component; import ubic.gemma.core.analysis.report.WhatsNewService; +import ubic.gemma.persistence.service.TableMaintenanceUtil; import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentService; import ubic.gemma.web.util.BaseSpringWebTest; import java.lang.reflect.InvocationTargetException; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.*; + /** * Tests security of methods run by Quartz. * @@ -42,7 +53,11 @@ public class SchedulerSecurityTest extends BaseSpringWebTest { private WhatsNewService whatsNewService; @Autowired - private ManualAuthenticationService manualAuthenticationService; + private TableMaintenanceUtil tableMaintenanceUtil; + + @Autowired + @Qualifier("groupAgentSecureInvoker") + private SecureInvoker secureInvoker; /* * Tests whether we can run a secured method that has been granted to GROUP_AGENT @@ -59,7 +74,7 @@ public void runSecuredMethodOnSchedule() throws Exception { jobDetail.setConcurrent( false ); jobDetail.setBeanName( jobName ); jobDetail.afterPropertiesSet(); // needed when we do this programatically. - jobDetail.setManualAuthenticationService( this.manualAuthenticationService ); + jobDetail.setSecureInvoker( this.secureInvoker ); jobDetail.invoke(); @@ -81,7 +96,7 @@ public void runSecuredMethodOnScheduleMultiGroup() throws Exception { jobDetail.setConcurrent( false ); jobDetail.setBeanName( jobName ); jobDetail.afterPropertiesSet(); // needed when we do this programatically. - jobDetail.setManualAuthenticationService( this.manualAuthenticationService ); + jobDetail.setSecureInvoker( this.secureInvoker ); jobDetail.invoke(); @@ -106,9 +121,44 @@ public void runUnauthorizedMethodOnSchedule() throws Exception { jobDetail.setConcurrent( false ); jobDetail.setBeanName( jobName ); jobDetail.afterPropertiesSet(); // needed when we do this programatically. - jobDetail.setManualAuthenticationService( this.manualAuthenticationService ); + jobDetail.setSecureInvoker( this.secureInvoker ); jobDetail.invoke(); } + @Component + public static class TestSecureJob extends SecureQuartzJobBean { + + private TableMaintenanceUtil tableMaintenanceUtil; + + @Override + protected void executeAsAgent( JobExecutionContext context ) { + assertNotNull( tableMaintenanceUtil ); + assertNotNull( SecurityContextHolder.getContext().getAuthentication() ); + assertTrue( SecurityContextHolder.getContext().getAuthentication().isAuthenticated() ); + assertTrue( SecurityContextHolder.getContext().getAuthentication().getAuthorities() + .contains( new SimpleGrantedAuthority( "GROUP_AGENT" ) ) ); + context.setResult( "Hello world!" ); + } + + @SuppressWarnings("unused") + public void setTableMaintenanceUtil( TableMaintenanceUtil tableMaintenanceUtil ) { + this.tableMaintenanceUtil = tableMaintenanceUtil; + } + } + + @Autowired + private TestSecureJob testSecureJob; + + @Test + public void testSecureJob() throws JobExecutionException { + JobExecutionContext context = mock(); + JobDataMap jdm = new JobDataMap(); + jdm.put( "tableMaintenanceUtil", tableMaintenanceUtil ); + jdm.put( "secureInvoker", secureInvoker ); + when( context.getScheduler() ).thenReturn( mock() ); + when( context.getMergedJobDataMap() ).thenReturn( jdm ); + testSecureJob.execute( context ); + verify( context ).setResult( "Hello world!" ); + } } From 1aaa15fafe9a60ce4f413ddfc222b583a7cb9402 Mon Sep 17 00:00:00 2001 From: Guillaume Poirier-Morency Date: Sat, 16 Mar 2024 16:12:15 -0700 Subject: [PATCH 10/61] Make the secure invoker a constructor argument --- .../SecureMethodInvokingJobDetailFactoryBean.java | 10 +++++----- .../ubic/gemma/applicationContext-schedule.xml | 14 +++++++------- .../gemma/web/scheduler/SchedulerSecurityTest.java | 9 +++------ 3 files changed, 15 insertions(+), 18 deletions(-) diff --git a/gemma-web/src/main/java/ubic/gemma/web/scheduler/SecureMethodInvokingJobDetailFactoryBean.java b/gemma-web/src/main/java/ubic/gemma/web/scheduler/SecureMethodInvokingJobDetailFactoryBean.java index de67a873c5..b5dce2a801 100644 --- a/gemma-web/src/main/java/ubic/gemma/web/scheduler/SecureMethodInvokingJobDetailFactoryBean.java +++ b/gemma-web/src/main/java/ubic/gemma/web/scheduler/SecureMethodInvokingJobDetailFactoryBean.java @@ -18,9 +18,7 @@ */ package ubic.gemma.web.scheduler; -import lombok.Setter; import org.springframework.scheduling.quartz.MethodInvokingJobDetailFactoryBean; -import org.springframework.util.Assert; import java.lang.reflect.InvocationTargetException; @@ -31,14 +29,16 @@ * @author paul * @see SecureQuartzJobBean */ -@Setter public class SecureMethodInvokingJobDetailFactoryBean extends MethodInvokingJobDetailFactoryBean { - private SecureInvoker secureInvoker; + private final SecureInvoker secureInvoker; + + public SecureMethodInvokingJobDetailFactoryBean( SecureInvoker secureInvoker ) { + this.secureInvoker = secureInvoker; + } @Override public Object invoke() throws InvocationTargetException, IllegalAccessException { - Assert.notNull( secureInvoker, "groupAgentInvoker is not set." ); try { return secureInvoker.invoke( super::invoke ); } catch ( InvocationTargetException | IllegalAccessException | RuntimeException e ) { diff --git a/gemma-web/src/main/resources/ubic/gemma/applicationContext-schedule.xml b/gemma-web/src/main/resources/ubic/gemma/applicationContext-schedule.xml index 5e4ac1fb79..9889328bee 100644 --- a/gemma-web/src/main/resources/ubic/gemma/applicationContext-schedule.xml +++ b/gemma-web/src/main/resources/ubic/gemma/applicationContext-schedule.xml @@ -51,10 +51,10 @@ + - @@ -64,10 +64,10 @@ + - @@ -77,10 +77,10 @@ + - @@ -90,10 +90,10 @@ + - @@ -101,12 +101,11 @@ - + + - @@ -181,6 +180,7 @@ + diff --git a/gemma-web/src/test/java/ubic/gemma/web/scheduler/SchedulerSecurityTest.java b/gemma-web/src/test/java/ubic/gemma/web/scheduler/SchedulerSecurityTest.java index 8c0f1e66bf..2295dcf189 100644 --- a/gemma-web/src/test/java/ubic/gemma/web/scheduler/SchedulerSecurityTest.java +++ b/gemma-web/src/test/java/ubic/gemma/web/scheduler/SchedulerSecurityTest.java @@ -68,13 +68,12 @@ public void runSecuredMethodOnSchedule() throws Exception { String jobName = "job_" + RandomStringUtils.randomAlphabetic( 10 ); - SecureMethodInvokingJobDetailFactoryBean jobDetail = new SecureMethodInvokingJobDetailFactoryBean(); + SecureMethodInvokingJobDetailFactoryBean jobDetail = new SecureMethodInvokingJobDetailFactoryBean( this.secureInvoker ); jobDetail.setTargetMethod( "generateWeeklyReport" ); jobDetail.setTargetObject( whatsNewService ); // access should be ok for GROUP_AGENT. jobDetail.setConcurrent( false ); jobDetail.setBeanName( jobName ); jobDetail.afterPropertiesSet(); // needed when we do this programatically. - jobDetail.setSecureInvoker( this.secureInvoker ); jobDetail.invoke(); @@ -89,14 +88,13 @@ public void runSecuredMethodOnScheduleMultiGroup() throws Exception { String jobName = "job_" + RandomStringUtils.randomAlphabetic( 10 ); - SecureMethodInvokingJobDetailFactoryBean jobDetail = new SecureMethodInvokingJobDetailFactoryBean(); + SecureMethodInvokingJobDetailFactoryBean jobDetail = new SecureMethodInvokingJobDetailFactoryBean( this.secureInvoker ); jobDetail.setTargetMethod( "findByUpdatedLimit" ); jobDetail.setArguments( new Object[] { 10 } ); jobDetail.setTargetObject( expressionExperimentService ); // access should be ok for GROUP_AGENT. jobDetail.setConcurrent( false ); jobDetail.setBeanName( jobName ); jobDetail.afterPropertiesSet(); // needed when we do this programatically. - jobDetail.setSecureInvoker( this.secureInvoker ); jobDetail.invoke(); @@ -114,14 +112,13 @@ public void runUnauthorizedMethodOnSchedule() throws Exception { /* * Mimics configuration in xml. */ - SecureMethodInvokingJobDetailFactoryBean jobDetail = new SecureMethodInvokingJobDetailFactoryBean(); + SecureMethodInvokingJobDetailFactoryBean jobDetail = new SecureMethodInvokingJobDetailFactoryBean( this.secureInvoker ); jobDetail.setTargetMethod( "remove" ); jobDetail.setArguments( new Object[] { null } ); jobDetail.setTargetObject( expressionExperimentService ); // no access jobDetail.setConcurrent( false ); jobDetail.setBeanName( jobName ); jobDetail.afterPropertiesSet(); // needed when we do this programatically. - jobDetail.setSecureInvoker( this.secureInvoker ); jobDetail.invoke(); } From f8054a829a4dd28f39428761b29ad959a24a6673 Mon Sep 17 00:00:00 2001 From: Paul Pavlidis Date: Fri, 3 May 2024 17:46:40 -0700 Subject: [PATCH 11/61] reverting GO loading and instead reading the label from the gene2go file I don't know why were weren't doing this before. --- .../apps/NCBIGene2GOAssociationLoaderCLI.java | 4 ---- .../NCBIGene2GOAssociationParser.java | 21 +++++++++---------- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/NCBIGene2GOAssociationLoaderCLI.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/NCBIGene2GOAssociationLoaderCLI.java index eb8d996e63..17ed5222f2 100755 --- a/gemma-cli/src/main/java/ubic/gemma/core/apps/NCBIGene2GOAssociationLoaderCLI.java +++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/NCBIGene2GOAssociationLoaderCLI.java @@ -63,9 +63,6 @@ public class NCBIGene2GOAssociationLoaderCLI extends AbstractAuthenticatedCLI { @Autowired private ExternalDatabaseService externalDatabaseService; - @Autowired - private GeneOntologyService goService; - private String filePath = null; @Override @@ -84,7 +81,6 @@ protected void buildOptions( Options options ) { @Override protected void doWork() throws Exception { NCBIGene2GOAssociationLoader gene2GOAssLoader = new NCBIGene2GOAssociationLoader(); - OntologyUtils.ensureInitialized( goService ); gene2GOAssLoader.setPersisterHelper( persisterHelper ); Collection taxa = taxonService.loadAll(); diff --git a/gemma-core/src/main/java/ubic/gemma/core/loader/association/NCBIGene2GOAssociationParser.java b/gemma-core/src/main/java/ubic/gemma/core/loader/association/NCBIGene2GOAssociationParser.java index d1b66abebc..f756a4a066 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/loader/association/NCBIGene2GOAssociationParser.java +++ b/gemma-core/src/main/java/ubic/gemma/core/loader/association/NCBIGene2GOAssociationParser.java @@ -38,7 +38,7 @@ /** * This parses GO annotations from NCBI. See readme. - * + * * 

  * tax_id:
  * the unique identifier provided by NCBI Taxonomy
@@ -66,8 +66,7 @@
  * @author keshav
  * @author pavlidis
  */
-public class NCBIGene2GOAssociationParser extends BasicLineParser
-        implements QueuingParser {
+public class NCBIGene2GOAssociationParser extends BasicLineParser implements QueuingParser {
 
     private static final String COMMENT_INDICATOR = "#";
     private static final Set ignoredEvidenceCodes = new HashSet<>();
@@ -85,6 +84,9 @@ public class NCBIGene2GOAssociationParser extends BasicLineParser queue;
 
     private int count = 0;
@@ -132,18 +134,16 @@ protected void addResult( Gene2GOAssociation obj ) {
      * Note that "-" means a missing value, which in practice only occurs in the "qualifier" and "pubmed" columns.
      *
      * @param  line line
-     * @return      Object
+     * @return Object
      */
     @SuppressWarnings({ "unused", "WeakerAccess" }) // Possible external use
     public Gene2GOAssociation mapFromGene2GO( String line ) {
 
         String[] values = StringUtils.splitPreserveAllTokens( line, "\t" );
 
-        if ( line.startsWith( NCBIGene2GOAssociationParser.COMMENT_INDICATOR ) )
-            return null;
+        if ( line.startsWith( NCBIGene2GOAssociationParser.COMMENT_INDICATOR ) ) return null;
 
-        if ( values.length < 8 )
-            return null;
+        if ( values.length < 8 ) return null;
 
         Integer taxonId;
         try {
@@ -163,7 +163,7 @@ public Gene2GOAssociation mapFromGene2GO( String line ) {
         Characteristic oe = Characteristic.Factory.newInstance();
         String value = values[GO_ID].replace( ":", "_" );
         oe.setValueUri( GeneOntologyService.BASE_GO_URI + value );
-        oe.setValue( value ); // NOTE: this is not the GO term label, it's the GO ID. It's OK because we do label lookups
+        oe.setValue( values[GO_TERM_LABEL] );
 
         // g2GOAss.setSource( ncbiGeneDb );
 
@@ -196,8 +196,7 @@ public Gene2GOAssociation mapFromGene2GO( String line ) {
 
     @Override
     public void parse( InputStream inputStream, BlockingQueue aqueue ) throws IOException {
-        if ( inputStream == null )
-            throw new IllegalArgumentException( "InputStream was null" );
+        if ( inputStream == null ) throw new IllegalArgumentException( "InputStream was null" );
         this.queue = aqueue;
         super.parse( inputStream );
 

From 73b869c6acb0fb7a740c388a629b98e076b481d1 Mon Sep 17 00:00:00 2001
From: Paul Pavlidis 
Date: Mon, 6 May 2024 11:27:16 -0700
Subject: [PATCH 12/61] minor

---
 .../loader/association/NCBIGene2GOAssociationLoader.java  | 8 ++++----
 .../model/association/Gene2OntologyEntryAssociation.java  | 6 ++++++
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/gemma-core/src/main/java/ubic/gemma/core/loader/association/NCBIGene2GOAssociationLoader.java b/gemma-core/src/main/java/ubic/gemma/core/loader/association/NCBIGene2GOAssociationLoader.java
index 2203821ae5..51e178548a 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/loader/association/NCBIGene2GOAssociationLoader.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/loader/association/NCBIGene2GOAssociationLoader.java
@@ -140,13 +140,13 @@ private void load( BlockingQueue queue ) {
         Collection itemsToPersist = new ArrayList<>();
         try {
             while ( !( producerDone.get() && queue.isEmpty() ) ) {
-                Gene2GOAssociation associations = queue.poll();
+                Gene2GOAssociation association = queue.poll();
 
-                if ( associations == null ) {
+                if ( association == null ) {
                     continue;
                 }
 
-                itemsToPersist.add( associations );
+                itemsToPersist.add( association );
                 if ( ++count % NCBIGene2GOAssociationLoader.BATCH_SIZE == 0 ) {
                     persisterHelper.persist( itemsToPersist );
                     itemsToPersist.clear();
@@ -160,7 +160,7 @@ private void load( BlockingQueue queue ) {
                     double meanspt = secspt / cpt;
 
                     String progString = "Processed and loaded " + count + " (" + secsperthousand
-                            + " seconds elapsed, average per thousand=" + String.format( "%.2f", meanspt ) + ")";
+                            + " seconds elapsed, average per thousand=" + String.format( "%.2f", meanspt ) + "), last was: " + association;
                     NCBIGene2GOAssociationLoader.log.info( progString );
                     millis = System.currentTimeMillis();
                 }
diff --git a/gemma-core/src/main/java/ubic/gemma/model/association/Gene2OntologyEntryAssociation.java b/gemma-core/src/main/java/ubic/gemma/model/association/Gene2OntologyEntryAssociation.java
index 21a3a61f60..26a2e30e76 100644
--- a/gemma-core/src/main/java/ubic/gemma/model/association/Gene2OntologyEntryAssociation.java
+++ b/gemma-core/src/main/java/ubic/gemma/model/association/Gene2OntologyEntryAssociation.java
@@ -45,4 +45,10 @@ public Characteristic getOntologyEntry() {
         return this.ontologyEntry;
     }
 
+    @Override
+    public String toString() {
+        if ( gene == null || ontologyEntry == null ) return "?";
+        return gene + " ---> " + ontologyEntry.getValue() + " [" + ontologyEntry.getValueUri() + "]";
+    }
+
 }
\ No newline at end of file

From 4ebebeb2bdecf72b50b86ddfd71f1a3212522b9a Mon Sep 17 00:00:00 2001
From: Guillaume Poirier-Morency 
Date: Mon, 6 May 2024 13:38:31 -0700
Subject: [PATCH 13/61] Honor JsonInclude.NON_NULL on the category when
 serializing a FactorValueBasicValueObject

---
 .../serializers/FactorValueBasicValueObjectSerializer.java    | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/gemma-rest/src/main/java/ubic/gemma/rest/serializers/FactorValueBasicValueObjectSerializer.java b/gemma-rest/src/main/java/ubic/gemma/rest/serializers/FactorValueBasicValueObjectSerializer.java
index d923b805a1..eecfb36a7d 100644
--- a/gemma-rest/src/main/java/ubic/gemma/rest/serializers/FactorValueBasicValueObjectSerializer.java
+++ b/gemma-rest/src/main/java/ubic/gemma/rest/serializers/FactorValueBasicValueObjectSerializer.java
@@ -19,7 +19,9 @@ public void serialize( FactorValueBasicValueObject factorValueBasicValueObject,
         jsonGenerator.writeObjectField( "id", factorValueBasicValueObject.getId() );
         jsonGenerator.writeStringField( "ontologyId", FactorValueOntologyUtils.getUri( factorValueBasicValueObject.getId() ) );
         jsonGenerator.writeObjectField( "experimentalFactorId", factorValueBasicValueObject.getExperimentalFactorId() );
-        jsonGenerator.writeObjectField( "experimentalFactorCategory", factorValueBasicValueObject.getExperimentalFactorCategory() );
+        if ( factorValueBasicValueObject.getExperimentalFactorCategory() != null ) {
+            jsonGenerator.writeObjectField( "experimentalFactorCategory", factorValueBasicValueObject.getExperimentalFactorCategory() );
+        }
         if ( factorValueBasicValueObject.getMeasurement() != null ) {
             jsonGenerator.writeObjectField( "measurement", factorValueBasicValueObject.getMeasurement() );
         }

From bc8aa1a6e934adb0031ae3fa4a8514583c048ae1 Mon Sep 17 00:00:00 2001
From: Paul Pavlidis 
Date: Mon, 6 May 2024 16:03:05 -0700
Subject: [PATCH 14/61] more fixes for logic of batch effect badges

Should more reliably show that batches with only one sample were encountered.
---
 .../BatchInfoPopulationHelperServiceImpl.java | 66 +++++++------------
 .../BatchInfoPopulationServiceImpl.java       |  2 +
 .../ExpressionExperimentServiceImpl.java      | 16 +++--
 .../RNASeqBatchInfoPopulationTest.java        | 10 +++
 .../GSE173615.fastq-headers-table.txt         | 18 +++++
 5 files changed, 64 insertions(+), 48 deletions(-)
 create mode 100644 gemma-core/src/test/resources/data/analysis/preprocess/batcheffects/fastqheaders/GSE173615.fastq-headers-table.txt

diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchInfoPopulationHelperServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchInfoPopulationHelperServiceImpl.java
index 44104053c2..b75b95bb43 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchInfoPopulationHelperServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchInfoPopulationHelperServiceImpl.java
@@ -33,6 +33,7 @@
 import ubic.gemma.persistence.service.expression.experiment.ExperimentalDesignService;
 import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentService;
 
+import java.beans.Expression;
 import java.text.DateFormat;
 import java.util.*;
 
@@ -81,15 +82,14 @@ public ExperimentalFactor createRnaSeqBatchFactor( ExpressionExperiment ee, Map<
          */
         Map> batchIdToHeaders;
         try {
-            batchIdToHeaders = this
-                    .convertHeadersToBatches( ee, headers.values() );
+            batchIdToHeaders = this.convertHeadersToBatches( ee, headers.values() );
         } catch ( FASTQHeadersPresentButNotUsableException e ) {
-            this.auditTrailService.addUpdateEvent( ee, UninformativeFASTQHeadersForBatchingEvent.class, "Batches unable to be determined",
-                    "RNA-seq experiment, FASTQ headers and platform not informative for batches" );
+            log.info( "Batches unable to be determined from headers: " + ee );
+            this.auditTrailService.addUpdateEvent( ee, UninformativeFASTQHeadersForBatchingEvent.class, "Batches unable to be determined", "RNA-seq experiment, FASTQ headers and platform not informative for batches" );
             return null;
         } catch ( SingletonBatchesException e ) {
-            this.auditTrailService.addUpdateEvent( ee, SingletonBatchInvalidEvent.class, "At least one singleton batch",
-                    "RNA-seq experiment, FASTQ headers indicate at least one batch of just one sample" );
+            log.info( "At least one singleton batch: " + ee + " " + e.getMessage() );
+            this.auditTrailService.addUpdateEvent( ee, SingletonBatchInvalidEvent.class, "At least one singleton batch", "RNA-seq experiment, FASTQ headers indicate at least one batch of just one sample" );
             return null;
         }
 
@@ -216,12 +216,7 @@ Map> convertDatesToBatches( List allDates, List> convertDatesToBatches( List allDates, List() );
                         mergedAnySingletons = true;
                     } else {
-                        BatchInfoPopulationHelperServiceImpl.log
-                                .warn( "Singleton resolved by adding to the last batch: gap is " + String
-                                        .format( "%.2f",
-                                                ( currentDate.getTime() - lastDate.getTime() ) / ( double ) ( 1000 * 60
-                                                        * 60 * 24 ) )
-                                        + " hours." );
+                        BatchInfoPopulationHelperServiceImpl.log.warn( "Singleton resolved by adding to the last batch: gap is " + String.format( "%.2f", ( currentDate.getTime() - lastDate.getTime() ) / ( double ) ( 1000 * 60 * 60 * 24 ) ) + " hours." );
                         // don't start a new batch, fall through.
                     }
 
@@ -267,8 +252,7 @@ Map> convertDatesToBatches( List allDates, List> convertDatesToBatches( List allDates, List> convertHeadersToBatches( ExpressionExperiment ee, Collection headers )
-            throws FASTQHeadersPresentButNotUsableException, SingletonBatchesException {
+    private Map> convertHeadersToBatches( ExpressionExperiment ee, Collection headers ) throws FASTQHeadersPresentButNotUsableException, SingletonBatchesException {
         Map> result = new LinkedHashMap<>();
 
         Map> goodHeaderSampleInfos = new HashMap<>();
@@ -351,7 +334,7 @@ private Map> convertHeadersToBatches( ExpressionExper
          */
         Map> batchInfos = new HashMap<>();
         if ( !goodHeaderSampleInfos.isEmpty() ) {
-            goodHeaderSampleInfos = batch( goodHeaderSampleInfos, headers.size() );
+            goodHeaderSampleInfos = batch( ee, goodHeaderSampleInfos, headers.size() );
             batchInfos.putAll( goodHeaderSampleInfos );
         }
 
@@ -427,12 +410,13 @@ Map> convertHeadersToBatches( Collection head
      * and involves recreating the map multiple times
      *
      *
+     * @param  ee experiment
      * @param  batchInfos only of samples that have "good" headers
      * @param  numSamples how many samples
      * @return Map of batches (represented by the appropriate FastqHeaderData) to samples that are in the
      *                    batch.
      */
-    private Map> batch( Map> batchInfos, int numSamples ) {
+    private Map> batch( ExpressionExperiment ee, Map> batchInfos, int numSamples ) {
 
         int numBatches = batchInfos.size();
 
@@ -458,12 +442,17 @@ private Map> batch( Map> updatedBatchInfos = dropResolution( batchInfos );
 
+            if ( updatedBatchInfos.size() == 1 ) {
+                log.info( "Could not resolve singleton batches despite dropping resolution" );
+                return batchInfos; // return the previous one.
+            }
+
             if ( updatedBatchInfos.size() == batchInfos.size() ) {
                 // we've reached the bottom
                 return updatedBatchInfos;
             }
 
-            return batch( updatedBatchInfos, numSamples ); // start over with lower resolution
+            return batch( ee, updatedBatchInfos, numSamples ); // start over with lower resolution
         }
         // reasonable number of samples per batch -- proceed. 
         return batchInfos;
@@ -872,8 +861,7 @@ public boolean equals( Object obj ) {
      * for microarrays, it a map of batchids to dates
      * d2fv is populated by this call to be a map of headers or dates to factor values
      */
-    private  ExperimentalFactor createExperimentalFactor( ExpressionExperiment ee,
-            Map> descriptorsToBatch, Map d2fv ) {
+    private  ExperimentalFactor createExperimentalFactor( ExpressionExperiment ee, Map> descriptorsToBatch, Map d2fv ) {
         ExperimentalFactor ef = null;
         if ( descriptorsToBatch == null || descriptorsToBatch.size() < 2 ) {
             if ( descriptorsToBatch != null ) {
@@ -939,8 +927,7 @@ private ExperimentalFactor makeFactorForBatch( ExpressionExperiment ee ) {
         ef.setCategory( this.getBatchFactorCategory() );
         ef.setExperimentalDesign( ed );
         ef.setName( ExperimentalDesignUtils.BATCH_FACTOR_NAME );
-        ef.setDescription(
-                "Scan date or similar proxy for 'batch'" + " extracted from the raw data files." );
+        ef.setDescription( "Scan date or similar proxy for 'batch'" + " extracted from the raw data files." );
 
         ef = this.persistFactor( ee, ef );
         return ef;
@@ -959,10 +946,7 @@ private ExperimentalFactor persistFactor( ExpressionExperiment ee, ExperimentalF
 
     private String formatBatchName( int batchNum, DateFormat df, Date d ) {
         String batchDateString;
-        batchDateString = ExperimentalDesignUtils.BATCH_FACTOR_NAME_PREFIX + StringUtils
-                .leftPad( Integer.toString( batchNum ), 2, "0" ) + "_"
-                + df
-                .format( DateUtils.truncate( d, Calendar.HOUR ) );
+        batchDateString = ExperimentalDesignUtils.BATCH_FACTOR_NAME_PREFIX + StringUtils.leftPad( Integer.toString( batchNum ), 2, "0" ) + "_" + df.format( DateUtils.truncate( d, Calendar.HOUR ) );
         return batchDateString;
     }
 
@@ -974,9 +958,7 @@ private String formatBatchName( int batchNum, DateFormat df, Date d ) {
      *                     separate batch.
      */
     private boolean gapIsLarge( Date earlierDate, Date date ) {
-        return !DateUtils.isSameDay( date, earlierDate ) && DateUtils
-                .addHours( earlierDate, BatchInfoPopulationHelperServiceImpl.MAX_GAP_BETWEEN_SAMPLES_TO_BE_SAME_BATCH )
-                .before( date );
+        return !DateUtils.isSameDay( date, earlierDate ) && DateUtils.addHours( earlierDate, BatchInfoPopulationHelperServiceImpl.MAX_GAP_BETWEEN_SAMPLES_TO_BE_SAME_BATCH ).before( date );
     }
 
     private Characteristic getBatchFactorCategory() {
diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchInfoPopulationServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchInfoPopulationServiceImpl.java
index dd114a3e82..a20adfd7d9 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchInfoPopulationServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchInfoPopulationServiceImpl.java
@@ -256,6 +256,8 @@ private void getBatchDataFromRawFiles( ExpressionExperiment ee, Collection h = bs.readFastqHeaders( "GSE173615" );
+
+        Map> batches = s.convertHeadersToBatches( h.values() );
+    }
+
     @After
     public void teardown() {
         if ( ee != null )
diff --git a/gemma-core/src/test/resources/data/analysis/preprocess/batcheffects/fastqheaders/GSE173615.fastq-headers-table.txt b/gemma-core/src/test/resources/data/analysis/preprocess/batcheffects/fastqheaders/GSE173615.fastq-headers-table.txt
new file mode 100644
index 0000000000..e3c866a5ac
--- /dev/null
+++ b/gemma-core/src/test/resources/data/analysis/preprocess/batcheffects/fastqheaders/GSE173615.fastq-headers-table.txt
@@ -0,0 +1,18 @@
+GSM5272654	SRR14366630_1	GPL24247	https://www.ncbi.nlm.nih.gov/sra?term=SRX10719174	@SRR14366630.1.1 A01045:206:HCJVKDSXY:3:1101:18358:1031 length=150
+GSM5272654	SRR14366630_2	GPL24247	https://www.ncbi.nlm.nih.gov/sra?term=SRX10719174	@SRR14366630.1.2 A01045:206:HCJVKDSXY:3:1101:18358:1031 length=150
+GSM5272652	SRR14366628_1	GPL24247	https://www.ncbi.nlm.nih.gov/sra?term=SRX10719172	@SRR14366628.1.1 A01045:206:HCJVKDSXY:3:1101:14235:1031 length=150
+GSM5272652	SRR14366628_2	GPL24247	https://www.ncbi.nlm.nih.gov/sra?term=SRX10719172	@SRR14366628.1.2 A01045:206:HCJVKDSXY:3:1101:14235:1031 length=150
+GSM5272657	SRR14366633_1	GPL24247	https://www.ncbi.nlm.nih.gov/sra?term=SRX10719177	@SRR14366633.1.1 A01045:206:HCJVKDSXY:3:1101:21956:1031 length=150
+GSM5272657	SRR14366633_2	GPL24247	https://www.ncbi.nlm.nih.gov/sra?term=SRX10719177	@SRR14366633.1.2 A01045:206:HCJVKDSXY:3:1101:21956:1031 length=150
+GSM5272660	SRR14366636_1	GPL24247	https://www.ncbi.nlm.nih.gov/sra?term=SRX10719180	@SRR14366636.1.1 A01045:206:HCJVKDSXY:4:1101:29686:1016 length=150
+GSM5272660	SRR14366636_2	GPL24247	https://www.ncbi.nlm.nih.gov/sra?term=SRX10719180	@SRR14366636.1.2 A01045:206:HCJVKDSXY:4:1101:29686:1016 length=150
+GSM5272658	SRR14366634_1	GPL24247	https://www.ncbi.nlm.nih.gov/sra?term=SRX10719178	@SRR14366634.1.1 A01045:206:HCJVKDSXY:3:1101:13530:1031 length=150
+GSM5272658	SRR14366634_2	GPL24247	https://www.ncbi.nlm.nih.gov/sra?term=SRX10719178	@SRR14366634.1.2 A01045:206:HCJVKDSXY:3:1101:13530:1031 length=150
+GSM5272653	SRR14366629_1	GPL24247	https://www.ncbi.nlm.nih.gov/sra?term=SRX10719173	@SRR14366629.1.1 A01045:206:HCJVKDSXY:3:1101:19515:1031 length=150
+GSM5272653	SRR14366629_2	GPL24247	https://www.ncbi.nlm.nih.gov/sra?term=SRX10719173	@SRR14366629.1.2 A01045:206:HCJVKDSXY:3:1101:19515:1031 length=150
+GSM5272655	SRR14366631_1	GPL24247	https://www.ncbi.nlm.nih.gov/sra?term=SRX10719175	@SRR14366631.1.1 A01045:206:HCJVKDSXY:3:1101:16947:1031 length=150
+GSM5272655	SRR14366631_2	GPL24247	https://www.ncbi.nlm.nih.gov/sra?term=SRX10719175	@SRR14366631.1.2 A01045:206:HCJVKDSXY:3:1101:16947:1031 length=150
+GSM5272656	SRR14366632_1	GPL24247	https://www.ncbi.nlm.nih.gov/sra?term=SRX10719176	@SRR14366632.1.1 A01045:206:HCJVKDSXY:3:1101:16152:1031 length=150
+GSM5272656	SRR14366632_2	GPL24247	https://www.ncbi.nlm.nih.gov/sra?term=SRX10719176	@SRR14366632.1.2 A01045:206:HCJVKDSXY:3:1101:16152:1031 length=150
+GSM5272659	SRR14366635_1	GPL24247	https://www.ncbi.nlm.nih.gov/sra?term=SRX10719179	@SRR14366635.1.1 A01045:206:HCJVKDSXY:3:1101:14507:1031 length=150
+GSM5272659	SRR14366635_2	GPL24247	https://www.ncbi.nlm.nih.gov/sra?term=SRX10719179	@SRR14366635.1.2 A01045:206:HCJVKDSXY:3:1101:14507:1031 length=150

From c4002208c78739241bbb5d66b109282c6d163953 Mon Sep 17 00:00:00 2001
From: Guillaume Poirier-Morency 
Date: Mon, 6 May 2024 16:34:39 -0700
Subject: [PATCH 15/61] Use more efficient writeStringField() instead of
 writeObjectField() in FV serialization

---
 ...tractFactorValueValueObjectSerializer.java | 30 +++++++++----------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/gemma-rest/src/main/java/ubic/gemma/rest/serializers/AbstractFactorValueValueObjectSerializer.java b/gemma-rest/src/main/java/ubic/gemma/rest/serializers/AbstractFactorValueValueObjectSerializer.java
index 593682c557..4869ec56ca 100644
--- a/gemma-rest/src/main/java/ubic/gemma/rest/serializers/AbstractFactorValueValueObjectSerializer.java
+++ b/gemma-rest/src/main/java/ubic/gemma/rest/serializers/AbstractFactorValueValueObjectSerializer.java
@@ -47,26 +47,26 @@ protected void writeStatements( Long factorValueId, Collection
Date: Mon, 6 May 2024 16:55:29 -0700
Subject: [PATCH 16/61] Add a column showing the FASTQ header examplar for each
 sample

This may be useful for confirming/QCing batching for RNAseq.
---
 .../biomaterial/BioMaterialValueObject.java   |  5 +++++
 .../api/annotation/BioMaterialEditor.js       | 22 ++++++++++++++-----
 2 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/gemma-core/src/main/java/ubic/gemma/model/expression/biomaterial/BioMaterialValueObject.java b/gemma-core/src/main/java/ubic/gemma/model/expression/biomaterial/BioMaterialValueObject.java
index 6cbfa7f5ea..d9c793f74c 100644
--- a/gemma-core/src/main/java/ubic/gemma/model/expression/biomaterial/BioMaterialValueObject.java
+++ b/gemma-core/src/main/java/ubic/gemma/model/expression/biomaterial/BioMaterialValueObject.java
@@ -53,6 +53,10 @@ public class BioMaterialValueObject extends IdentifiableValueObject
     private String assayName;
     @GemmaWebOnly
     private String assayDescription;
+
+    @GemmaWebOnly
+    private String fastqHeaders = null;
+
     /**
      * Related {@link BioAssay} IDs.
      */
@@ -178,6 +182,7 @@ public BioMaterialValueObject( BioMaterial bm, BioAssay ba ) {
         this.assayName = ba.getName();
         this.assayDescription = ba.getDescription();
         this.assayProcessingDate = ba.getProcessingDate();
+        this.fastqHeaders = ba.getFastqHeaders() == null ? "" : ba.getFastqHeaders();
     }
 
     @JsonProperty("factorValues")
diff --git a/gemma-web/src/main/webapp/scripts/api/annotation/BioMaterialEditor.js b/gemma-web/src/main/webapp/scripts/api/annotation/BioMaterialEditor.js
index 20019fb8b9..a443e6da26 100755
--- a/gemma-web/src/main/webapp/scripts/api/annotation/BioMaterialEditor.js
+++ b/gemma-web/src/main/webapp/scripts/api/annotation/BioMaterialEditor.js
@@ -116,7 +116,14 @@ Gemma.BioMaterialGrid = Ext.extend( Gemma.GemmaGridPanel, {
          width : 40,
          dataIndex : "baDate",
          sortable : true,
-         tooltip : 'BioAssay processing date (primarily available for microarrays only)'
+         tooltip : 'BioAssay processing date (available for microarrays only, may be absent)'
+      }, {
+         id : "fastq-column",
+         header : "BA FASTQ",
+         width : 40,
+         dataIndex : "baFastq",
+         sortable : true,
+         tooltip : 'BioAssay FASTQ header exemplar (available for RNA-seq only, may be absent)'
       } ];
 
       this.factorValueEditors = [];
@@ -262,6 +269,9 @@ Gemma.BioMaterialGrid = Ext.extend( Gemma.GemmaGridPanel, {
       }, {
          name : "baDate",
          type : "string"
+      }, {
+         name : "baFastq",
+         type : "string"
       } ];
 
       // Add one slot in the record per factor. The name of the fields will be like
@@ -305,11 +315,11 @@ Gemma.BioMaterialGrid = Ext.extend( Gemma.GemmaGridPanel, {
    changeNonFactorDisplay : function( hide ) {
       var colModel = this.getColumnModel();
       var columns = colModel.config;
-     // Ext.suspendLayouts(); alas, this is a extjs 4 feature
+      // Ext.suspendLayouts(); alas, this is a extjs 4 feature
       for ( var i = 0; i < columns.length; i++ ) {
          var column = columns[i];
-         if (!hide) {
-            colModel.setHidden(i, false);
+         if ( !hide ) {
+            colModel.setHidden( i, false );
             continue;
          }
          if ( column.isRaw ) {
@@ -519,7 +529,7 @@ Gemma.BioMaterialGrid = Ext.extend( Gemma.GemmaGridPanel, {
           * This order must match the record!
           */
          data[i] = [ bmvo.id, bmvo.name, bmvo.description, chars, bmvo.assayName,
-            bmvo.assayDescription, Gemma.Renderers.dateRenderer( bmvo.assayProcessingDate ) ];
+            bmvo.assayDescription, Gemma.Renderers.dateRenderer( bmvo.assayProcessingDate ), bmvo.fastqHeaders ];
 
          var factors = bmvo.factors;
 
@@ -545,7 +555,7 @@ Gemma.BioMaterialGrid = Ext.extend( Gemma.GemmaGridPanel, {
             }
 
             var cval = bmvo.characteristicValues[c];
-             if ( cval ) {
+            if ( cval ) {
                data[i].push( cval );
             } else {
                data[i].push( "" ); // shouldn't happen if we picked useful characteristics well.

From 64b74feeaa8fd4f7da6279988492f8c23cd93af3 Mon Sep 17 00:00:00 2001
From: Guillaume Poirier-Morency 
Date: Wed, 8 May 2024 10:28:50 -0700
Subject: [PATCH 17/61] Fix last updated clause for H2

---
 .../service/TableMaintenanceUtilImpl.java           | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/TableMaintenanceUtilImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/TableMaintenanceUtilImpl.java
index 1fdceb01b6..1bffb46d47 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/TableMaintenanceUtilImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/TableMaintenanceUtilImpl.java
@@ -89,13 +89,18 @@ public class TableMaintenanceUtilImpl implements TableMaintenanceUtil {
                     + "and ACE.SID_FK = (select ACLSID.ID from ACLSID where ACLSID.GRANTED_AUTHORITY = 'IS_AUTHENTICATED_ANONYMOUSLY') "
                     + "group by AOI.ID), 0)";
 
+    /**
+     * Clause for selecting entities updated since a given date.
+     */
+    private static final String CD_LAST_UPDATED_SINCE = "(CD.LAST_UPDATED is null or :since is null or CD.LAST_UPDATED >= :since)";
+
     private static final String EE2C_EE_QUERY =
             "select MIN(C.ID), C.NAME, C.DESCRIPTION, C.CATEGORY, C.CATEGORY_URI, C.`VALUE`, C.VALUE_URI, C.ORIGINAL_VALUE, C.EVIDENCE_CODE, I.ID, (" + SELECT_ANONYMOUS_MASK + "), cast(? as char(255)) "
                     + "from INVESTIGATION I "
                     + "join CURATION_DETAILS CD on I.CURATION_DETAILS_FK = CD.ID "
                     + "join CHARACTERISTIC C on I.ID = C.INVESTIGATION_FK "
                     + "where I.class = 'ExpressionExperiment' "
-                    + "and COALESCE(CD.LAST_UPDATED, 0) >= COALESCE(:since, 0) "
+                    + "and " + CD_LAST_UPDATED_SINCE + " "
                     + "group by I.ID, COALESCE(C.CATEGORY_URI, C.CATEGORY), COALESCE(C.VALUE_URI, C.`VALUE`)";
 
     private static final String EE2C_BM_QUERY =
@@ -106,7 +111,7 @@ public class TableMaintenanceUtilImpl implements TableMaintenanceUtil {
                     + "join BIO_MATERIAL BM on BA.SAMPLE_USED_FK = BM.ID "
                     + "join CHARACTERISTIC C on BM.ID = C.BIO_MATERIAL_FK "
                     + "where I.class = 'ExpressionExperiment' "
-                    + "and COALESCE(CD.LAST_UPDATED, 0) >= COALESCE(:since, 0) "
+                    + "and " + CD_LAST_UPDATED_SINCE + " "
                     + "group by I.ID, COALESCE(C.CATEGORY_URI, C.CATEGORY), COALESCE(C.VALUE_URI, C.`VALUE`)";
 
     private static final String EE2C_ED_QUERY =
@@ -120,7 +125,7 @@ public class TableMaintenanceUtilImpl implements TableMaintenanceUtil {
                     + "where I.class = 'ExpressionExperiment' "
                     // remove C.class = 'Statement' once the old-style characteristics are removed (see https://github.com/PavlidisLab/Gemma/issues/929 for details)
                     + "and C.class = 'Statement' "
-                    + "and COALESCE(CD.LAST_UPDATED, 0) >= COALESCE(:since, 0) "
+                    + "and " + CD_LAST_UPDATED_SINCE + " "
                     + "group by I.ID, COALESCE(C.CATEGORY_URI, C.CATEGORY), COALESCE(C.VALUE_URI, C.`VALUE`)";
 
     private static final String EE2AD_QUERY = "insert into EXPRESSION_EXPERIMENT2ARRAY_DESIGN (EXPRESSION_EXPERIMENT_FK, ARRAY_DESIGN_FK, IS_ORIGINAL_PLATFORM, ACL_IS_AUTHENTICATED_ANONYMOUSLY_MASK) "
@@ -137,7 +142,7 @@ public class TableMaintenanceUtilImpl implements TableMaintenanceUtil {
             + "join BIO_ASSAY BA on I.ID = BA.EXPRESSION_EXPERIMENT_FK "
             + "join ARRAY_DESIGN AD on BA.ORIGINAL_PLATFORM_FK = AD.ID "
             + "where I.class = 'ExpressionExperiment' "
-            + "and COALESCE(CD.LAST_UPDATED, 0) >= COALESCE(:since, 0) "
+            + "and " + CD_LAST_UPDATED_SINCE + " "
             + "group by I.ID, AD.ID "
             + "on duplicate key update ACL_IS_AUTHENTICATED_ANONYMOUSLY_MASK = VALUES(ACL_IS_AUTHENTICATED_ANONYMOUSLY_MASK)";
 

From 3716f44ee9981fcdc213c82a7b37c81b9b48160d Mon Sep 17 00:00:00 2001
From: Guillaume Poirier-Morency 
Date: Wed, 8 May 2024 10:43:35 -0700
Subject: [PATCH 18/61] Use a simpler approach for secure jobs

Simply provide a SecurityContext to execute Quartz jobs and methods
invocations.
---
 .../gemma/web/scheduler/Ee2AdUpdateJob.java   |  2 +-
 .../gemma/web/scheduler/Ee2cUpdateJob.java    |  2 +-
 ...ionServiceBasedSecurityContextFactory.java | 63 +++++++++++++++++++
 .../gemma/web/scheduler/SecureInvoker.java    | 58 -----------------
 ...ureMethodInvokingJobDetailFactoryBean.java | 16 +++--
 .../web/scheduler/SecureQuartzJobBean.java    | 21 ++++---
 .../gemma/applicationContext-schedule.xml     | 22 +++----
 .../web/scheduler/SchedulerSecurityTest.java  | 18 +++---
 8 files changed, 110 insertions(+), 92 deletions(-)
 create mode 100644 gemma-web/src/main/java/ubic/gemma/web/scheduler/ManualAuthenticationServiceBasedSecurityContextFactory.java
 delete mode 100644 gemma-web/src/main/java/ubic/gemma/web/scheduler/SecureInvoker.java

diff --git a/gemma-web/src/main/java/ubic/gemma/web/scheduler/Ee2AdUpdateJob.java b/gemma-web/src/main/java/ubic/gemma/web/scheduler/Ee2AdUpdateJob.java
index cc6ed1f5f7..c7c1527af1 100644
--- a/gemma-web/src/main/java/ubic/gemma/web/scheduler/Ee2AdUpdateJob.java
+++ b/gemma-web/src/main/java/ubic/gemma/web/scheduler/Ee2AdUpdateJob.java
@@ -15,7 +15,7 @@ public class Ee2AdUpdateJob extends SecureQuartzJobBean implements StatefulJob {
     private TableMaintenanceUtil tableMaintenanceUtil;
 
     @Override
-    protected void executeAsAgent( JobExecutionContext context ) {
+    protected void executeAs( JobExecutionContext context ) {
         Assert.notNull( tableMaintenanceUtil, "The tableMaintenanceUtil bean was not set." );
         tableMaintenanceUtil.updateExpressionExperiment2ArrayDesignEntries( context.getPreviousFireTime() );
     }
diff --git a/gemma-web/src/main/java/ubic/gemma/web/scheduler/Ee2cUpdateJob.java b/gemma-web/src/main/java/ubic/gemma/web/scheduler/Ee2cUpdateJob.java
index 7f444de293..514707a363 100644
--- a/gemma-web/src/main/java/ubic/gemma/web/scheduler/Ee2cUpdateJob.java
+++ b/gemma-web/src/main/java/ubic/gemma/web/scheduler/Ee2cUpdateJob.java
@@ -17,7 +17,7 @@ public class Ee2cUpdateJob extends SecureQuartzJobBean implements StatefulJob {
     private Class level = null;
 
     @Override
-    public void executeAsAgent( JobExecutionContext context ) {
+    public void executeAs( JobExecutionContext context ) {
         Assert.notNull( tableMaintenanceUtil, "The tableMaintenanceUtil bean was not set." );
         if ( level == null ) {
             tableMaintenanceUtil.updateExpressionExperiment2CharacteristicEntries( context.getPreviousFireTime(), false );
diff --git a/gemma-web/src/main/java/ubic/gemma/web/scheduler/ManualAuthenticationServiceBasedSecurityContextFactory.java b/gemma-web/src/main/java/ubic/gemma/web/scheduler/ManualAuthenticationServiceBasedSecurityContextFactory.java
new file mode 100644
index 0000000000..ed0bcf2337
--- /dev/null
+++ b/gemma-web/src/main/java/ubic/gemma/web/scheduler/ManualAuthenticationServiceBasedSecurityContextFactory.java
@@ -0,0 +1,63 @@
+package ubic.gemma.web.scheduler;
+
+import gemma.gsec.authentication.ManualAuthenticationService;
+import lombok.Setter;
+import lombok.extern.apachecommons.CommonsLog;
+import org.springframework.beans.factory.config.AbstractFactoryBean;
+import org.springframework.security.core.AuthenticationException;
+import org.springframework.security.core.context.SecurityContext;
+import org.springframework.security.core.context.SecurityContextHolder;
+
+/**
+ * Creates a security context based using manual authentication.
+ * @author poirigui
+ */
+@Setter
+@CommonsLog
+public class ManualAuthenticationServiceBasedSecurityContextFactory extends AbstractFactoryBean {
+
+    private final ManualAuthenticationService manualAuthenticationService;
+
+    private String userName;
+    private String password;
+
+    /**
+     * Fallback to an anonymous authentication if the authentication fails.
+     */
+    private boolean fallbackToAnonymous = false;
+
+    public ManualAuthenticationServiceBasedSecurityContextFactory( ManualAuthenticationService manualAuthenticationService ) {
+        this.manualAuthenticationService = manualAuthenticationService;
+    }
+
+    @Override
+    public Class getObjectType() {
+        return SecurityContext.class;
+    }
+
+    @Override
+    protected SecurityContext createInstance() {
+        try {
+            SecurityContext securityContext = SecurityContextHolder.createEmptyContext();
+            securityContext.setAuthentication( manualAuthenticationService.attemptAuthentication( userName, password ) );
+            return securityContext;
+        } catch ( AuthenticationException e ) {
+            if ( fallbackToAnonymous ) {
+                log.error( "Failed to authenticate schedule job, jobs probably won't work, but trying anonymous." );
+                SecurityContext securityContext = SecurityContextHolder.createEmptyContext();
+                SecurityContextHolder.setContext( securityContext );
+                // gsec will call SecurityContextHolder.getContext().setAuthentication()
+                SecurityContext previousSecurityContext = SecurityContextHolder.getContext();
+                try {
+                    manualAuthenticationService.authenticateAnonymously();
+                    return securityContext;
+                } finally {
+                    SecurityContextHolder.clearContext();
+                    SecurityContextHolder.setContext( previousSecurityContext );
+                }
+            } else {
+                throw e;
+            }
+        }
+    }
+}
diff --git a/gemma-web/src/main/java/ubic/gemma/web/scheduler/SecureInvoker.java b/gemma-web/src/main/java/ubic/gemma/web/scheduler/SecureInvoker.java
deleted file mode 100644
index 268ae8233c..0000000000
--- a/gemma-web/src/main/java/ubic/gemma/web/scheduler/SecureInvoker.java
+++ /dev/null
@@ -1,58 +0,0 @@
-package ubic.gemma.web.scheduler;
-
-import gemma.gsec.authentication.ManualAuthenticationService;
-import lombok.Setter;
-import lombok.extern.apachecommons.CommonsLog;
-import org.springframework.security.core.AuthenticationException;
-import org.springframework.security.core.context.SecurityContext;
-import org.springframework.security.core.context.SecurityContextHolder;
-
-import java.util.concurrent.Callable;
-
-/**
- * Invoke a callable as an authenticated user with given credentials.
- * @author poirigui
- */
-@Setter
-@CommonsLog
-public class SecureInvoker {
-
-    private final ManualAuthenticationService manualAuthenticationService;
-
-    private String userName;
-    private String password;
-
-    /**
-     * Fallback to an anonymous authentication if the authentication fails.
-     */
-    private boolean fallbackToAnonymous = false;
-
-    public SecureInvoker( ManualAuthenticationService manualAuthenticationService ) {
-        this.manualAuthenticationService = manualAuthenticationService;
-    }
-
-    public  T invoke( Callable callable ) throws Exception {
-        SecurityContext previousSecurityContext = SecurityContextHolder.getContext();
-        try {
-            try {
-                SecurityContext securityContext = SecurityContextHolder.createEmptyContext();
-                securityContext.setAuthentication( manualAuthenticationService.attemptAuthentication( userName, password ) );
-                SecurityContextHolder.setContext( securityContext );
-            } catch ( AuthenticationException e ) {
-                if ( fallbackToAnonymous ) {
-                    log.error( "Failed to authenticate schedule job, jobs probably won't work, but trying anonymous" );
-                    SecurityContext securityContext = SecurityContextHolder.createEmptyContext();
-                    SecurityContextHolder.setContext( securityContext );
-                    // gsec will call SecurityContextHolder.getContext().setAuthentication()
-                    manualAuthenticationService.authenticateAnonymously();
-                } else {
-                    throw e;
-                }
-            }
-            assert SecurityContextHolder.getContext().getAuthentication() != null;
-            return callable.call();
-        } finally {
-            SecurityContextHolder.setContext( previousSecurityContext );
-        }
-    }
-}
diff --git a/gemma-web/src/main/java/ubic/gemma/web/scheduler/SecureMethodInvokingJobDetailFactoryBean.java b/gemma-web/src/main/java/ubic/gemma/web/scheduler/SecureMethodInvokingJobDetailFactoryBean.java
index b5dce2a801..c7afdc6d11 100644
--- a/gemma-web/src/main/java/ubic/gemma/web/scheduler/SecureMethodInvokingJobDetailFactoryBean.java
+++ b/gemma-web/src/main/java/ubic/gemma/web/scheduler/SecureMethodInvokingJobDetailFactoryBean.java
@@ -19,6 +19,9 @@
 package ubic.gemma.web.scheduler;
 
 import org.springframework.scheduling.quartz.MethodInvokingJobDetailFactoryBean;
+import org.springframework.security.concurrent.DelegatingSecurityContextCallable;
+import org.springframework.security.core.context.SecurityContext;
+import org.springframework.util.Assert;
 
 import java.lang.reflect.InvocationTargetException;
 
@@ -31,20 +34,21 @@
  */
 public class SecureMethodInvokingJobDetailFactoryBean extends MethodInvokingJobDetailFactoryBean {
 
-    private final SecureInvoker secureInvoker;
+    private final SecurityContext securityContext;
 
-    public SecureMethodInvokingJobDetailFactoryBean( SecureInvoker secureInvoker ) {
-        this.secureInvoker = secureInvoker;
+    public SecureMethodInvokingJobDetailFactoryBean( SecurityContext securityContext ) {
+        Assert.notNull( securityContext, "A security context must be provided." );
+        this.securityContext = securityContext;
     }
 
     @Override
     public Object invoke() throws InvocationTargetException, IllegalAccessException {
         try {
-            return secureInvoker.invoke( super::invoke );
-        } catch ( InvocationTargetException | IllegalAccessException | RuntimeException e ) {
+            return DelegatingSecurityContextCallable.create( super::invoke, securityContext ).call();
+        } catch ( InvocationTargetException | RuntimeException e ) {
             throw e;
         } catch ( Exception e ) {
-            throw new InvocationTargetException( e );
+            throw new RuntimeException( e );
         }
     }
 }
diff --git a/gemma-web/src/main/java/ubic/gemma/web/scheduler/SecureQuartzJobBean.java b/gemma-web/src/main/java/ubic/gemma/web/scheduler/SecureQuartzJobBean.java
index 02287a1480..d3a855ddf4 100644
--- a/gemma-web/src/main/java/ubic/gemma/web/scheduler/SecureQuartzJobBean.java
+++ b/gemma-web/src/main/java/ubic/gemma/web/scheduler/SecureQuartzJobBean.java
@@ -4,29 +4,34 @@
 import org.quartz.JobExecutionContext;
 import org.quartz.JobExecutionException;
 import org.springframework.scheduling.quartz.QuartzJobBean;
+import org.springframework.security.concurrent.DelegatingSecurityContextCallable;
+import org.springframework.security.core.context.SecurityContext;
 import org.springframework.util.Assert;
 
 /**
- *
+ * A secure Quartz job bean that executes with a given security context.
+ * @author poirigui
  */
 @Setter
 public abstract class SecureQuartzJobBean extends QuartzJobBean {
 
-    private SecureInvoker secureInvoker;
+    private SecurityContext securityContext;
 
     @Override
     protected final void executeInternal( JobExecutionContext context ) throws JobExecutionException {
-        Assert.notNull( secureInvoker, "The secureInvoker bean is not set." );
+        Assert.notNull( securityContext, "A security context is not set." );
         try {
-            secureInvoker.invoke( () -> {
-                executeAsAgent( context );
+            DelegatingSecurityContextCallable.create( () -> {
+                executeAs( context );
                 return null;
-            } );
+            }, securityContext ).call();
+        } catch ( JobExecutionException | RuntimeException e ) {
+            throw e;
         } catch ( Exception e ) {
-            throw new JobExecutionException( e );
+            throw new RuntimeException( e );
         }
     }
 
-    protected abstract void executeAsAgent( JobExecutionContext context ) throws JobExecutionException;
+    protected abstract void executeAs( JobExecutionContext context ) throws JobExecutionException;
 }
 
diff --git a/gemma-web/src/main/resources/ubic/gemma/applicationContext-schedule.xml b/gemma-web/src/main/resources/ubic/gemma/applicationContext-schedule.xml
index 9889328bee..6c36c500c2 100644
--- a/gemma-web/src/main/resources/ubic/gemma/applicationContext-schedule.xml
+++ b/gemma-web/src/main/resources/ubic/gemma/applicationContext-schedule.xml
@@ -36,7 +36,7 @@
     
 
     
-    
+    
         
         
         
@@ -51,7 +51,7 @@
     
         
             
-                
+                
                 
                 
                 
@@ -64,7 +64,7 @@
         
             
-                
+                
                 
                 
                 
@@ -77,7 +77,7 @@
         
             
-                
+                
                 
                 
                 
@@ -90,7 +90,7 @@
         
             
-                
+                
                 
                 
                 
@@ -102,7 +102,7 @@
     
         
             
-                
+                
                 
                 
                 
@@ -119,7 +119,7 @@
                     
                         
                         
-                        
+                        
                     
                 
             
@@ -135,7 +135,7 @@
                     
                         
                         
-                        
+                        
                     
                 
             
@@ -152,7 +152,7 @@
                     
                         
                         
-                        
+                        
                     
                 
             
@@ -168,7 +168,7 @@
                 
                     
                         
-                        
+                        
                     
                 
             
@@ -180,7 +180,7 @@
     
         
             
-                
+                
                 
                 
                 
diff --git a/gemma-web/src/test/java/ubic/gemma/web/scheduler/SchedulerSecurityTest.java b/gemma-web/src/test/java/ubic/gemma/web/scheduler/SchedulerSecurityTest.java
index 2295dcf189..b7c67d6976 100644
--- a/gemma-web/src/test/java/ubic/gemma/web/scheduler/SchedulerSecurityTest.java
+++ b/gemma-web/src/test/java/ubic/gemma/web/scheduler/SchedulerSecurityTest.java
@@ -26,6 +26,7 @@
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.security.core.authority.SimpleGrantedAuthority;
+import org.springframework.security.core.context.SecurityContext;
 import org.springframework.security.core.context.SecurityContextHolder;
 import org.springframework.stereotype.Component;
 import ubic.gemma.core.analysis.report.WhatsNewService;
@@ -35,6 +36,7 @@
 
 import java.lang.reflect.InvocationTargetException;
 
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.mockito.Mockito.*;
@@ -56,8 +58,8 @@ public class SchedulerSecurityTest extends BaseSpringWebTest {
     private TableMaintenanceUtil tableMaintenanceUtil;
 
     @Autowired
-    @Qualifier("groupAgentSecureInvoker")
-    private SecureInvoker secureInvoker;
+    @Qualifier("groupAgentSecurityContext")
+    private SecurityContext securityContext;
 
     /*
      * Tests whether we can run a secured method that has been granted to GROUP_AGENT
@@ -68,7 +70,7 @@ public void runSecuredMethodOnSchedule() throws Exception {
 
         String jobName = "job_" + RandomStringUtils.randomAlphabetic( 10 );
 
-        SecureMethodInvokingJobDetailFactoryBean jobDetail = new SecureMethodInvokingJobDetailFactoryBean( this.secureInvoker );
+        SecureMethodInvokingJobDetailFactoryBean jobDetail = new SecureMethodInvokingJobDetailFactoryBean( this.securityContext );
         jobDetail.setTargetMethod( "generateWeeklyReport" );
         jobDetail.setTargetObject( whatsNewService ); // access should be ok for GROUP_AGENT.
         jobDetail.setConcurrent( false );
@@ -88,7 +90,7 @@ public void runSecuredMethodOnScheduleMultiGroup() throws Exception {
 
         String jobName = "job_" + RandomStringUtils.randomAlphabetic( 10 );
 
-        SecureMethodInvokingJobDetailFactoryBean jobDetail = new SecureMethodInvokingJobDetailFactoryBean( this.secureInvoker );
+        SecureMethodInvokingJobDetailFactoryBean jobDetail = new SecureMethodInvokingJobDetailFactoryBean( this.securityContext );
         jobDetail.setTargetMethod( "findByUpdatedLimit" );
         jobDetail.setArguments( new Object[] { 10 } );
         jobDetail.setTargetObject( expressionExperimentService ); // access should be ok for GROUP_AGENT.
@@ -112,7 +114,7 @@ public void runUnauthorizedMethodOnSchedule() throws Exception {
         /*
          * Mimics configuration in xml.
          */
-        SecureMethodInvokingJobDetailFactoryBean jobDetail = new SecureMethodInvokingJobDetailFactoryBean( this.secureInvoker );
+        SecureMethodInvokingJobDetailFactoryBean jobDetail = new SecureMethodInvokingJobDetailFactoryBean( this.securityContext );
         jobDetail.setTargetMethod( "remove" );
         jobDetail.setArguments( new Object[] { null } );
         jobDetail.setTargetObject( expressionExperimentService ); // no access
@@ -129,7 +131,7 @@ public static class TestSecureJob extends SecureQuartzJobBean {
         private TableMaintenanceUtil tableMaintenanceUtil;
 
         @Override
-        protected void executeAsAgent( JobExecutionContext context ) {
+        protected void executeAs( JobExecutionContext context ) {
             assertNotNull( tableMaintenanceUtil );
             assertNotNull( SecurityContextHolder.getContext().getAuthentication() );
             assertTrue( SecurityContextHolder.getContext().getAuthentication().isAuthenticated() );
@@ -152,10 +154,12 @@ public void testSecureJob() throws JobExecutionException {
         JobExecutionContext context = mock();
         JobDataMap jdm = new JobDataMap();
         jdm.put( "tableMaintenanceUtil", tableMaintenanceUtil );
-        jdm.put( "secureInvoker", secureInvoker );
+        jdm.put( "securityContext", securityContext );
         when( context.getScheduler() ).thenReturn( mock() );
         when( context.getMergedJobDataMap() ).thenReturn( jdm );
+        SecurityContext previousContext = SecurityContextHolder.getContext();
         testSecureJob.execute( context );
+        assertThat( SecurityContextHolder.getContext() ).isSameAs( previousContext );
         verify( context ).setResult( "Hello world!" );
     }
 }

From c184f046b2ea9c7cd5a90f9cf4b89b0ee7e22281 Mon Sep 17 00:00:00 2001
From: Guillaume Poirier-Morency 
Date: Wed, 8 May 2024 12:21:49 -0700
Subject: [PATCH 19/61] Use AbstractFactoryBean for
 TextResourceToSetOfLinesFactoryBean

---
 .../util/TextResourceToSetOfLinesFactoryBean.java     | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/gemma-core/src/main/java/ubic/gemma/core/util/TextResourceToSetOfLinesFactoryBean.java b/gemma-core/src/main/java/ubic/gemma/core/util/TextResourceToSetOfLinesFactoryBean.java
index 2ee5640aab..6cefaf7208 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/util/TextResourceToSetOfLinesFactoryBean.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/util/TextResourceToSetOfLinesFactoryBean.java
@@ -1,6 +1,6 @@
 package ubic.gemma.core.util;
 
-import org.springframework.beans.factory.FactoryBean;
+import org.springframework.beans.factory.config.AbstractFactoryBean;
 import org.springframework.core.io.Resource;
 
 import java.io.BufferedReader;
@@ -15,7 +15,7 @@
  * Lines starting with '#' are ignored.
  * @author poirigui
  */
-public class TextResourceToSetOfLinesFactoryBean implements FactoryBean> {
+public class TextResourceToSetOfLinesFactoryBean extends AbstractFactoryBean> {
 
     private final Resource resource;
 
@@ -24,7 +24,7 @@ public TextResourceToSetOfLinesFactoryBean( Resource resource ) {
     }
 
     @Override
-    public Set getObject() throws Exception {
+    protected Set createInstance() throws Exception {
         return new BufferedReader( new InputStreamReader( resource.getInputStream(), StandardCharsets.UTF_8 ) )
                 .lines()
                 .filter( line -> !line.startsWith( "#" ) )
@@ -35,9 +35,4 @@ public Set getObject() throws Exception {
     public Class getObjectType() {
         return Set.class;
     }
-
-    @Override
-    public boolean isSingleton() {
-        return true;
-    }
 }

From 5a9a0daebbf16c2b431063d38fba43a99223cf88 Mon Sep 17 00:00:00 2001
From: Paul Pavlidis 
Date: Wed, 8 May 2024 17:18:24 -0700
Subject: [PATCH 20/61] handle PROBLEMATIC_BATCH_INFO_FAILURE in display (treat
 as "no batch info")

Note: This flag is somewhat different from UNINFORMATIVE_HEADERS_FAILURE - the headers were present, but we failed to get batch information from them, but more likely to be due to a problem with the FASTQ header file missing rows or something like that, vs UNINFORMATIVE_HEADERS_FAILURE is used when the only problem is the headers don't contain lane etc. info.
---
 .../api/entities/experiment/ExpressionExperimentPage.js       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/gemma-web/src/main/webapp/scripts/api/entities/experiment/ExpressionExperimentPage.js b/gemma-web/src/main/webapp/scripts/api/entities/experiment/ExpressionExperimentPage.js
index 23e703b879..8a4451293b 100755
--- a/gemma-web/src/main/webapp/scripts/api/entities/experiment/ExpressionExperimentPage.js
+++ b/gemma-web/src/main/webapp/scripts/api/entities/experiment/ExpressionExperimentPage.js
@@ -88,7 +88,7 @@ function getBatchInfoBadges(ee) {
     if (!hasBatchConfound && ee.batchEffect !== null) {
 		if (ee.batchEffect === "SINGLETON_BATCHES_FAILURE") {
 			result = result + getStatusBadge('exclamation-triangle', 'dark-yellow', 'unable to batch', Gemma.HelpText.WidgetDefaults.ExpressionExperimentDetails.noBatchesSingletons);
-		} else if (ee.batchEffect === "UNINFORMATIVE_HEADERS_FAILURE") {
+		} else if (ee.batchEffect === "UNINFORMATIVE_HEADERS_FAILURE" || ee.batchEffect === "PROBLEMATIC_BATCH_INFO_FAILURE") {
 			result = result + getStatusBadge('exclamation-triangle', 'dark-yellow', 'no batch info', Gemma.HelpText.WidgetDefaults.ExpressionExperimentDetails.noBatchesBadHeaders);
 		} else if (ee.batchEffect === "BATCH_CORRECTED_SUCCESS") { // ExpressionExperimentServiceImpl::getBatchEffectDescription()
             result = result + getStatusBadge('cogs', 'green', 'batch corrected', ee.batchEffectStatistics)
@@ -110,7 +110,7 @@ function getBatchInfoBadges(ee) {
             result = result + getStatusBadge( 'exclamation-triangle', 'dark-yellow', 'undetermined batch effect', 'Batch effect could not be determined.');
         } else {
             // unsupported batch effect type
-            result = result + getStatusBadge('exclamation-triangle', 'dark-yellow', ee.batchEffect, 'Unsupported batch effect type')
+            result = result + getStatusBadge('exclamation-triangle', 'dark-yellow', ee.batchEffect, 'Some other batch effect situation')
         }
     }
 

From c2f4dfc8fc7c995f5462a5909c58c2f5934570a4 Mon Sep 17 00:00:00 2001
From: Guillaume Poirier-Morency 
Date: Wed, 8 May 2024 13:31:53 -0700
Subject: [PATCH 21/61] Populate first and second baseline groups for result
 set with an interaction term

---
 ...xpressionAnalysisResultSetValueObject.java | 29 ++++++-
 .../ExpressionAnalysisResultSetDaoImpl.java   | 86 +++++++++++++++++++
 2 files changed, 114 insertions(+), 1 deletion(-)

diff --git a/gemma-core/src/main/java/ubic/gemma/model/analysis/expression/diff/DifferentialExpressionAnalysisResultSetValueObject.java b/gemma-core/src/main/java/ubic/gemma/model/analysis/expression/diff/DifferentialExpressionAnalysisResultSetValueObject.java
index 2c6551ccc1..cfb3bab107 100644
--- a/gemma-core/src/main/java/ubic/gemma/model/analysis/expression/diff/DifferentialExpressionAnalysisResultSetValueObject.java
+++ b/gemma-core/src/main/java/ubic/gemma/model/analysis/expression/diff/DifferentialExpressionAnalysisResultSetValueObject.java
@@ -3,6 +3,7 @@
 import com.fasterxml.jackson.annotation.JsonInclude;
 import ubic.gemma.model.analysis.AnalysisResultSetValueObject;
 import ubic.gemma.model.expression.experiment.ExperimentalFactorValueObject;
+import ubic.gemma.model.expression.experiment.FactorValue;
 import ubic.gemma.model.expression.experiment.FactorValueBasicValueObject;
 import ubic.gemma.model.genome.Gene;
 
@@ -20,6 +21,7 @@ public class DifferentialExpressionAnalysisResultSetValueObject extends Analysis
     private DifferentialExpressionAnalysisValueObject analysis;
     private Collection experimentalFactors;
     private FactorValueBasicValueObject baselineGroup;
+    private FactorValueBasicValueObject secondBaselineGroup;
 
     /**
      * Related analysis results.
@@ -50,9 +52,21 @@ public DifferentialExpressionAnalysisResultSetValueObject( ExpressionAnalysisRes
         }
     }
 
+    /**
+     * Create a simple analysis results set VO for a result set containing an interaction term.
+     * 

+     * The baseline group is not populated for result sets containing an interaction term, so it has to be provided.
+     * See #1119 for details
+     */
+    public DifferentialExpressionAnalysisResultSetValueObject( ExpressionAnalysisResultSet analysisResultSet, FactorValue baselineGroup, FactorValue secondBaselineGroup ) {
+        this( analysisResultSet );
+        this.baselineGroup = new FactorValueBasicValueObject( baselineGroup );
+        this.secondBaselineGroup = new FactorValueBasicValueObject( secondBaselineGroup );
+    }
+
     /**
      * Create an expression analysis result set VO with all its associated results.
-     *
+     * 

      * Note: this constructor assumes that {@link ExpressionAnalysisResultSet#getResults()} has already been initialized.
      */
     public DifferentialExpressionAnalysisResultSetValueObject( ExpressionAnalysisResultSet analysisResultSet, Map> result2Genes ) {
@@ -87,6 +101,14 @@ public void setBaselineGroup( FactorValueBasicValueObject baselineGroup ) {
         this.baselineGroup = baselineGroup;
     }
 
+    public FactorValueBasicValueObject getSecondBaselineGroup() {
+        return secondBaselineGroup;
+    }
+
+    public void setSecondBaselineGroup( FactorValueBasicValueObject secondBaselineGroup ) {
+        this.secondBaselineGroup = secondBaselineGroup;
+    }
+
     @Override
     public Collection getResults() {
         return results;
@@ -95,4 +117,9 @@ public Collection getResults()
     public void setResults( Collection results ) {
         this.results = results;
     }
+
+    @Override
+    public String toString() {
+        return getClass().getSimpleName() + " Id=" + getId();
+    }
 }
diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/ExpressionAnalysisResultSetDaoImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/ExpressionAnalysisResultSetDaoImpl.java
index aea15139e5..c5c95082dd 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/ExpressionAnalysisResultSetDaoImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/ExpressionAnalysisResultSetDaoImpl.java
@@ -35,6 +35,8 @@
 import ubic.gemma.model.expression.designElement.CompositeSequence;
 import ubic.gemma.model.expression.experiment.BioAssaySet;
 import ubic.gemma.model.expression.experiment.ExpressionExperiment;
+import ubic.gemma.model.expression.experiment.FactorValue;
+import ubic.gemma.model.expression.experiment.FactorValueBasicValueObject;
 import ubic.gemma.model.genome.Gene;
 import ubic.gemma.persistence.service.AbstractCriteriaFilteringVoEnabledDao;
 import ubic.gemma.persistence.util.*;
@@ -43,9 +45,11 @@
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.stream.Collectors;
 
 import static ubic.gemma.persistence.service.TableMaintenanceUtil.GENE2CS_QUERY_SPACE;
+import static ubic.gemma.persistence.util.QueryUtils.optimizeParameterList;
 
 /**
  * @author Paul
@@ -135,6 +139,12 @@ public Slice findByBioAssayS
         return new Slice<>( super.loadValueObjects( data ), sort, offset, limit, totalElements );
     }
 
+    @Override
+    protected void postProcessValueObjects( List differentialExpressionAnalysisResultSetValueObjects ) {
+        populateBaselines( differentialExpressionAnalysisResultSetValueObjects );
+    }
+
+
     @Override
     public void thaw( ExpressionAnalysisResultSet ears ) {
         // this drastically reduces the number of columns fetched which would anyway be repeated
@@ -271,4 +281,80 @@ public Map> loadResultToGenesMap( ExpressionAnalysisResultSet r
                                         .map( l -> ( Gene ) l[1] )
                                         .collect( Collectors.toList() ) ) ) );
     }
+
+    /**
+     * Populate baseline groups for results sets with interactions.
+     * 

+     * Those are not being populated in the database because there is no storage for a "second" baseline group. For more
+     * details, read #1119.
+     */
+    private void populateBaselines( List vos ) {
+        Collection vosWithMissingBaselines = vos.stream()
+                .filter( vo -> vo.getBaselineGroup() == null )
+                .collect( Collectors.toList() );
+        if ( vosWithMissingBaselines.isEmpty() ) {
+            return;
+        }
+        // pick baseline groups from other result sets from the same analysis
+        Collection rsIds = optimizeParameterList( EntityUtils.getIds( vosWithMissingBaselines ) );
+        //noinspection unchecked
+        List otherBaselineGroups = getSessionFactory().getCurrentSession()
+                .createQuery( "select rs.id, otherBg from ExpressionAnalysisResultSet rs "
+                        + "join rs.analysis a "
+                        + "join a.resultSets otherRs "
+                        + "join otherRs.baselineGroup otherBg "
+                        + "where rs.id in :rsIds and otherRs.id not in :rsIds" )
+                .setParameterList( "rsIds", rsIds )
+                .list();
+        // pick one representative contrasts to order the first and second baseline group consistently
+        //noinspection unchecked
+        List representativeContrasts = getSessionFactory().getCurrentSession()
+                .createQuery( "select rs.id, c from ExpressionAnalysisResultSet rs "
+                        + "join rs.results r join r.contrasts c "
+                        + "where rs.id in :rsIds "
+                        + "group by rs" )
+                .setParameterList( "rsIds", rsIds )
+                .list();
+        Map> baselineMapping = otherBaselineGroups.stream()
+                .collect( Collectors.groupingBy( row -> ( Long ) row[0], Collectors.mapping( row -> ( FactorValue ) row[1], Collectors.toSet() ) ) );
+        Map contrastsMapping = representativeContrasts.stream()
+                .collect( Collectors.toMap( row -> ( Long ) row[0], row -> ( ContrastResult ) row[1] ) );
+        for ( DifferentialExpressionAnalysisResultSetValueObject vo : vosWithMissingBaselines ) {
+            ContrastResult contrast = contrastsMapping.get( vo.getId() );
+            if ( contrast == null ) {
+                log.warn( "Could ont find a representative contrast for " + vo + " to populate its baseline groups." );
+                continue;
+            }
+            if ( contrast.getFactorValue() == null || contrast.getSecondFactorValue() == null ) {
+                log.warn( "Could not populate baselines for " + vo + " as its contrasts lack factor values. This is likely a continuous factor." );
+                // very likely a continuous factor, it does not have a baseline
+                continue;
+            }
+            // I don't think this is allowed
+            if ( contrast.getFactorValue().getExperimentalFactor().equals( contrast.getSecondFactorValue().getExperimentalFactor() ) ) {
+                log.warn( "Could not populate baselines for " + vo + ", its representative contrast uses the same experimental factor for its first and second factor value." );
+                continue;
+            }
+            Set baselines = baselineMapping.get( vo.getId() );
+            if ( baselines == null || baselines.size() != 2 ) {
+                log.warn( "Could not find two other result sets with baseline for " + vo + " to populate its baseline groups." );
+                continue;
+            }
+            FactorValue firstBaseline = null, secondBaseline = null;
+            for ( FactorValue fv : baselines ) {
+                if ( fv.getExperimentalFactor().equals( contrast.getFactorValue().getExperimentalFactor() ) ) {
+                    firstBaseline = fv;
+                }
+                if ( fv.getExperimentalFactor().equals( contrast.getSecondFactorValue().getExperimentalFactor() ) ) {
+                    secondBaseline = fv;
+                }
+            }
+            if ( firstBaseline != null && secondBaseline != null ) {
+                vo.setBaselineGroup( new FactorValueBasicValueObject( firstBaseline ) );
+                vo.setSecondBaselineGroup( new FactorValueBasicValueObject( secondBaseline ) );
+            } else {
+                log.warn( "Could not fill the baseline groups for " + vo + ": one or more baselines were not found in other result sets from the same analysis." );
+            }
+        }
+    }
 }
\ No newline at end of file

From 60c53f284ec8f271bedd45b9036a7359ea171431 Mon Sep 17 00:00:00 2001
From: Guillaume Poirier-Morency 
Date: Wed, 8 May 2024 19:17:57 -0700
Subject: [PATCH 22/61] Remove unused constructor for DEARSVO

---
 ...ntialExpressionAnalysisResultSetValueObject.java | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/gemma-core/src/main/java/ubic/gemma/model/analysis/expression/diff/DifferentialExpressionAnalysisResultSetValueObject.java b/gemma-core/src/main/java/ubic/gemma/model/analysis/expression/diff/DifferentialExpressionAnalysisResultSetValueObject.java
index cfb3bab107..ad90bcb767 100644
--- a/gemma-core/src/main/java/ubic/gemma/model/analysis/expression/diff/DifferentialExpressionAnalysisResultSetValueObject.java
+++ b/gemma-core/src/main/java/ubic/gemma/model/analysis/expression/diff/DifferentialExpressionAnalysisResultSetValueObject.java
@@ -3,7 +3,6 @@
 import com.fasterxml.jackson.annotation.JsonInclude;
 import ubic.gemma.model.analysis.AnalysisResultSetValueObject;
 import ubic.gemma.model.expression.experiment.ExperimentalFactorValueObject;
-import ubic.gemma.model.expression.experiment.FactorValue;
 import ubic.gemma.model.expression.experiment.FactorValueBasicValueObject;
 import ubic.gemma.model.genome.Gene;
 
@@ -52,18 +51,6 @@ public DifferentialExpressionAnalysisResultSetValueObject( ExpressionAnalysisRes
         }
     }
 
-    /**
-     * Create a simple analysis results set VO for a result set containing an interaction term.
-     * 

-     * The baseline group is not populated for result sets containing an interaction term, so it has to be provided.
-     * See #1119 for details
-     */
-    public DifferentialExpressionAnalysisResultSetValueObject( ExpressionAnalysisResultSet analysisResultSet, FactorValue baselineGroup, FactorValue secondBaselineGroup ) {
-        this( analysisResultSet );
-        this.baselineGroup = new FactorValueBasicValueObject( baselineGroup );
-        this.secondBaselineGroup = new FactorValueBasicValueObject( secondBaselineGroup );
-    }
-
     /**
      * Create an expression analysis result set VO with all its associated results.
      * 


From a5c1992fe6807f8644f5028986df9f1727442332 Mon Sep 17 00:00:00 2001
From: Guillaume Poirier-Morency 
Date: Thu, 9 May 2024 09:55:43 -0700
Subject: [PATCH 23/61] Add an IntelliJ configuration for generating the test
 database

---
 .idea/runConfigurations/Generate_testdb.xml | 36 +++++++++++++++++++++
 1 file changed, 36 insertions(+)
 create mode 100644 .idea/runConfigurations/Generate_testdb.xml

diff --git a/.idea/runConfigurations/Generate_testdb.xml b/.idea/runConfigurations/Generate_testdb.xml
new file mode 100644
index 0000000000..c0a06f2205
--- /dev/null
+++ b/.idea/runConfigurations/Generate_testdb.xml
@@ -0,0 +1,36 @@
+
+  
+    
+      
+    
+    
+  
+
\ No newline at end of file

From f4a85ab173905b3a0f6d008e0ba4683d6565dc56 Mon Sep 17 00:00:00 2001
From: Guillaume Poirier-Morency 
Date: Thu, 9 May 2024 13:27:56 -0700
Subject: [PATCH 24/61] Use BaseJerseyIntegrationTest for all integration tests
 in gemma-rest

Prevent subclasses from overriding JerseyTest.tearDown().
---
 .../rest/AnalysisResultSetsJerseyTest.java    |  2 +-
 .../AnalysisResultSetsWebServiceTest.java     | 21 +++++++++------
 .../ubic/gemma/rest/DatasetsRestTest.java     | 19 +++++++-------
 .../gemma/rest/DatasetsWebServiceTest.java    |  3 +--
 .../gemma/rest/PlatformsWebServiceTest.java   | 26 +++++++++++--------
 .../ubic/gemma/rest/RootWebServiceTest.java   |  8 ++----
 .../ubic/gemma/rest/SearchWebServiceTest.java |  4 +--
 .../ubic/gemma/rest/TaxaWebServiceTest.java   | 15 +++++------
 .../ubic/gemma/rest/util/BaseJerseyTest.java  |  8 +++++-
 9 files changed, 56 insertions(+), 50 deletions(-)

diff --git a/gemma-rest/src/test/java/ubic/gemma/rest/AnalysisResultSetsJerseyTest.java b/gemma-rest/src/test/java/ubic/gemma/rest/AnalysisResultSetsJerseyTest.java
index 59f69fbcd6..8eb5da2969 100644
--- a/gemma-rest/src/test/java/ubic/gemma/rest/AnalysisResultSetsJerseyTest.java
+++ b/gemma-rest/src/test/java/ubic/gemma/rest/AnalysisResultSetsJerseyTest.java
@@ -43,7 +43,7 @@ public void setUpMocks() {
     }
 
     @After
-    public void tearDown() {
+    public void removeFixtures() {
         expressionExperimentService.remove( ee );
     }
 
diff --git a/gemma-rest/src/test/java/ubic/gemma/rest/AnalysisResultSetsWebServiceTest.java b/gemma-rest/src/test/java/ubic/gemma/rest/AnalysisResultSetsWebServiceTest.java
index f88020c768..b045cdb492 100644
--- a/gemma-rest/src/test/java/ubic/gemma/rest/AnalysisResultSetsWebServiceTest.java
+++ b/gemma-rest/src/test/java/ubic/gemma/rest/AnalysisResultSetsWebServiceTest.java
@@ -9,7 +9,7 @@
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.test.context.ActiveProfiles;
 import org.springframework.test.context.web.WebAppConfiguration;
-import ubic.gemma.core.util.test.BaseSpringContextTest;
+import ubic.gemma.core.util.test.PersistentDummyObjectHelper;
 import ubic.gemma.model.analysis.expression.diff.*;
 import ubic.gemma.model.common.description.DatabaseEntry;
 import ubic.gemma.model.common.description.ExternalDatabase;
@@ -17,10 +17,11 @@
 import ubic.gemma.model.expression.designElement.CompositeSequence;
 import ubic.gemma.model.expression.experiment.ExpressionExperiment;
 import ubic.gemma.persistence.service.analysis.expression.diff.DifferentialExpressionAnalysisService;
-import ubic.gemma.persistence.service.analysis.expression.diff.ExpressionAnalysisResultSetService;
 import ubic.gemma.persistence.service.common.description.DatabaseEntryService;
+import ubic.gemma.persistence.service.common.description.ExternalDatabaseService;
 import ubic.gemma.persistence.service.expression.arrayDesign.ArrayDesignService;
 import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentService;
+import ubic.gemma.rest.util.BaseJerseyIntegrationTest;
 import ubic.gemma.rest.util.MalformedArgException;
 import ubic.gemma.rest.util.ResponseDataObject;
 import ubic.gemma.rest.util.args.*;
@@ -40,9 +41,7 @@
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.*;
 
-@ActiveProfiles("web")
-@WebAppConfiguration
-public class AnalysisResultSetsWebServiceTest extends BaseSpringContextTest {
+public class AnalysisResultSetsWebServiceTest extends BaseJerseyIntegrationTest {
 
     @Autowired
     private AnalysisResultSetsWebService service;
@@ -59,6 +58,9 @@ public class AnalysisResultSetsWebServiceTest extends BaseSpringContextTest {
     @Autowired
     private ArrayDesignService arrayDesignService;
 
+    @Autowired
+    private ExternalDatabaseService externalDatabaseService;
+
     /* fixtures */
     private ArrayDesign arrayDesign;
     private ExpressionExperiment ee;
@@ -66,10 +68,13 @@ public class AnalysisResultSetsWebServiceTest extends BaseSpringContextTest {
     private ExpressionAnalysisResultSet dears;
     private DatabaseEntry databaseEntry2;
 
+    @Autowired
+    private PersistentDummyObjectHelper testHelper;
+
     @Before
-    public void setUp() throws Exception {
+    public void setupMocks() {
 
-        ee = getTestPersistentBasicExpressionExperiment();
+        ee = testHelper.getTestPersistentBasicExpressionExperiment();
 
         arrayDesign = testHelper.getTestPersistentArrayDesign( 1, true, true );
         CompositeSequence probe = arrayDesign.getCompositeSequences().stream().findFirst().orElse( null );
@@ -112,7 +117,7 @@ public void setUp() throws Exception {
     }
 
     @After
-    public void tearDown() {
+    public void removeFixtures() {
         differentialExpressionAnalysisService.remove( dea );
         expressionExperimentService.remove( ee );
         databaseEntryService.remove( databaseEntry2 );
diff --git a/gemma-rest/src/test/java/ubic/gemma/rest/DatasetsRestTest.java b/gemma-rest/src/test/java/ubic/gemma/rest/DatasetsRestTest.java
index 81b9754209..ca0c3926d7 100644
--- a/gemma-rest/src/test/java/ubic/gemma/rest/DatasetsRestTest.java
+++ b/gemma-rest/src/test/java/ubic/gemma/rest/DatasetsRestTest.java
@@ -5,14 +5,13 @@
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.test.context.ActiveProfiles;
-import org.springframework.test.context.web.WebAppConfiguration;
-import ubic.gemma.core.util.test.BaseSpringContextTest;
+import ubic.gemma.core.util.test.PersistentDummyObjectHelper;
 import ubic.gemma.core.util.test.category.SlowTest;
 import ubic.gemma.model.expression.experiment.ExpressionExperiment;
 import ubic.gemma.model.expression.experiment.ExpressionExperimentValueObject;
 import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentDao;
 import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentService;
+import ubic.gemma.rest.util.BaseJerseyIntegrationTest;
 import ubic.gemma.rest.util.MalformedArgException;
 import ubic.gemma.rest.util.QueriedAndFilteredAndPaginatedResponseDataObject;
 import ubic.gemma.rest.util.ResponseDataObject;
@@ -34,9 +33,7 @@
  * @author tesarst
  */
 @Category(SlowTest.class)
-@ActiveProfiles("web")
-@WebAppConfiguration
-public class DatasetsRestTest extends BaseSpringContextTest {
+public class DatasetsRestTest extends BaseJerseyIntegrationTest {
 
     @Autowired
     private DatasetsWebService datasetsWebService;
@@ -47,18 +44,22 @@ public class DatasetsRestTest extends BaseSpringContextTest {
     @Autowired
     private ExpressionExperimentService expressionExperimentService;
 
+    @Autowired
+    private PersistentDummyObjectHelper testHelper;
+
     /* fixtures */
     private final ArrayList ees = new ArrayList<>( 10 );
 
     @Before
-    public void setUp() throws Exception {
+    public void setUpMocks() {
         for ( int i = 0; i < 10; i++ ) {
-            ees.add( this.getNewTestPersistentCompleteExpressionExperiment() );
+            testHelper.resetSeed();
+            ees.add( testHelper.getTestExpressionExperimentWithAllDependencies( false ) );
         }
     }
 
     @After
-    public void tearDown() {
+    public void resetMocks() {
         expressionExperimentService.remove( ees );
     }
 
diff --git a/gemma-rest/src/test/java/ubic/gemma/rest/DatasetsWebServiceTest.java b/gemma-rest/src/test/java/ubic/gemma/rest/DatasetsWebServiceTest.java
index 992ffcf73d..76e41b5dc7 100644
--- a/gemma-rest/src/test/java/ubic/gemma/rest/DatasetsWebServiceTest.java
+++ b/gemma-rest/src/test/java/ubic/gemma/rest/DatasetsWebServiceTest.java
@@ -162,8 +162,7 @@ public void setUpMocks() throws TimeoutException {
     }
 
     @After
-    public void tearDown() throws Exception {
-        super.tearDown();
+    public void resetMocks() throws Exception {
         reset( expressionExperimentService, quantitationTypeService, analyticsProvider, expressionDataFileService );
     }
 
diff --git a/gemma-rest/src/test/java/ubic/gemma/rest/PlatformsWebServiceTest.java b/gemma-rest/src/test/java/ubic/gemma/rest/PlatformsWebServiceTest.java
index dbf79acb60..1234d49626 100644
--- a/gemma-rest/src/test/java/ubic/gemma/rest/PlatformsWebServiceTest.java
+++ b/gemma-rest/src/test/java/ubic/gemma/rest/PlatformsWebServiceTest.java
@@ -5,9 +5,8 @@
 import org.junit.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.security.access.AccessDeniedException;
-import org.springframework.test.context.ActiveProfiles;
-import org.springframework.test.context.web.WebAppConfiguration;
-import ubic.gemma.core.util.test.BaseSpringContextTest;
+import ubic.gemma.core.util.test.PersistentDummyObjectHelper;
+import ubic.gemma.core.util.test.TestAuthenticationUtils;
 import ubic.gemma.model.expression.arrayDesign.ArrayDesign;
 import ubic.gemma.model.expression.arrayDesign.ArrayDesignValueObject;
 import ubic.gemma.model.expression.arrayDesign.BlacklistedPlatform;
@@ -17,6 +16,7 @@
 import ubic.gemma.persistence.service.expression.arrayDesign.ArrayDesignService;
 import ubic.gemma.persistence.service.expression.experiment.BlacklistedEntityService;
 import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentService;
+import ubic.gemma.rest.util.BaseJerseyIntegrationTest;
 import ubic.gemma.rest.util.FilteredAndPaginatedResponseDataObject;
 import ubic.gemma.rest.util.PaginatedResponseDataObject;
 import ubic.gemma.rest.util.args.*;
@@ -24,9 +24,7 @@
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 
-@ActiveProfiles("web")
-@WebAppConfiguration
-public class PlatformsWebServiceTest extends BaseSpringContextTest {
+public class PlatformsWebServiceTest extends BaseJerseyIntegrationTest {
 
     @Autowired
     private PlatformsWebService platformsWebService;
@@ -40,18 +38,24 @@ public class PlatformsWebServiceTest extends BaseSpringContextTest {
     @Autowired
     private BlacklistedEntityService blacklistedEntityService;
 
+    @Autowired
+    private PersistentDummyObjectHelper testHelper;
+
+    @Autowired
+    private TestAuthenticationUtils testAuthenticationUtils;
+
     /* fixtures */
     private ExpressionExperiment expressionExperiment;
     private ArrayDesign arrayDesign;
 
     @Before
-    public void setUp() throws Exception {
-        expressionExperiment = getTestPersistentBasicExpressionExperiment();
+    public void setUpMocks() {
+        expressionExperiment = testHelper.getTestPersistentBasicExpressionExperiment();
         arrayDesign = expressionExperiment.getBioAssays().iterator().next().getArrayDesignUsed();
     }
 
     @After
-    public void tearDown() {
+    public void removeFixtures() {
         eeService.remove( expressionExperiment );
         arrayDesignService.remove( arrayDesign );
         blacklistedEntityService.removeAll();
@@ -112,11 +116,11 @@ public void testGetBlacklistedPlatformsAsNonAdmin() {
         assertThat( blacklistedEntityService.isBlacklisted( arrayDesign ) ).isTrue();
         assertThat( bp.getShortName() ).isEqualTo( arrayDesign.getShortName() );
         try {
-            runAsUser( "bob" );
+            testAuthenticationUtils.runAsUser( "bob", true );
             assertThatThrownBy( () -> platformsWebService.getBlacklistedPlatforms( FilterArg.valueOf( "" ), SortArg.valueOf( "+id" ), OffsetArg.valueOf( "0" ), LimitArg.valueOf( "20" ) ) )
                     .isInstanceOf( AccessDeniedException.class );
         } finally {
-            runAsAdmin();
+            testAuthenticationUtils.runAsAdmin();
         }
     }
 }
diff --git a/gemma-rest/src/test/java/ubic/gemma/rest/RootWebServiceTest.java b/gemma-rest/src/test/java/ubic/gemma/rest/RootWebServiceTest.java
index 21b9b65fa6..6bd3dbe070 100644
--- a/gemma-rest/src/test/java/ubic/gemma/rest/RootWebServiceTest.java
+++ b/gemma-rest/src/test/java/ubic/gemma/rest/RootWebServiceTest.java
@@ -4,10 +4,8 @@
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.mock.web.MockHttpServletRequest;
 import org.springframework.mock.web.MockServletConfig;
-import org.springframework.test.context.ActiveProfiles;
-import org.springframework.test.context.web.WebAppConfiguration;
-import ubic.gemma.core.util.test.BaseSpringContextTest;
 import ubic.gemma.persistence.util.Settings;
+import ubic.gemma.rest.util.BaseJerseyIntegrationTest;
 import ubic.gemma.rest.util.OpenApiUtils;
 import ubic.gemma.rest.util.ResponseDataObject;
 
@@ -15,9 +13,7 @@
 
 import static org.assertj.core.api.Assertions.assertThat;
 
-@ActiveProfiles("web")
-@WebAppConfiguration
-public class RootWebServiceTest extends BaseSpringContextTest {
+public class RootWebServiceTest extends BaseJerseyIntegrationTest {
 
     @Autowired
     private RootWebService rootWebService;
diff --git a/gemma-rest/src/test/java/ubic/gemma/rest/SearchWebServiceTest.java b/gemma-rest/src/test/java/ubic/gemma/rest/SearchWebServiceTest.java
index c5b240fc63..c1e67d5672 100644
--- a/gemma-rest/src/test/java/ubic/gemma/rest/SearchWebServiceTest.java
+++ b/gemma-rest/src/test/java/ubic/gemma/rest/SearchWebServiceTest.java
@@ -49,8 +49,6 @@
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.*;
 
-@ActiveProfiles("web")
-@WebAppConfiguration
 @ContextConfiguration
 public class SearchWebServiceTest extends BaseJerseyTest {
 
@@ -127,7 +125,7 @@ public void setUpMocks() {
     }
 
     @After
-    public void tearDown() {
+    public void resetMocks() {
         reset( searchService, taxonService, arrayDesignService );
     }
 
diff --git a/gemma-rest/src/test/java/ubic/gemma/rest/TaxaWebServiceTest.java b/gemma-rest/src/test/java/ubic/gemma/rest/TaxaWebServiceTest.java
index 01ab969e1a..f76d3d16aa 100644
--- a/gemma-rest/src/test/java/ubic/gemma/rest/TaxaWebServiceTest.java
+++ b/gemma-rest/src/test/java/ubic/gemma/rest/TaxaWebServiceTest.java
@@ -1,17 +1,16 @@
 package ubic.gemma.rest;
 
+import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.commons.lang3.RandomUtils;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.test.context.ActiveProfiles;
-import org.springframework.test.context.web.WebAppConfiguration;
-import ubic.gemma.core.util.test.BaseSpringContextTest;
 import ubic.gemma.model.expression.experiment.ExpressionExperimentValueObject;
 import ubic.gemma.model.genome.Taxon;
 import ubic.gemma.model.genome.TaxonValueObject;
 import ubic.gemma.persistence.service.genome.taxon.TaxonService;
+import ubic.gemma.rest.util.BaseJerseyIntegrationTest;
 import ubic.gemma.rest.util.FilteredAndPaginatedResponseDataObject;
 import ubic.gemma.rest.util.ResponseDataObject;
 import ubic.gemma.rest.util.args.*;
@@ -20,9 +19,7 @@
 
 import static org.assertj.core.api.Assertions.assertThat;
 
-@ActiveProfiles("web")
-@WebAppConfiguration
-public class TaxaWebServiceTest extends BaseSpringContextTest {
+public class TaxaWebServiceTest extends BaseJerseyIntegrationTest {
 
     @Autowired
     private TaxonService taxonService;
@@ -34,17 +31,17 @@ public class TaxaWebServiceTest extends BaseSpringContextTest {
     private Taxon taxon;
 
     @Before
-    public void setUp() throws Exception {
+    public void createFixtures() {
         taxon = new Taxon();
         taxon.setNcbiId( RandomUtils.nextInt() );
         taxon.setCommonName( "common_name_" + RandomUtils.nextInt() );
-        taxon.setScientificName( "scientific_name_" + randomName() );
+        taxon.setScientificName( "scientific_name_" + RandomStringUtils.randomAlphabetic( 10 ) );
         taxon.setIsGenesUsable( false );
         taxon = taxonService.create( taxon );
     }
 
     @After
-    public void tearDown() {
+    public void removeFixtures() {
         taxonService.remove( taxon );
     }
 
diff --git a/gemma-rest/src/test/java/ubic/gemma/rest/util/BaseJerseyTest.java b/gemma-rest/src/test/java/ubic/gemma/rest/util/BaseJerseyTest.java
index 1e3ede1f8e..08f326a10e 100644
--- a/gemma-rest/src/test/java/ubic/gemma/rest/util/BaseJerseyTest.java
+++ b/gemma-rest/src/test/java/ubic/gemma/rest/util/BaseJerseyTest.java
@@ -69,15 +69,21 @@ protected final void configureClient( ClientConfig config ) {
         config.register( GZipEncoder.class );
     }
 
+    /**
+     * This is intentionally made final to prevent subclasses from overriding.
+     */
     @Before
     @Override
     public final void setUp() throws Exception {
         super.setUp();
     }
 
+    /**
+     * This is intentionally made final to prevent subclasses from overriding.
+     */
     @After
     @Override
-    public void tearDown() throws Exception {
+    public final void tearDown() throws Exception {
         super.tearDown();
     }
 }

From f6f119c9cbf5ee8b4653f0e34fce18fa4c0762f5 Mon Sep 17 00:00:00 2001
From: Guillaume Poirier-Morency 
Date: Thu, 9 May 2024 14:05:59 -0700
Subject: [PATCH 25/61] Fix buggy interpolation of mixed named and numbered
 parameters

---
 .../service/TableMaintenanceUtilImpl.java      | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/TableMaintenanceUtilImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/TableMaintenanceUtilImpl.java
index 1bffb46d47..1e6100e404 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/TableMaintenanceUtilImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/TableMaintenanceUtilImpl.java
@@ -95,7 +95,7 @@ public class TableMaintenanceUtilImpl implements TableMaintenanceUtil {
     private static final String CD_LAST_UPDATED_SINCE = "(CD.LAST_UPDATED is null or :since is null or CD.LAST_UPDATED >= :since)";
 
     private static final String EE2C_EE_QUERY =
-            "select MIN(C.ID), C.NAME, C.DESCRIPTION, C.CATEGORY, C.CATEGORY_URI, C.`VALUE`, C.VALUE_URI, C.ORIGINAL_VALUE, C.EVIDENCE_CODE, I.ID, (" + SELECT_ANONYMOUS_MASK + "), cast(? as char(255)) "
+            "select MIN(C.ID), C.NAME, C.DESCRIPTION, C.CATEGORY, C.CATEGORY_URI, C.`VALUE`, C.VALUE_URI, C.ORIGINAL_VALUE, C.EVIDENCE_CODE, I.ID, (" + SELECT_ANONYMOUS_MASK + "), cast(:eeClass as char(255)) "
                     + "from INVESTIGATION I "
                     + "join CURATION_DETAILS CD on I.CURATION_DETAILS_FK = CD.ID "
                     + "join CHARACTERISTIC C on I.ID = C.INVESTIGATION_FK "
@@ -104,7 +104,7 @@ public class TableMaintenanceUtilImpl implements TableMaintenanceUtil {
                     + "group by I.ID, COALESCE(C.CATEGORY_URI, C.CATEGORY), COALESCE(C.VALUE_URI, C.`VALUE`)";
 
     private static final String EE2C_BM_QUERY =
-            "select MIN(C.ID), C.NAME, C.DESCRIPTION, C.CATEGORY, C.CATEGORY_URI, C.`VALUE`, C.VALUE_URI, C.ORIGINAL_VALUE, C.EVIDENCE_CODE, I.ID, (" + SELECT_ANONYMOUS_MASK + "), cast(? as char(255)) "
+            "select MIN(C.ID), C.NAME, C.DESCRIPTION, C.CATEGORY, C.CATEGORY_URI, C.`VALUE`, C.VALUE_URI, C.ORIGINAL_VALUE, C.EVIDENCE_CODE, I.ID, (" + SELECT_ANONYMOUS_MASK + "), cast(:bmClass as char(255)) "
                     + "from INVESTIGATION I "
                     + "join CURATION_DETAILS CD on I.CURATION_DETAILS_FK = CD.ID "
                     + "join BIO_ASSAY BA on I.ID = BA.EXPRESSION_EXPERIMENT_FK "
@@ -115,7 +115,7 @@ public class TableMaintenanceUtilImpl implements TableMaintenanceUtil {
                     + "group by I.ID, COALESCE(C.CATEGORY_URI, C.CATEGORY), COALESCE(C.VALUE_URI, C.`VALUE`)";
 
     private static final String EE2C_ED_QUERY =
-            "select MIN(C.ID), C.NAME, C.DESCRIPTION, C.CATEGORY, C.CATEGORY_URI, C.`VALUE`, C.VALUE_URI, C.ORIGINAL_VALUE, C.EVIDENCE_CODE, I.ID, (" + SELECT_ANONYMOUS_MASK + "), cast(? as char(255)) "
+            "select MIN(C.ID), C.NAME, C.DESCRIPTION, C.CATEGORY, C.CATEGORY_URI, C.`VALUE`, C.VALUE_URI, C.ORIGINAL_VALUE, C.EVIDENCE_CODE, I.ID, (" + SELECT_ANONYMOUS_MASK + "), cast(:edClass as char(255)) "
                     + "from INVESTIGATION I "
                     + "join CURATION_DETAILS CD on I.CURATION_DETAILS_FK = CD.ID "
                     + "join EXPERIMENTAL_DESIGN on I.EXPERIMENTAL_DESIGN_FK = EXPERIMENTAL_DESIGN.ID "
@@ -255,9 +255,9 @@ public int updateExpressionExperiment2CharacteristicEntries( @Nullable Date sinc
                                 + EE2C_ED_QUERY + " "
                                 + "on duplicate key update NAME = VALUES(NAME), DESCRIPTION = VALUES(DESCRIPTION), CATEGORY = VALUES(CATEGORY), CATEGORY_URI = VALUES(CATEGORY_URI), `VALUE` = VALUES(`VALUE`), VALUE_URI = VALUES(VALUE_URI), ORIGINAL_VALUE = VALUES(ORIGINAL_VALUE), EVIDENCE_CODE = VALUES(EVIDENCE_CODE), ACL_IS_AUTHENTICATED_ANONYMOUSLY_MASK = VALUES(ACL_IS_AUTHENTICATED_ANONYMOUSLY_MASK), LEVEL = VALUES(LEVEL)" )
                 .addSynchronizedQuerySpace( EE2C_QUERY_SPACE )
-                .setParameter( 0, ExpressionExperiment.class )
-                .setParameter( 1, BioMaterial.class )
-                .setParameter( 2, ExperimentalDesign.class )
+                .setParameter( "eeClass", ExpressionExperiment.class )
+                .setParameter( "bmClass", BioMaterial.class )
+                .setParameter( "edClass", ExperimentalDesign.class )
                 .setParameter( "since", sinceLastUpdate )
                 .executeUpdate();
         log.info( String.format( "Done updating the EXPRESSION_EXPERIMENT2CHARACTERISTIC table; %d entries were updated%s.",
@@ -271,12 +271,16 @@ public int updateExpressionExperiment2CharacteristicEntries( @Nullable Date sinc
     @Transactional
     public int updateExpressionExperiment2CharacteristicEntries( Class level, @Nullable Date sinceLastUpdate, boolean truncate ) {
         Assert.isTrue( !( sinceLastUpdate != null && truncate ), "Cannot perform a partial update with sinceLastUpdate with truncate." );
+        String levelParamName;
         String query;
         if ( level.equals( ExpressionExperiment.class ) ) {
+            levelParamName = "eeClass";
             query = EE2C_EE_QUERY;
         } else if ( level.equals( BioMaterial.class ) ) {
+            levelParamName = "bmClass";
             query = EE2C_BM_QUERY;
         } else if ( level.equals( ExperimentalDesign.class ) ) {
+            levelParamName = "edClass";
             query = EE2C_ED_QUERY;
         } else {
             throw new IllegalArgumentException( "Level must be one of ExpressionExperiment.class, BioMaterial.class or ExperimentalDesign.class." );
@@ -297,7 +301,7 @@ public int updateExpressionExperiment2CharacteristicEntries( Class level, @Nu
                                 + query + " "
                                 + "on duplicate key update NAME = VALUES(NAME), DESCRIPTION = VALUES(DESCRIPTION), CATEGORY = VALUES(CATEGORY), CATEGORY_URI = VALUES(CATEGORY_URI), `VALUE` = VALUES(`VALUE`), VALUE_URI = VALUES(VALUE_URI), ORIGINAL_VALUE = VALUES(ORIGINAL_VALUE), EVIDENCE_CODE = VALUES(EVIDENCE_CODE), ACL_IS_AUTHENTICATED_ANONYMOUSLY_MASK = VALUES(ACL_IS_AUTHENTICATED_ANONYMOUSLY_MASK), LEVEL = VALUES(LEVEL)" )
                 .addSynchronizedQuerySpace( EE2C_QUERY_SPACE )
-                .setParameter( 0, level )
+                .setParameter( levelParamName, level )
                 .setParameter( "since", sinceLastUpdate )
                 .executeUpdate();
         log.info( String.format( "Done updating the EXPRESSION_EXPERIMENT2CHARACTERISTIC table at %s level; %d entries were updated%s.",

From 4aba8b104d30aa32a20e00ee8cb79ce166eb8278 Mon Sep 17 00:00:00 2001
From: Guillaume Poirier-Morency 
Date: Thu, 9 May 2024 14:21:13 -0700
Subject: [PATCH 26/61] Remove deprecated usage of AssertJ's asList()

---
 .../java/ubic/gemma/rest/DatasetsRestTest.java   | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/gemma-rest/src/test/java/ubic/gemma/rest/DatasetsRestTest.java b/gemma-rest/src/test/java/ubic/gemma/rest/DatasetsRestTest.java
index ca0c3926d7..a609993cdd 100644
--- a/gemma-rest/src/test/java/ubic/gemma/rest/DatasetsRestTest.java
+++ b/gemma-rest/src/test/java/ubic/gemma/rest/DatasetsRestTest.java
@@ -75,7 +75,7 @@ public void testAll() {
                 .hasFieldOrProperty( "totalElements" ); // FIXME: cannot test because of leftovers from other tests but should be 10
         assertThat( response.getData() )
                 .isNotNull()
-                .asList().hasSize( 5 );
+                .hasSize( 5 );
     }
 
     @Test
@@ -89,7 +89,7 @@ public void testSome() {
         assertThat( ee.getAccession() ).isNotNull();
         assertThat( response.getData() )
                 .isNotNull()
-                .asList().hasSize( 2 )
+                .hasSize( 2 )
                 .first()
                 .hasFieldOrPropertyWithValue( "accession", ee.getAccession().getAccession() )
                 .hasFieldOrPropertyWithValue( "externalDatabase", ee.getAccession().getExternalDatabase().getName() )
@@ -107,7 +107,7 @@ public void testSomeById() {
         assertThat( ee.getAccession() ).isNotNull();
         assertThat( response.getData() )
                 .isNotNull()
-                .asList().hasSize( 2 )
+                .hasSize( 2 )
                 .first()
                 .hasFieldOrPropertyWithValue( "accession", ee.getAccession().getAccession() )
                 .hasFieldOrPropertyWithValue( "externalDatabase", ee.getAccession().getExternalDatabase().getName() )
@@ -127,7 +127,7 @@ public void testAllFilterById() {
         );
         assertThat( response.getData() )
                 .isNotNull()
-                .asList().hasSize( 1 )
+                .hasSize( 1 )
                 .first()
                 .hasFieldOrPropertyWithValue( "id", ees.get( 0 ).getId() );
     }
@@ -150,7 +150,7 @@ public void testAllFilterByIdIn() {
         );
         assertThat( response.getData() )
                 .isNotNull()
-                .asList().hasSize( 1 )
+                .hasSize( 1 )
                 .first()
                 .hasFieldOrPropertyWithValue( "id", ees.get( 0 ).getId() )
                 .hasFieldOrPropertyWithValue( "shortName", ees.get( 0 ).getShortName() );
@@ -174,7 +174,7 @@ public void testAllFilterByShortName() {
         );
         assertThat( response.getData() )
                 .isNotNull()
-                .asList().hasSize( 1 )
+                .hasSize( 1 )
                 .first()
                 .hasFieldOrPropertyWithValue( "id", ees.get( 0 ).getId() )
                 .hasFieldOrPropertyWithValue( "shortName", ees.get( 0 ).getShortName() );
@@ -198,7 +198,7 @@ public void testAllFilterByShortNameIn() {
         );
         assertThat( response.getData() )
                 .isNotNull()
-                .asList().hasSize( 1 )
+                .hasSize( 1 )
                 .first()
                 .hasFieldOrPropertyWithValue( "id", ees.get( 0 ).getId() )
                 .hasFieldOrPropertyWithValue( "shortName", ees.get( 0 ).getShortName() );
@@ -230,7 +230,7 @@ public void testAllFilterByIdInOrShortNameIn() {
         );
         assertThat( response.getData() )
                 .isNotNull()
-                .asList().hasSize( 2 );
+                .hasSize( 2 );
         assertThat( response.getData() )
                 .extracting( "id" )
                 .containsExactlyInAnyOrder( ees.get( 0 ).getId(), ees.get( 1 ).getId() );

From a3bc9765a1fa848dc447d1c7855ece1652a84d86 Mon Sep 17 00:00:00 2001
From: Guillaume Poirier-Morency 
Date: Thu, 9 May 2024 14:32:24 -0700
Subject: [PATCH 27/61] rest: Add a human-readable description for jaxrs
 response in assertions

---
 .../ubic/gemma/rest/util/ResponseAssert.java  | 27 +++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/gemma-rest/src/test/java/ubic/gemma/rest/util/ResponseAssert.java b/gemma-rest/src/test/java/ubic/gemma/rest/util/ResponseAssert.java
index 304f532f67..59af4b7261 100644
--- a/gemma-rest/src/test/java/ubic/gemma/rest/util/ResponseAssert.java
+++ b/gemma-rest/src/test/java/ubic/gemma/rest/util/ResponseAssert.java
@@ -1,5 +1,6 @@
 package ubic.gemma.rest.util;
 
+import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.assertj.core.api.*;
 import org.assertj.core.internal.Maps;
@@ -8,10 +9,15 @@
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.NewCookie;
 import javax.ws.rs.core.Response;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
 import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 import java.util.function.Consumer;
+import java.util.stream.Collectors;
 
 /**
  * Assertions for jax-rs {@link Response}.
@@ -23,6 +29,27 @@ public class ResponseAssert extends AbstractAssert {
 
     public ResponseAssert( Response actual ) {
         super( actual, ResponseAssert.class );
+        info.description( "\nHTTP/1.1 %d %s\n%s\n\n%s\n",
+                actual.getStatus(), actual.getStatusInfo().getReasonPhrase(),
+                actual.getStringHeaders().entrySet().stream()
+                        .sorted( Map.Entry.comparingByKey() )
+                        .map( e -> e.getKey() + ": " + String.join( ", ", e.getValue() ) )
+                        .collect( Collectors.joining( "\n" ) ),
+                formatEntity( actual.getEntity() ) );
+    }
+
+    private String formatEntity( Object entity ) {
+        if ( entity instanceof ByteArrayInputStream ) {
+            try {
+                return IOUtils.toString( ( InputStream ) entity, StandardCharsets.UTF_8 );
+            } catch ( IOException e ) {
+                throw new RuntimeException( e );
+            } finally {
+                ( ( ByteArrayInputStream ) entity ).reset();
+            }
+        } else {
+            return entity.toString();
+        }
     }
 
     /**

From 8afa50795c64a91eb096a8388d71bb52a7afd0f5 Mon Sep 17 00:00:00 2001
From: Guillaume Poirier-Morency 
Date: Fri, 10 May 2024 10:25:28 -0700
Subject: [PATCH 28/61] Refactor test classe

Intrduce two base classes: BaseIntegrationTest and BaseWebIntegrationTest
which are slimmed down version of BaseSpringContextTest and BaseSpringWebTest.

Avoid relying on InitializingBean in abstract classes and instead setup
test resources on-demand
---
 .../links/LinkAnalysisServiceTest.java        |  4 +-
 .../core/security/audit/AuditAdviceTest.java  |  4 +-
 .../core/util/test/BaseIntegrationTest.java   | 49 ++++++++++++++++
 .../core/util/test/BaseSpringContextTest.java | 58 ++++---------------
 .../gemma/persistence/retry/RetryTest.java    |  4 +-
 .../rest/util/BaseJerseyIntegrationTest.java  |  6 +-
 .../ubic/gemma/rest/util/BaseJerseyTest.java  | 25 +++-----
 .../gemma/web/controller/ErrorPagesTest.java  | 14 ++---
 .../controller/OntologyControllerTest.java    | 12 ++--
 .../SignupControllerTest.java                 | 54 ++++++++---------
 .../bibref/BibRefControllerTest.java          |  2 +-
 .../bibref/PubMedQueryControllerTest.java     |  8 +--
 ...ntialExpressionAnalysisControllerTest.java | 16 ++---
 .../ArrayDesignControllerTest.java            |  5 +-
 ...mentalDesignControllerIntegrationTest.java |  6 +-
 .../ExpressionExperimentControllerTest.java   |  6 +-
 .../web/scheduler/SchedulerSecurityTest.java  |  4 +-
 .../gemma/web/util/BaseSpringWebTest.java     | 36 +++++-------
 .../web/util/BaseWebIntegrationTest.java      | 36 ++++++++++++
 .../java/ubic/gemma/web/util/BaseWebTest.java | 20 ++++---
 .../gemma/web/util/MessageSourceTest.java     |  2 +-
 21 files changed, 209 insertions(+), 162 deletions(-)
 create mode 100644 gemma-core/src/test/java/ubic/gemma/core/util/test/BaseIntegrationTest.java
 create mode 100644 gemma-web/src/test/java/ubic/gemma/web/util/BaseWebIntegrationTest.java

diff --git a/gemma-core/src/test/java/ubic/gemma/core/analysis/expression/coexpression/links/LinkAnalysisServiceTest.java b/gemma-core/src/test/java/ubic/gemma/core/analysis/expression/coexpression/links/LinkAnalysisServiceTest.java
index f833b60729..9ff585cc94 100644
--- a/gemma-core/src/test/java/ubic/gemma/core/analysis/expression/coexpression/links/LinkAnalysisServiceTest.java
+++ b/gemma-core/src/test/java/ubic/gemma/core/analysis/expression/coexpression/links/LinkAnalysisServiceTest.java
@@ -174,7 +174,7 @@ public void testLoadAnalyzeSaveAndCoexpSearch() {
     }
 
     private void checkUnsupportedLinksHaveNoSupport() {
-        JdbcTemplate jt = jdbcTemplate;
+        JdbcTemplate jt = getJdbcTemplate();
 
         // see SupportDetailsTest for validation that these strings represent empty byte arrays. I think the 1 at
         // position 12 is important.
@@ -306,7 +306,7 @@ private int checkResults( Collection ees, int expectedMinimumMaxSup
                         .findCoexpressionRelationships( mouse, EntityUtils.getIds( genesWithLinks ),
                                 EntityUtils.getIds( ees ), 100, false );
 
-                if( !multiGeneResults.isEmpty() ) {
+                if ( !multiGeneResults.isEmpty() ) {
 
                     for ( Long id : multiGeneResults.keySet() ) {
                         for ( CoexpressionValueObject coex : multiGeneResults.get( id ) ) {
diff --git a/gemma-core/src/test/java/ubic/gemma/core/security/audit/AuditAdviceTest.java b/gemma-core/src/test/java/ubic/gemma/core/security/audit/AuditAdviceTest.java
index 7d520018ff..b1fc96603e 100644
--- a/gemma-core/src/test/java/ubic/gemma/core/security/audit/AuditAdviceTest.java
+++ b/gemma-core/src/test/java/ubic/gemma/core/security/audit/AuditAdviceTest.java
@@ -239,12 +239,12 @@ private void checkAuditTrail( Auditable c, Collection trailIds, Collection
     }
 
     private boolean checkDeletedAuditTrail( Long atid ) {
-        return this.jdbcTemplate
+        return getJdbcTemplate()
                 .queryForObject( "SELECT COUNT(*) FROM AUDIT_TRAIL WHERE ID = ?", Integer.class, atid ) == 0;
     }
 
     private boolean checkDeletedEvent( Long i ) {
-        return this.jdbcTemplate
+        return getJdbcTemplate()
                 .queryForObject( "SELECT COUNT(*) FROM AUDIT_EVENT WHERE ID = ?", Integer.class, i ) == 0;
     }
 
diff --git a/gemma-core/src/test/java/ubic/gemma/core/util/test/BaseIntegrationTest.java b/gemma-core/src/test/java/ubic/gemma/core/util/test/BaseIntegrationTest.java
new file mode 100644
index 0000000000..f1a559b23b
--- /dev/null
+++ b/gemma-core/src/test/java/ubic/gemma/core/util/test/BaseIntegrationTest.java
@@ -0,0 +1,49 @@
+package ubic.gemma.core.util.test;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.experimental.categories.Category;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.security.core.context.SecurityContextHolder;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.junit4.AbstractJUnit4SpringContextTests;
+import ubic.gemma.core.util.test.category.IntegrationTest;
+import ubic.gemma.persistence.util.EnvironmentProfiles;
+
+/**
+ * Base class for integration tests.
+ * @author poirigui
+ */
+@ActiveProfiles(EnvironmentProfiles.TEST)
+@Category(IntegrationTest.class)
+@ContextConfiguration(locations = { "classpath*:ubic/gemma/applicationContext-*.xml" })
+public abstract class BaseIntegrationTest extends AbstractJUnit4SpringContextTests {
+
+    @Autowired
+    private TestAuthenticationUtils testAuthenticationUtils;
+
+    @BeforeClass
+    public static void setUpSecurityContextHolderStrategy() {
+        SecurityContextHolder.setStrategyName( SecurityContextHolder.MODE_INHERITABLETHREADLOCAL );
+    }
+
+    /**
+     * Setup the authentication for the test.
+     * 

+     * The default is to grant an administrator authority to the current user.
+     */
+    @Before
+    public final void setUpAuthentication() {
+        testAuthenticationUtils.runAsAdmin();
+    }
+
+    /**
+     * Clear the {@link SecurityContextHolder} so that subsequent tests don't inherit authentication.
+     */
+    @After
+    public final void tearDownSecurityContext() {
+        SecurityContextHolder.clearContext();
+    }
+}
diff --git a/gemma-core/src/test/java/ubic/gemma/core/util/test/BaseSpringContextTest.java b/gemma-core/src/test/java/ubic/gemma/core/util/test/BaseSpringContextTest.java
index 395b0ae86d..14d3a5d6e3 100644
--- a/gemma-core/src/test/java/ubic/gemma/core/util/test/BaseSpringContextTest.java
+++ b/gemma-core/src/test/java/ubic/gemma/core/util/test/BaseSpringContextTest.java
@@ -22,25 +22,15 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.hibernate.SessionFactory;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.BeforeClass;
 import org.junit.Rule;
-import org.junit.experimental.categories.Category;
 import org.mockito.junit.MockitoJUnit;
 import org.mockito.junit.MockitoRule;
-import org.springframework.beans.factory.InitializingBean;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.io.Resource;
 import org.springframework.core.io.support.EncodedResource;
 import org.springframework.dao.DataAccessException;
 import org.springframework.jdbc.core.JdbcTemplate;
-import org.springframework.security.core.context.SecurityContextHolder;
-import org.springframework.test.context.ActiveProfiles;
-import org.springframework.test.context.ContextConfiguration;
-import org.springframework.test.context.junit4.AbstractJUnit4SpringContextTests;
 import org.springframework.test.jdbc.JdbcTestUtils;
-import ubic.gemma.core.util.test.category.IntegrationTest;
 import ubic.gemma.model.analysis.Analysis;
 import ubic.gemma.model.association.BioSequence2GeneProduct;
 import ubic.gemma.model.common.auditAndSecurity.Contact;
@@ -62,9 +52,7 @@
 import ubic.gemma.persistence.persister.PersisterHelper;
 import ubic.gemma.persistence.service.common.description.ExternalDatabaseService;
 import ubic.gemma.persistence.service.genome.taxon.TaxonService;
-import ubic.gemma.persistence.util.EnvironmentProfiles;
 
-import javax.annotation.OverridingMethodsMustInvokeSuper;
 import javax.sql.DataSource;
 import java.nio.charset.StandardCharsets;
 import java.util.Collection;
@@ -74,15 +62,13 @@
 import static java.util.Objects.requireNonNull;
 
 /**
- * subclass for tests that need the container and use the database
- *
+ * Add a few utilities on top of {@link BaseIntegrationTest}.
  * @author pavlidis
+ * @deprecated favour the simpler {@link BaseIntegrationTest} for new tests
  */
-@ActiveProfiles(EnvironmentProfiles.TEST)
-@Category(IntegrationTest.class)
+@Deprecated
 @SuppressWarnings({ "WeakerAccess", "SameParameterValue", "unused" }) // Better left as is for future convenience
-@ContextConfiguration(locations = { "classpath*:ubic/gemma/applicationContext-*.xml" })
-public abstract class BaseSpringContextTest extends AbstractJUnit4SpringContextTests implements InitializingBean {
+public abstract class BaseSpringContextTest extends BaseIntegrationTest {
 
     /* shared fixtures */
     private static ArrayDesign readOnlyAd = null;
@@ -96,11 +82,6 @@ public abstract class BaseSpringContextTest extends AbstractJUnit4SpringContextT
     @Rule
     public MockitoRule rule = MockitoJUnit.rule();
 
-    /**
-     * The SimpleJdbcTemplate that this base class manages, available to subclasses. (Datasource; autowired at setter)
-     */
-    protected JdbcTemplate jdbcTemplate;
-
     /**
      * The data source as defined in ubic/gemma/applicationContext-dataSource.xml
      */
@@ -120,33 +101,16 @@ public abstract class BaseSpringContextTest extends AbstractJUnit4SpringContextT
     @Autowired
     private TestAuthenticationUtils testAuthenticationUtils;
 
-    @Override
-    @OverridingMethodsMustInvokeSuper
-    public void afterPropertiesSet() {
-        this.jdbcTemplate = new JdbcTemplate( dataSource );
-    }
-
-    @BeforeClass
-    public static void setUpSecurityContextHolderStrategy() {
-        SecurityContextHolder.setStrategyName( SecurityContextHolder.MODE_INHERITABLETHREADLOCAL );
-    }
-
     /**
-     * Setup the authentication for the test.
-     * 

-     * The default is to grant an administrator authority to the current user.
+     * The SimpleJdbcTemplate that this base class manages, available to subclasses. (Datasource; autowired at setter)
      */
-    @Before
-    public void setUpAuthentication() {
-        testAuthenticationUtils.runAsAdmin();
-    }
+    private JdbcTemplate jdbcTemplate;
 
-    /**
-     * Clear the {@link SecurityContextHolder} so that subsequent tests don't inherit authentication.
-     */
-    @After
-    public final void tearDownSecurityContext() {
-        SecurityContextHolder.clearContext();
+    protected JdbcTemplate getJdbcTemplate() {
+        if ( jdbcTemplate == null ) {
+            jdbcTemplate = new JdbcTemplate( dataSource );
+        }
+        return jdbcTemplate;
     }
 
     /**
diff --git a/gemma-core/src/test/java/ubic/gemma/persistence/retry/RetryTest.java b/gemma-core/src/test/java/ubic/gemma/persistence/retry/RetryTest.java
index 49cf7971c5..1db6ab5fad 100644
--- a/gemma-core/src/test/java/ubic/gemma/persistence/retry/RetryTest.java
+++ b/gemma-core/src/test/java/ubic/gemma/persistence/retry/RetryTest.java
@@ -3,6 +3,7 @@
 import org.junit.After;
 import org.junit.Test;
 import org.mockito.internal.verification.VerificationModeFactory;
+import org.springframework.beans.factory.InitializingBean;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.dao.PessimisticLockingFailureException;
 import org.springframework.retry.policy.SimpleRetryPolicy;
@@ -13,7 +14,7 @@
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.mockito.Mockito.*;
 
-public class RetryTest extends BaseSpringContextTest {
+public class RetryTest extends BaseSpringContextTest implements InitializingBean {
 
     public interface TestRetryDao {
 
@@ -54,7 +55,6 @@ public void setTestRetryDao( TestRetryDao testRetryDao ) {
 
     @Override
     public void afterPropertiesSet() {
-        super.afterPropertiesSet();
         testRetryDao = mock( TestRetryDao.class );
         testRetryService.setTestRetryDao( testRetryDao );
         // 10 is too slow for testing
diff --git a/gemma-rest/src/test/java/ubic/gemma/rest/util/BaseJerseyIntegrationTest.java b/gemma-rest/src/test/java/ubic/gemma/rest/util/BaseJerseyIntegrationTest.java
index 5062301717..e2448a7836 100644
--- a/gemma-rest/src/test/java/ubic/gemma/rest/util/BaseJerseyIntegrationTest.java
+++ b/gemma-rest/src/test/java/ubic/gemma/rest/util/BaseJerseyIntegrationTest.java
@@ -12,8 +12,12 @@
 
 /**
  * Base class for Jersey-based integration tests.
+ * 

+ * Unfortunately, it's not possible to inherit from {@link ubic.gemma.core.util.test.BaseIntegrationTest} so we have to
+ * duplicate some of the setup and teardown code.
  *
  * @author poirigui
+ * @see ubic.gemma.core.util.test.BaseIntegrationTest
  */
 @Category(IntegrationTest.class)
 @ContextConfiguration(locations = { "classpath*:ubic/gemma/applicationContext-*.xml" })
@@ -28,7 +32,7 @@ public static void setUpSecurityContextHolderStrategy() {
     }
 
     @Before
-    public void setUpAuthentication() {
+    public final void setUpAuthentication() {
         testAuthenticationUtils.runAsAdmin();
     }
 
diff --git a/gemma-rest/src/test/java/ubic/gemma/rest/util/BaseJerseyTest.java b/gemma-rest/src/test/java/ubic/gemma/rest/util/BaseJerseyTest.java
index 08f326a10e..af72152818 100644
--- a/gemma-rest/src/test/java/ubic/gemma/rest/util/BaseJerseyTest.java
+++ b/gemma-rest/src/test/java/ubic/gemma/rest/util/BaseJerseyTest.java
@@ -10,8 +10,8 @@
 import org.junit.After;
 import org.junit.Before;
 import org.junit.runner.RunWith;
-import org.springframework.beans.factory.InitializingBean;
-import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.ApplicationContextAware;
 import org.springframework.security.core.context.SecurityContextHolder;
 import org.springframework.test.context.ActiveProfiles;
 import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
@@ -30,39 +30,32 @@
 @ActiveProfiles({ "web", EnvironmentProfiles.TEST })
 @RunWith(SpringJUnit4ClassRunner.class)
 @WebAppConfiguration
-public abstract class BaseJerseyTest extends JerseyTest implements InitializingBean {
+public abstract class BaseJerseyTest extends JerseyTest implements ApplicationContextAware {
 
     private ResourceConfig application;
 
-    /**
-     * The {@link WebApplicationContext} that is being used by the container. You can use it to inject specific beans
-     * for testing purposes.
-     */
-    @Autowired
-    private WebApplicationContext applicationContext;
-
     @Override
     protected final TestContainerFactory getTestContainerFactory() throws TestContainerException {
         return new InMemoryTestContainerFactory();
     }
 
-    @Override
-    public final void afterPropertiesSet() {
-        application.property( "contextConfig", applicationContext );
-    }
-
     @Override
     protected final Application configure() {
         SecurityContextHolder.setStrategyName( SecurityContextHolder.MODE_INHERITABLETHREADLOCAL );
         application = new ResourceConfig()
                 .packages( "io.swagger.v3.jaxrs2.integration.resources", "ubic.gemma.rest" )
                 .registerClasses( GZipEncoder.class )
-                // use a generic context for now, it will be replaced when this bean is fully initialized in afterPropertiesSet()
+                // use a generic context for now, it will be replaced when this bean is fully initialized in setApplicationContext()
                 .property( "contextConfig", new GenericWebApplicationContext() )
                 .property( "openApi.configuration.location", "/WEB-INF/classes/openapi-configuration.yaml" );
         return application;
     }
 
+    @Override
+    public final void setApplicationContext( ApplicationContext applicationContext ) {
+        application.property( "contextConfig", applicationContext );
+    }
+
     @Override
     protected final void configureClient( ClientConfig config ) {
         // ensures that the test client can decompress gzipped payloads
diff --git a/gemma-web/src/test/java/ubic/gemma/web/controller/ErrorPagesTest.java b/gemma-web/src/test/java/ubic/gemma/web/controller/ErrorPagesTest.java
index 138a31f02f..13c1742caa 100644
--- a/gemma-web/src/test/java/ubic/gemma/web/controller/ErrorPagesTest.java
+++ b/gemma-web/src/test/java/ubic/gemma/web/controller/ErrorPagesTest.java
@@ -4,14 +4,14 @@
 import org.springframework.security.access.AccessDeniedException;
 import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.RequestMapping;
-import ubic.gemma.web.util.BaseSpringWebTest;
+import ubic.gemma.web.util.BaseWebIntegrationTest;
 import ubic.gemma.web.util.EntityNotFoundException;
 
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
 import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*;
 
-public class ErrorPagesTest extends BaseSpringWebTest {
+public class ErrorPagesTest extends BaseWebIntegrationTest {
 
     @Controller
     static class ErrorController {
@@ -39,21 +39,21 @@ public String error500FromException() {
 
     @Test
     public void test() throws Exception {
-        mvc.perform( get( "/test/error/400" ) )
+        perform( get( "/test/error/400" ) )
                 .andExpect( status().isBadRequest() )
                 .andExpect( view().name( "error/400" ) )
                 .andExpect( model().attribute( "exception", instanceOf( IllegalArgumentException.class ) ) );
-        mvc.perform( get( "/test/error/403/from_access_denied" ) )
+        perform( get( "/test/error/403/from_access_denied" ) )
                 .andExpect( status().isForbidden() )
                 .andExpect( view().name( "error/403" ) )
                 .andExpect( model().attribute( "exception", instanceOf( AccessDeniedException.class ) ) );
-        mvc.perform( get( "/test/error/404" ) )
+        perform( get( "/test/error/404" ) )
                 .andExpect( status().isNotFound() )
                 .andExpect( view().name( "error/404" ) )
                 .andExpect( model().attribute( "exception", instanceOf( EntityNotFoundException.class ) ) );
-        mvc.perform( get( "/test/error/404/built_in_spring_mvc_error" ) )
+        perform( get( "/test/error/404/built_in_spring_mvc_error" ) )
                 .andExpect( status().isNotFound() );
-        mvc.perform( get( "/test/error/500" ) )
+        perform( get( "/test/error/500" ) )
                 .andExpect( status().isInternalServerError() )
                 .andExpect( view().name( "error/500" ) )
                 .andExpect( model().attribute( "exception", instanceOf( RuntimeException.class ) ) );
diff --git a/gemma-web/src/test/java/ubic/gemma/web/controller/OntologyControllerTest.java b/gemma-web/src/test/java/ubic/gemma/web/controller/OntologyControllerTest.java
index 762a3ef7d4..220767d9cc 100644
--- a/gemma-web/src/test/java/ubic/gemma/web/controller/OntologyControllerTest.java
+++ b/gemma-web/src/test/java/ubic/gemma/web/controller/OntologyControllerTest.java
@@ -80,27 +80,27 @@ public void tearDown() {
     @Test
     public void testGemmaOntologyUnavailable() throws Exception {
         when( gemmaOntology.isOntologyLoaded() ).thenReturn( false );
-        mvc.perform( get( "/ont/TGEMO_00001" ) )
+        perform( get( "/ont/TGEMO_00001" ) )
                 .andExpect( status().isServiceUnavailable() );
     }
 
     @Test
     public void testGetObo() throws Exception {
-        mvc.perform( get( "/ont/TGEMO.OWL" ) )
+        perform( get( "/ont/TGEMO.OWL" ) )
                 .andExpect( status().isFound() )
                 .andExpect( redirectedUrl( gemmaOntologyUrl ) );
     }
 
     @Test
     public void testGetTerm() throws Exception {
-        mvc.perform( get( "/ont/TGEMO_00001" ) )
+        perform( get( "/ont/TGEMO_00001" ) )
                 .andExpect( status().isFound() )
                 .andExpect( redirectedUrl( gemmaOntologyUrl + "#http://gemma.msl.ubc.ca/ont/TGEMO_00001" ) );
     }
 
     @Test
     public void testGetMissingTerm() throws Exception {
-        mvc.perform( get( "/ont/TGEMO_02312312" ) )
+        perform( get( "/ont/TGEMO_02312312" ) )
                 .andExpect( status().isNotFound() )
                 .andExpect( view().name( "error/404" ) )
                 .andExpect( model().attribute( "exception", instanceOf( EntityNotFoundException.class ) ) );
@@ -112,7 +112,7 @@ public void testGetTgfvoAsHtml() throws Exception {
         OntologyIndividual oi = new OntologyIndividualSimple( "http://gemma.msl.ubc.ca/ont/TGFVO/1", "foo", fvClass );
         when( factorValueOntologyService.getIndividual( "http://gemma.msl.ubc.ca/ont/TGFVO/1" ) )
                 .thenReturn( oi );
-        mvc.perform( get( "/ont/TGFVO/1" ) )
+        perform( get( "/ont/TGFVO/1" ) )
                 .andExpect( status().isOk() )
                 .andExpect( content().string( containsString( "FactorValue #1: foo" ) ) )
                 .andExpect( content().string( containsString( "instance of" ) ) )
@@ -129,7 +129,7 @@ public void testGetTgfvoAsRdf() throws Exception {
         OntologyIndividual oi = new OntologyIndividualSimple( "http://gemma.msl.ubc.ca/ont/TGFVO/1", "foo", fvClass );
         when( factorValueOntologyService.getIndividual( "http://gemma.msl.ubc.ca/ont/TGFVO/1" ) )
                 .thenReturn( oi );
-        mvc.perform( get( "/ont/TGFVO/1" ).accept( MediaType.parseMediaType( "application/rdf+xml" ) ) )
+        perform( get( "/ont/TGFVO/1" ).accept( MediaType.parseMediaType( "application/rdf+xml" ) ) )
                 .andExpect( status().isOk() )
                 .andExpect( content().contentTypeCompatibleWith( "application/rdf+xml" ) );
         verify( factorValueOntologyService ).getIndividual( "http://gemma.msl.ubc.ca/ont/TGFVO/1" );
diff --git a/gemma-web/src/test/java/ubic/gemma/web/controller/common/auditAndSecurity/SignupControllerTest.java b/gemma-web/src/test/java/ubic/gemma/web/controller/common/auditAndSecurity/SignupControllerTest.java
index e7e9f0e799..759c549427 100644
--- a/gemma-web/src/test/java/ubic/gemma/web/controller/common/auditAndSecurity/SignupControllerTest.java
+++ b/gemma-web/src/test/java/ubic/gemma/web/controller/common/auditAndSecurity/SignupControllerTest.java
@@ -18,6 +18,7 @@
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
+import org.springframework.beans.factory.InitializingBean;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.MediaType;
 import org.springframework.security.authentication.encoding.PasswordEncoder;
@@ -25,7 +26,7 @@
 import ubic.gemma.core.security.authentication.UserManager;
 import ubic.gemma.web.controller.common.auditAndSecurity.recaptcha.ReCaptcha;
 import ubic.gemma.web.controller.common.auditAndSecurity.recaptcha.ReCaptchaResponse;
-import ubic.gemma.web.util.BaseSpringWebTest;
+import ubic.gemma.web.util.BaseWebIntegrationTest;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.ArgumentMatchers.any;
@@ -40,7 +41,7 @@
  * @author Paul
  */
 @DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS)
-public class SignupControllerTest extends BaseSpringWebTest {
+public class SignupControllerTest extends BaseWebIntegrationTest implements InitializingBean {
 
     @Autowired
     private SignupController suc;
@@ -55,7 +56,6 @@ public class SignupControllerTest extends BaseSpringWebTest {
 
     @Override
     public void afterPropertiesSet() {
-        super.afterPropertiesSet();
         suc.setRecaptchaTester( mockReCaptcha );
     }
 
@@ -76,12 +76,12 @@ public void testSignup() throws Exception {
         String uname = RandomStringUtils.randomAlphabetic( 10 );
         String password = RandomStringUtils.randomAlphabetic( 40 );
         String email = "foo@" + RandomStringUtils.randomAlphabetic( 10 ) + ".edu";
-        mvc.perform( post( "/signup.html" )
-                        .param( "password", password )
-                        .param( "passwordConfirm", password )
-                        .param( "username", uname )
-                        .param( "email", email )
-                        .param( "emailConfirm", email ) )
+        perform( post( "/signup.html" )
+                .param( "password", password )
+                .param( "passwordConfirm", password )
+                .param( "username", uname )
+                .param( "email", email )
+                .param( "emailConfirm", email ) )
                 .andExpect( status().isOk() )
                 .andExpect( content().contentTypeCompatibleWith( MediaType.APPLICATION_JSON ) )
                 .andExpect( jsonPath( "$.success" ).value( true ) );
@@ -102,12 +102,12 @@ public void testSignupWhenRecaptchaIsDisabled() throws Exception {
         String uname = RandomStringUtils.randomAlphabetic( 10 );
         String password = RandomStringUtils.randomAlphabetic( 40 );
         String email = "foo@" + RandomStringUtils.randomAlphabetic( 10 ) + ".edu";
-        mvc.perform( post( "/signup.html" )
-                        .param( "password", password )
-                        .param( "passwordConfirm", password )
-                        .param( "username", uname )
-                        .param( "email", email )
-                        .param( "emailConfirm", email ) )
+        perform( post( "/signup.html" )
+                .param( "password", password )
+                .param( "passwordConfirm", password )
+                .param( "username", uname )
+                .param( "email", email )
+                .param( "emailConfirm", email ) )
                 .andExpect( status().isOk() )
                 .andExpect( content().contentTypeCompatibleWith( MediaType.APPLICATION_JSON ) )
                 .andExpect( jsonPath( "$.success" ).value( true ) );
@@ -129,12 +129,12 @@ public void testSignupWithRecaptchaIsInvalid() throws Exception {
         String uname = RandomStringUtils.randomAlphabetic( 10 );
         String password = RandomStringUtils.randomAlphabetic( 40 );
         String email = "foo@" + RandomStringUtils.randomAlphabetic( 10 ) + ".edu";
-        mvc.perform( post( "/signup.html" )
-                        .param( "password", password )
-                        .param( "passwordConfirm", password )
-                        .param( "username", uname )
-                        .param( "email", email )
-                        .param( "emailConfirm", email ) )
+        perform( post( "/signup.html" )
+                .param( "password", password )
+                .param( "passwordConfirm", password )
+                .param( "username", uname )
+                .param( "email", email )
+                .param( "emailConfirm", email ) )
                 .andExpect( status().isBadRequest() )
                 .andExpect( content().contentTypeCompatibleWith( MediaType.APPLICATION_JSON ) )
                 .andExpect( jsonPath( "$.success" ).value( false ) );
@@ -151,12 +151,12 @@ public void testSignupWithPasswordDosentMatch() throws Exception {
         String uname = RandomStringUtils.randomAlphabetic( 10 );
         String password = RandomStringUtils.randomAlphabetic( 40 );
         String email = "foo@" + RandomStringUtils.randomAlphabetic( 10 ) + ".edu";
-        mvc.perform( post( "/signup.html" )
-                        .param( "password", password )
-                        .param( "passwordConfirm", password )
-                        .param( "username", uname )
-                        .param( "email", email )
-                        .param( "emailConfirm", email ) )
+        perform( post( "/signup.html" )
+                .param( "password", password )
+                .param( "passwordConfirm", password )
+                .param( "username", uname )
+                .param( "email", email )
+                .param( "emailConfirm", email ) )
                 .andExpect( status().isBadRequest() )
                 .andExpect( content().contentTypeCompatibleWith( MediaType.APPLICATION_JSON ) )
                 .andExpect( jsonPath( "$.success" ).value( false ) );
diff --git a/gemma-web/src/test/java/ubic/gemma/web/controller/common/description/bibref/BibRefControllerTest.java b/gemma-web/src/test/java/ubic/gemma/web/controller/common/description/bibref/BibRefControllerTest.java
index d5a1c240ff..01ac04ad60 100644
--- a/gemma-web/src/test/java/ubic/gemma/web/controller/common/description/bibref/BibRefControllerTest.java
+++ b/gemma-web/src/test/java/ubic/gemma/web/controller/common/description/bibref/BibRefControllerTest.java
@@ -164,7 +164,7 @@ public void testShow() throws Exception {
     @Test
     public void testShowAllForExperiments() {
         ModelAndView mv = brc
-                .showAllForExperiments( this.newGet( "/bibRef/showAllEeBibRefs.html" ), ( HttpServletResponse ) null );
+                .showAllForExperiments( new MockHttpServletRequest( "GET", "/bibRef/showAllEeBibRefs.html" ), ( HttpServletResponse ) null );
         @SuppressWarnings("unchecked") Map> citationToEEs = ( Map> ) mv
                 .getModel().get( "citationToEEs" );
         assertNotNull( citationToEEs );
diff --git a/gemma-web/src/test/java/ubic/gemma/web/controller/common/description/bibref/PubMedQueryControllerTest.java b/gemma-web/src/test/java/ubic/gemma/web/controller/common/description/bibref/PubMedQueryControllerTest.java
index 2dc60969d7..5ed06d4c81 100644
--- a/gemma-web/src/test/java/ubic/gemma/web/controller/common/description/bibref/PubMedQueryControllerTest.java
+++ b/gemma-web/src/test/java/ubic/gemma/web/controller/common/description/bibref/PubMedQueryControllerTest.java
@@ -47,7 +47,7 @@ public void testDisplayForm() {
 
     @Test
     public final void testOnSubmit() throws Exception {
-        MockHttpServletRequest request = this.newPost( "/pubMedSearch.html" );
+        MockHttpServletRequest request = new MockHttpServletRequest( "POST", "/pubMedSearch.html" );
         request.addParameter( "accession", "134444" );
 
         try {
@@ -79,7 +79,7 @@ public final void testOnSubmitAlreadyExists() throws Exception {
         // put it in the system.
         this.getTestPersistentBibliographicReference( "12299" );
 
-        MockHttpServletRequest request = this.newPost( "/pubMedSearch.html" );
+        MockHttpServletRequest request = new MockHttpServletRequest( "POST", "/pubMedSearch.html" );
 
         ModelAndView mv = controller.onSubmit( request, new PubMedSearchCommand( "12299" ),
                 new BeanPropertyBindingResult( new PubMedSearchCommand( "12299" ), "searchCriteria" ),
@@ -93,7 +93,7 @@ public final void testOnSubmitAlreadyExists() throws Exception {
 
     @Test
     public final void testOnSubmitInvalidValue() throws Exception {
-        MockHttpServletRequest request = this.newPost( "/pubMedSearch.html" );
+        MockHttpServletRequest request = new MockHttpServletRequest( "POST", "/pubMedSearch.html" );
         ModelAndView mv = controller.onSubmit( request, new PubMedSearchCommand( "bad idea" ),
                 new BeanPropertyBindingResult( new PubMedSearchCommand( "bad idea" ), "searchCriteria" ),
                 new SimpleSessionStatus() );
@@ -104,7 +104,7 @@ public final void testOnSubmitInvalidValue() throws Exception {
 
     @Test
     public final void testOnSubmitNotFound() throws Exception {
-        MockHttpServletRequest request = this.newPost( "/pubMedSearch.html" );
+        MockHttpServletRequest request = new MockHttpServletRequest( "POST", "/pubMedSearch.html" );
         ModelAndView mv = controller.onSubmit( request, new PubMedSearchCommand( "13133333314444" ),
                 new BeanPropertyBindingResult( new PubMedSearchCommand( "13133333314444" ), "searchCriteria" ),
                 new SimpleSessionStatus() );
diff --git a/gemma-web/src/test/java/ubic/gemma/web/controller/diff/DifferentialExpressionAnalysisControllerTest.java b/gemma-web/src/test/java/ubic/gemma/web/controller/diff/DifferentialExpressionAnalysisControllerTest.java
index 49b2d3be5c..0149d1df94 100644
--- a/gemma-web/src/test/java/ubic/gemma/web/controller/diff/DifferentialExpressionAnalysisControllerTest.java
+++ b/gemma-web/src/test/java/ubic/gemma/web/controller/diff/DifferentialExpressionAnalysisControllerTest.java
@@ -74,14 +74,14 @@ public void resetMocks() {
 
     @Test
     public void testIndex() throws Exception {
-        mvc.perform( dwrStaticPage( "/index.html" ) )
+        perform( dwrStaticPage( "/index.html" ) )
                 .andExpect( status().isOk() )
                 .andExpect( content().contentType( MediaType.TEXT_HTML ) );
     }
 
     @Test
     public void testDiffExAnalysisControllerTestPage() throws Exception {
-        mvc.perform( dwrStaticPage( "/test/DifferentialExpressionAnalysisController" ) )
+        perform( dwrStaticPage( "/test/DifferentialExpressionAnalysisController" ) )
                 .andExpect( status().isOk() )
                 .andExpect( content().contentType( MediaType.TEXT_HTML ) );
     }
@@ -92,13 +92,13 @@ public void testDiffExAnalysisControllerTestPage() throws Exception {
      */
     @Test
     public void testUndefinedTestPage() throws Exception {
-        mvc.perform( dwrStaticPage( "/test/bleh" ) )
+        perform( dwrStaticPage( "/test/bleh" ) )
                 .andExpect( status().isNotImplemented() );
     }
 
     @Test
     public void testJsEngine() throws Exception {
-        mvc.perform( dwrStaticPage( "/engine.js" ) )
+        perform( dwrStaticPage( "/engine.js" ) )
                 .andExpect( status().isOk() )
                 .andExpect( content().contentType( "text/javascript;charset=utf-8" ) );
     }
@@ -109,7 +109,7 @@ public void test() throws Exception {
         ee.setExperimentalDesign( new ExperimentalDesign() );
         when( expressionExperimentService.loadAndThawLiteOrFail( eq( 1L ), any(), any() ) ).thenReturn( ee );
         when( taskRunningService.submitTaskCommand( any() ) ).thenReturn( "23" );
-        mvc.perform( dwr( DifferentialExpressionAnalysisController.class, "run", 1L ) )
+        perform( dwr( DifferentialExpressionAnalysisController.class, "run", 1L ) )
                 .andExpect( callback().value( "23" ) );
         verify( taskRunningService ).submitTaskCommand( any() );
     }
@@ -119,7 +119,7 @@ public void testBatchCall() throws Exception {
         ExpressionExperiment ee = ExpressionExperiment.Factory.newInstance();
         ee.setExperimentalDesign( new ExperimentalDesign() );
         when( expressionExperimentService.loadAndThawLiteOrFail( eq( 1L ), any(), any() ) ).thenReturn( ee );
-        mvc.perform( dwrBatch( 1 ).dwr( DifferentialExpressionAnalysisController.class, "run", 1L ) )
+        perform( dwrBatch( 1 ).dwr( DifferentialExpressionAnalysisController.class, "run", 1L ) )
                 .andExpect( batch( 0 ).callback().doesNotExist() )
                 .andExpect( batch( 1 ).callback().value( nullValue() ) );
         verify( taskRunningService ).submitTaskCommand( any() );
@@ -131,7 +131,7 @@ public void testMultipleCalls() throws Exception {
         ee.setExperimentalDesign( new ExperimentalDesign() );
         when( expressionExperimentService.loadAndThawLiteOrFail( eq( 1L ), any(), any() ) ).thenReturn( ee );
         when( expressionExperimentService.loadAndThawLiteOrFail( eq( 2L ), any(), any() ) ).thenReturn( ee );
-        mvc.perform( dwr( DifferentialExpressionAnalysisController.class, "run", 1L )
+        perform( dwr( DifferentialExpressionAnalysisController.class, "run", 1L )
                         .and( 2L ) )
                 .andExpect( callback( 0 ).value( nullValue() ) )
                 .andExpect( callback( 1 ).value( nullValue() ) )
@@ -142,7 +142,7 @@ public void testMultipleCalls() throws Exception {
 
     @Test
     public void testMissingEndpoint() throws Exception {
-        mvc.perform( dwr( DifferentialExpressionAnalysisController.class, "run2", 1L ) )
+        perform( dwr( DifferentialExpressionAnalysisController.class, "run2", 1L ) )
                 .andExpect( exception().javaClassName( "java.lang.Throwable" ) )
                 .andExpect( exception().message( "Error" ) );
         verifyNoInteractions( taskRunningService );
diff --git a/gemma-web/src/test/java/ubic/gemma/web/controller/expression/arrayDesign/ArrayDesignControllerTest.java b/gemma-web/src/test/java/ubic/gemma/web/controller/expression/arrayDesign/ArrayDesignControllerTest.java
index 28839a8896..8a7d4dd3e3 100644
--- a/gemma-web/src/test/java/ubic/gemma/web/controller/expression/arrayDesign/ArrayDesignControllerTest.java
+++ b/gemma-web/src/test/java/ubic/gemma/web/controller/expression/arrayDesign/ArrayDesignControllerTest.java
@@ -21,7 +21,8 @@
 import org.junit.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.servlet.ModelAndView;
-import ubic.gemma.web.util.BaseSpringWebTest;
+import ubic.gemma.core.util.test.BaseIntegrationTest;
+import ubic.gemma.web.util.BaseWebIntegrationTest;
 
 import java.util.Collection;
 
@@ -31,7 +32,7 @@
 /**
  * @author keshav
  */
-public class ArrayDesignControllerTest extends BaseSpringWebTest {
+public class ArrayDesignControllerTest extends BaseWebIntegrationTest {
 
     @Autowired
     private ArrayDesignController arrayDesignController;
diff --git a/gemma-web/src/test/java/ubic/gemma/web/controller/expression/experiment/ExperimentalDesignControllerIntegrationTest.java b/gemma-web/src/test/java/ubic/gemma/web/controller/expression/experiment/ExperimentalDesignControllerIntegrationTest.java
index 18e887a4d1..fe04768200 100644
--- a/gemma-web/src/test/java/ubic/gemma/web/controller/expression/experiment/ExperimentalDesignControllerIntegrationTest.java
+++ b/gemma-web/src/test/java/ubic/gemma/web/controller/expression/experiment/ExperimentalDesignControllerIntegrationTest.java
@@ -71,7 +71,7 @@ public void testShowExperimentalDesign() throws Exception {
         ExperimentalDesign ed = ee.getExperimentalDesign();
         assertNotNull( ed );
         assertNotNull( ed.getId() );
-        mvc.perform( get( "/experimentalDesign/showExperimentalDesign.html" )
+        perform( get( "/experimentalDesign/showExperimentalDesign.html" )
                         .param( "edid", ed.getId().toString() ) )
                 .andExpect( status().isOk() )
                 .andExpect( request().attribute( "id", ed.getId() ) )
@@ -84,7 +84,7 @@ public void testShowExperimentalDesign() throws Exception {
     public void testShowExperimentalDesignByExperimentId() throws Exception {
         ExperimentalDesign ed = ee.getExperimentalDesign();
         assertTrue( ed != null && ee.getId() != null );
-        mvc.perform( get( "/experimentalDesign/showExperimentalDesign.html" )
+        perform( get( "/experimentalDesign/showExperimentalDesign.html" )
                         .param( "eeid", ee.getId().toString() ) )
                 .andExpect( status().isOk() )
                 .andExpect( request().attribute( "id", ed.getId() ) )
@@ -96,7 +96,7 @@ public void testShowExperimentalDesignByExperimentShortName() throws Exception {
         ExperimentalDesign ed = ee.getExperimentalDesign();
         assertNotNull( ee.getShortName() );
         assertTrue( ed != null && ee.getId() != null );
-        mvc.perform( get( "/experimentalDesign/showExperimentalDesign.html" )
+        perform( get( "/experimentalDesign/showExperimentalDesign.html" )
                         .param( "shortName", ee.getShortName() ) )
                 .andExpect( status().isOk() )
                 .andExpect( view().name( "experimentalDesign.detail" ) );
diff --git a/gemma-web/src/test/java/ubic/gemma/web/controller/expression/experiment/ExpressionExperimentControllerTest.java b/gemma-web/src/test/java/ubic/gemma/web/controller/expression/experiment/ExpressionExperimentControllerTest.java
index de998a8178..04823d3d5a 100644
--- a/gemma-web/src/test/java/ubic/gemma/web/controller/expression/experiment/ExpressionExperimentControllerTest.java
+++ b/gemma-web/src/test/java/ubic/gemma/web/controller/expression/experiment/ExpressionExperimentControllerTest.java
@@ -99,7 +99,7 @@ public void testUpdatePubMed() throws Exception {
         ExpressionExperiment ee = getTestPersistentExpressionExperiment();
         ees.add( ee );
 
-        mvc.perform( dwr( ExpressionExperimentController.class, "updatePubMed", ee.getId(), "1" ) )
+        perform( dwr( ExpressionExperimentController.class, "updatePubMed", ee.getId(), "1" ) )
                 .andExpect( callback().exist() )
                 .andDo( getCallback( ( String taskId ) -> {
                     SubmittedTask st = taskRunningService.getSubmittedTask( taskId );
@@ -114,7 +114,7 @@ public void testUpdatePubMed() throws Exception {
                 } ) );
 
 
-        mvc.perform( dwr( ExpressionExperimentController.class, "updatePubMed", ee.getId(), "2" ) )
+        perform( dwr( ExpressionExperimentController.class, "updatePubMed", ee.getId(), "2" ) )
                 .andExpect( callback().exist() )
                 .andDo( getCallback( ( String taskId ) -> {
                     SubmittedTask st = taskRunningService.getSubmittedTask( taskId );
@@ -135,7 +135,7 @@ public void testUpdatePubMedAsAnonymousUser() throws Exception {
         ees.add( ee );
         runAsAnonymous();
         try {
-            mvc.perform( dwr( ExpressionExperimentController.class, "updatePubMed", ee.getId(), "1" ) )
+            perform( dwr( ExpressionExperimentController.class, "updatePubMed", ee.getId(), "1" ) )
                     .andExpect( status().isOk() )
                     .andExpect( exception().exist() )
                     .andExpect( callback().doesNotExist() )
diff --git a/gemma-web/src/test/java/ubic/gemma/web/scheduler/SchedulerSecurityTest.java b/gemma-web/src/test/java/ubic/gemma/web/scheduler/SchedulerSecurityTest.java
index b7c67d6976..6fc6609608 100644
--- a/gemma-web/src/test/java/ubic/gemma/web/scheduler/SchedulerSecurityTest.java
+++ b/gemma-web/src/test/java/ubic/gemma/web/scheduler/SchedulerSecurityTest.java
@@ -32,7 +32,7 @@
 import ubic.gemma.core.analysis.report.WhatsNewService;
 import ubic.gemma.persistence.service.TableMaintenanceUtil;
 import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentService;
-import ubic.gemma.web.util.BaseSpringWebTest;
+import ubic.gemma.web.util.BaseWebIntegrationTest;
 
 import java.lang.reflect.InvocationTargetException;
 
@@ -46,7 +46,7 @@
  *
  * @author keshav
  */
-public class SchedulerSecurityTest extends BaseSpringWebTest {
+public class SchedulerSecurityTest extends BaseWebIntegrationTest {
 
     @Autowired
     private ExpressionExperimentService expressionExperimentService;
diff --git a/gemma-web/src/test/java/ubic/gemma/web/util/BaseSpringWebTest.java b/gemma-web/src/test/java/ubic/gemma/web/util/BaseSpringWebTest.java
index 0811c144d0..69a122c287 100644
--- a/gemma-web/src/test/java/ubic/gemma/web/util/BaseSpringWebTest.java
+++ b/gemma-web/src/test/java/ubic/gemma/web/util/BaseSpringWebTest.java
@@ -18,55 +18,49 @@
  */
 package ubic.gemma.web.util;
 
+import org.apache.commons.logging.Log;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.mock.web.MockHttpServletRequest;
 import org.springframework.test.context.ActiveProfiles;
 import org.springframework.test.context.ContextConfiguration;
 import org.springframework.test.context.web.WebAppConfiguration;
 import org.springframework.test.web.servlet.MockMvc;
+import org.springframework.test.web.servlet.RequestBuilder;
+import org.springframework.test.web.servlet.ResultActions;
 import org.springframework.test.web.servlet.setup.MockMvcBuilders;
 import org.springframework.web.context.WebApplicationContext;
 import ubic.gemma.core.util.test.BaseSpringContextTest;
 
-import javax.annotation.OverridingMethodsMustInvokeSuper;
+import java.util.logging.Logger;
 
 /**
  * Class to extend for tests of controllers et al. that need a spring context. Provides convenience methods for dealing
  * with mock requests and responses. Also provides a safe port to send email on for testing (for example, using
  * dumbster)
- *
+ * 

  * This is meant for integration tests, if you want to perform unit tests, consider using {@link WebAppConfiguration}
  * and {@link ContextConfiguration} with a static inner class annotated with {@link org.springframework.context.annotation.Configuration}.
  * See {@link ubic.gemma.web.services.rest.SearchWebServiceTest} for a complete example.
- *
  * @author pavlidis
+ * @deprecated favour the simpler {@link BaseWebIntegrationTest} for new tests
  */
+@Deprecated
 @ActiveProfiles("web")
 @WebAppConfiguration
 @ContextConfiguration(locations = { "classpath*:WEB-INF/gemma-servlet.xml" })
 public abstract class BaseSpringWebTest extends BaseSpringContextTest {
 
     @Autowired
-    protected WebApplicationContext applicationContext;
+    private WebApplicationContext applicationContext;
 
-    protected MockMvc mvc;
-
-    @Override
-    @OverridingMethodsMustInvokeSuper
-    public void afterPropertiesSet() {
-        super.afterPropertiesSet();
-        mvc = MockMvcBuilders.webAppContextSetup( applicationContext ).build();
-    }
-
-    public MockHttpServletRequest newGet( String url ) {
-        return new MockHttpServletRequest( "GET", url );
-    }
+    private MockMvc mvc;
 
     /**
-     * Convenience methods to make tests simpler
+     * @see MockMvc#perform(RequestBuilder)
      */
-    public MockHttpServletRequest newPost( String url ) {
-        return new MockHttpServletRequest( "POST", url );
+    protected final ResultActions perform( RequestBuilder requestBuilder ) throws Exception {
+        if ( mvc == null ) {
+            mvc = MockMvcBuilders.webAppContextSetup( applicationContext ).build();
+        }
+        return mvc.perform( requestBuilder );
     }
-
 }
diff --git a/gemma-web/src/test/java/ubic/gemma/web/util/BaseWebIntegrationTest.java b/gemma-web/src/test/java/ubic/gemma/web/util/BaseWebIntegrationTest.java
new file mode 100644
index 0000000000..e55a595830
--- /dev/null
+++ b/gemma-web/src/test/java/ubic/gemma/web/util/BaseWebIntegrationTest.java
@@ -0,0 +1,36 @@
+package ubic.gemma.web.util;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.web.WebAppConfiguration;
+import org.springframework.test.web.servlet.MockMvc;
+import org.springframework.test.web.servlet.RequestBuilder;
+import org.springframework.test.web.servlet.ResultActions;
+import org.springframework.test.web.servlet.setup.MockMvcBuilders;
+import org.springframework.web.context.WebApplicationContext;
+import ubic.gemma.core.util.test.BaseIntegrationTest;
+
+/**
+ * Base class for web integration tests.
+ * 

+ * For a unit-test web test, use {@link BaseWebTest}.
+ * @author poirigui
+ */
+@ActiveProfiles("web")
+@WebAppConfiguration
+@ContextConfiguration(locations = { "classpath*:WEB-INF/gemma-servlet.xml" })
+public abstract class BaseWebIntegrationTest extends BaseIntegrationTest {
+
+    @Autowired
+    private WebApplicationContext applicationContext;
+
+    private MockMvc mvc;
+
+    protected final ResultActions perform( RequestBuilder requestBuilder ) throws Exception {
+        if ( mvc == null ) {
+            mvc = MockMvcBuilders.webAppContextSetup( applicationContext ).build();
+        }
+        return mvc.perform( requestBuilder );
+    }
+}
diff --git a/gemma-web/src/test/java/ubic/gemma/web/util/BaseWebTest.java b/gemma-web/src/test/java/ubic/gemma/web/util/BaseWebTest.java
index fc6fd08ea0..1cd72a8863 100644
--- a/gemma-web/src/test/java/ubic/gemma/web/util/BaseWebTest.java
+++ b/gemma-web/src/test/java/ubic/gemma/web/util/BaseWebTest.java
@@ -1,6 +1,5 @@
 package ubic.gemma.web.util;
 
-import org.springframework.beans.factory.InitializingBean;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.annotation.Bean;
 import org.springframework.security.access.AccessDeniedException;
@@ -8,6 +7,8 @@
 import org.springframework.test.context.junit4.AbstractJUnit4SpringContextTests;
 import org.springframework.test.context.web.WebAppConfiguration;
 import org.springframework.test.web.servlet.MockMvc;
+import org.springframework.test.web.servlet.RequestBuilder;
+import org.springframework.test.web.servlet.ResultActions;
 import org.springframework.test.web.servlet.setup.MockMvcBuilders;
 import org.springframework.web.context.WebApplicationContext;
 import org.springframework.web.servlet.HandlerExceptionResolver;
@@ -25,12 +26,12 @@
 /**
  * Base class for a Web-based unit test.
  * 

- * For a full integration test base class, use {@link ubic.gemma.web.util.BaseSpringWebTest}.
+ * For a full integration test base class, use {@link ubic.gemma.web.util.BaseWebIntegrationTest}.
  * @author poirigui
  */
 @ActiveProfiles({ "web", EnvironmentProfiles.TEST })
 @WebAppConfiguration
-public abstract class BaseWebTest extends AbstractJUnit4SpringContextTests implements InitializingBean {
+public abstract class BaseWebTest extends AbstractJUnit4SpringContextTests {
 
     public abstract static class BaseWebTestContextConfiguration {
 
@@ -84,10 +85,15 @@ public MailEngine mailEngine() {
     @Autowired
     private WebApplicationContext applicationContext;
 
-    protected MockMvc mvc;
+    private MockMvc mvc;
 
-    @Override
-    public final void afterPropertiesSet() {
-        mvc = MockMvcBuilders.webAppContextSetup( applicationContext ).build();
+    /**
+     * @see MockMvc#perform(RequestBuilder)
+     */
+    protected final ResultActions perform( RequestBuilder requestBuilder ) throws Exception {
+        if ( mvc == null ) {
+            mvc = MockMvcBuilders.webAppContextSetup( applicationContext ).build();
+        }
+        return mvc.perform( requestBuilder );
     }
 }
diff --git a/gemma-web/src/test/java/ubic/gemma/web/util/MessageSourceTest.java b/gemma-web/src/test/java/ubic/gemma/web/util/MessageSourceTest.java
index 3c02bb0d57..d9f256fda3 100644
--- a/gemma-web/src/test/java/ubic/gemma/web/util/MessageSourceTest.java
+++ b/gemma-web/src/test/java/ubic/gemma/web/util/MessageSourceTest.java
@@ -8,7 +8,7 @@
 
 import static org.junit.Assert.assertEquals;
 
-public class MessageSourceTest extends BaseSpringWebTest {
+public class MessageSourceTest extends BaseWebIntegrationTest {
 
     @Autowired
     private MessageSource messageSource;

From 799b045bdf6bda3dfbda7e5295f24a75dc974c12 Mon Sep 17 00:00:00 2001
From: Guillaume Poirier-Morency 
Date: Wed, 8 May 2024 19:54:10 -0700
Subject: [PATCH 29/61] rest: Add endpoints to disambiguate gene symbols (fix
 #575)

Add endpoints for retrieving all genes with pagination as well as all
the gene-related endpoint with a taxon argument.

Produce an error when an ambiguous gene identifier is supplied.

Remove unnecessary regex in getDatasetExpressionForGenesInTaxa()

Fix getTaxonDatasets(), its sort argument was incorrectly refeering to
Taxon instead of ExpressionExperiment.

rest: Update REST API version to 2.7.5

rest: Update changelogs
---
 .../core/genome/gene/service/GeneService.java |   8 --
 .../genome/gene/service/GeneServiceImpl.java  |   8 +-
 .../persistence/service/genome/GeneDao.java   |   6 -
 .../service/genome/GeneDaoImpl.java           |  11 --
 .../genome/taxon/TaxonServiceImpl.java        |   1 -
 .../ubic/gemma/rest/DatasetsWebService.java   |  31 ++++-
 .../java/ubic/gemma/rest/GeneWebService.java  |  44 +++----
 .../java/ubic/gemma/rest/TaxaWebService.java  |  92 ++++++++-----
 .../gemma/rest/util/args/AbstractArg.java     |   7 +-
 .../rest/util/args/AbstractEntityArg.java     |   3 +-
 .../util/args/AbstractEntityArgService.java   |  10 +-
 .../rest/util/args/EntityArgService.java      |  40 ++++++
 .../gemma/rest/util/args/GeneAnyIdArg.java    |  32 -----
 .../ubic/gemma/rest/util/args/GeneArg.java    |  14 +-
 .../gemma/rest/util/args/GeneArgService.java  | 121 ++++++++++++++----
 .../rest/util/args/GeneEnsemblIdArg.java      |  11 +-
 .../gemma/rest/util/args/GeneNcbiIdArg.java   |   2 +-
 .../gemma/rest/util/args/GeneSymbolArg.java   |   9 +-
 .../main/resources/openapi-configuration.yaml |   2 +-
 .../main/resources/restapidocs/CHANGELOG.md   |   6 +
 .../gemma/rest/DatasetsWebServiceTest.java    |   6 +
 .../ubic/gemma/rest/GeneWebServiceTest.java   |  82 ++++++++++++
 .../ubic/gemma/rest/TaxaWebServiceTest.java   |  48 ++++++-
 .../java/ubic/gemma/rest/util/Assertions.java |   2 +-
 .../rest/util/args/GeneArgServiceTest.java    |  10 +-
 25 files changed, 418 insertions(+), 188 deletions(-)
 delete mode 100644 gemma-rest/src/main/java/ubic/gemma/rest/util/args/GeneAnyIdArg.java
 create mode 100644 gemma-rest/src/test/java/ubic/gemma/rest/GeneWebServiceTest.java

diff --git a/gemma-core/src/main/java/ubic/gemma/core/genome/gene/service/GeneService.java b/gemma-core/src/main/java/ubic/gemma/core/genome/gene/service/GeneService.java
index be06f9f84e..ad7e341e4d 100755
--- a/gemma-core/src/main/java/ubic/gemma/core/genome/gene/service/GeneService.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/genome/gene/service/GeneService.java
@@ -131,14 +131,6 @@ public interface GeneService extends BaseService, FilteringVoEnabledServic
     @Secured({ "IS_AUTHENTICATED_ANONYMOUSLY", "AFTER_ACL_ARRAYDESIGN_COLLECTION_READ" })
     Collection getCompositeSequencesById( Long id );
 
-    /**
-     * Gets all the genes for a given taxon
-     *
-     * @param taxon taxon
-     * @return genes for given taxon
-     */
-    Collection getGenesByTaxon( Taxon taxon );
-
     List getPhysicalLocationsValueObjects( Gene gene );
 
     /**
diff --git a/gemma-core/src/main/java/ubic/gemma/core/genome/gene/service/GeneServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/genome/gene/service/GeneServiceImpl.java
index 718a4ce54f..6ffe54c9fd 100755
--- a/gemma-core/src/main/java/ubic/gemma/core/genome/gene/service/GeneServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/genome/gene/service/GeneServiceImpl.java
@@ -34,6 +34,7 @@
 import ubic.gemma.model.association.coexpression.GeneCoexpressionNodeDegreeValueObject;
 import ubic.gemma.model.association.phenotype.PhenotypeAssociation;
 import ubic.gemma.model.common.description.AnnotationValueObject;
+import ubic.gemma.model.common.description.CharacteristicValueObject;
 import ubic.gemma.model.common.description.ExternalDatabase;
 import ubic.gemma.model.common.search.SearchSettings;
 import ubic.gemma.model.expression.arrayDesign.ArrayDesign;
@@ -44,7 +45,6 @@
 import ubic.gemma.model.genome.PhysicalLocationValueObject;
 import ubic.gemma.model.genome.Taxon;
 import ubic.gemma.model.genome.gene.*;
-import ubic.gemma.model.common.description.CharacteristicValueObject;
 import ubic.gemma.persistence.service.AbstractFilteringVoEnabledService;
 import ubic.gemma.persistence.service.AbstractService;
 import ubic.gemma.persistence.service.association.Gene2GOAssociationService;
@@ -230,12 +230,6 @@ public Collection getCompositeSequencesById( final Long id )
         return this.geneDao.getCompositeSequencesById( id );
     }
 
-    @Override
-    @Transactional(readOnly = true)
-    public Collection getGenesByTaxon( final Taxon taxon ) {
-        return this.geneDao.getGenesByTaxon( taxon );
-    }
-
     @Override
     @Transactional(readOnly = true)
     public List getPhysicalLocationsValueObjects( Gene gene ) {
diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/genome/GeneDao.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/genome/GeneDao.java
index f68c7c6ae0..a9a8e821fb 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/genome/GeneDao.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/genome/GeneDao.java
@@ -102,12 +102,6 @@ public interface GeneDao extends FilteringVoEnabledDao {
 
     Collection getCompositeSequencesById( long id );
 
-    /**
-     * @param taxon taxon
-     * @return a collections of genes that match the given taxon
-     */
-    Collection getGenesByTaxon( Taxon taxon );
-
     /**
      * @param taxon taxon
      * @return a collection of genes that are actually MicroRNA for a given taxon
diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/genome/GeneDaoImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/genome/GeneDaoImpl.java
index 98610cad04..85a31564f1 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/genome/GeneDaoImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/genome/GeneDaoImpl.java
@@ -281,17 +281,6 @@ public Collection getCompositeSequencesById( long id ) {
                 .list();
     }
 
-    @Override
-    public Collection getGenesByTaxon( Taxon taxon ) {
-        //language=HQL
-        final String queryString = "select gene from Gene as gene where gene.taxon = :taxon ";
-        //noinspection unchecked
-        return this.getSessionFactory().getCurrentSession()
-                .createQuery( queryString )
-                .setParameter( "taxon", taxon )
-                .list();
-    }
-
     @Override
     public Collection getMicroRnaByTaxon( Taxon taxon ) {
         //language=HQL
diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/genome/taxon/TaxonServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/genome/taxon/TaxonServiceImpl.java
index a3626db7cc..76a43e3101 100755
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/genome/taxon/TaxonServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/genome/taxon/TaxonServiceImpl.java
@@ -25,7 +25,6 @@
 import ubic.gemma.model.genome.TaxonValueObject;
 import ubic.gemma.persistence.service.AbstractFilteringVoEnabledService;
 import ubic.gemma.persistence.service.AbstractService;
-import ubic.gemma.persistence.service.AbstractVoEnabledService;
 import ubic.gemma.persistence.service.expression.arrayDesign.ArrayDesignService;
 import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentService;
 
diff --git a/gemma-rest/src/main/java/ubic/gemma/rest/DatasetsWebService.java b/gemma-rest/src/main/java/ubic/gemma/rest/DatasetsWebService.java
index ef53fdb713..9d00c6664f 100644
--- a/gemma-rest/src/main/java/ubic/gemma/rest/DatasetsWebService.java
+++ b/gemma-rest/src/main/java/ubic/gemma/rest/DatasetsWebService.java
@@ -124,14 +124,17 @@ public class DatasetsWebService {
     private DifferentialExpressionAnalysisService differentialExpressionAnalysisService;
     @Autowired
     private AuditEventService auditEventService;
+    @Autowired
+    private QuantitationTypeArgService quantitationTypeArgService;
+    @Autowired
+    private OntologyService ontologyService;
+
     @Autowired
     private DatasetArgService datasetArgService;
     @Autowired
     private GeneArgService geneArgService;
     @Autowired
-    private QuantitationTypeArgService quantitationTypeArgService;
-    @Autowired
-    private OntologyService ontologyService;
+    private TaxonArgService taxonArgService;
 
     @Autowired
     private HttpServletRequest request;
@@ -928,6 +931,7 @@ public ResponseDataObject getDatasetSvd( // Params:
      *                        You can combine various identifiers in one query, but an invalid identifier will cause the
      *                        call to yield an error.
      *                        

+     * @param taxonArg        a taxon to retrieve gene identifiers from
      * @param genes           a list of gene identifiers, separated by commas (','). Identifiers can be one of
      *                        NCBI ID, Ensembl ID or official symbol. NCBI ID is the most efficient (and
      *                        guaranteed to be unique) identifier. Official symbol will return a random homologue. Use
@@ -951,7 +955,26 @@ public ResponseDataObject getDatasetSvd( // Params:
      *                        
      */
     @GET
-    @Path("/{datasets}/expressions/genes/{genes: [^/]+}")
+    @Path("/{datasets}/expressions/taxa/{taxa}/genes/{genes}")
+    @Produces(MediaType.APPLICATION_JSON)
+    @Operation(summary = "Retrieve the expression data matrix of a set of datasets and genes")
+    public ResponseDataObject> getDatasetExpressionForGenesInTaxa( // Params:
+            @PathParam("datasets") DatasetArrayArg datasets, // Required
+            @PathParam("taxa") TaxonArg taxonArg, // Required
+            @PathParam("genes") GeneArrayArg genes, // Required
+            @QueryParam("keepNonSpecific") @DefaultValue("false") Boolean keepNonSpecific, // Optional, default false
+            @QueryParam("consolidate") ExpLevelConsolidationArg consolidate // Optional, default everything is returned
+    ) {
+        return Responder.respond( processedExpressionDataVectorService
+                .getExpressionLevels( datasetArgService.getEntities( datasets ),
+                        geneArgService.getEntitiesWithTaxon( genes, taxonArgService.getEntity( taxonArg ) ),
+                        keepNonSpecific,
+                        consolidate == null ? null : consolidate.getValue() )
+        );
+    }
+
+    @GET
+    @Path("/{datasets}/expressions/genes/{genes}")
     @Produces(MediaType.APPLICATION_JSON)
     @Operation(summary = "Retrieve the expression data matrix of a set of datasets and genes")
     public ResponseDataObject> getDatasetExpressionForGenes( // Params:
diff --git a/gemma-rest/src/main/java/ubic/gemma/rest/GeneWebService.java b/gemma-rest/src/main/java/ubic/gemma/rest/GeneWebService.java
index a010f15117..5b9e510200 100644
--- a/gemma-rest/src/main/java/ubic/gemma/rest/GeneWebService.java
+++ b/gemma-rest/src/main/java/ubic/gemma/rest/GeneWebService.java
@@ -15,9 +15,6 @@
 package ubic.gemma.rest;
 
 import io.swagger.v3.oas.annotations.Operation;
-import io.swagger.v3.oas.annotations.media.Content;
-import io.swagger.v3.oas.annotations.media.Schema;
-import io.swagger.v3.oas.annotations.responses.ApiResponse;
 import org.apache.commons.lang3.time.DateUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
@@ -33,12 +30,10 @@
 import ubic.gemma.model.genome.PhysicalLocationValueObject;
 import ubic.gemma.model.genome.gene.GeneValueObject;
 import ubic.gemma.model.genome.gene.phenotype.valueObject.GeneEvidenceValueObject;
-import ubic.gemma.persistence.service.expression.designElement.CompositeSequenceService;
 import ubic.gemma.persistence.util.Filters;
-import ubic.gemma.rest.util.FilteredAndPaginatedResponseDataObject;
+import ubic.gemma.rest.util.PaginatedResponseDataObject;
 import ubic.gemma.rest.util.Responder;
 import ubic.gemma.rest.util.ResponseDataObject;
-import ubic.gemma.rest.util.ResponseErrorObject;
 import ubic.gemma.rest.util.args.*;
 
 import javax.ws.rs.*;
@@ -59,29 +54,31 @@
 @Path("/genes")
 public class GeneWebService {
 
-    private GeneService geneService;
-    private CompositeSequenceService compositeSequenceService;
-    private GeneCoexpressionSearchService geneCoexpressionSearchService;
-    private GeneArgService geneArgService;
-
-    /**
-     * Required by spring
-     */
-    public GeneWebService() {
-    }
+    private final GeneService geneService;
+    private final GeneCoexpressionSearchService geneCoexpressionSearchService;
+    private final GeneArgService geneArgService;
 
     /**
      * Constructor for service autowiring
      */
     @Autowired
-    public GeneWebService( GeneService geneService, CompositeSequenceService compositeSequenceService,
-            GeneCoexpressionSearchService geneCoexpressionSearchService, GeneArgService geneArgService ) {
+    public GeneWebService( GeneService geneService, GeneCoexpressionSearchService geneCoexpressionSearchService, GeneArgService geneArgService ) {
         this.geneService = geneService;
-        this.compositeSequenceService = compositeSequenceService;
         this.geneCoexpressionSearchService = geneCoexpressionSearchService;
         this.geneArgService = geneArgService;
     }
 
+    @GET
+    @Path("/")
+    @Produces(MediaType.APPLICATION_JSON)
+    @Operation(summary = "Retrieve all genes")
+    public PaginatedResponseDataObject getGenes(
+            @QueryParam("offset") @DefaultValue("0") OffsetArg offsetArg,
+            @QueryParam("limit") @DefaultValue("20") LimitArg limitArg
+    ) {
+        return Responder.paginate( geneArgService.getGenes( offsetArg.getValue(), limitArg.getValue() ), new String[] { "id" } );
+    }
+
     /**
      * Retrieves all genes matching the identifier.
      *
@@ -156,14 +153,12 @@ public ResponseDataObject> getGeneLocations( /
     @Path("/{gene}/probes")
     @Produces(MediaType.APPLICATION_JSON)
     @Operation(summary = "Retrieve the probes associated to a genes across all platforms")
-    public FilteredAndPaginatedResponseDataObject getGeneProbes( // Params:
+    public PaginatedResponseDataObject getGeneProbes( // Params:
             @PathParam("gene") GeneArg geneArg, // Required
             @QueryParam("offset") @DefaultValue("0") OffsetArg offset, // Optional, default 0
             @QueryParam("limit") @DefaultValue("20") LimitArg limit // Optional, default 20
     ) {
-        return Responder.paginate( compositeSequenceService
-                .loadValueObjectsForGene( geneArgService.getEntity( geneArg ), offset.getValue(),
-                        limit.getValue() ), geneArgService.getFilters( geneArg ), new String[] { "id" } );
+        return Responder.paginate( geneArgService.getGeneProbes( geneArg, offset.getValue(), limit.getValue() ), new String[] { "id" } );
     }
 
     /**
@@ -179,7 +174,7 @@ public FilteredAndPaginatedResponseDataObject getG
     public ResponseDataObject> getGeneGoTerms( // Params:
             @PathParam("gene") GeneArg geneArg // Required
     ) {
-        return Responder.respond( geneArgService.getGoTerms( geneArg ) );
+        return Responder.respond( geneArgService.getGeneGoTerms( geneArg ) );
     }
 
     /**
@@ -206,5 +201,4 @@ public ResponseDataObject> getGeneGeneCoexpress
                     this.add( geneArgService.getEntity( with ).getId() );
                 }}, 1, limit.getValueNoMaximum(), false ).getResults() );
     }
-
 }
diff --git a/gemma-rest/src/main/java/ubic/gemma/rest/TaxaWebService.java b/gemma-rest/src/main/java/ubic/gemma/rest/TaxaWebService.java
index 8afc5362fe..cfe871389b 100644
--- a/gemma-rest/src/main/java/ubic/gemma/rest/TaxaWebService.java
+++ b/gemma-rest/src/main/java/ubic/gemma/rest/TaxaWebService.java
@@ -27,8 +27,10 @@
 import ubic.gemma.core.search.ParseSearchException;
 import ubic.gemma.core.search.SearchException;
 import ubic.gemma.core.search.SearchTimeoutException;
+import ubic.gemma.model.expression.designElement.CompositeSequenceValueObject;
 import ubic.gemma.model.expression.experiment.ExpressionExperiment;
 import ubic.gemma.model.expression.experiment.ExpressionExperimentValueObject;
+import ubic.gemma.model.genome.GeneOntologyTermValueObject;
 import ubic.gemma.model.genome.PhysicalLocationValueObject;
 import ubic.gemma.model.genome.Taxon;
 import ubic.gemma.model.genome.TaxonValueObject;
@@ -37,9 +39,11 @@
 import ubic.gemma.model.genome.gene.phenotype.valueObject.GeneEvidenceValueObject;
 import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentService;
 import ubic.gemma.persistence.service.genome.taxon.TaxonService;
+import ubic.gemma.persistence.util.Filter;
 import ubic.gemma.persistence.util.Filters;
 import ubic.gemma.persistence.util.Sort;
 import ubic.gemma.rest.util.FilteredAndPaginatedResponseDataObject;
+import ubic.gemma.rest.util.PaginatedResponseDataObject;
 import ubic.gemma.rest.util.Responder;
 import ubic.gemma.rest.util.ResponseDataObject;
 import ubic.gemma.rest.util.args.*;
@@ -58,26 +62,19 @@
 public class TaxaWebService {
 
     protected static final Log log = LogFactory.getLog( TaxaWebService.class.getName() );
-    private TaxonService taxonService;
-    private ExpressionExperimentService expressionExperimentService;
-    private PhenotypeAssociationManagerService phenotypeAssociationManagerService;
-    private TaxonArgService taxonArgService;
-    private DatasetArgService datasetArgService;
-    private GeneArgService geneArgService;
 
-    /**
-     * Required by spring
-     */
-    public TaxaWebService() {
-    }
+    private final TaxonService taxonService;
+    private final ExpressionExperimentService expressionExperimentService;
+    private final PhenotypeAssociationManagerService phenotypeAssociationManagerService;
+    private final TaxonArgService taxonArgService;
+    private final DatasetArgService datasetArgService;
+    private final GeneArgService geneArgService;
 
     /**
      * Constructor for service autowiring
      */
     @Autowired
-    public TaxaWebService( TaxonService taxonService, ExpressionExperimentService expressionExperimentService,
-            PhenotypeAssociationManagerService phenotypeAssociationManagerService, TaxonArgService taxonArgService,
-            DatasetArgService datasetArgService, GeneArgService geneArgService ) {
+    public TaxaWebService( TaxonService taxonService, ExpressionExperimentService expressionExperimentService, PhenotypeAssociationManagerService phenotypeAssociationManagerService, TaxonArgService taxonArgService, DatasetArgService datasetArgService, GeneArgService geneArgService ) {
         this.taxonService = taxonService;
         this.expressionExperimentService = expressionExperimentService;
         this.phenotypeAssociationManagerService = phenotypeAssociationManagerService;
@@ -153,6 +150,18 @@ public ResponseDataObject> getTaxonGenesOverlappingChromos
         return Responder.respond( taxonArgService.getGenesOnChromosome( taxonArg, chromosomeName, strand, start, size ) );
     }
 
+    @GET
+    @Path("/{taxon}/genes")
+    @Produces(MediaType.APPLICATION_JSON)
+    @Operation(summary = "Retrieve all genes in a given taxon")
+    public PaginatedResponseDataObject getTaxonGenes(
+            @PathParam("taxon") TaxonArg taxonArg,
+            @QueryParam("offset") @DefaultValue("0") OffsetArg offsetArg,
+            @QueryParam("limit") @DefaultValue("20") LimitArg limitArg
+    ) {
+        return Responder.paginate( geneArgService.getGenesInTaxon( taxonArgService.getEntity( taxonArg ), offsetArg.getValue(), limitArg.getValue() ), new String[] { "id" } );
+    }
+
     /**
      * Retrieves genes matching the identifier on the given taxon.
      *
@@ -160,16 +169,39 @@ public ResponseDataObject> getTaxonGenesOverlappingChromos
      *                 scientific name, common name. It is recommended to use the ID for efficiency.
      * @param geneArg  can either be the NCBI ID, Ensembl ID or official symbol. NCBI ID is most efficient (and
      *                 guaranteed to be unique). Official symbol returns a gene homologue on a random taxon.
+     * @see GeneWebService#getGenes(GeneArrayArg)
      */
     @GET
     @Path("/{taxon}/genes/{gene}")
     @Produces(MediaType.APPLICATION_JSON)
-    @Operation(summary = "Retrieve all genes in a given taxon")
-    public ResponseDataObject> getTaxonGenes( // Params:
+    @Operation(summary = "Retrieve genes matching gene identifiers in a given taxon")
+    public ResponseDataObject> getTaxonGenesByIds( // Params:
             @PathParam("taxon") TaxonArg taxonArg, // Required
-            @PathParam("gene") GeneArg geneArg // Required
+            @PathParam("gene") GeneArrayArg geneArg // Required
     ) {
-        return Responder.respond( geneArgService.getGenesOnTaxon( geneArg, taxonArgService.getEntity( taxonArg ) ) );
+        return Responder.respond( geneArgService.getGenesInTaxon( geneArg, taxonArgService.getEntity( taxonArg ) ) );
+    }
+
+    /**
+     * @see GeneWebService#getGeneProbes(GeneArg, OffsetArg, LimitArg)
+     */
+    @GET
+    @Path("/{taxon}/genes/{gene}/probes")
+    @Produces(MediaType.APPLICATION_JSON)
+    @Operation(summary = "Retrieve the probes associated to a genes across all platforms in a given taxon")
+    public PaginatedResponseDataObject getTaxonGeneProbes( @PathParam("taxon") TaxonArg taxonArg, @PathParam("gene") GeneArg geneArg, @QueryParam("offset") @DefaultValue("0") OffsetArg offsetArg, @QueryParam("limit") @DefaultValue("20") LimitArg limitArg ) {
+        return Responder.paginate( geneArgService.getGeneProbesInTaxon( geneArg, taxonArgService.getEntity( taxonArg ), offsetArg.getValue(), limitArg.getValue() ), new String[] { "id" } );
+    }
+
+    /**
+     * @see GeneWebService#getGeneGoTerms(GeneArg)
+     */
+    @GET
+    @Path("/{taxon}/genes/{gene}/goTerms")
+    @Produces(MediaType.APPLICATION_JSON)
+    @Operation(summary = "Retrieve the GO terms associated to a gene in a given taxon")
+    public ResponseDataObject> getTaxonGeneGoTerms( @PathParam("taxon") TaxonArg taxonArg, @PathParam("gene") GeneArg geneArg ) {
+        return Responder.respond( geneArgService.getGeneGoTermsInTaxon( geneArg, taxonArgService.getEntity( taxonArg ) ) );
     }
 
     /**
@@ -213,12 +245,11 @@ public ResponseDataObject> getGenesEvidenceInTaxon
     @Path("/{taxon}/genes/{gene}/locations")
     @Produces(MediaType.APPLICATION_JSON)
     @Operation(summary = "Retrieve physical locations for a given gene and taxon")
-    public ResponseDataObject> getGeneLocationsInTaxon( // Params:
+    public ResponseDataObject> getTaxonGeneLocations( // Params:
             @PathParam("taxon") TaxonArg taxonArg, // Required
             @PathParam("gene") GeneArg geneArg // Required
     ) {
-        Taxon taxon = taxonArgService.getEntity( taxonArg );
-        return Responder.respond( geneArgService.getGeneLocation( geneArg, taxon ) );
+        return Responder.respond( geneArgService.getGeneLocationInTaxon( geneArg, taxonArgService.getEntity( taxonArg ) ) );
     }
 
     /**
@@ -236,14 +267,13 @@ public FilteredAndPaginatedResponseDataObject g
             @QueryParam("filter") @DefaultValue("") FilterArg filter, // Optional, default null
             @QueryParam("offset") @DefaultValue("0") OffsetArg offset, // Optional, default 0
             @QueryParam("limit") @DefaultValue("20") LimitArg limit, // Optional, default 20
-            @QueryParam("sort") @DefaultValue("+id") SortArg sort // Optional, default +id
+            @QueryParam("sort") @DefaultValue("+id") SortArg sort // Optional, default +id
     ) {
         // will raise a NotFoundException if the taxon is not found
-        taxonArgService.getEntity( taxonArg );
+        Taxon taxon = taxonArgService.getEntity( taxonArg );
         Filters filters = datasetArgService.getFilters( filter )
-                .and( taxonArgService.getFilters( taxonArg ) );
-        return Responder.paginate( expressionExperimentService::loadValueObjects, filters, new String[] { "id" },
-                taxonArgService.getSort( sort ), offset.getValue(), limit.getValue() );
+                .and( expressionExperimentService.getFilter( "taxon.id", Long.class, Filter.Operator.eq, taxon.getId() ) );
+        return Responder.paginate( expressionExperimentService::loadValueObjects, filters, new String[] { "id" }, datasetArgService.getSort( sort ), offset.getValue(), limit.getValue() );
     }
 
     /**
@@ -272,11 +302,9 @@ public ResponseDataObject> getTaxonPhenotypes( // Params:
     ) {
         Taxon taxon = taxonArgService.getEntity( taxonArg );
         if ( tree ) {
-            return Responder.respond( phenotypeAssociationManagerService
-                    .loadAllPhenotypesAsTree( new EvidenceFilter( taxon.getId(), editableOnly ) ) );
+            return Responder.respond( phenotypeAssociationManagerService.loadAllPhenotypesAsTree( new EvidenceFilter( taxon.getId(), editableOnly ) ) );
         }
-        return Responder.respond( phenotypeAssociationManagerService
-                .loadAllPhenotypesByTree( new EvidenceFilter( taxon.getId(), editableOnly ) ) );
+        return Responder.respond( phenotypeAssociationManagerService.loadAllPhenotypesByTree( new EvidenceFilter( taxon.getId(), editableOnly ) ) );
     }
 
     /**
@@ -298,9 +326,7 @@ public ResponseDataObject> findCandidateGenesInTaxo
             @QueryParam("editableOnly") @DefaultValue("false") Boolean editableOnly // Optional, default false
     ) {
         Set response;
-        response = this.phenotypeAssociationManagerService.findCandidateGenes(
-                new EvidenceFilter( taxonArgService.getEntity( taxonArg ).getId(), editableOnly ),
-                new HashSet<>( phenotypes.getValue() ) );
+        response = this.phenotypeAssociationManagerService.findCandidateGenes( new EvidenceFilter( taxonArgService.getEntity( taxonArg ).getId(), editableOnly ), new HashSet<>( phenotypes.getValue() ) );
         return Responder.respond( response );
     }
 
diff --git a/gemma-rest/src/main/java/ubic/gemma/rest/util/args/AbstractArg.java b/gemma-rest/src/main/java/ubic/gemma/rest/util/args/AbstractArg.java
index a2ee986f99..1cc67170af 100644
--- a/gemma-rest/src/main/java/ubic/gemma/rest/util/args/AbstractArg.java
+++ b/gemma-rest/src/main/java/ubic/gemma/rest/util/args/AbstractArg.java
@@ -8,8 +8,8 @@
 /**
  * Base class for non Object-specific functionality argument types, that can be malformed on input (E.g an argument
  * representing a number was a non-numeric string in the request).
- *
- * The {@link Schema} annotation used in sub-classes ensures that custom args are represented by a string in the OpenAPI
+ * 

+ * The {@link Schema} annotation used in subclasses ensures that custom args are represented by a string in the OpenAPI
  * specification.
  *
  * @author tesarst
@@ -20,7 +20,7 @@ public abstract class AbstractArg implements Arg {
 
     /**
      * Constructor for well-formed value.
-     *
+     * 

      * Note that well-formed values can never be null. Although, the argument itself can be null to represent a value
      * that is omitted by the request.
      *
@@ -46,5 +46,4 @@ public T getValue() {
     public String toString() {
         return String.valueOf( this.value );
     }
-
 }
diff --git a/gemma-rest/src/main/java/ubic/gemma/rest/util/args/AbstractEntityArg.java b/gemma-rest/src/main/java/ubic/gemma/rest/util/args/AbstractEntityArg.java
index 9e8cc97927..ca3335a780 100644
--- a/gemma-rest/src/main/java/ubic/gemma/rest/util/args/AbstractEntityArg.java
+++ b/gemma-rest/src/main/java/ubic/gemma/rest/util/args/AbstractEntityArg.java
@@ -4,7 +4,6 @@
 import ubic.gemma.persistence.service.FilteringService;
 
 import javax.annotation.Nullable;
-import javax.ws.rs.BadRequestException;
 import java.util.Collections;
 import java.util.List;
 
@@ -55,7 +54,7 @@ Class getPropertyType() {
     List getEntities( S service ) {
         O entity = getEntity( service );
         if ( entity != null ) {
-            return Collections.singletonList( getEntity( service ) );
+            return Collections.singletonList( entity );
         } else {
             return Collections.emptyList();
         }
diff --git a/gemma-rest/src/main/java/ubic/gemma/rest/util/args/AbstractEntityArgService.java b/gemma-rest/src/main/java/ubic/gemma/rest/util/args/AbstractEntityArgService.java
index ef7c941633..d5fcab33e0 100644
--- a/gemma-rest/src/main/java/ubic/gemma/rest/util/args/AbstractEntityArgService.java
+++ b/gemma-rest/src/main/java/ubic/gemma/rest/util/args/AbstractEntityArgService.java
@@ -78,15 +78,19 @@ public T getEntity( AbstractEntityArg entityArg ) throws NotFoundExcept
 
     @Override
     public List getEntities( AbstractEntityArg entityArg ) throws NotFoundException, BadRequestException {
-        return entityArg.getEntities( service );
+        List result = entityArg.getEntities( service );
+        if ( result.isEmpty() ) {
+            // will raise a NotFoundException
+            checkEntity( entityArg, null );
+        }
+        return result;
     }
 
     @Override
     public List getEntities( AbstractEntityArrayArg entitiesArg ) throws NotFoundException, BadRequestException {
         List objects = new ArrayList<>( entitiesArg.getValue().size() );
         for ( String s : entitiesArg.getValue() ) {
-            AbstractEntityArg arg = entityArgValueOf( entitiesArg.getEntityArgClass(), s );
-            objects.add( checkEntity( arg, arg.getEntity( service ) ) );
+            objects.add( getEntity( entityArgValueOf( entitiesArg.getEntityArgClass(), s ) ) );
         }
         return objects;
     }
diff --git a/gemma-rest/src/main/java/ubic/gemma/rest/util/args/EntityArgService.java b/gemma-rest/src/main/java/ubic/gemma/rest/util/args/EntityArgService.java
index a8e822e62e..9cba8323ee 100644
--- a/gemma-rest/src/main/java/ubic/gemma/rest/util/args/EntityArgService.java
+++ b/gemma-rest/src/main/java/ubic/gemma/rest/util/args/EntityArgService.java
@@ -57,18 +57,58 @@ public interface EntityArgService getFilterablePropertyResolvableAllowedValuesLabels( String p ) throws BadRequestException;
 
+    /**
+     * Retrieve the entity represented by this argument.
+     * @throws NotFoundException   if the entity does not exist
+     * @throws BadRequestException if the argument is malformed
+     */
     @Nonnull
     T getEntity( AbstractEntityArg entityArg ) throws NotFoundException, BadRequestException;
 
+    /**
+     * Retrieve the entities represented by this argument.
+     * 

+     * Note that this will never return an empty array.
+     * 

+     * This is intended for cases where an argument could match more than one entity.
+     * @throws NotFoundException   if no entity matching the argument exist
+     * @throws BadRequestException if the argument is malformed
+     */
     List getEntities( AbstractEntityArg entityArg ) throws NotFoundException, BadRequestException;
 
+    /**
+     * Retrieve each entity represented by the array argument, raising a {@link NotFoundException} if any of them is
+     * missing.
+     * @throws NotFoundException   if any entity is missing
+     * @throws BadRequestException if the argument is malformed
+     */
     List getEntities( AbstractEntityArrayArg entitiesArg ) throws NotFoundException, BadRequestException;
 
+    /**
+     * Translate the provided entity argument into a {@link Filters}.
+     * 

+     * This will generate clause in the form of {@code property = value}.
+     * @throws BadRequestException if the argument is malformed
+     */
      Filters getFilters( AbstractEntityArg entityArg ) throws BadRequestException;
 
+    /**
+     * Translate the provided entity argument into a {@link Filters}.
+     * 

+     * This will generate clause in the form of {@code property in (values...)}.
+     * @throws BadRequestException if the argument is malformed
+     */
     Filters getFilters( AbstractEntityArrayArg entitiesArg ) throws BadRequestException;
 
+    /**
+     * Obtain a {@link Filters} from a filter argument.
+     * @throws BadRequestException if the argument is malformed
+     */
     Filters getFilters( FilterArg filterArg ) throws BadRequestException;
 
+    /**
+     * Obtain a {@link Sort} from a sort argument.
+     * @throws BadRequestException if the argument is malformed
+     */
     Sort getSort( SortArg sortArg ) throws BadRequestException;
 }
diff --git a/gemma-rest/src/main/java/ubic/gemma/rest/util/args/GeneAnyIdArg.java b/gemma-rest/src/main/java/ubic/gemma/rest/util/args/GeneAnyIdArg.java
deleted file mode 100644
index 31abe80262..0000000000
--- a/gemma-rest/src/main/java/ubic/gemma/rest/util/args/GeneAnyIdArg.java
+++ /dev/null
@@ -1,32 +0,0 @@
-package ubic.gemma.rest.util.args;
-
-import ubic.gemma.core.genome.gene.service.GeneService;
-import ubic.gemma.model.genome.Gene;
-import ubic.gemma.model.genome.Taxon;
-
-import javax.annotation.Nullable;
-import javax.ws.rs.BadRequestException;
-
-/**
- * Base class for GeneArg representing any of the identifiers of a Gene.
- *
- * @param  the type of the Gene Identifier.
- * @author tesarst
- */
-public abstract class GeneAnyIdArg extends GeneArg {
-
-    protected GeneAnyIdArg( String propertyName, Class propertyType, T value ) {
-        super( propertyName, propertyType, value );
-    }
-
-    @Nullable
-    @Override
-    Gene getEntityWithTaxon( GeneService geneService, Taxon taxon ) {
-        // gene retrieved by ID are unambiguous
-        Gene gene = getEntity( geneService );
-        if ( gene != null && !gene.getTaxon().equals( taxon ) ) {
-            throw new BadRequestException( String.format( "The gene %s does not belong to taxon %s.", gene, taxon ) );
-        }
-        return gene;
-    }
-}
diff --git a/gemma-rest/src/main/java/ubic/gemma/rest/util/args/GeneArg.java b/gemma-rest/src/main/java/ubic/gemma/rest/util/args/GeneArg.java
index 975007f723..b205d50648 100644
--- a/gemma-rest/src/main/java/ubic/gemma/rest/util/args/GeneArg.java
+++ b/gemma-rest/src/main/java/ubic/gemma/rest/util/args/GeneArg.java
@@ -7,7 +7,8 @@
 import ubic.gemma.model.genome.Taxon;
 import ubic.gemma.rest.util.MalformedArgException;
 
-import javax.annotation.Nullable;
+import java.util.List;
+import java.util.stream.Collectors;
 
 /**
  * Mutable argument type base class for Gene API.
@@ -30,11 +31,12 @@ protected GeneArg( String propertyName, Class propertyType, T value ) {
         super( propertyName, propertyType, value );
     }
 
-    /**
-     * Retrieve a gene entity from the given service for the given taxon.
-     */
-    @Nullable
-    abstract Gene getEntityWithTaxon( GeneService geneService, Taxon taxon );
+
+    List getEntitiesWithTaxon( GeneService service, Taxon taxon ) {
+        return getEntities( service ).stream()
+                .filter( gene -> gene.getTaxon().equals( taxon ) )
+                .collect( Collectors.toList() );
+    }
 
     /**
      * Used by RS to parse value of request parameters.
diff --git a/gemma-rest/src/main/java/ubic/gemma/rest/util/args/GeneArgService.java b/gemma-rest/src/main/java/ubic/gemma/rest/util/args/GeneArgService.java
index 1ca5b008e0..e23f4d1ce5 100644
--- a/gemma-rest/src/main/java/ubic/gemma/rest/util/args/GeneArgService.java
+++ b/gemma-rest/src/main/java/ubic/gemma/rest/util/args/GeneArgService.java
@@ -6,42 +6,106 @@
 import ubic.gemma.core.genome.gene.service.GeneService;
 import ubic.gemma.core.ontology.providers.GeneOntologyService;
 import ubic.gemma.core.search.SearchException;
+import ubic.gemma.model.expression.designElement.CompositeSequenceValueObject;
 import ubic.gemma.model.genome.Gene;
 import ubic.gemma.model.genome.GeneOntologyTermValueObject;
 import ubic.gemma.model.genome.PhysicalLocationValueObject;
 import ubic.gemma.model.genome.Taxon;
 import ubic.gemma.model.genome.gene.GeneValueObject;
 import ubic.gemma.model.genome.gene.phenotype.valueObject.GeneEvidenceValueObject;
+import ubic.gemma.persistence.service.expression.designElement.CompositeSequenceService;
+import ubic.gemma.persistence.util.Filter;
+import ubic.gemma.persistence.util.Filters;
+import ubic.gemma.persistence.util.Slice;
+import ubic.gemma.persistence.util.Sort;
 
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
+import javax.ws.rs.BadRequestException;
+import javax.ws.rs.NotFoundException;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Objects;
-import java.util.stream.Collectors;
 
 @Service
 public class GeneArgService extends AbstractEntityArgService {
 
     private final PhenotypeAssociationManagerService phenotypeAssociationManagerService;
     private final GeneOntologyService geneOntologyService;
-
+    private final CompositeSequenceService compositeSequenceService;
 
     @Autowired
-    public GeneArgService( GeneService service, PhenotypeAssociationManagerService phenotypeAssociationManagerService, GeneOntologyService geneOntologyService ) {
+    public GeneArgService( GeneService service, PhenotypeAssociationManagerService phenotypeAssociationManagerService, GeneOntologyService geneOntologyService, CompositeSequenceService compositeSequenceService ) {
         super( service );
         this.phenotypeAssociationManagerService = phenotypeAssociationManagerService;
         this.geneOntologyService = geneOntologyService;
+        this.compositeSequenceService = compositeSequenceService;
+    }
+
+    /**
+     * {@inheritDoc}
+     * @throws BadRequestException if more than one gene match the supplied gene argument
+     */
+    @Nonnull
+    @Override
+    public Gene getEntity( AbstractEntityArg entityArg ) throws NotFoundException, BadRequestException {
+        List matchedGenes = getEntities( entityArg );
+        if ( matchedGenes.isEmpty() ) {
+            return checkEntity( entityArg, null );
+        } else if ( matchedGenes.size() > 1 ) {
+            throw new BadRequestException( "Gene identifier " + entityArg + " matches more than one gene, supply a taxon to disambiguate." );
+        } else {
+            return matchedGenes.iterator().next();
+        }
+    }
+
+    /**
+     * Obtain a gene from a specific taxon.
+     * @throws BadRequestException if more than one gene match the supplied gene argumen in the given taxon
+     */
+    @Nonnull
+    public Gene getEntityWithTaxon( GeneArg entityArg, Taxon taxon ) {
+        List matchedGenes = getEntitiesWithTaxon( entityArg, taxon );
+        if ( matchedGenes.isEmpty() ) {
+            return checkEntity( entityArg, null );
+        } else if ( matchedGenes.size() > 1 ) {
+            throw new BadRequestException( "Gene identifier " + entityArg + " matches more than one gene in " + taxon.getCommonName() + ", use a different type of identifier such as an NCBI or Ensembl ID." );
+        } else {
+            return matchedGenes.iterator().next();
+        }
     }
 
-    public Gene getEntityWithTaxon( GeneArg arg, Taxon taxon ) {
-        return checkEntity( arg, arg.getEntityWithTaxon( service, taxon ) );
+    /**
+     * Obtain genes from a specific taxon.
+     */
+    public List getEntitiesWithTaxon( GeneArg genes, Taxon taxon ) {
+        return genes.getEntitiesWithTaxon( service, taxon );
     }
 
     /**
-     * @return all genes that match the value of the GeneArg.
+     * Obtain genes from a specific taxon.
      */
-    public List getValueObjects( GeneArg arg ) {
-        return service.loadValueObjects( getEntities( arg ) );
+    public List getEntitiesWithTaxon( GeneArrayArg genes, Taxon taxon ) {
+        List objects = new ArrayList<>( genes.getValue().size() );
+        for ( String s : genes.getValue() ) {
+            GeneArg arg = ( GeneArg ) entityArgValueOf( genes.getEntityArgClass(), s );
+            objects.add( getEntityWithTaxon( arg, taxon ) );
+        }
+        return objects;
+    }
+
+    public Slice getGenes( int offset, int limit ) {
+        return service.loadValueObjects( null, service.getSort( "id", Sort.Direction.ASC ), offset, limit );
+    }
+
+    public Slice getGenesInTaxon( Taxon taxon, int offset, int limit ) {
+        return service.loadValueObjects( Filters.by( service.getFilter( "taxon.id", Long.class, Filter.Operator.eq, taxon.getId() ) ), service.getSort( "id", Sort.Direction.ASC ), offset, limit );
+    }
+
+    /**
+     * @return a collection of Gene value objects..
+     */
+    public List getGenesInTaxon( GeneArrayArg arg, Taxon taxon ) {
+        return service.loadValueObjects( getEntitiesWithTaxon( arg, taxon ) );
     }
 
     /**
@@ -64,19 +128,10 @@ public List getGeneLocation( GeneArg geneArg ) {
      * @param taxon       the taxon to limit the search to. Can be null.
      * @return collection of physical location objects.
      */
-    public List getGeneLocation( GeneArg arg, Taxon taxon ) {
+    public List getGeneLocationInTaxon( GeneArg arg, Taxon taxon ) {
         return service.getPhysicalLocationsValueObjects( getEntityWithTaxon( arg, taxon ) );
     }
 
-    /**
-     * @return a collection of Gene value objects..
-     */
-    public List getGenesOnTaxon( GeneArg arg, Taxon taxon ) {
-        return this.getValueObjects( arg ).stream()
-                .filter( vo -> Objects.equals( vo.getTaxonId(), taxon.getId() ) )
-                .collect( Collectors.toList() );
-    }
-
     /**
      * Searches for gene evidence of the gene that this GeneArg represents, on the given taxon.
      *
@@ -92,11 +147,29 @@ public List getGeneEvidence( GeneArg arg, @Nullable
 
     /**
      * Returns GO terms for the gene that this GeneArg represents.
-     *
-     * @return collection of physical location objects.
      */
-    public List getGoTerms( GeneArg arg ) {
-        Gene gene = this.getEntity( arg );
-        return geneOntologyService.getValueObjects( gene );
+    public List getGeneGoTerms( GeneArg arg ) {
+        return geneOntologyService.getValueObjects( this.getEntity( arg ) );
+    }
+
+    /**
+     * Obtain GO terms for the gene in the given taxon.
+     */
+    public List getGeneGoTermsInTaxon( GeneArg geneArg, Taxon taxon ) {
+        return geneOntologyService.getValueObjects( this.getEntityWithTaxon( geneArg, taxon ) );
+    }
+
+    /**
+     * Obtain probes for the gene across all platforms.
+     */
+    public Slice getGeneProbes( GeneArg geneArg, int offset, int limit ) {
+        return compositeSequenceService.loadValueObjectsForGene( getEntity( geneArg ), offset, limit );
+    }
+
+    /**
+     * Obtain probes for the gene in the given taxon across all platforms.
+     */
+    public Slice getGeneProbesInTaxon( GeneArg geneArg, Taxon taxon, int offset, int limit ) {
+        return compositeSequenceService.loadValueObjectsForGene( getEntityWithTaxon( geneArg, taxon ), offset, limit );
     }
 }
diff --git a/gemma-rest/src/main/java/ubic/gemma/rest/util/args/GeneEnsemblIdArg.java b/gemma-rest/src/main/java/ubic/gemma/rest/util/args/GeneEnsemblIdArg.java
index 18c42bde2e..7677623872 100644
--- a/gemma-rest/src/main/java/ubic/gemma/rest/util/args/GeneEnsemblIdArg.java
+++ b/gemma-rest/src/main/java/ubic/gemma/rest/util/args/GeneEnsemblIdArg.java
@@ -4,9 +4,6 @@
 import io.swagger.v3.oas.annotations.media.Schema;
 import ubic.gemma.core.genome.gene.service.GeneService;
 import ubic.gemma.model.genome.Gene;
-import ubic.gemma.model.genome.Taxon;
-
-import javax.annotation.Nullable;
 
 /**
  * Long argument type for Gene API, referencing the Gene Ensembl ID.
@@ -15,7 +12,7 @@
  */
 @Schema(type = "string", description = "An Ensembl gene identifier which typically starts with 'ENSG'.",
         externalDocs = @ExternalDocumentation(url = "https://www.ensembl.org/"))
-public class GeneEnsemblIdArg extends GeneAnyIdArg {
+public class GeneEnsemblIdArg extends GeneArg {
 
     /**
      * @param s intentionally primitive type, so the value property can never be null.
@@ -28,10 +25,4 @@ public class GeneEnsemblIdArg extends GeneAnyIdArg {
     Gene getEntity( GeneService service ) {
         return service.findByEnsemblId( this.getValue() );
     }
-
-    @Nullable
-    @Override
-    protected Gene getEntityWithTaxon( GeneService service, Taxon taxon ) {
-        return getEntity( service );
-    }
 }
diff --git a/gemma-rest/src/main/java/ubic/gemma/rest/util/args/GeneNcbiIdArg.java b/gemma-rest/src/main/java/ubic/gemma/rest/util/args/GeneNcbiIdArg.java
index 0d520d2cbf..f179a51db0 100644
--- a/gemma-rest/src/main/java/ubic/gemma/rest/util/args/GeneNcbiIdArg.java
+++ b/gemma-rest/src/main/java/ubic/gemma/rest/util/args/GeneNcbiIdArg.java
@@ -14,7 +14,7 @@
  */
 @Schema(type = "string", description = "An NCBI gene identifier.",
         externalDocs = @ExternalDocumentation(url = "https://www.ncbi.nlm.nih.gov/gene"))
-public class GeneNcbiIdArg extends GeneAnyIdArg {
+public class GeneNcbiIdArg extends GeneArg {
 
     /**
      * @param l intentionally primitive type, so the value property can never be null.
diff --git a/gemma-rest/src/main/java/ubic/gemma/rest/util/args/GeneSymbolArg.java b/gemma-rest/src/main/java/ubic/gemma/rest/util/args/GeneSymbolArg.java
index 932bfc9f08..93b6e86ee2 100644
--- a/gemma-rest/src/main/java/ubic/gemma/rest/util/args/GeneSymbolArg.java
+++ b/gemma-rest/src/main/java/ubic/gemma/rest/util/args/GeneSymbolArg.java
@@ -9,6 +9,7 @@
 import javax.ws.rs.BadRequestException;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 
 /**
@@ -39,12 +40,12 @@ Gene getEntity( GeneService service ) {
     }
 
     @Override
-    protected Gene getEntityWithTaxon( GeneService service, Taxon taxon ) {
-        return service.findByOfficialSymbol( getValue(), taxon );
+    List getEntities( GeneService service ) throws BadRequestException {
+        return new ArrayList<>( service.findByOfficialSymbol( getValue() ) );
     }
 
     @Override
-    List getEntities( GeneService service ) throws BadRequestException {
-        return new ArrayList<>( service.findByOfficialSymbol( getValue() ) );
+    List getEntitiesWithTaxon( GeneService service, Taxon taxon ) {
+        return Collections.singletonList( service.findByOfficialSymbol( getValue(), taxon ) );
     }
 }
diff --git a/gemma-rest/src/main/resources/openapi-configuration.yaml b/gemma-rest/src/main/resources/openapi-configuration.yaml
index 6e01e1c87a..1fe4634bba 100644
--- a/gemma-rest/src/main/resources/openapi-configuration.yaml
+++ b/gemma-rest/src/main/resources/openapi-configuration.yaml
@@ -8,7 +8,7 @@ openAPI:
       url: https://dev.gemma.msl.ubc.ca/rest/v2
   info:
     title: Gemma RESTful API
-    version: 2.7.4
+    version: 2.7.5
     description: |
       This website documents the usage of the [Gemma RESTful API](https://gemma.msl.ubc.ca/rest/v2/). Here you can find
       example script usage of the API, as well as graphical interface for each endpoint, with description of its
diff --git a/gemma-rest/src/main/resources/restapidocs/CHANGELOG.md b/gemma-rest/src/main/resources/restapidocs/CHANGELOG.md
index 3be718efa3..e5bd861fe5 100644
--- a/gemma-rest/src/main/resources/restapidocs/CHANGELOG.md
+++ b/gemma-rest/src/main/resources/restapidocs/CHANGELOG.md
@@ -1,5 +1,11 @@
 ## Updates
 
+### Update 2.7.5
+
+- fix a bug in `getTaxonDatasets` sorting parameter, it was indicating `Taxon` instead of `ExpressionExperiment`
+- disambiguate all endpoints that expect a gene identifier with a taxon argument
+- add endpoints to retrieve all genes with pagination
+
 ### Update 2.7.4
 
 - indicate 503 status codes for endpoints that could timeout due to a long-running search
diff --git a/gemma-rest/src/test/java/ubic/gemma/rest/DatasetsWebServiceTest.java b/gemma-rest/src/test/java/ubic/gemma/rest/DatasetsWebServiceTest.java
index 76e41b5dc7..60d7bbbb56 100644
--- a/gemma-rest/src/test/java/ubic/gemma/rest/DatasetsWebServiceTest.java
+++ b/gemma-rest/src/test/java/ubic/gemma/rest/DatasetsWebServiceTest.java
@@ -36,6 +36,7 @@
 import ubic.gemma.rest.util.args.DatasetArgService;
 import ubic.gemma.rest.util.args.GeneArgService;
 import ubic.gemma.rest.util.args.QuantitationTypeArgService;
+import ubic.gemma.rest.util.args.TaxonArgService;
 
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
@@ -111,6 +112,11 @@ public QuantitationTypeArgService quantitationTypeArgService( QuantitationTypeSe
             return new QuantitationTypeArgService( quantitationTypeService );
         }
 
+        @Bean
+        public TaxonArgService taxonArgService() {
+            return mock();
+        }
+
         @Bean
         public GeneArgService geneArgService() {
             return mock( GeneArgService.class );
diff --git a/gemma-rest/src/test/java/ubic/gemma/rest/GeneWebServiceTest.java b/gemma-rest/src/test/java/ubic/gemma/rest/GeneWebServiceTest.java
new file mode 100644
index 0000000000..72fb75a7f9
--- /dev/null
+++ b/gemma-rest/src/test/java/ubic/gemma/rest/GeneWebServiceTest.java
@@ -0,0 +1,82 @@
+package ubic.gemma.rest;
+
+import org.apache.commons.lang.math.RandomUtils;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import ubic.gemma.core.genome.gene.service.GeneService;
+import ubic.gemma.model.genome.Gene;
+import ubic.gemma.model.genome.Taxon;
+import ubic.gemma.persistence.service.genome.taxon.TaxonService;
+import ubic.gemma.rest.util.BaseJerseyIntegrationTest;
+
+import javax.ws.rs.core.Response;
+
+import static ubic.gemma.rest.util.Assertions.assertThat;
+
+public class GeneWebServiceTest extends BaseJerseyIntegrationTest {
+
+    @Autowired
+    private TaxonService taxonService;
+
+    @Autowired
+    private GeneService geneService;
+
+    /* fixtures */
+    private Taxon taxon;
+    private Gene gene;
+
+    @Before
+    public void createFixtures() {
+        taxon = new Taxon();
+        taxon.setNcbiId( RandomUtils.nextInt() );
+        taxon.setCommonName( "common_name_" + RandomUtils.nextInt() );
+        taxon.setScientificName( "scientific_name_" + RandomStringUtils.randomAlphabetic( 10 ) );
+        taxon.setIsGenesUsable( false );
+        taxon = taxonService.create( taxon );
+        gene = new Gene();
+        gene.setTaxon( taxon );
+        gene.setNcbiGeneId( RandomUtils.nextInt() );
+        gene.setEnsemblId( "ensembl_id_" + RandomStringUtils.randomAlphabetic( 10 ) );
+        gene.setOfficialSymbol( "official_symbol_" + RandomStringUtils.randomAlphabetic( 10 ) );
+        gene = geneService.create( gene );
+    }
+
+    @After
+    public void removeFixtures() {
+        geneService.remove( gene );
+        taxonService.remove( taxon );
+    }
+
+    @Test
+    public void testGenes() {
+        assertThat( target( "/genes" ).request().get() )
+                .hasStatus( Response.Status.OK );
+    }
+
+    @Test
+    public void testGenesByIds() {
+        assertThat( target( "/genes/" + gene.getOfficialSymbol() ).request().get() )
+                .hasStatus( Response.Status.OK );
+    }
+
+    @Test
+    public void testGeneProbes() {
+        assertThat( target( "/genes/" + gene.getOfficialSymbol() + "/probes" ).request().get() )
+                .hasStatus( Response.Status.OK );
+    }
+
+    @Test
+    public void testGeneGoTerms() {
+        assertThat( target( "/genes/" + gene.getOfficialSymbol() + "/goTerms" ).request().get() )
+                .hasStatus( Response.Status.OK );
+    }
+
+    @Test
+    public void testGeneLocations() {
+        assertThat( target( "/genes/" + gene.getOfficialSymbol() + "/locations" ).request().get() )
+                .hasStatus( Response.Status.OK );
+    }
+}
\ No newline at end of file
diff --git a/gemma-rest/src/test/java/ubic/gemma/rest/TaxaWebServiceTest.java b/gemma-rest/src/test/java/ubic/gemma/rest/TaxaWebServiceTest.java
index f76d3d16aa..12080efd2e 100644
--- a/gemma-rest/src/test/java/ubic/gemma/rest/TaxaWebServiceTest.java
+++ b/gemma-rest/src/test/java/ubic/gemma/rest/TaxaWebServiceTest.java
@@ -1,12 +1,14 @@
 package ubic.gemma.rest;
 
+import org.apache.commons.lang.math.RandomUtils;
 import org.apache.commons.lang3.RandomStringUtils;
-import org.apache.commons.lang3.RandomUtils;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 import org.springframework.beans.factory.annotation.Autowired;
+import ubic.gemma.core.genome.gene.service.GeneService;
 import ubic.gemma.model.expression.experiment.ExpressionExperimentValueObject;
+import ubic.gemma.model.genome.Gene;
 import ubic.gemma.model.genome.Taxon;
 import ubic.gemma.model.genome.TaxonValueObject;
 import ubic.gemma.persistence.service.genome.taxon.TaxonService;
@@ -15,20 +17,25 @@
 import ubic.gemma.rest.util.ResponseDataObject;
 import ubic.gemma.rest.util.args.*;
 
+import javax.ws.rs.core.Response;
 import java.util.List;
 
-import static org.assertj.core.api.Assertions.assertThat;
+import static ubic.gemma.rest.util.Assertions.assertThat;
 
 public class TaxaWebServiceTest extends BaseJerseyIntegrationTest {
 
     @Autowired
     private TaxonService taxonService;
 
+    @Autowired
+    private GeneService geneService;
+
     @Autowired
     private TaxaWebService taxaWebService;
 
     /* fixtures */
     private Taxon taxon;
+    private Gene gene;
 
     @Before
     public void createFixtures() {
@@ -38,10 +45,17 @@ public void createFixtures() {
         taxon.setScientificName( "scientific_name_" + RandomStringUtils.randomAlphabetic( 10 ) );
         taxon.setIsGenesUsable( false );
         taxon = taxonService.create( taxon );
+        gene = new Gene();
+        gene.setTaxon( taxon );
+        gene.setNcbiGeneId( RandomUtils.nextInt() );
+        gene.setEnsemblId( "ensembl_id_" + RandomStringUtils.randomAlphabetic( 10 ) );
+        gene.setOfficialSymbol( "official_symbol_" + RandomStringUtils.randomAlphabetic( 10 ) );
+        gene = geneService.create( gene );
     }
 
     @After
     public void removeFixtures() {
+        geneService.remove( gene );
         taxonService.remove( taxon );
     }
 
@@ -79,4 +93,34 @@ public void testTaxonDatasetsByNcbiId() {
                 SortArg.valueOf( "+id" ) );
         assertThat( response.getData() ).isEmpty();
     }
+
+    @Test
+    public void testTaxonGenes() {
+        assertThat( target( "/taxa/" + taxon.getCommonName() + "/genes" ).request().get() )
+                .hasStatus( Response.Status.OK );
+    }
+
+    @Test
+    public void testTaxonGenesByIds() {
+        assertThat( target( "/taxa/" + taxon.getCommonName() + "/genes/" + gene.getOfficialSymbol() ).request().get() )
+                .hasStatus( Response.Status.OK );
+    }
+
+    @Test
+    public void testTaxonGeneProbes() {
+        assertThat( target( "/taxa/" + taxon.getCommonName() + "/genes/" + gene.getOfficialSymbol() + "/probes" ).request().get() )
+                .hasStatus( Response.Status.OK );
+    }
+
+    @Test
+    public void testTaxonGeneGoTerms() {
+        assertThat( target( "/taxa/" + taxon.getCommonName() + "/genes/" + gene.getOfficialSymbol() + "/goTerms" ).request().get() )
+                .hasStatus( Response.Status.OK );
+    }
+
+    @Test
+    public void testTaxonGeneLocations() {
+        assertThat( target( "/taxa/" + taxon.getCommonName() + "/genes/" + gene.getOfficialSymbol() + "/locations" ).request().get() )
+                .hasStatus( Response.Status.OK );
+    }
 }
diff --git a/gemma-rest/src/test/java/ubic/gemma/rest/util/Assertions.java b/gemma-rest/src/test/java/ubic/gemma/rest/util/Assertions.java
index fd128fbc0b..0a39eb9921 100644
--- a/gemma-rest/src/test/java/ubic/gemma/rest/util/Assertions.java
+++ b/gemma-rest/src/test/java/ubic/gemma/rest/util/Assertions.java
@@ -15,6 +15,6 @@ public static ResponseAssert assertThat( Response response ) {
     }
 
     public static ErrorsAssert assertThat( Errors errors ) {
-        return new ErrorsAssert(errors);
+        return new ErrorsAssert( errors );
     }
 }
diff --git a/gemma-rest/src/test/java/ubic/gemma/rest/util/args/GeneArgServiceTest.java b/gemma-rest/src/test/java/ubic/gemma/rest/util/args/GeneArgServiceTest.java
index 9e6d61c16b..12d3c26a1d 100644
--- a/gemma-rest/src/test/java/ubic/gemma/rest/util/args/GeneArgServiceTest.java
+++ b/gemma-rest/src/test/java/ubic/gemma/rest/util/args/GeneArgServiceTest.java
@@ -1,5 +1,6 @@
 package ubic.gemma.rest.util.args;
 
+import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -7,9 +8,7 @@
 import org.springframework.context.annotation.Configuration;
 import org.springframework.test.context.ContextConfiguration;
 import org.springframework.test.context.junit4.AbstractJUnit4SpringContextTests;
-import ubic.gemma.core.association.phenotype.PhenotypeAssociationManagerService;
 import ubic.gemma.core.genome.gene.service.GeneService;
-import ubic.gemma.core.ontology.providers.GeneOntologyService;
 import ubic.gemma.model.genome.Gene;
 import ubic.gemma.model.genome.Taxon;
 import ubic.gemma.persistence.util.TestComponent;
@@ -27,7 +26,7 @@ static class GeneArgServiceTestTestContextConfig {
 
         @Bean
         public GeneArgService geneArgService( GeneService geneService ) {
-            return new GeneArgService( geneService, mock( PhenotypeAssociationManagerService.class ), mock( GeneOntologyService.class ) );
+            return new GeneArgService( geneService, mock(), mock(), mock() );
         }
 
         @Bean
@@ -50,6 +49,11 @@ public void setupMocks() {
         when( geneService.findByOfficialSymbol( any(), any() ) ).thenReturn( new Gene() );
     }
 
+    @After
+    public void resetMocks() {
+        reset( geneService );
+    }
+
     @Test
     public void testNcbiId() {
         geneArgService.getEntity( new GeneNcbiIdArg( 1 ) );

From ae920438e4275e4fa70e28601eb709a0c0960b8d Mon Sep 17 00:00:00 2001
From: Guillaume Poirier-Morency 
Date: Fri, 10 May 2024 10:51:05 -0700
Subject: [PATCH 30/61] Improve test for getTaxonDatasets()

---
 .../java/ubic/gemma/rest/TaxaWebService.java  |  4 +++-
 .../ubic/gemma/rest/TaxaWebServiceTest.java   | 21 ++++++++++---------
 2 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/gemma-rest/src/main/java/ubic/gemma/rest/TaxaWebService.java b/gemma-rest/src/main/java/ubic/gemma/rest/TaxaWebService.java
index cfe871389b..d0fd9923e7 100644
--- a/gemma-rest/src/main/java/ubic/gemma/rest/TaxaWebService.java
+++ b/gemma-rest/src/main/java/ubic/gemma/rest/TaxaWebService.java
@@ -253,7 +253,9 @@ public ResponseDataObject> getTaxonGeneLocatio
     }
 
     /**
-     * Retrieves datasets for the given taxon. Filtering allowed exactly like in {@link DatasetsWebService#getDatasets(QueryArg, FilterArg, OffsetArg, LimitArg, SortArg)}.
+     * Retrieves datasets for the given taxon.
+     * 

+     * Filtering allowed exactly like in {@link DatasetsWebService#getDatasets(QueryArg, FilterArg, OffsetArg, LimitArg, SortArg)}.
      *
      * @param taxonArg can either be Taxon ID, Taxon NCBI ID, or one of its string identifiers:
      *                 scientific name, common name. It is recommended to use the ID for efficiency.
diff --git a/gemma-rest/src/test/java/ubic/gemma/rest/TaxaWebServiceTest.java b/gemma-rest/src/test/java/ubic/gemma/rest/TaxaWebServiceTest.java
index 12080efd2e..f0b72b804b 100644
--- a/gemma-rest/src/test/java/ubic/gemma/rest/TaxaWebServiceTest.java
+++ b/gemma-rest/src/test/java/ubic/gemma/rest/TaxaWebServiceTest.java
@@ -7,17 +7,17 @@
 import org.junit.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 import ubic.gemma.core.genome.gene.service.GeneService;
-import ubic.gemma.model.expression.experiment.ExpressionExperimentValueObject;
 import ubic.gemma.model.genome.Gene;
 import ubic.gemma.model.genome.Taxon;
 import ubic.gemma.model.genome.TaxonValueObject;
 import ubic.gemma.persistence.service.genome.taxon.TaxonService;
 import ubic.gemma.rest.util.BaseJerseyIntegrationTest;
-import ubic.gemma.rest.util.FilteredAndPaginatedResponseDataObject;
 import ubic.gemma.rest.util.ResponseDataObject;
-import ubic.gemma.rest.util.args.*;
+import ubic.gemma.rest.util.args.TaxonArrayArg;
 
+import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
+import java.util.Collections;
 import java.util.List;
 
 import static ubic.gemma.rest.util.Assertions.assertThat;
@@ -85,13 +85,14 @@ public void testTaxonByScientificName() {
 
     @Test
     public void testTaxonDatasetsByNcbiId() {
-        FilteredAndPaginatedResponseDataObject response = taxaWebService.getTaxonDatasets(
-                TaxonArg.valueOf( taxon.getNcbiId().toString() ),
-                FilterArg.valueOf( "" ),
-                OffsetArg.valueOf( "0" ),
-                LimitArg.valueOf( "20" ),
-                SortArg.valueOf( "+id" ) );
-        assertThat( response.getData() ).isEmpty();
+        assertThat( target( "/taxa/" + taxon.getNcbiId() + "/datasets" ).queryParam( "filter", "id > 100" ).request().get() )
+                .hasStatus( Response.Status.OK )
+                .hasMediaTypeCompatibleWith( MediaType.APPLICATION_JSON_TYPE )
+                .entity()
+                .hasFieldOrPropertyWithValue( "filter", "id > 100 and taxon.id = " + taxon.getId() )
+                .hasFieldOrPropertyWithValue( "offset", 0 )
+                .hasFieldOrPropertyWithValue( "limit", 20 )
+                .hasFieldOrPropertyWithValue( "data", Collections.emptyList() );
     }
 
     @Test

From 010495ee2200fef8e1b0afeedd7d64627d06251e Mon Sep 17 00:00:00 2001
From: Guillaume Poirier-Morency 
Date: Fri, 10 May 2024 10:53:35 -0700
Subject: [PATCH 31/61] Also suggest to use a better identifier if a gene
 symbol is ambiguous

---
 .../src/main/java/ubic/gemma/rest/util/args/GeneArgService.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gemma-rest/src/main/java/ubic/gemma/rest/util/args/GeneArgService.java b/gemma-rest/src/main/java/ubic/gemma/rest/util/args/GeneArgService.java
index e23f4d1ce5..eb38d29b62 100644
--- a/gemma-rest/src/main/java/ubic/gemma/rest/util/args/GeneArgService.java
+++ b/gemma-rest/src/main/java/ubic/gemma/rest/util/args/GeneArgService.java
@@ -52,7 +52,7 @@ public Gene getEntity( AbstractEntityArg entityArg ) throw
         if ( matchedGenes.isEmpty() ) {
             return checkEntity( entityArg, null );
         } else if ( matchedGenes.size() > 1 ) {
-            throw new BadRequestException( "Gene identifier " + entityArg + " matches more than one gene, supply a taxon to disambiguate." );
+            throw new BadRequestException( "Gene identifier " + entityArg + " matches more than one gene, supply a taxon to disambiguate or use a different type of identifier such as an NCBI or Ensembl ID." );
         } else {
             return matchedGenes.iterator().next();
         }

From 33eab1aee52ddeca635d2a46ca2131fbd61b5e08 Mon Sep 17 00:00:00 2001
From: Guillaume Poirier-Morency 
Date: Fri, 10 May 2024 10:56:58 -0700
Subject: [PATCH 32/61] rest: Add a test for ambiguous gene symbol

---
 .../java/ubic/gemma/rest/GeneWebServiceTest.java | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/gemma-rest/src/test/java/ubic/gemma/rest/GeneWebServiceTest.java b/gemma-rest/src/test/java/ubic/gemma/rest/GeneWebServiceTest.java
index 72fb75a7f9..80b833fc48 100644
--- a/gemma-rest/src/test/java/ubic/gemma/rest/GeneWebServiceTest.java
+++ b/gemma-rest/src/test/java/ubic/gemma/rest/GeneWebServiceTest.java
@@ -26,7 +26,7 @@ public class GeneWebServiceTest extends BaseJerseyIntegrationTest {
 
     /* fixtures */
     private Taxon taxon;
-    private Gene gene;
+    private Gene gene, gene2;
 
     @Before
     public void createFixtures() {
@@ -47,6 +47,9 @@ public void createFixtures() {
     @After
     public void removeFixtures() {
         geneService.remove( gene );
+        if ( gene2 != null ) {
+            geneService.remove( gene2 );
+        }
         taxonService.remove( taxon );
     }
 
@@ -68,6 +71,17 @@ public void testGeneProbes() {
                 .hasStatus( Response.Status.OK );
     }
 
+    @Test
+    public void testGeneProbesWhenIdentifierIsAmbiguous() {
+        gene2 = new Gene();
+        gene2.setOfficialSymbol( gene.getOfficialSymbol() );
+        gene2 = geneService.create( gene2 );
+        assertThat( target( "/genes/" + gene.getOfficialSymbol() + "/probes" ).request().get() )
+                .hasStatus( Response.Status.BAD_REQUEST )
+                .entity()
+                .hasFieldOrPropertyWithValue( "error.message", "Gene identifier " + gene.getOfficialSymbol() + " matches more than one gene, supply a taxon to disambiguate or use a different type of identifier such as an NCBI or Ensembl ID." );
+    }
+
     @Test
     public void testGeneGoTerms() {
         assertThat( target( "/genes/" + gene.getOfficialSymbol() + "/goTerms" ).request().get() )

From 81d6b88b7e5d767ec28957fe5c5b9af064024fa2 Mon Sep 17 00:00:00 2001
From: Guillaume Poirier-Morency 
Date: Fri, 10 May 2024 11:12:37 -0700
Subject: [PATCH 33/61] Set spring.security.strategy in the JVM options

Add a check in prepareContext() to ensure that the context is set
properly.
---
 .idea/runConfigurations/Gemma_Web.xml         |  2 +-
 gemma-cli/pom.xml                             |  2 +-
 .../java/ubic/gemma/core/apps/GemmaCLI.java   |  5 ---
 .../persistence/util/SpringContextUtil.java   |  6 ++++
 .../core/util/test/BaseIntegrationTest.java   |  6 ----
 .../util/SpringContextUtilTest.java           | 35 +++++++++++++++++++
 .../groovy/framework/SpringSupport.groovy     |  1 -
 .../rest/util/BaseJerseyIntegrationTest.java  |  6 ----
 .../ubic/gemma/rest/util/BaseJerseyTest.java  |  1 -
 .../gemma/web/listener/StartupListener.java   |  2 --
 pom.xml                                       |  6 ++--
 11 files changed, 47 insertions(+), 25 deletions(-)
 create mode 100644 gemma-core/src/test/java/ubic/gemma/persistence/util/SpringContextUtilTest.java

diff --git a/.idea/runConfigurations/Gemma_Web.xml b/.idea/runConfigurations/Gemma_Web.xml
index bae9002fff..2cedff5233 100644
--- a/.idea/runConfigurations/Gemma_Web.xml
+++ b/.idea/runConfigurations/Gemma_Web.xml
@@ -1,6 +1,6 @@
 
   
-    
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/GemmaCLI.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/GemmaCLI.java
index 395062c7fd..e7300c15b9 100644
--- a/gemma-cli/src/main/java/ubic/gemma/core/apps/GemmaCLI.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/GemmaCLI.java
@@ -143,11 +143,6 @@ public static void main( String[] args ) {
 
         loggingConfigurer.apply();
 
-        /*
-         * Guarantee that the security settings are uniform throughout the application (all threads).
-         */
-        SecurityContextHolder.setStrategyName( SecurityContextHolder.MODE_INHERITABLETHREADLOCAL );
-
         List profiles = new ArrayList<>();
         profiles.add( "cli" );
 
diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/util/SpringContextUtil.java b/gemma-core/src/main/java/ubic/gemma/persistence/util/SpringContextUtil.java
index 584a3e741d..2f8ae5f406 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/util/SpringContextUtil.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/util/SpringContextUtil.java
@@ -25,6 +25,7 @@
 import org.springframework.context.ApplicationContext;
 import org.springframework.context.ConfigurableApplicationContext;
 import org.springframework.context.support.ClassPathXmlApplicationContext;
+import org.springframework.security.core.context.SecurityContextHolder;
 import ubic.gemma.core.util.BuildInfo;
 
 import java.util.ArrayList;
@@ -93,12 +94,17 @@ public static ApplicationContext getApplicationContext( boolean testing, boolean
      * 

      * Perform the following steps:
      * 

      */
     public static void prepareContext( ApplicationContext context ) {
+        if ( !SecurityContextHolder.getContextHolderStrategy().getClass().getName().equals( "org.springframework.security.core.context.InheritableThreadLocalSecurityContextHolderStrategy" ) ) {
+            throw new IllegalStateException( String.format( "The security context holder strategy is not set to be inherited in new threads, make sure -D%s=%s is set in your JVM options.",
+                    SecurityContextHolder.SYSTEM_PROPERTY, SecurityContextHolder.MODE_INHERITABLETHREADLOCAL ) );
+        }
         if ( context instanceof ConfigurableApplicationContext ) {
             ConfigurableApplicationContext cac = ( ConfigurableApplicationContext ) context;
             if ( !cac.getEnvironment().acceptsProfiles( EnvironmentProfiles.PRODUCTION, EnvironmentProfiles.DEV, EnvironmentProfiles.TEST ) ) {
diff --git a/gemma-core/src/test/java/ubic/gemma/core/util/test/BaseIntegrationTest.java b/gemma-core/src/test/java/ubic/gemma/core/util/test/BaseIntegrationTest.java
index f1a559b23b..5b991e81c0 100644
--- a/gemma-core/src/test/java/ubic/gemma/core/util/test/BaseIntegrationTest.java
+++ b/gemma-core/src/test/java/ubic/gemma/core/util/test/BaseIntegrationTest.java
@@ -2,7 +2,6 @@
 
 import org.junit.After;
 import org.junit.Before;
-import org.junit.BeforeClass;
 import org.junit.experimental.categories.Category;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.security.core.context.SecurityContextHolder;
@@ -24,11 +23,6 @@ public abstract class BaseIntegrationTest extends AbstractJUnit4SpringContextTes
     @Autowired
     private TestAuthenticationUtils testAuthenticationUtils;
 
-    @BeforeClass
-    public static void setUpSecurityContextHolderStrategy() {
-        SecurityContextHolder.setStrategyName( SecurityContextHolder.MODE_INHERITABLETHREADLOCAL );
-    }
-
     /**
      * Setup the authentication for the test.
      * 

diff --git a/gemma-core/src/test/java/ubic/gemma/persistence/util/SpringContextUtilTest.java b/gemma-core/src/test/java/ubic/gemma/persistence/util/SpringContextUtilTest.java
new file mode 100644
index 0000000000..07a13d2393
--- /dev/null
+++ b/gemma-core/src/test/java/ubic/gemma/persistence/util/SpringContextUtilTest.java
@@ -0,0 +1,35 @@
+package ubic.gemma.persistence.util;
+
+import org.junit.Test;
+import org.springframework.context.support.GenericApplicationContext;
+import org.springframework.security.core.context.SecurityContextHolder;
+
+public class SpringContextUtilTest {
+
+    @Test
+    public void testPrepareContext() {
+        GenericApplicationContext context = new GenericApplicationContext();
+        context.getEnvironment().addActiveProfile( EnvironmentProfiles.TEST );
+        SpringContextUtil.prepareContext( context );
+    }
+
+    @Test(expected = IllegalStateException.class)
+    public void testPrepareContextWhenMoreThanOneEnvironmentProfileIsActive() {
+        GenericApplicationContext context = new GenericApplicationContext();
+        context.getEnvironment().addActiveProfile( EnvironmentProfiles.TEST );
+        context.getEnvironment().addActiveProfile( EnvironmentProfiles.DEV );
+        SpringContextUtil.prepareContext( context );
+    }
+
+    @Test(expected = IllegalStateException.class)
+    public void testPrepareContextWithIncorrectSecurityContextHolderStrategyIsSet() {
+        GenericApplicationContext context = new GenericApplicationContext();
+        context.getEnvironment().addActiveProfile( EnvironmentProfiles.TEST );
+        try {
+            SecurityContextHolder.setStrategyName( SecurityContextHolder.MODE_THREADLOCAL );
+            SpringContextUtil.prepareContext( context );
+        } finally {
+            SecurityContextHolder.setStrategyName( SecurityContextHolder.MODE_INHERITABLETHREADLOCAL );
+        }
+    }
+}
\ No newline at end of file
diff --git a/gemma-groovy-support/src/main/groovy/ubic/gemma/groovy/framework/SpringSupport.groovy b/gemma-groovy-support/src/main/groovy/ubic/gemma/groovy/framework/SpringSupport.groovy
index 59b831c5fb..28fb3af125 100644
--- a/gemma-groovy-support/src/main/groovy/ubic/gemma/groovy/framework/SpringSupport.groovy
+++ b/gemma-groovy-support/src/main/groovy/ubic/gemma/groovy/framework/SpringSupport.groovy
@@ -40,7 +40,6 @@ class SpringSupport {
     }
 
     SpringSupport(String userName, String password, List activeProfiles) {
-        SecurityContextHolder.setStrategyName(SecurityContextHolder.MODE_INHERITABLETHREADLOCAL)
         ctx = SpringContextUtil.getApplicationContext(activeProfiles as String[])
         authenticate(userName, password);
     }
diff --git a/gemma-rest/src/test/java/ubic/gemma/rest/util/BaseJerseyIntegrationTest.java b/gemma-rest/src/test/java/ubic/gemma/rest/util/BaseJerseyIntegrationTest.java
index e2448a7836..86bec25f75 100644
--- a/gemma-rest/src/test/java/ubic/gemma/rest/util/BaseJerseyIntegrationTest.java
+++ b/gemma-rest/src/test/java/ubic/gemma/rest/util/BaseJerseyIntegrationTest.java
@@ -2,7 +2,6 @@
 
 import org.junit.After;
 import org.junit.Before;
-import org.junit.BeforeClass;
 import org.junit.experimental.categories.Category;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.security.core.context.SecurityContextHolder;
@@ -26,11 +25,6 @@ public abstract class BaseJerseyIntegrationTest extends BaseJerseyTest {
     @Autowired
     private TestAuthenticationUtils testAuthenticationUtils;
 
-    @BeforeClass
-    public static void setUpSecurityContextHolderStrategy() {
-        SecurityContextHolder.setStrategyName( SecurityContextHolder.MODE_INHERITABLETHREADLOCAL );
-    }
-
     @Before
     public final void setUpAuthentication() {
         testAuthenticationUtils.runAsAdmin();
diff --git a/gemma-rest/src/test/java/ubic/gemma/rest/util/BaseJerseyTest.java b/gemma-rest/src/test/java/ubic/gemma/rest/util/BaseJerseyTest.java
index af72152818..eca25914e8 100644
--- a/gemma-rest/src/test/java/ubic/gemma/rest/util/BaseJerseyTest.java
+++ b/gemma-rest/src/test/java/ubic/gemma/rest/util/BaseJerseyTest.java
@@ -41,7 +41,6 @@ protected final TestContainerFactory getTestContainerFactory() throws TestContai
 
     @Override
     protected final Application configure() {
-        SecurityContextHolder.setStrategyName( SecurityContextHolder.MODE_INHERITABLETHREADLOCAL );
         application = new ResourceConfig()
                 .packages( "io.swagger.v3.jaxrs2.integration.resources", "ubic.gemma.rest" )
                 .registerClasses( GZipEncoder.class )
diff --git a/gemma-web/src/main/java/ubic/gemma/web/listener/StartupListener.java b/gemma-web/src/main/java/ubic/gemma/web/listener/StartupListener.java
index 3711983d58..a47e17b54d 100644
--- a/gemma-web/src/main/java/ubic/gemma/web/listener/StartupListener.java
+++ b/gemma-web/src/main/java/ubic/gemma/web/listener/StartupListener.java
@@ -89,8 +89,6 @@ public void contextInitialized( ServletContextEvent event ) {
         StopWatch sw = new StopWatch();
         sw.start();
 
-        SecurityContextHolder.setStrategyName( SecurityContextHolder.MODE_INHERITABLETHREADLOCAL );
-
         // call Spring's context ContextLoaderListener to initialize
         // all the context files specified in web.xml
         super.contextInitialized( event );
diff --git a/pom.xml b/pom.xml
index 9d7baa68c6..75905df968 100644
--- a/pom.xml
+++ b/pom.xml
@@ -493,7 +493,7 @@
 				maven-surefire-plugin
 				2.22.2
 				
-					-Dlog4j1.compatibility=true -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager
+					${jvmOptions}
 					${redirectTestOutputToFile}
 					
 						**/*Test.java
@@ -508,7 +508,7 @@
 				maven-failsafe-plugin
 				2.22.2
 				
-					-Dlog4j1.compatibility=true -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager
+					${jvmOptions}
 					${redirectTestOutputToFile}
 					
 						**/*Test.java
@@ -723,5 +723,7 @@
 		
 		${skipTests}
 		true
+		
+		-Dlog4j1.compatibility=true -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Djava.awt.headless=true -Dspring.security.strategy=MODE_INHERITABLETHREADLOCAL
 	
 

From 403bbea468b4af6c61456c88e31956be6fe84a9e Mon Sep 17 00:00:00 2001
From: Guillaume Poirier-Morency 
Date: Fri, 10 May 2024 11:28:27 -0700
Subject: [PATCH 34/61] Eliminate SortArgTaxon, it was caused by a bug in
 getTaxonDatasets()

---
 gemma-rest/src/test/java/ubic/gemma/rest/OpenApiTest.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gemma-rest/src/test/java/ubic/gemma/rest/OpenApiTest.java b/gemma-rest/src/test/java/ubic/gemma/rest/OpenApiTest.java
index 4dc9a7ff66..d1d88ba74d 100644
--- a/gemma-rest/src/test/java/ubic/gemma/rest/OpenApiTest.java
+++ b/gemma-rest/src/test/java/ubic/gemma/rest/OpenApiTest.java
@@ -209,7 +209,7 @@ public void testSortArgSchemas() {
         assertThat( spec.getComponents().getSchemas() )
                 // FIXME: remove the dangling 'Sort'
                 // .doesNotContainKey( "Sort" )
-                .containsKeys( "SortArgExpressionExperiment", "SortArgArrayDesign", "SortArgExpressionAnalysisResultSet", "SortArgTaxon" );
+                .containsKeys( "SortArgExpressionExperiment", "SortArgArrayDesign", "SortArgExpressionAnalysisResultSet" );
         Schema schema = spec.getComponents().getSchemas().get( "SortArgExpressionExperiment" );
         assertThat( schema.getType() )
                 .isEqualTo( "string" );

From 073433a33fb34fbb5bd734d1dab361b8cc8a3e0c Mon Sep 17 00:00:00 2001
From: Guillaume Poirier-Morency 
Date: Fri, 10 May 2024 11:37:49 -0700
Subject: [PATCH 35/61] Borrow all annotations from
 AbstractJUnit4SpringContextTests in BaseJerseyTest

---
 .../java/ubic/gemma/rest/util/BaseJerseyTest.java    | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/gemma-rest/src/test/java/ubic/gemma/rest/util/BaseJerseyTest.java b/gemma-rest/src/test/java/ubic/gemma/rest/util/BaseJerseyTest.java
index eca25914e8..9edc606340 100644
--- a/gemma-rest/src/test/java/ubic/gemma/rest/util/BaseJerseyTest.java
+++ b/gemma-rest/src/test/java/ubic/gemma/rest/util/BaseJerseyTest.java
@@ -12,9 +12,12 @@
 import org.junit.runner.RunWith;
 import org.springframework.context.ApplicationContext;
 import org.springframework.context.ApplicationContextAware;
-import org.springframework.security.core.context.SecurityContextHolder;
 import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.TestExecutionListeners;
 import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+import org.springframework.test.context.support.DependencyInjectionTestExecutionListener;
+import org.springframework.test.context.support.DirtiesContextTestExecutionListener;
+import org.springframework.test.context.web.ServletTestExecutionListener;
 import org.springframework.test.context.web.WebAppConfiguration;
 import org.springframework.web.context.WebApplicationContext;
 import org.springframework.web.context.support.GenericWebApplicationContext;
@@ -25,11 +28,16 @@
 /**
  * Base class for Jersey-based tests that needs a {@link WebApplicationContext} for loading and configuring or mocking
  * Spring components.
+ * 

+ * Unfortunately, it is not possible to inherit from {@link org.springframework.test.context.junit4.AbstractJUnit4SpringContextTests},
+ * so we have to borrow some its annotations here.
  * @author poirigui
  */
 @ActiveProfiles({ "web", EnvironmentProfiles.TEST })
-@RunWith(SpringJUnit4ClassRunner.class)
 @WebAppConfiguration
+@RunWith(SpringJUnit4ClassRunner.class)
+@TestExecutionListeners({ ServletTestExecutionListener.class, DependencyInjectionTestExecutionListener.class,
+        DirtiesContextTestExecutionListener.class })
 public abstract class BaseJerseyTest extends JerseyTest implements ApplicationContextAware {
 
     private ResourceConfig application;

From cd01b54419532e602cf6b0390096f933d7fe408f Mon Sep 17 00:00:00 2001
From: Guillaume Poirier-Morency 
Date: Fri, 10 May 2024 11:45:06 -0700
Subject: [PATCH 36/61] Add missing spring.security.strategy in Gemma CLI
 IntelliJ configuration

---
 .idea/runConfigurations/Gemma_CLI.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.idea/runConfigurations/Gemma_CLI.xml b/.idea/runConfigurations/Gemma_CLI.xml
index 08f48e2a11..b482d4f1aa 100644
--- a/.idea/runConfigurations/Gemma_CLI.xml
+++ b/.idea/runConfigurations/Gemma_CLI.xml
@@ -2,7 +2,7 @@
   
      ("
-                  + Ext.util.Format.ellipsis( eevo.name, 35 ) + ")";
+                  + Ext.util.Format.ellipsis( eevo.name, 65 ) + ")";
 
                if ( this.vizWindow && this.vizWindow.originalTitle ) {
-                  this.vizWindow.setTitle( this.vizWindow.originalTitle + " In: " + eeInfoTitle );
+                  this.vizWindow.setTitle( this.vizWindow.originalTitle + " in: " + eeInfoTitle );
                } else {
                   this.setTitle( eeInfoTitle );
                }

From 8e5fbee9caf313730ef964c9a74fcbedf613b106 Mon Sep 17 00:00:00 2001
From: Guillaume Poirier-Morency 
Date: Fri, 10 May 2024 14:05:20 -0700
Subject: [PATCH 40/61] Add missed Jena warn log exclusions

---
 gemma-core/src/test/resources/log4j-test.properties | 3 ++-
 gemma-web/src/main/config/log4j-dev.properties      | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/gemma-core/src/test/resources/log4j-test.properties b/gemma-core/src/test/resources/log4j-test.properties
index 855937ec79..11d74cf0fc 100644
--- a/gemma-core/src/test/resources/log4j-test.properties
+++ b/gemma-core/src/test/resources/log4j-test.properties
@@ -12,4 +12,5 @@ log4j.appender.progressUpdate=ubic.gemma.core.job.executor.common.ProgressUpdate
 log4j.rootLogger=WARN,stderr
 log4j.logger.ubic.gemma=INFO,progressUpdate
 log4j.logger.net.sf.ehcache.hibernate.strategy.AbstractReadWriteEhcacheAccessStrategy=ERROR
-log4j.logger.org.hibernate.cache.ehcache.internal.strategy.AbstractReadWriteEhcacheAccessStrategy=ERROR
\ No newline at end of file
+log4j.logger.org.hibernate.cache.ehcache.internal.strategy.AbstractReadWriteEhcacheAccessStrategy=ERROR
+log4j.logger.com.hp.hpl.jena.rdf.model.impl.RDFDefaultErrorHandler=ERROR
\ No newline at end of file
diff --git a/gemma-web/src/main/config/log4j-dev.properties b/gemma-web/src/main/config/log4j-dev.properties
index f8bd5b1506..ea8b384d61 100644
--- a/gemma-web/src/main/config/log4j-dev.properties
+++ b/gemma-web/src/main/config/log4j-dev.properties
@@ -107,4 +107,4 @@ log4j.logger.org.hibernate.search.impl.SimpleIndexingProgressMonitor=INFO,progre
 log4j.additivity.org.hibernate.search.impl.SimpleIndexingProgressMonitor=false
 
 # Jena
-lo4j.logger.com.hp.hpl.jena.rdf.model.impl.RDFDefaultErrorHandler=ERROR
\ No newline at end of file
+log4j.logger.com.hp.hpl.jena.rdf.model.impl.RDFDefaultErrorHandler=ERROR
\ No newline at end of file

From 24fc678858cb1b7099e2a0bb23d9c0c790ed4f41 Mon Sep 17 00:00:00 2001
From: Guillaume Poirier-Morency 
Date: Mon, 13 May 2024 10:27:31 -0700
Subject: [PATCH 41/61] Update baseCode to 1.1.23-SNAPSHOT

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 75905df968..2767275438 100644
--- a/pom.xml
+++ b/pom.xml
@@ -140,7 +140,7 @@
 		
 			baseCode
 			baseCode
-			1.1.22
+			1.1.23-SNAPSHOT
 		
 
 		

From 27281b24b07ddb790035917dc527aad019c4a80e Mon Sep 17 00:00:00 2001
From: Guillaume Poirier-Morency 
Date: Mon, 13 May 2024 12:53:02 -0700
Subject: [PATCH 42/61] Fix typo in rewriteBatchedStatements

---
 .../main/resources/ubic/gemma/applicationContext-dataSource.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gemma-core/src/main/resources/ubic/gemma/applicationContext-dataSource.xml b/gemma-core/src/main/resources/ubic/gemma/applicationContext-dataSource.xml
index 5c50319099..25504aef63 100644
--- a/gemma-core/src/main/resources/ubic/gemma/applicationContext-dataSource.xml
+++ b/gemma-core/src/main/resources/ubic/gemma/applicationContext-dataSource.xml
@@ -5,7 +5,7 @@
    http://www.springframework.org/schema/beans/spring-beans-3.2.xsd http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
 
     
-        true
+        true
         
         America/Vancouver
         

From 4b68326b2005d2eefc4a49ff8c0bfd521d91e745 Mon Sep 17 00:00:00 2001
From: Guillaume Poirier-Morency 
Date: Mon, 13 May 2024 13:10:34 -0700
Subject: [PATCH 43/61] Only retrieve the best BAD once when converting matrix
 to vectors

---
 .../matrix/ExpressionDataDoubleMatrix.java       | 16 ++++------------
 1 file changed, 4 insertions(+), 12 deletions(-)

diff --git a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataDoubleMatrix.java b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataDoubleMatrix.java
index 4348066e17..ccb3adc962 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataDoubleMatrix.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataDoubleMatrix.java
@@ -36,7 +36,6 @@
 import ubic.gemma.model.expression.designElement.CompositeSequence;
 import ubic.gemma.model.expression.experiment.ExpressionExperiment;
 
-import javax.annotation.Nullable;
 import java.text.NumberFormat;
 import java.util.*;
 
@@ -403,27 +402,22 @@ public void set( int row, int column, Double value ) {
     public Collection toProcessedDataVectors() {
         Collection result = new HashSet<>();
         QuantitationType qt = this.getQuantitationTypes().iterator().next();
-
         ByteArrayConverter bac = new ByteArrayConverter();
         if ( this.getQuantitationTypes().size() > 1 ) {
             throw new UnsupportedOperationException( "Cannot convert matrix that has more than one quantitation type" );
         }
-
+        BioAssayDimension bad = this.getBestBioAssayDimension();
         for ( int i = 0; i < this.rows(); i++ ) {
-
             Double[] data = this.getRow( i );
-
             ProcessedExpressionDataVector v = ProcessedExpressionDataVector.Factory.newInstance();
-            v.setBioAssayDimension( this.getBestBioAssayDimension() );
+            v.setBioAssayDimension( bad );
             v.setDesignElement( this.getRowNames().get( i ) );
             v.setQuantitationType( qt );
             v.setData( bac.doubleArrayToBytes( data ) );
             v.setExpressionExperiment( this.expressionExperiment );
             // we don't fill in the ranks because we only have the mean value here.
-
             result.add( v );
         }
-
         return result;
     }
 
@@ -441,18 +435,16 @@ public Collection toRawDataVectors() {
             throw new UnsupportedOperationException( "Cannot convert matrix that has more than one quantitation type" );
         }
 
+        BioAssayDimension bad = this.getBestBioAssayDimension();
         for ( int i = 0; i < this.rows(); i++ ) {
-
             Double[] data = this.getRow( i );
-
             RawExpressionDataVector v = RawExpressionDataVector.Factory.newInstance();
-            v.setBioAssayDimension( this.getBestBioAssayDimension() );
+            v.setBioAssayDimension( bad );
             v.setDesignElement( this.getRowNames().get( i ) );
             v.setQuantitationType( qt );
             v.setData( bac.doubleArrayToBytes( data ) );
             v.setExpressionExperiment( this.expressionExperiment );
             // we don't fill in the ranks because we only have the mean value here.
-
             result.add( v );
         }
 

From 8f453954a902bcd13648416f4744430279e63154 Mon Sep 17 00:00:00 2001
From: Guillaume Poirier-Morency 
Date: Mon, 13 May 2024 14:56:18 -0700
Subject: [PATCH 44/61] rest: Include inferred terms in DatasetsWebService
 endpoints

Revamp payloads to reuse as much fields as possible.
---
 .../ExpressionExperimentService.java          |   6 +-
 .../ExpressionExperimentServiceImpl.java      |   9 +-
 .../ExpressionExperimentServiceTest.java      |   4 +-
 .../rest/AnalysisResultSetsWebService.java    |  11 +-
 .../gemma/rest/AnnotationsWebService.java     |  14 +-
 .../ubic/gemma/rest/DatasetsWebService.java   | 180 +++++++++++++-----
 .../java/ubic/gemma/rest/GeneWebService.java  |  25 +--
 .../ubic/gemma/rest/PhenotypeWebService.java  |  11 +-
 .../ubic/gemma/rest/PlatformsWebService.java  |  19 +-
 .../java/ubic/gemma/rest/RootWebService.java  |  10 +-
 .../java/ubic/gemma/rest/TaxaWebService.java  |  30 +--
 .../FilteredAndLimitedResponseDataObject.java |  19 ++
 ...ilteredAndPaginatedResponseDataObject.java |  23 +--
 .../rest/util/FilteredResponseDataObject.java |  22 ++-
 .../rest/util/LimitedResponseDataObject.java  |  33 ----
 .../util/PaginatedResponseDataObject.java     |  20 +-
 ...dFilteredAndLimitedResponseDataObject.java |  24 +++
 ...ilteredAndPaginatedResponseDataObject.java |  25 +--
 .../QueriedAndFilteredResponseDataObject.java |  21 +-
 .../util/{Responder.java => Responders.java}  |  53 +++---
 .../gemma/rest/util/ResponseDataObject.java   |   7 +-
 .../rest/util/args/DatasetArgService.java     |   6 +-
 .../gemma/rest/AnnotationsWebServiceTest.java |   2 +-
 .../ubic/gemma/rest/DatasetsRestTest.java     |  17 +-
 .../gemma/rest/DatasetsWebServiceTest.java    |  12 +-
 .../java/ubic/gemma/rest/OpenApiTest.java     |   2 +-
 26 files changed, 344 insertions(+), 261 deletions(-)
 create mode 100644 gemma-rest/src/main/java/ubic/gemma/rest/util/FilteredAndLimitedResponseDataObject.java
 delete mode 100644 gemma-rest/src/main/java/ubic/gemma/rest/util/LimitedResponseDataObject.java
 create mode 100644 gemma-rest/src/main/java/ubic/gemma/rest/util/QueriedAndFilteredAndLimitedResponseDataObject.java
 rename gemma-rest/src/main/java/ubic/gemma/rest/util/{Responder.java => Responders.java} (57%)

diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentService.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentService.java
index 2590d187fc..e10bc20592 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentService.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentService.java
@@ -262,10 +262,12 @@ ExpressionExperiment addRawVectors( ExpressionExperiment eeToUpdate,
 
     /**
      * Apply ontological inference to augment a filter with additional terms.
+     *
      * @param mentionedTerms if non-null, all the terms explicitly mentioned in the filters are added to the collection.
-     *                       The returned filter might contain terms that have been inferred.
+     * @param inferredTerms  if non-null, all the terms inferred from those mentioned in the filters are added to the
+     *                       collection
      */
-    Filters getFiltersWithInferredAnnotations( Filters f, @Nullable Collection mentionedTerms, long timeout, TimeUnit timeUnit ) throws TimeoutException;
+    Filters getFiltersWithInferredAnnotations( Filters f, @Nullable Collection mentionedTerms, @Nullable Collection inferredTerms, long timeout, TimeUnit timeUnit ) throws TimeoutException;
 
     @Value
     class CharacteristicWithUsageStatisticsAndOntologyTerm {
diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentServiceImpl.java
index 0f844dccec..671253bf42 100755
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentServiceImpl.java
@@ -608,7 +608,7 @@ public Set getAnnotationsById( Long eeId ) {
      * For example, {@code characteristics.termUri = a or characteristics.termUri = b} will be transformed into {@code characteristics.termUri in (a, b, children of a and b...)}.
      */
     @Override
-    public Filters getFiltersWithInferredAnnotations( Filters f, @Nullable Collection mentionedTerms, long timeout, TimeUnit timeUnit ) throws TimeoutException {
+    public Filters getFiltersWithInferredAnnotations( Filters f, @Nullable Collection mentionedTerms, @Nullable Collection inferredTerms, long timeout, TimeUnit timeUnit ) throws TimeoutException {
         StopWatch timer = StopWatch.createStarted();
         Filters f2 = Filters.empty();
         // apply inference to terms
@@ -641,9 +641,14 @@ public Filters getFiltersWithInferredAnnotations( Filters f, @Nullable Collectio
                 if ( mentionedTerms != null ) {
                     mentionedTerms.addAll( terms );
                 }
+                Set c = ontologyService.getChildren( terms, false, true, Math.max( timeUnit.toMillis( timeout ) - timer.getTime(), 0 ), TimeUnit.MILLISECONDS );
+                if ( inferredTerms != null ) {
+                    inferredTerms.addAll( terms );
+                    inferredTerms.addAll( c );
+                }
                 Set termAndChildrenUris = new TreeSet<>( String.CASE_INSENSITIVE_ORDER );
                 termAndChildrenUris.addAll( e.getValue() );
-                termAndChildrenUris.addAll( ontologyService.getChildren( terms, false, true, Math.max( timeUnit.toMillis( timeout ) - timer.getTime(), 0 ), TimeUnit.MILLISECONDS ).stream()
+                termAndChildrenUris.addAll( c.stream()
                         .map( OntologyTerm::getUri )
                         .collect( Collectors.toList() ) );
                 for ( List termAndChildrenUrisBatch : org.apache.commons.collections4.ListUtils.partition( new ArrayList<>( termAndChildrenUris ), QueryUtils.MAX_PARAMETER_LIST_SIZE ) ) {
diff --git a/gemma-core/src/test/java/ubic/gemma/model/expression/experiment/ExpressionExperimentServiceTest.java b/gemma-core/src/test/java/ubic/gemma/model/expression/experiment/ExpressionExperimentServiceTest.java
index 2590264573..060e5eaefb 100644
--- a/gemma-core/src/test/java/ubic/gemma/model/expression/experiment/ExpressionExperimentServiceTest.java
+++ b/gemma-core/src/test/java/ubic/gemma/model/expression/experiment/ExpressionExperimentServiceTest.java
@@ -168,7 +168,7 @@ public void testGetFiltersWithInferredAnnotations() throws TimeoutException {
         OntologyTerm term = mock( OntologyTerm.class );
         when( ontologyService.getTerms( Collections.singleton( "http://example.com/T00001" ) ) ).thenReturn( Collections.singleton( term ) );
         Filters f = Filters.by( "c", "valueUri", String.class, Filter.Operator.eq, "http://example.com/T00001", "characteristics.valueUri" );
-        Filters inferredFilters = expressionExperimentService.getFiltersWithInferredAnnotations( f, null, 30, TimeUnit.SECONDS );
+        Filters inferredFilters = expressionExperimentService.getFiltersWithInferredAnnotations( f, null, null, 30, TimeUnit.SECONDS );
         verify( ontologyService ).getTerms( Collections.singleton( "http://example.com/T00001" ) );
         verify( ontologyService ).getChildren( eq( Collections.singleton( term ) ), eq( false ), eq( true ), longThat( l -> l <= 30000L ), eq( TimeUnit.MILLISECONDS ) );
     }
@@ -178,7 +178,7 @@ public void testGetFiltersWithCategories() throws TimeoutException {
         OntologyTerm term = mock( OntologyTerm.class );
         when( ontologyService.getTerms( Collections.singleton( "http://example.com/T00001" ) ) ).thenReturn( Collections.singleton( term ) );
         Filters f = Filters.by( "c", "categoryUri", String.class, Filter.Operator.eq, "http://example.com/T00001", "characteristics.categoryUri" );
-        expressionExperimentService.getFiltersWithInferredAnnotations( f, null, 30, TimeUnit.SECONDS );
+        expressionExperimentService.getFiltersWithInferredAnnotations( f, null, null, 30, TimeUnit.SECONDS );
         verifyNoInteractions( ontologyService );
     }
 
diff --git a/gemma-rest/src/main/java/ubic/gemma/rest/AnalysisResultSetsWebService.java b/gemma-rest/src/main/java/ubic/gemma/rest/AnalysisResultSetsWebService.java
index bf534d6d49..e96a27d711 100644
--- a/gemma-rest/src/main/java/ubic/gemma/rest/AnalysisResultSetsWebService.java
+++ b/gemma-rest/src/main/java/ubic/gemma/rest/AnalysisResultSetsWebService.java
@@ -51,6 +51,9 @@
 import java.util.List;
 import java.util.Map;
 
+import static ubic.gemma.rest.util.Responders.paginate;
+import static ubic.gemma.rest.util.Responders.respond;
+
 /**
  * Endpoint for {@link ubic.gemma.model.analysis.AnalysisResultSet}
  */
@@ -107,7 +110,7 @@ public FilteredAndPaginatedResponseDataObject getNumberOfResultSets(
             @QueryParam("filter") @DefaultValue("") FilterArg filter ) {
-        return Responder.respond( expressionAnalysisResultSetService.count( expressionAnalysisResultSetArgService.getFilters( filter ) ) );
+        return respond( expressionAnalysisResultSetService.count( expressionAnalysisResultSetArgService.getFilters( filter ) ) );
     }
 
     private static final String TSV_EXAMPLE = "# If you use this file for your research, please cite:\n" +
@@ -152,13 +155,13 @@ public ResponseDataObject ge
             @Parameter(hidden = true) @QueryParam("excludeResults") @DefaultValue("false") Boolean excludeResults ) {
         if ( excludeResults ) {
             ExpressionAnalysisResultSet ears = expressionAnalysisResultSetArgService.getEntity( analysisResultSet );
-            return Responder.respond( expressionAnalysisResultSetService.loadValueObject( ears ) );
+            return respond( expressionAnalysisResultSetService.loadValueObject( ears ) );
         } else {
             ExpressionAnalysisResultSet ears = analysisResultSet.getEntityWithContrastsAndResults( expressionAnalysisResultSetService );
             if ( ears == null ) {
                 throw new NotFoundException( "Could not find ExpressionAnalysisResultSet for " + analysisResultSet + "." );
             }
-            return Responder.respond( expressionAnalysisResultSetService.loadValueObjectWithResults( ears ) );
+            return respond( expressionAnalysisResultSetService.loadValueObjectWithResults( ears ) );
         }
     }
 
diff --git a/gemma-rest/src/main/java/ubic/gemma/rest/AnnotationsWebService.java b/gemma-rest/src/main/java/ubic/gemma/rest/AnnotationsWebService.java
index 1a1471575b..c64ae8da4b 100644
--- a/gemma-rest/src/main/java/ubic/gemma/rest/AnnotationsWebService.java
+++ b/gemma-rest/src/main/java/ubic/gemma/rest/AnnotationsWebService.java
@@ -46,7 +46,6 @@
 import ubic.gemma.persistence.util.Slice;
 import ubic.gemma.persistence.util.Sort;
 import ubic.gemma.rest.util.QueriedAndFilteredAndPaginatedResponseDataObject;
-import ubic.gemma.rest.util.Responder;
 import ubic.gemma.rest.util.ResponseDataObject;
 import ubic.gemma.rest.util.ResponseErrorObject;
 import ubic.gemma.rest.util.args.*;
@@ -58,6 +57,9 @@
 import java.util.*;
 import java.util.concurrent.TimeUnit;
 
+import static ubic.gemma.rest.util.Responders.paginate;
+import static ubic.gemma.rest.util.Responders.respond;
+
 /**
  * RESTful interface for annotations.
  *
@@ -128,7 +130,7 @@ public ResponseDataObject> searchAnnotat
             throw new BadRequestException( "Search query cannot be empty." );
         }
         try {
-            return Responder.respond( new ArrayList<>( this.getTerms( query, FIND_CHARACTERISTICS_TIMEOUT_MS ) ) );
+            return respond( new ArrayList<>( this.getTerms( query, FIND_CHARACTERISTICS_TIMEOUT_MS ) ) );
         } catch ( SearchTimeoutException e ) {
             throw new ServiceUnavailableException( e.getMessage(), DateUtils.addSeconds( new Date(), 30 ), e.getCause() );
         } catch ( ParseSearchException e ) {
@@ -219,11 +221,7 @@ public QueriedAndFilteredAndPaginatedResponseDataObject highlightDocument( Document document, org.apache.luce
             @ApiResponse(useReturnTypeSchema = true, content = @Content()),
             @ApiResponse(responseCode = "503", description = SEARCH_TIMEOUT_DESCRIPTION, content = @Content(schema = @Schema(implementation = ResponseErrorObject.class)))
     })
-    public QueriedAndFilteredAndPaginatedResponseDataObject getDatasets( // Params:
+    public QueriedAndFilteredAndInferredAndPaginatedResponseDataObject getDatasets( // Params:
             @QueryParam("query") QueryArg query,
             @QueryParam("filter") @DefaultValue("") FilterArg filterArg, // Optional, default null
             @QueryParam("offset") @DefaultValue("0") OffsetArg offsetArg, // Optional, default 0
             @QueryParam("limit") @DefaultValue("20") LimitArg limitArg, // Optional, default 20
             @QueryParam("sort") @DefaultValue("+id") SortArg sortArg // Optional, default +id
     ) {
-        Filters filters = datasetArgService.getFilters( filterArg );
+        Collection inferredTerms = new HashSet<>();
+        Filters filters = datasetArgService.getFilters( filterArg, null, inferredTerms );
         Sort sort = datasetArgService.getSort( sortArg );
         int offset = offsetArg.getValue();
         int limit = limitArg.getValue();
+        Slice payload;
         if ( query != null ) {
             List ids = new ArrayList<>( expressionExperimentService.loadIdsWithCache( filters, sort ) );
             Map scoreById = new HashMap<>();
@@ -214,15 +220,13 @@ public QueriedAndFilteredAndPaginatedResponseDataObject> resultById = results.stream().collect( Collectors.toMap( SearchResult::getResultId, e -> e ) );
 
             List vos = expressionExperimentService.loadValueObjectsByIdsWithRelationsAndCache( idsSlice );
-            return Responder.queryAndPaginate(
-                    new Slice<>( vos, Sort.by( null, "searchResult.score", Sort.Direction.DESC ), offset, limit, ( long ) ids.size() )
-                            .map( vo -> new ExpressionExperimentWithSearchResultValueObject( vo, resultById.get( vo.getId() ) ) ),
-                    query.getValue(), filters, new String[] { "id" } );
+            payload = new Slice<>( vos, Sort.by( null, "searchResult.score", Sort.Direction.DESC ), offset, limit, ( long ) ids.size() )
+                    .map( vo -> new ExpressionExperimentWithSearchResultValueObject( vo, resultById.get( vo.getId() ) ) );
         } else {
-            return Responder.queryAndPaginate(
-                    expressionExperimentService.loadValueObjectsWithCache( filters, sort, offset, limit ).map( vo -> new ExpressionExperimentWithSearchResultValueObject( vo, null ) ),
-                    null, filters, new String[] { "id" } );
+            payload = expressionExperimentService.loadValueObjectsWithCache( filters, sort, offset, limit )
+                    .map( vo -> new ExpressionExperimentWithSearchResultValueObject( vo, null ) );
         }
+        return paginate( payload, query != null ? query.getValue() : null, filters, new String[] { "id" }, inferredTerms );
     }
 
     @Value
@@ -261,7 +265,7 @@ public ResponseDataObject getNumberOfDatasets(
         } else {
             extraIds = null;
         }
-        return Responder.respond( expressionExperimentService.countWithCache( filters, extraIds ) );
+        return respond( expressionExperimentService.countWithCache( filters, extraIds ) );
     }
 
     public interface UsageStatistics {
@@ -279,12 +283,13 @@ public interface UsageStatistics {
             @ApiResponse(useReturnTypeSchema = true, content = @Content()),
             @ApiResponse(responseCode = "503", description = SEARCH_TIMEOUT_DESCRIPTION, content = @Content(schema = @Schema(implementation = ResponseErrorObject.class)))
     })
-    public LimitedResponseDataObject getDatasetsPlatformsUsageStatistics(
+    public QueriedAndFilteredAndInferredAndLimitedResponseDataObject getDatasetsPlatformsUsageStatistics(
             @QueryParam("query") QueryArg query,
             @QueryParam("filter") @DefaultValue("") FilterArg filter,
             @QueryParam("limit") @DefaultValue("50") LimitArg limit
     ) {
-        Filters filters = datasetArgService.getFilters( filter );
+        Collection inferredTerms = new HashSet<>();
+        Filters filters = datasetArgService.getFilters( filter, null, inferredTerms );
         Set extraIds;
         if ( query != null ) {
             extraIds = datasetArgService.getIdsForSearchQuery( query );
@@ -301,7 +306,7 @@ public LimitedResponseDataObject getD
                         .map( e -> new ArrayDesignWithUsageStatisticsValueObject( e, countsById.get( e.getId() ), tts.getOrDefault( TechnologyType.valueOf( e.getTechnologyType() ), 0L ) ) )
                         .sorted( Comparator.comparing( UsageStatistics::getNumberOfExpressionExperiments, Comparator.reverseOrder() ) )
                         .collect( Collectors.toList() );
-        return Responder.limit( results, query != null ? query.getValue() : null, filters, new String[] { "id" }, Sort.by( null, "numberOfExpressionExperiments", Sort.Direction.DESC, "numberOfExpressionExperiments" ), l );
+        return top( results, query != null ? query.getValue() : null, filters, new String[] { "id" }, Sort.by( null, "numberOfExpressionExperiments", Sort.Direction.DESC, "numberOfExpressionExperiments" ), l, inferredTerms );
     }
 
     @Value
@@ -322,7 +327,7 @@ public static class CategoryWithUsageStatisticsValueObject implements UsageStati
                     @ApiResponse(useReturnTypeSchema = true, content = @Content()),
                     @ApiResponse(responseCode = "503", description = SEARCH_TIMEOUT_DESCRIPTION, content = @Content(schema = @Schema(implementation = ResponseErrorObject.class)))
             })
-    public QueriedAndFilteredResponseDataObject getDatasetsCategoriesUsageStatistics(
+    public QueriedAndFilteredAndInferredAndLimitedResponseDataObject getDatasetsCategoriesUsageStatistics(
             @QueryParam("query") QueryArg query,
             @QueryParam("filter") @DefaultValue("") FilterArg filter,
             @QueryParam("limit") @DefaultValue("20") LimitArg limit,
@@ -335,7 +340,8 @@ public QueriedAndFilteredResponseDataObject mentionedTerms = retainMentionedTerms ? new HashSet<>() : null;
-        Filters filters = datasetArgService.getFilters( filter, mentionedTerms );
+        Collection inferredTerms = new HashSet<>();
+        Filters filters = datasetArgService.getFilters( filter, mentionedTerms, inferredTerms );
         Set extraIds;
         if ( query != null ) {
             extraIds = datasetArgService.getIdsForSearchQuery( query );
@@ -355,7 +361,7 @@ public QueriedAndFilteredResponseDataObject new CategoryWithUsageStatisticsValueObject( e.getKey().getCategoryUri(), e.getKey().getCategory(), e.getValue() ) )
                 .sorted( Comparator.comparing( UsageStatistics::getNumberOfExpressionExperiments, Comparator.reverseOrder() ) )
                 .collect( Collectors.toList() );
-        return Responder.queryAndFilter( results, query != null ? query.getValue() : null, filters, new String[] { "classUri", "className" }, Sort.by( null, "numberOfExpressionExperiments", Sort.Direction.DESC, "numberOfExpressionExperiments" ) );
+        return top( results, query != null ? query.getValue() : null, filters, new String[] { "classUri", "className" }, Sort.by( null, "numberOfExpressionExperiments", Sort.Direction.DESC, "numberOfExpressionExperiments" ), maxResults, inferredTerms );
     }
 
     @Value
@@ -386,7 +392,7 @@ public ArrayDesignWithUsageStatisticsValueObject( ArrayDesignValueObject arrayDe
                     @ApiResponse(useReturnTypeSchema = true, content = @Content()),
                     @ApiResponse(responseCode = "503", description = SEARCH_TIMEOUT_DESCRIPTION, content = @Content(schema = @Schema(implementation = ResponseErrorObject.class)))
             })
-    public LimitedResponseDataObject getDatasetsAnnotationsUsageStatistics(
+    public QueriedAndFilteredAndInferredAndLimitedResponseDataObject getDatasetsAnnotationsUsageStatistics(
             @QueryParam("query") QueryArg query,
             @QueryParam("filter") @DefaultValue("") FilterArg filter,
             @Parameter(description = "List of fields to exclude from the payload. Only `parentTerms` can be excluded.") @QueryParam("exclude") ExcludeArg exclude,
@@ -407,7 +413,8 @@ public LimitedResponseDataObject getDa
         }
         // ensure that implied terms are retained in the usage frequency
         Collection mentionedTerms = retainMentionedTerms ? new HashSet<>() : null;
-        Filters filters = datasetArgService.getFilters( filter, mentionedTerms );
+        Collection inferredTerms = new HashSet<>();
+        Filters filters = datasetArgService.getFilters( filter, mentionedTerms, inferredTerms );
         Set extraIds;
         if ( query != null ) {
             extraIds = datasetArgService.getIdsForSearchQuery( query );
@@ -451,9 +458,14 @@ public LimitedResponseDataObject getDa
                 results.add( new AnnotationWithUsageStatisticsValueObject( e.getCharacteristic(), e.getNumberOfExpressionExperiments(), null ) );
             }
         }
-        return Responder.limit( results, query != null ? query.getValue() : null, filters, new String[] { "classUri", "className", "termUri", "termName" },
+        return top(
+                results,
+                query != null ? query.getValue() : null,
+                filters,
+                new String[] { "classUri", "className", "termUri", "termName" },
                 Sort.by( null, "numberOfExpressionExperiments", Sort.Direction.DESC, "numberOfExpressionExperiments" ),
-                limit );
+                limit,
+                inferredTerms );
     }
 
     private Set getExcludedFields( @Nullable ExcludeArg exclude ) {
@@ -544,22 +556,30 @@ public AnnotationWithUsageStatisticsValueObject( Characteristic c, Long numberOf
             @ApiResponse(useReturnTypeSchema = true, content = @Content()),
             @ApiResponse(responseCode = "503", description = SEARCH_TIMEOUT_DESCRIPTION, content = @Content(schema = @Schema(implementation = ResponseErrorObject.class)))
     })
-    public QueriedAndFilteredResponseDataObject getDatasetsTaxaUsageStatistics(
+    public QueriedAndFilteredAndInferredResponseDataObject getDatasetsTaxaUsageStatistics(
             @QueryParam("query") QueryArg query,
             @QueryParam("filter") @DefaultValue("") FilterArg filterArg
     ) {
-        Filters filters = datasetArgService.getFilters( filterArg );
+        Collection inferredTerms = new HashSet<>();
+        Filters filters = datasetArgService.getFilters( filterArg, null, inferredTerms );
         Set extraIds;
         if ( query != null ) {
             extraIds = datasetArgService.getIdsForSearchQuery( query );
         } else {
             extraIds = null;
         }
-        return Responder.queryAndFilter( expressionExperimentService.getTaxaUsageFrequency( filters, extraIds )
+        List payload = expressionExperimentService.getTaxaUsageFrequency( filters, extraIds )
                 .entrySet().stream()
                 .sorted( Map.Entry.comparingByValue( Comparator.reverseOrder() ) )
                 .map( e -> new TaxonWithUsageStatisticsValueObject( e.getKey(), e.getValue() ) )
-                .collect( Collectors.toList() ), query != null ? query.getValue() : null, filters, new String[] { "id" }, Sort.by( null, "numberOfExpressionExperiments", Sort.Direction.DESC, "numberOfExpressionExperiments" ) );
+                .collect( Collectors.toList() );
+        return all(
+                payload,
+                query != null ? query.getValue() : null,
+                filters,
+                new String[] { "id" },
+                Sort.by( null, "numberOfExpressionExperiments", Sort.Direction.DESC, "numberOfExpressionExperiments" ),
+                inferredTerms );
     }
 
     @Value
@@ -591,16 +611,17 @@ public TaxonWithUsageStatisticsValueObject( Taxon taxon, Long numberOfExpression
     @Path("/{dataset}")
     @Produces(MediaType.APPLICATION_JSON)
     @Operation(summary = "Retrieve datasets by their identifiers")
-    public FilteredAndPaginatedResponseDataObject getDatasetsByIds( // Params:
+    public FilteredAndInferredAndPaginatedResponseDataObject getDatasetsByIds( // Params:
             @PathParam("dataset") DatasetArrayArg datasetsArg, // Optional
             @QueryParam("filter") @DefaultValue("") FilterArg filter, // Optional, default null
             @QueryParam("offset") @DefaultValue("0") OffsetArg offset, // Optional, default 0
             @QueryParam("limit") @DefaultValue("20") LimitArg limit, // Optional, default 20
             @QueryParam("sort") @DefaultValue("+id") SortArg sort // Optional, default +id
     ) {
-        Filters filters = datasetArgService.getFilters( filter ).and( datasetArgService.getFilters( datasetsArg ) );
-        return Responder.paginate( expressionExperimentService::loadValueObjectsWithCache, filters, new String[] { "id" },
-                datasetArgService.getSort( sort ), offset.getValue(), limit.getValue() );
+        Collection inferredTerms = new HashSet<>();
+        Filters filters = datasetArgService.getFilters( filter, null, inferredTerms ).and( datasetArgService.getFilters( datasetsArg ) );
+        return paginate( expressionExperimentService::loadValueObjectsWithCache, filters, new String[] { "id" },
+                datasetArgService.getSort( sort ), offset.getValue(), limit.getValue(), inferredTerms );
     }
 
     /**
@@ -611,14 +632,15 @@ public FilteredAndPaginatedResponseDataObject g
     @Produces(MediaType.APPLICATION_JSON)
     @Secured("GROUP_ADMIN")
     @Operation(summary = "Retrieve all blacklisted datasets", hidden = true)
-    public FilteredAndPaginatedResponseDataObject getBlacklistedDatasets(
+    public FilteredAndInferredAndPaginatedResponseDataObject getBlacklistedDatasets(
             @QueryParam("filter") @DefaultValue("") FilterArg filterArg,
             @QueryParam("sort") @DefaultValue("+id") SortArg sortArg,
             @QueryParam("offset") @DefaultValue("0") OffsetArg offset,
             @QueryParam("limit") @DefaultValue("20") LimitArg limit ) {
-        return Responder.paginate( expressionExperimentService::loadBlacklistedValueObjects,
-                datasetArgService.getFilters( filterArg ), new String[] { "id" }, datasetArgService.getSort( sortArg ),
-                offset.getValue(), limit.getValue() );
+        Collection inferredTerms = new HashSet<>();
+        return paginate( expressionExperimentService::loadBlacklistedValueObjects,
+                datasetArgService.getFilters( filterArg, null, inferredTerms ), new String[] { "id" }, datasetArgService.getSort( sortArg ),
+                offset.getValue(), limit.getValue(), inferredTerms );
     }
 
     /**
@@ -637,7 +659,7 @@ public FilteredAndPaginatedResponseDataObject g
     public ResponseDataObject> getDatasetPlatforms( // Params:
             @PathParam("dataset") DatasetArg datasetArg // Required
     ) {
-        return Responder.respond( datasetArgService.getPlatforms( datasetArg ) );
+        return respond( datasetArgService.getPlatforms( datasetArg ) );
     }
 
     /**
@@ -656,7 +678,7 @@ public ResponseDataObject> getDatasetPlatforms( //
     public ResponseDataObject> getDatasetSamples( // Params:
             @PathParam("dataset") DatasetArg datasetArg // Required
     ) {
-        return Responder.respond( datasetArgService.getSamples( datasetArg ) );
+        return respond( datasetArgService.getSamples( datasetArg ) );
     }
 
     /**
@@ -677,7 +699,7 @@ public ResponseDataObject> getDa
             @QueryParam("offset") @DefaultValue("0") OffsetArg offset, // Optional, default 0
             @QueryParam("limit") @DefaultValue("20") LimitArg limit // Optional, default 20
     ) {
-        return Responder.respond(
+        return respond(
                 this.getDiffExVos( datasetArgService.getEntity( datasetArg ).getId(),
                         offset.getValue(), limit.getValue() )
         );
@@ -734,7 +756,7 @@ public Response getDatasetDifferentialExpressionAnalysesResultSets(
     public ResponseDataObject> getDatasetAnnotations( // Params:
             @PathParam("dataset") DatasetArg datasetArg // Required
     ) {
-        return Responder.respond( datasetArgService.getAnnotations( datasetArg ) );
+        return respond( datasetArgService.getAnnotations( datasetArg ) );
     }
 
     /**
@@ -745,7 +767,7 @@ public ResponseDataObject> getDatasetAnnotations( //
     @Produces(MediaType.APPLICATION_JSON)
     @Operation(summary = "Retrieve quantitation types of a dataset")
     public ResponseDataObject> getDatasetQuantitationTypes( @PathParam("dataset") DatasetArg datasetArg ) {
-        return Responder.respond( datasetArgService.getQuantitationTypes( datasetArg ) );
+        return respond( datasetArgService.getQuantitationTypes( datasetArg ) );
     }
 
     /**
@@ -897,7 +919,7 @@ public ResponseDataObject getDatasetHasBatchInformation( // Params:
             @PathParam("dataset") DatasetArg datasetArg // Required
     ) {
         ExpressionExperiment ee = datasetArgService.getEntity( datasetArg );
-        return Responder.respond( this.auditEventService.hasEvent( ee, BatchInformationFetchingEvent.class ) );
+        return respond( this.auditEventService.hasEvent( ee, BatchInformationFetchingEvent.class ) );
     }
 
     /**
@@ -917,7 +939,7 @@ public ResponseDataObject getDatasetSvd( // Params:
             @PathParam("dataset") DatasetArg datasetArg // Required
     ) {
         SVDValueObject svd = svdService.getSvd( datasetArgService.getEntity( datasetArg ).getId() );
-        return Responder.respond( svd == null ? null : new SimpleSVDValueObject( Arrays.asList( svd.getBioMaterialIds() ), svd.getVariances(), svd.getvMatrix().getRawMatrix() )
+        return respond( svd == null ? null : new SimpleSVDValueObject( Arrays.asList( svd.getBioMaterialIds() ), svd.getVariances(), svd.getvMatrix().getRawMatrix() )
         );
     }
 
@@ -965,7 +987,7 @@ public ResponseDataObject> getDatase
             @QueryParam("keepNonSpecific") @DefaultValue("false") Boolean keepNonSpecific, // Optional, default false
             @QueryParam("consolidate") ExpLevelConsolidationArg consolidate // Optional, default everything is returned
     ) {
-        return Responder.respond( processedExpressionDataVectorService
+        return respond( processedExpressionDataVectorService
                 .getExpressionLevels( datasetArgService.getEntities( datasets ),
                         geneArgService.getEntitiesWithTaxon( genes, taxonArgService.getEntity( taxonArg ) ),
                         keepNonSpecific,
@@ -983,7 +1005,7 @@ public ResponseDataObject> getDatase
             @QueryParam("keepNonSpecific") @DefaultValue("false") Boolean keepNonSpecific, // Optional, default false
             @QueryParam("consolidate") ExpLevelConsolidationArg consolidate // Optional, default everything is returned
     ) {
-        return Responder.respond( processedExpressionDataVectorService
+        return respond( processedExpressionDataVectorService
                 .getExpressionLevels( datasetArgService.getEntities( datasets ),
                         geneArgService.getEntities( genes ), keepNonSpecific,
                         consolidate == null ? null : consolidate.getValue() )
@@ -1024,7 +1046,7 @@ public ResponseDataObject> getDatase
             @QueryParam("keepNonSpecific") @DefaultValue("false") Boolean keepNonSpecific, // Optional, default false
             @QueryParam("consolidate") ExpLevelConsolidationArg consolidate // Optional, default everything is returned
     ) {
-        return Responder.respond( processedExpressionDataVectorService
+        return respond( processedExpressionDataVectorService
                 .getExpressionLevelsPca( datasetArgService.getEntities( datasets ), limit.getValueNoMaximum(),
                         component, keepNonSpecific,
                         consolidate == null ? null : consolidate.getValue() )
@@ -1070,7 +1092,7 @@ public ResponseDataObject> getDatase
         if ( diffExSet == null ) {
             throw new BadRequestException( "The 'diffExSet' query parameter must be supplied." );
         }
-        return Responder.respond( processedExpressionDataVectorService
+        return respond( processedExpressionDataVectorService
                 .getExpressionLevelsDiffEx( datasetArgService.getEntities( datasets ),
                         diffExSet, threshold, limit.getValueNoMaximum(), keepNonSpecific,
                         consolidate == null ? null : consolidate.getValue() )
@@ -1122,4 +1144,76 @@ private static class SimpleSVDValueObject {
         double[] variances;
         double[][] vMatrix;
     }
+
+    private  QueriedAndFilteredAndInferredResponseDataObject all( List results, String query, @Nullable Filters filters, String[] groupBy, @Nullable Sort by, Collection inferredTerms ) {
+        return new QueriedAndFilteredAndInferredResponseDataObject<>( results, query, filters, groupBy, by, inferredTerms );
+    }
+
+    private  QueriedAndFilteredAndInferredAndLimitedResponseDataObject top( List payload, @Nullable String query, @Nullable Filters filters, String[] groupBy, @Nullable Sort sort, @Nullable Integer limit, Collection inferredTerms ) {
+        return new QueriedAndFilteredAndInferredAndLimitedResponseDataObject<>( payload, query, filters, groupBy, sort, limit, inferredTerms );
+    }
+
+    private  FilteredAndInferredAndPaginatedResponseDataObject paginate( Slice payload, @Nullable Filters filters, String[] groupBy, Collection inferredTerms ) throws NotFoundException {
+        return new FilteredAndInferredAndPaginatedResponseDataObject<>( payload, filters, groupBy, inferredTerms );
+    }
+
+    private  QueriedAndFilteredAndInferredAndPaginatedResponseDataObject paginate( Slice payload, String query, Filters filters, String[] groupBy, Collection inferredTerms ) {
+        return new QueriedAndFilteredAndInferredAndPaginatedResponseDataObject<>( payload, query, filters, groupBy, inferredTerms );
+    }
+
+    private  FilteredAndInferredAndPaginatedResponseDataObject paginate( Responders.FilterMethod filterMethod, @Nullable Filters filters, String[] groupBy, @Nullable Sort sort, int offset, int limit, Collection inferredTerms ) throws NotFoundException {
+        return paginate( filterMethod.load( filters, sort, offset, limit ), filters, groupBy, inferredTerms );
+    }
+
+    @Getter
+    public static class QueriedAndFilteredAndInferredResponseDataObject extends QueriedAndFilteredResponseDataObject {
+
+        private final List inferredTerms;
+
+        public QueriedAndFilteredAndInferredResponseDataObject( List payload, @Nullable String query, @Nullable Filters filters, String[] groupBy, @Nullable Sort sort, Collection inferredTerms ) {
+            super( payload, query, filters, groupBy, sort );
+            this.inferredTerms = inferredTerms.stream()
+                    .map( t -> new CharacteristicValueObject( t.getLabel(), t.getUri() ) )
+                    .collect( Collectors.toList() );
+        }
+    }
+
+    @Getter
+    public static class QueriedAndFilteredAndInferredAndLimitedResponseDataObject extends QueriedAndFilteredAndLimitedResponseDataObject {
+
+        private final List inferredTerms;
+
+        public QueriedAndFilteredAndInferredAndLimitedResponseDataObject( List payload, @Nullable String query, @Nullable Filters filters, String[] groupBy, @Nullable Sort sort, @Nullable Integer limit, Collection inferredTerms ) {
+            super( payload, query, filters, groupBy, sort, limit );
+            this.inferredTerms = inferredTerms.stream()
+                    .map( t -> new CharacteristicValueObject( t.getLabel(), t.getUri() ) )
+                    .collect( Collectors.toList() );
+        }
+    }
+
+    @Getter
+    public static class FilteredAndInferredAndPaginatedResponseDataObject extends FilteredAndPaginatedResponseDataObject {
+
+        private final List inferredTerms;
+
+        public FilteredAndInferredAndPaginatedResponseDataObject( Slice payload, @Nullable Filters filters, @Nullable String[] groupBy, Collection inferredTerms ) {
+            super( payload, filters, groupBy );
+            this.inferredTerms = inferredTerms.stream()
+                    .map( t -> new CharacteristicValueObject( t.getLabel(), t.getUri() ) )
+                    .collect( Collectors.toList() );
+        }
+    }
+
+    @Getter
+    public static class QueriedAndFilteredAndInferredAndPaginatedResponseDataObject extends QueriedAndFilteredAndPaginatedResponseDataObject {
+
+        private final List inferredTerms;
+
+        public QueriedAndFilteredAndInferredAndPaginatedResponseDataObject( Slice payload, @Nullable String query, @Nullable Filters filters, String[] groupBy, Collection inferredTerms ) {
+            super( payload, query, filters, groupBy );
+            this.inferredTerms = inferredTerms.stream()
+                    .map( t -> new CharacteristicValueObject( t.getLabel(), t.getUri() ) )
+                    .collect( Collectors.toList() );
+        }
+    }
 }
diff --git a/gemma-rest/src/main/java/ubic/gemma/rest/GeneWebService.java b/gemma-rest/src/main/java/ubic/gemma/rest/GeneWebService.java
index 2da6ef01ce..99e10da874 100644
--- a/gemma-rest/src/main/java/ubic/gemma/rest/GeneWebService.java
+++ b/gemma-rest/src/main/java/ubic/gemma/rest/GeneWebService.java
@@ -32,7 +32,6 @@
 import ubic.gemma.model.genome.gene.phenotype.valueObject.GeneEvidenceValueObject;
 import ubic.gemma.persistence.util.Filters;
 import ubic.gemma.rest.util.PaginatedResponseDataObject;
-import ubic.gemma.rest.util.Responder;
 import ubic.gemma.rest.util.ResponseDataObject;
 import ubic.gemma.rest.util.args.*;
 
@@ -42,6 +41,9 @@
 import java.util.Date;
 import java.util.List;
 
+import static ubic.gemma.rest.util.Responders.paginate;
+import static ubic.gemma.rest.util.Responders.respond;
+
 /**
  * RESTful interface for genes.
  * Does not have an 'all' endpoint (no use-cases).
@@ -75,7 +77,7 @@ public PaginatedResponseDataObject getGenes(
             @QueryParam("offset") @DefaultValue("0") OffsetArg offsetArg,
             @QueryParam("limit") @DefaultValue("20") LimitArg limitArg
     ) {
-        return Responder.paginate( geneArgService.getGenes( offsetArg.getValue(), limitArg.getValue() ), new String[] { "id" } );
+        return paginate( geneArgService.getGenes( offsetArg.getValue(), limitArg.getValue() ), new String[] { "id" } );
     }
 
     /**
@@ -98,7 +100,7 @@ public ResponseDataObject> getGenes( // Params:
         SortArg sort = SortArg.valueOf( "+id" );
         Filters filters = Filters.empty();
         filters.and( geneArgService.getFilters( genes ) );
-        return Responder.respond( geneService.loadValueObjects( filters, geneArgService.getSort( sort ), 0, -1 ) );
+        return respond( geneService.loadValueObjects( filters, geneArgService.getSort( sort ), 0, -1 ) );
     }
 
     /**
@@ -116,7 +118,7 @@ public ResponseDataObject> getGeneEvidence( // Par
             @PathParam("gene") GeneArg geneArg // Required
     ) {
         try {
-            return Responder.respond( geneArgService.getGeneEvidence( geneArg, null ) );
+            return respond( geneArgService.getGeneEvidence( geneArg, null ) );
         } catch ( ParseSearchException e ) {
             throw new BadRequestException( "Invalid search query: " + e.getQuery() );
         } catch ( SearchTimeoutException e ) {
@@ -139,7 +141,7 @@ public ResponseDataObject> getGeneEvidence( // Par
     public ResponseDataObject> getGeneLocations( // Params:
             @PathParam("gene") GeneArg geneArg // Required
     ) {
-        return Responder.respond( geneArgService.getGeneLocation( geneArg ) );
+        return respond( geneArgService.getGeneLocation( geneArg ) );
     }
 
     /**
@@ -157,7 +159,7 @@ public PaginatedResponseDataObject getGeneProbes(
             @QueryParam("offset") @DefaultValue("0") OffsetArg offset, // Optional, default 0
             @QueryParam("limit") @DefaultValue("20") LimitArg limit // Optional, default 20
     ) {
-        return Responder.paginate( geneArgService.getGeneProbes( geneArg, offset.getValue(), limit.getValue() ), new String[] { "id" } );
+        return paginate( geneArgService.getGeneProbes( geneArg, offset.getValue(), limit.getValue() ), new String[] { "id" } );
     }
 
     /**
@@ -173,7 +175,7 @@ public PaginatedResponseDataObject getGeneProbes(
     public ResponseDataObject> getGeneGoTerms( // Params:
             @PathParam("gene") GeneArg geneArg // Required
     ) {
-        return Responder.respond( geneArgService.getGeneGoTerms( geneArg ) );
+        return respond( geneArgService.getGeneGoTerms( geneArg ) );
     }
 
     /**
@@ -194,10 +196,9 @@ public ResponseDataObject> getGeneGeneCoexpress
             @QueryParam("limit") @DefaultValue("100") LimitArg limit, // Optional, default 100
             @QueryParam("stringency") @DefaultValue("1") Integer stringency // Optional, default 1
     ) {
-        return Responder
-                .respond( geneCoexpressionSearchService.coexpressionSearchQuick( null, new ArrayList( 2 ) {{
-                    this.add( geneArgService.getEntity( geneArg ).getId() );
-                    this.add( geneArgService.getEntity( with ).getId() );
-                }}, 1, limit.getValueNoMaximum(), false ).getResults() );
+        return respond( geneCoexpressionSearchService.coexpressionSearchQuick( null, new ArrayList( 2 ) {{
+            this.add( geneArgService.getEntity( geneArg ).getId() );
+            this.add( geneArgService.getEntity( with ).getId() );
+        }}, 1, limit.getValueNoMaximum(), false ).getResults() );
     }
 }
diff --git a/gemma-rest/src/main/java/ubic/gemma/rest/PhenotypeWebService.java b/gemma-rest/src/main/java/ubic/gemma/rest/PhenotypeWebService.java
index 913c5c3712..709689b655 100644
--- a/gemma-rest/src/main/java/ubic/gemma/rest/PhenotypeWebService.java
+++ b/gemma-rest/src/main/java/ubic/gemma/rest/PhenotypeWebService.java
@@ -22,14 +22,17 @@
 import ubic.gemma.model.genome.gene.phenotype.valueObject.DumpsValueObject;
 import ubic.gemma.model.genome.gene.phenotype.valueObject.EvidenceValueObject;
 import ubic.gemma.persistence.service.association.phenotype.PhenotypeAssociationDaoImpl;
-import ubic.gemma.rest.util.Responder;
 import ubic.gemma.rest.util.ResponseDataObject;
-import ubic.gemma.rest.util.args.*;
+import ubic.gemma.rest.util.args.LimitArg;
+import ubic.gemma.rest.util.args.OffsetArg;
+import ubic.gemma.rest.util.args.TaxonArg;
 
 import javax.ws.rs.*;
 import javax.ws.rs.core.MediaType;
 import java.util.Set;
 
+import static ubic.gemma.rest.util.Responders.respond;
+
 /**
  * RESTful interface for phenotypes.
  * Does not have an 'all' endpoint (no use-cases). To list all phenotypes on a specific taxon,
@@ -78,7 +81,7 @@ public ResponseDataObject
Retrieve this in RDF/XML:
" );
-        s.append( String.format( "
curl -X Accept:application/rdf+xml %s/ont/TGFVO/%d
", hostUrl, factorValueId ) );
+        s.append( String.format( "
curl -H Accept:application/rdf+xml %s/ont/TGFVO/%d
", hostUrl, factorValueId ) );
         s.append( "" );
         return s.toString();
     }
@@ -152,7 +152,7 @@ public String getFactorValueAnnotation( @PathVariable("factorValueId") Long fact
         }
         s.append( "" );
         s.append( "
Retrieve this in RDF/XML:
" );
-        s.append( String.format( "
curl -X Accept:application/rdf+xml %s/ont/TGFVO/%d/%d
", hostUrl, factorValueId, annotationId ) );
+        s.append( String.format( "
curl -H Accept:application/rdf+xml %s/ont/TGFVO/%d/%d
", hostUrl, factorValueId, annotationId ) );
         s.append( "" );
         return s.toString();
     }
diff --git a/gemma-web/src/test/java/ubic/gemma/web/controller/OntologyControllerTest.java b/gemma-web/src/test/java/ubic/gemma/web/controller/OntologyControllerTest.java
index 220767d9cc..b6cee88a6b 100644
--- a/gemma-web/src/test/java/ubic/gemma/web/controller/OntologyControllerTest.java
+++ b/gemma-web/src/test/java/ubic/gemma/web/controller/OntologyControllerTest.java
@@ -118,7 +118,7 @@ public void testGetTgfvoAsHtml() throws Exception {
                 .andExpect( content().string( containsString( "instance of" ) ) )
                 .andExpect( content().string( containsString( "TGEMO_0000001" ) ) )
                 .andExpect( content().string( containsString( "bar" ) ) )
-                .andExpect( content().string( containsString( "curl -X Accept:application/rdf+xml https://gemma.msl.ubc.ca/ont/TGFVO/1" ) ) );
+                .andExpect( content().string( containsString( "curl -H Accept:application/rdf+xml https://gemma.msl.ubc.ca/ont/TGFVO/1" ) ) );
         verify( factorValueOntologyService ).getIndividual( "http://gemma.msl.ubc.ca/ont/TGFVO/1" );
         verify( factorValueOntologyService ).getFactorValueAnnotations( "http://gemma.msl.ubc.ca/ont/TGFVO/1" );
     }

From d74ff4808bf6270b8e41cb43a64354ed4cf23ce7 Mon Sep 17 00:00:00 2001
From: Guillaume Poirier-Morency 
Date: Tue, 14 May 2024 15:08:54 -0700
Subject: [PATCH 46/61] Omit secondBaselineGroup for non-interaction terms

---
 .../diff/DifferentialExpressionAnalysisResultSetValueObject.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/gemma-core/src/main/java/ubic/gemma/model/analysis/expression/diff/DifferentialExpressionAnalysisResultSetValueObject.java b/gemma-core/src/main/java/ubic/gemma/model/analysis/expression/diff/DifferentialExpressionAnalysisResultSetValueObject.java
index ad90bcb767..5390bc9f4e 100644
--- a/gemma-core/src/main/java/ubic/gemma/model/analysis/expression/diff/DifferentialExpressionAnalysisResultSetValueObject.java
+++ b/gemma-core/src/main/java/ubic/gemma/model/analysis/expression/diff/DifferentialExpressionAnalysisResultSetValueObject.java
@@ -20,6 +20,7 @@ public class DifferentialExpressionAnalysisResultSetValueObject extends Analysis
     private DifferentialExpressionAnalysisValueObject analysis;
     private Collection experimentalFactors;
     private FactorValueBasicValueObject baselineGroup;
+    @JsonInclude(JsonInclude.Include.NON_NULL)
     private FactorValueBasicValueObject secondBaselineGroup;
 
     /**

From f90ef6a14d48d65e82385ad164ef2793cdbb9408 Mon Sep 17 00:00:00 2001
From: Guillaume Poirier-Morency 
Date: Tue, 14 May 2024 15:20:19 -0700
Subject: [PATCH 47/61] Remove unnecessary extends for Assertions

---
 .../src/test/java/ubic/gemma/rest/DatasetsWebServiceTest.java   | 1 +
 gemma-rest/src/test/java/ubic/gemma/rest/OpenApiTest.java       | 1 +
 gemma-rest/src/test/java/ubic/gemma/rest/util/Assertions.java   | 2 +-
 3 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/gemma-rest/src/test/java/ubic/gemma/rest/DatasetsWebServiceTest.java b/gemma-rest/src/test/java/ubic/gemma/rest/DatasetsWebServiceTest.java
index 8f3d892f0a..6ab02f7937 100644
--- a/gemma-rest/src/test/java/ubic/gemma/rest/DatasetsWebServiceTest.java
+++ b/gemma-rest/src/test/java/ubic/gemma/rest/DatasetsWebServiceTest.java
@@ -46,6 +46,7 @@
 import java.util.concurrent.TimeoutException;
 import java.util.stream.Collectors;
 
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.Mockito.*;
 import static ubic.gemma.rest.util.Assertions.assertThat;
 
diff --git a/gemma-rest/src/test/java/ubic/gemma/rest/OpenApiTest.java b/gemma-rest/src/test/java/ubic/gemma/rest/OpenApiTest.java
index 93aaf55d67..639a380dcd 100644
--- a/gemma-rest/src/test/java/ubic/gemma/rest/OpenApiTest.java
+++ b/gemma-rest/src/test/java/ubic/gemma/rest/OpenApiTest.java
@@ -38,6 +38,7 @@
 import java.io.InputStream;
 import java.util.Collections;
 
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 import static ubic.gemma.rest.util.Assertions.assertThat;
diff --git a/gemma-rest/src/test/java/ubic/gemma/rest/util/Assertions.java b/gemma-rest/src/test/java/ubic/gemma/rest/util/Assertions.java
index 0a39eb9921..c829b58270 100644
--- a/gemma-rest/src/test/java/ubic/gemma/rest/util/Assertions.java
+++ b/gemma-rest/src/test/java/ubic/gemma/rest/util/Assertions.java
@@ -8,7 +8,7 @@
  * Entrypoint for custom AssertJ assertions.
  * @author poirigui
  */
-public class Assertions extends org.assertj.core.api.Assertions {
+public class Assertions {
 
     public static ResponseAssert assertThat( Response response ) {
         return new ResponseAssert( response );

From 6f20d6c18efbbd88aa1cb72fbc1ffce61f33ff01 Mon Sep 17 00:00:00 2001
From: Guillaume Poirier-Morency 
Date: Tue, 14 May 2024 10:32:01 -0700
Subject: [PATCH 48/61] Add support for paginating results

Unify the getResultSets TSV and JSON endpoints
---
 .../AnalysisResultSetValueObject.java         |   5 -
 ...xpressionAnalysisResultSetValueObject.java |   6 +-
 .../diff/ExpressionAnalysisResultSetDao.java  |  47 +++++-
 .../ExpressionAnalysisResultSetDaoImpl.java   |  91 ++++++++++-
 .../ExpressionAnalysisResultSetService.java   |  10 +-
 ...xpressionAnalysisResultSetServiceImpl.java |  26 ++++
 .../rest/AnalysisResultSetsWebService.java    | 144 ++++++++++++++----
 .../ubic/gemma/rest/util/MediaTypeUtils.java  |  54 ++++++-
 .../args/ExpressionAnalysisResultSetArg.java  |   4 -
 ...ExpressionAnalysisResultSetArgService.java |  12 ++
 .../AnalysisResultSetsWebServiceTest.java     |  76 ++++-----
 .../ubic/gemma/rest/util/ResponseAssert.java  |  24 +--
 12 files changed, 391 insertions(+), 108 deletions(-)

diff --git a/gemma-core/src/main/java/ubic/gemma/model/analysis/AnalysisResultSetValueObject.java b/gemma-core/src/main/java/ubic/gemma/model/analysis/AnalysisResultSetValueObject.java
index 03fea30c2d..a7173d6f30 100644
--- a/gemma-core/src/main/java/ubic/gemma/model/analysis/AnalysisResultSetValueObject.java
+++ b/gemma-core/src/main/java/ubic/gemma/model/analysis/AnalysisResultSetValueObject.java
@@ -19,7 +19,6 @@
 package ubic.gemma.model.analysis;
 
 import ubic.gemma.model.IdentifiableValueObject;
-import ubic.gemma.model.analysis.expression.diff.DifferentialExpressionAnalysisResultValueObject;
 
 import java.util.Collection;
 
@@ -31,10 +30,6 @@
  */
 public abstract class AnalysisResultSetValueObject> extends IdentifiableValueObject {
 
-    protected AnalysisResultSetValueObject() {
-        super();
-    }
-
     protected AnalysisResultSetValueObject( R analysisResultSet ) {
         super( analysisResultSet );
     }
diff --git a/gemma-core/src/main/java/ubic/gemma/model/analysis/expression/diff/DifferentialExpressionAnalysisResultSetValueObject.java b/gemma-core/src/main/java/ubic/gemma/model/analysis/expression/diff/DifferentialExpressionAnalysisResultSetValueObject.java
index 5390bc9f4e..9865a3f7de 100644
--- a/gemma-core/src/main/java/ubic/gemma/model/analysis/expression/diff/DifferentialExpressionAnalysisResultSetValueObject.java
+++ b/gemma-core/src/main/java/ubic/gemma/model/analysis/expression/diff/DifferentialExpressionAnalysisResultSetValueObject.java
@@ -25,16 +25,12 @@ public class DifferentialExpressionAnalysisResultSetValueObject extends Analysis
 
     /**
      * Related analysis results.
-     *
+     * 

      * Note that this field is excluded from the JSON serialization if left unset.
      */
     @JsonInclude(JsonInclude.Include.NON_NULL)
     private Collection results;
 
-    public DifferentialExpressionAnalysisResultSetValueObject() {
-        super();
-    }
-
     /**
      * Create a simple analysis results set VO with limited data.
      */
diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/ExpressionAnalysisResultSetDao.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/ExpressionAnalysisResultSetDao.java
index 4bd6c795ff..bf0ec96122 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/ExpressionAnalysisResultSetDao.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/ExpressionAnalysisResultSetDao.java
@@ -43,13 +43,42 @@
  */
 public interface ExpressionAnalysisResultSetDao extends AnalysisResultSetDao, FilteringVoEnabledDao {
 
+    /**
+     * Load an analysis result set with its all of its associated results.
+     *
+     * @param id the ID of the analysis result set
+     * @return the analysis result set with its associated results, or null if not found
+     */
     @Nullable
     ExpressionAnalysisResultSet loadWithResultsAndContrasts( Long id );
 
+    /**
+     * Load a slice of an analysis result set.
+     * 

+     * Results are sorted by ascending correct P-value.
+     *
+     * @param offset    an offset of results to load
+     * @param limit     a limit of results to load, or -1 to load all results starting at offset
+     * @see #loadWithResultsAndContrasts(Long)
+     */
+    @Nullable
+    ExpressionAnalysisResultSet loadWithResultsAndContrasts( Long id, int offset, int limit );
+
+    /**
+     * Load a slice of an analysis result set with a corrected P-value threshold.
+     * 

+     * Important note: when using a threshold, results with null P-values will not be included, thus setting the
+     * threshold to {@code 1.0} is not equivalent to {@link #loadWithResultsAndContrasts(Long, int, int)}.
+     * @param threshold corrected P-value maximum threshold (inclusive)
+     */
+    @Nullable
+    ExpressionAnalysisResultSet loadWithResultsAndContrasts( Long id, double threshold, int offset, int limit );
+
     boolean canDelete( DifferentialExpressionAnalysis differentialExpressionAnalysis );
 
     /**
      * Load an analysis result set with its all of its associated results.
+     *
      * @see #loadValueObject(Identifiable)
      */
     DifferentialExpressionAnalysisResultSetValueObject loadValueObjectWithResults( ExpressionAnalysisResultSet resultSet );
@@ -68,11 +97,11 @@ public interface ExpressionAnalysisResultSetDao extends AnalysisResultSetDao findByBioAssaySetInAndDatabaseEntryInLimit( @Nullable Collection bioAssaySets, @Nullable Collection databaseEntries, @Nullable Filters filters, int offset, int limit, @Nullable Sort sort );
 
@@ -80,4 +109,14 @@ public interface ExpressionAnalysisResultSetDao extends AnalysisResultSetDao results = ( List ) getSessionFactory().getCurrentSession()
+                    .createQuery( "select res from DifferentialExpressionAnalysisResult res "
+                            + "where res.resultSet = :ears "
+                            + "order by res.correctedPvalue" )
+                    .setParameter( "ears", ears )
+                    .setFirstResult( offset )
+                    .setMaxResults( limit )
+                    .list();
+            for ( DifferentialExpressionAnalysisResult r : results ) {
+                Hibernate.initialize( r.getProbe() );
+            }
+            for ( DifferentialExpressionAnalysisResult r : results ) {
+                Hibernate.initialize( r.getContrasts() );
+            }
+            // preserve order of results
+            ears.setResults( new LinkedHashSet<>( results ) );
+        }
+        if ( timer.getTime() > 1000 ) {
+            log.info( String.format( "Loaded [%s id=%d] with results, probes and contrasts in %d ms.",
+                    elementClass.getName(), id, timer.getTime() ) );
+        }
+        return ears;
+    }
+
+    @Override
+    public ExpressionAnalysisResultSet loadWithResultsAndContrasts( Long id, double threshold, int offset, int limit ) {
+        Assert.isTrue( threshold >= 0 && threshold <= 1, "Corrected P-value threshold must be in the [0, 1] interval." );
+        if ( offset == 0 && id == -1 ) {
+            return loadWithResultsAndContrasts( id );
+        }
+        StopWatch timer = StopWatch.createStarted();
+        ExpressionAnalysisResultSet ears = load( id );
+        if ( ears != null ) {
+            //noinspection unchecked
+            List results = ( List ) getSessionFactory().getCurrentSession()
+                    .createQuery( "select res from DifferentialExpressionAnalysisResult res "
+                            + "where res.resultSet = :ears and res.correctedPvalue <= :threshold "
+                            + "order by res.correctedPvalue" )
+                    .setParameter( "ears", ears )
+                    .setParameter( "threshold", threshold )
+                    .setFirstResult( offset )
+                    .setMaxResults( limit )
+                    .list();
+            for ( DifferentialExpressionAnalysisResult r : results ) {
+                Hibernate.initialize( r.getProbe() );
+            }
+            for ( DifferentialExpressionAnalysisResult r : results ) {
+                Hibernate.initialize( r.getContrasts() );
+            }
+            // preserve order of results
+            ears.setResults( new LinkedHashSet<>( results ) );
+        }
+        if ( timer.getTime() > 1000 ) {
+            log.info( String.format( "Loaded [%s id=%d] with results, probes and contrasts in %d ms.",
+                    elementClass.getName(), id, timer.getTime() ) );
+        }
+        return ears;
+    }
+
     @Override
     public boolean canDelete( DifferentialExpressionAnalysis differentialExpressionAnalysis ) {
         return this.getSessionFactory().getCurrentSession().createQuery(
@@ -166,6 +232,23 @@ public void thaw( ExpressionAnalysisResultSet ears ) {
         }
     }
 
+    @Override
+    public long countResults( ExpressionAnalysisResultSet ears ) {
+        return ( Long ) getSessionFactory().getCurrentSession()
+                .createQuery( "select count(*) from ExpressionAnalysisResultSet ears join ears.results where ears = :ears" )
+                .setParameter( "ears", ears )
+                .uniqueResult();
+    }
+
+    @Override
+    public long countResults( ExpressionAnalysisResultSet ears, double threshold ) {
+        return ( Long ) getSessionFactory().getCurrentSession()
+                .createQuery( "select count(*) from ExpressionAnalysisResultSet ears join ears.results r where ears = :ears and r.correctedPvalue <= :threshold" )
+                .setParameter( "ears", ears )
+                .setParameter( "threshold", threshold )
+                .uniqueResult();
+    }
+
     @Override
     protected Criteria getFilteringCriteria( @Nullable Filters filters ) {
         Criteria query = this.getSessionFactory().getCurrentSession()
diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/ExpressionAnalysisResultSetService.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/ExpressionAnalysisResultSetService.java
index 6beee6caff..f4fe4b6071 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/ExpressionAnalysisResultSetService.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/ExpressionAnalysisResultSetService.java
@@ -1,8 +1,8 @@
 package ubic.gemma.persistence.service.analysis.expression.diff;
 
 import ubic.gemma.model.analysis.expression.diff.DifferentialExpressionAnalysisResult;
-import ubic.gemma.model.analysis.expression.diff.ExpressionAnalysisResultSet;
 import ubic.gemma.model.analysis.expression.diff.DifferentialExpressionAnalysisResultSetValueObject;
+import ubic.gemma.model.analysis.expression.diff.ExpressionAnalysisResultSet;
 import ubic.gemma.model.common.description.DatabaseEntry;
 import ubic.gemma.model.expression.experiment.BioAssaySet;
 import ubic.gemma.model.genome.Gene;
@@ -22,6 +22,14 @@ public interface ExpressionAnalysisResultSetService extends AnalysisResultSetSer
 
     ExpressionAnalysisResultSet loadWithResultsAndContrasts( Long value );
 
+    ExpressionAnalysisResultSet loadWithResultsAndContrasts( Long value, int offset, int limit );
+
+    ExpressionAnalysisResultSet loadWithResultsAndContrasts( Long value, double threshold, int offset, int limit );
+
+    long countResults( ExpressionAnalysisResultSet ears );
+
+    long countResults( ExpressionAnalysisResultSet ears, double threshold );
+
     @CheckReturnValue
     ExpressionAnalysisResultSet thaw( ExpressionAnalysisResultSet e );
 
diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/ExpressionAnalysisResultSetServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/ExpressionAnalysisResultSetServiceImpl.java
index ecab9fa8b2..d462e692ff 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/ExpressionAnalysisResultSetServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/ExpressionAnalysisResultSetServiceImpl.java
@@ -36,6 +36,32 @@ public ExpressionAnalysisResultSet loadWithResultsAndContrasts( Long value ) {
         return result != null ? thaw( result ) : null;
     }
 
+    @Override
+    @Transactional(readOnly = true)
+    public ExpressionAnalysisResultSet loadWithResultsAndContrasts( Long value, int offset, int limit ) {
+        ExpressionAnalysisResultSet result = voDao.loadWithResultsAndContrasts( value, offset, limit );
+        return result != null ? thaw( result ) : null;
+    }
+
+    @Override
+    @Transactional(readOnly = true)
+    public ExpressionAnalysisResultSet loadWithResultsAndContrasts( Long value, double threshold, int offset, int limit ) {
+        ExpressionAnalysisResultSet result = voDao.loadWithResultsAndContrasts( value, threshold, offset, limit );
+        return result != null ? thaw( result ) : null;
+    }
+
+    @Override
+    @Transactional(readOnly = true)
+    public long countResults( ExpressionAnalysisResultSet ears ) {
+        return voDao.countResults( ears );
+    }
+
+    @Override
+    @Transactional(readOnly = true)
+    public long countResults( ExpressionAnalysisResultSet ears, double threshold ) {
+        return voDao.countResults( ears, threshold );
+    }
+
     @Override
     @Transactional(readOnly = true)
     public ExpressionAnalysisResultSet thaw( ExpressionAnalysisResultSet ears ) {
diff --git a/gemma-rest/src/main/java/ubic/gemma/rest/AnalysisResultSetsWebService.java b/gemma-rest/src/main/java/ubic/gemma/rest/AnalysisResultSetsWebService.java
index e96a27d711..e6c368788e 100644
--- a/gemma-rest/src/main/java/ubic/gemma/rest/AnalysisResultSetsWebService.java
+++ b/gemma-rest/src/main/java/ubic/gemma/rest/AnalysisResultSetsWebService.java
@@ -25,6 +25,7 @@
 import io.swagger.v3.oas.annotations.media.ExampleObject;
 import io.swagger.v3.oas.annotations.media.Schema;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import lombok.Getter;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import ubic.gemma.core.analysis.service.ExpressionAnalysisResultSetFileService;
@@ -38,11 +39,15 @@
 import ubic.gemma.persistence.service.analysis.expression.diff.ExpressionAnalysisResultSetService;
 import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentService;
 import ubic.gemma.persistence.util.Filters;
+import ubic.gemma.persistence.util.Sort;
 import ubic.gemma.rest.annotations.GZIP;
 import ubic.gemma.rest.util.*;
 import ubic.gemma.rest.util.args.*;
 
+import javax.annotation.Nullable;
 import javax.ws.rs.*;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.HttpHeaders;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.StreamingOutput;
 import java.io.OutputStreamWriter;
@@ -51,6 +56,8 @@
 import java.util.List;
 import java.util.Map;
 
+import static ubic.gemma.rest.util.MediaTypeUtils.TEXT_TAB_SEPARATED_VALUES_UTF8;
+import static ubic.gemma.rest.util.MediaTypeUtils.negotiate;
 import static ubic.gemma.rest.util.Responders.paginate;
 import static ubic.gemma.rest.util.Responders.respond;
 
@@ -61,7 +68,9 @@
 @Path("/resultSets")
 public class AnalysisResultSetsWebService {
 
-    private static final String TEXT_TAB_SEPARATED_VALUE_Q9_MEDIA_TYPE = MediaTypeUtils.TEXT_TAB_SEPARATED_VALUES_UTF8 + "; q=0.9";
+    private static final String TEXT_TAB_SEPARATED_VALUES_Q9 = TEXT_TAB_SEPARATED_VALUES_UTF8 + "; q=0.9";
+
+    private static final MediaType TEXT_TAB_SEPARATED_VALUES_Q9_TYPE = MediaTypeUtils.withQuality( MediaTypeUtils.TEXT_TAB_SEPARATED_VALUES_UTF8_TYPE, 0.9 );
 
     @Autowired
     private ExpressionAnalysisResultSetService expressionAnalysisResultSetService;
@@ -145,19 +154,67 @@ public ResponseDataObject getNumberOfResultSets(
     @GZIP
     @GET
     @Path("/{resultSet}")
-    @Produces(MediaType.APPLICATION_JSON)
-    @Operation(summary = "Retrieve a single analysis result set by its identifier", responses = {
-            @ApiResponse(useReturnTypeSchema = true, content = @Content()),
-            @ApiResponse(responseCode = "404", description = "The analysis result set could not be found.",
-                    content = @Content(schema = @Schema(implementation = ResponseErrorObject.class))) })
-    public ResponseDataObject getResultSet(
+    @Produces({ MediaType.APPLICATION_JSON, TEXT_TAB_SEPARATED_VALUES_Q9 })
+    @Operation(summary = "Retrieve a single analysis result set by its identifier",
+            description = "A slice or results can be retrieved by specifying the `offset` and `limit` parameters. This is only applicable to the JSON representation.",
+            responses = {
+                    @ApiResponse(content = {
+                            @Content(mediaType = MediaType.APPLICATION_JSON,
+                                    schema = @Schema(implementation = PaginatedResultsResponseDataObjectDifferentialExpressionAnalysisResultSetValueObject.class)),
+                            @Content(mediaType = TEXT_TAB_SEPARATED_VALUES_UTF8, /* no need to expose the q-value */
+                                    schema = @Schema(type = "string", format = "binary"),
+                                    examples = { @ExampleObject(value = TSV_EXAMPLE) })
+                    }),
+                    @ApiResponse(responseCode = "404", description = "The analysis result set could not be found.",
+                            content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ResponseErrorObject.class))) })
+    public Object getResultSet(
             @PathParam("resultSet") ExpressionAnalysisResultSetArg analysisResultSet,
-            @Parameter(hidden = true) @QueryParam("excludeResults") @DefaultValue("false") Boolean excludeResults ) {
+            @QueryParam("threshold") Double threshold,
+            @QueryParam("offset") OffsetArg offsetArg,
+            @QueryParam("limit") LimitArg limitArg,
+            @Parameter(hidden = true) @QueryParam("excludeResults") @DefaultValue("false") Boolean excludeResults,
+            @Context HttpHeaders headers ) {
+        MediaType acceptedMediaType = negotiate( headers, MediaType.APPLICATION_JSON_TYPE, TEXT_TAB_SEPARATED_VALUES_Q9_TYPE );
+        if ( acceptedMediaType.equals( MediaType.APPLICATION_JSON_TYPE ) ) {
+            if ( offsetArg != null || limitArg != null || threshold != null ) {
+                if ( excludeResults ) {
+                    throw new BadRequestException( "The excludeResults parameter cannot be used with offset/limit or threshold parameters." );
+                }
+                int offset = 0, limit = LimitArg.MAXIMUM;
+                if ( offsetArg != null ) {
+                    offset = offsetArg.getValue();
+                }
+                if ( limitArg != null ) {
+                    limit = limitArg.getValue();
+                }
+                if ( threshold != null ) {
+                    if ( threshold < 0.0 || threshold > 1.0 ) {
+                        throw new BadRequestException( "The threshold must be between 0 and 1." );
+                    }
+                    return getResultSetAsJson( analysisResultSet, threshold, offset, limit );
+                } else {
+                    return getResultSetAsJson( analysisResultSet, offset, limit );
+                }
+            } else {
+                return getResultSetAsJson( analysisResultSet, excludeResults );
+            }
+        } else {
+            if ( excludeResults ) {
+                throw new BadRequestException( "The excludeResults parameter cannot be used with the TSV representation." );
+            }
+            if ( threshold != null ) {
+                throw new BadRequestException( "The threshold parameter cannot be used with the TSV representation." );
+            }
+            return getResultSetAsTsv( analysisResultSet );
+        }
+    }
+
+    private ResponseDataObject getResultSetAsJson( ExpressionAnalysisResultSetArg analysisResultSet, boolean excludeResults ) {
         if ( excludeResults ) {
             ExpressionAnalysisResultSet ears = expressionAnalysisResultSetArgService.getEntity( analysisResultSet );
             return respond( expressionAnalysisResultSetService.loadValueObject( ears ) );
         } else {
-            ExpressionAnalysisResultSet ears = analysisResultSet.getEntityWithContrastsAndResults( expressionAnalysisResultSetService );
+            ExpressionAnalysisResultSet ears = expressionAnalysisResultSetArgService.getEntityWithContrastsAndResults( analysisResultSet );
             if ( ears == null ) {
                 throw new NotFoundException( "Could not find ExpressionAnalysisResultSet for " + analysisResultSet + "." );
             }
@@ -165,26 +222,26 @@ public ResponseDataObject ge
         }
     }
 
-    /**
-     * Retrieve an {@link AnalysisResultSet} in a tabular format.
-     * 

-     * This is intentionally using a slightly different parameter name for the {@link Path} to create a distinct entry
-     * in the OpenAPI specification as a workaround to Swagger's codegen incapability to treat multiple media types per
-     * endpoint.
-     */
-    @GZIP
-    @GET
-    @Path("/{resultSet_}")
-    @Produces(TEXT_TAB_SEPARATED_VALUE_Q9_MEDIA_TYPE)
-    @Operation(summary = "Retrieve a single analysis result set by its identifier as a tab-separated values", responses = {
-            @ApiResponse(content = @Content(mediaType = MediaTypeUtils.TEXT_TAB_SEPARATED_VALUES_UTF8, /* no need to show the q-value in the endpoint */
-                    schema = @Schema(type = "string", format = "binary"),
-                    examples = { @ExampleObject(value = TSV_EXAMPLE) })),
-            @ApiResponse(responseCode = "404", description = "The analysis result set could not be found.",
-                    content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ResponseErrorObject.class))) })
-    public StreamingOutput getResultSetAsTsv(
-            @PathParam("resultSet_") ExpressionAnalysisResultSetArg analysisResultSet ) {
-        final ExpressionAnalysisResultSet ears = analysisResultSet.getEntityWithContrastsAndResults( expressionAnalysisResultSetService );
+    private PaginatedResultsResponseDataObjectDifferentialExpressionAnalysisResultSetValueObject getResultSetAsJson( ExpressionAnalysisResultSetArg analysisResultSet, int offset, int limit ) {
+        ExpressionAnalysisResultSet ears = expressionAnalysisResultSetArgService.getEntityWithContrastsAndResults( analysisResultSet, offset, limit );
+        if ( ears == null ) {
+            throw new NotFoundException( "Could not find ExpressionAnalysisResultSet for " + analysisResultSet + "." );
+        }
+        long totalElements = expressionAnalysisResultSetService.countResults( ears );
+        return paginateResults( expressionAnalysisResultSetService.loadValueObjectWithResults( ears ), null, offset, limit, totalElements );
+    }
+
+    private PaginatedResultsResponseDataObjectDifferentialExpressionAnalysisResultSetValueObject getResultSetAsJson( ExpressionAnalysisResultSetArg analysisResultSet, double threshold, int offset, int limit ) {
+        ExpressionAnalysisResultSet ears = expressionAnalysisResultSetArgService.getEntityWithContrastsAndResults( analysisResultSet, threshold, offset, limit );
+        if ( ears == null ) {
+            throw new NotFoundException( "Could not find ExpressionAnalysisResultSet for " + analysisResultSet + "." );
+        }
+        long totalElements = expressionAnalysisResultSetService.countResults( ears, threshold );
+        return paginateResults( expressionAnalysisResultSetService.loadValueObjectWithResults( ears ), threshold, offset, limit, totalElements );
+    }
+
+    private StreamingOutput getResultSetAsTsv( ExpressionAnalysisResultSetArg analysisResultSet ) {
+        final ExpressionAnalysisResultSet ears = expressionAnalysisResultSetArgService.getEntityWithContrastsAndResults( analysisResultSet );
         if ( ears == null ) {
             throw new NotFoundException( "Could not find ExpressionAnalysisResultSet for " + analysisResultSet + "." );
         }
@@ -195,4 +252,33 @@ public StreamingOutput getResultSetAsTsv(
             }
         };
     }
+
+    private PaginatedResultsResponseDataObjectDifferentialExpressionAnalysisResultSetValueObject paginateResults( DifferentialExpressionAnalysisResultSetValueObject resultSet, @Nullable Double threshold, int offset, int limit, long totalElements ) {
+        return new PaginatedResultsResponseDataObjectDifferentialExpressionAnalysisResultSetValueObject( resultSet, threshold, offset, limit, totalElements );
+    }
+
+    /**
+     * Similar to {@link ubic.gemma.rest.util.PaginatedResponseDataObject}, but the {@code data.results} is paginated
+     * instead of {@code data}
+     */
+    @Getter
+    public static class PaginatedResultsResponseDataObjectDifferentialExpressionAnalysisResultSetValueObject extends ResponseDataObject {
+
+        private final String filter;
+        private final SortValueObject sort;
+        private final String[] groupBy;
+        private final Integer offset;
+        private final Integer limit;
+        private final Long totalElements;
+
+        public PaginatedResultsResponseDataObjectDifferentialExpressionAnalysisResultSetValueObject( DifferentialExpressionAnalysisResultSetValueObject resultSet, @Nullable Double threshold, int offset, int limit, long totalElements ) {
+            super( resultSet );
+            this.filter = threshold != null ? "results.correctedPvalue <= " + threshold : "";
+            this.sort = new SortValueObject( Sort.by( null, "correctedPvalue", Sort.Direction.ASC, "results.correctedPvalue" ) );
+            this.groupBy = new String[] { "results.id" };
+            this.offset = offset;
+            this.limit = limit;
+            this.totalElements = totalElements;
+        }
+    }
 }
diff --git a/gemma-rest/src/main/java/ubic/gemma/rest/util/MediaTypeUtils.java b/gemma-rest/src/main/java/ubic/gemma/rest/util/MediaTypeUtils.java
index 0c08cc3522..5da13c1bc5 100644
--- a/gemma-rest/src/main/java/ubic/gemma/rest/util/MediaTypeUtils.java
+++ b/gemma-rest/src/main/java/ubic/gemma/rest/util/MediaTypeUtils.java
@@ -1,6 +1,14 @@
 package ubic.gemma.rest.util;
 
+import javax.ws.rs.BadRequestException;
+import javax.ws.rs.NotAcceptableException;
+import javax.ws.rs.core.HttpHeaders;
 import javax.ws.rs.core.MediaType;
+import java.util.Collections;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 /**
  * Utilities for {@link MediaType}.
@@ -11,6 +19,50 @@ public class MediaTypeUtils {
 
     public static final String TEXT_TAB_SEPARATED_VALUES_UTF8 = "text/tab-separated-values; charset=UTF-8";
 
-    @SuppressWarnings("unused")
     public static final MediaType TEXT_TAB_SEPARATED_VALUES_UTF8_TYPE = new MediaType( "text", "tab-separated-values", "UTF-8" );
+
+    public static MediaType withQuality( MediaType mediaType, double quality ) {
+        Map parameters;
+        if ( mediaType.getParameters() != null ) {
+            parameters = new TreeMap<>( String.CASE_INSENSITIVE_ORDER );
+            parameters.putAll( mediaType.getParameters() );
+            parameters.put( "q", String.valueOf( quality ) );
+        } else {
+            parameters = Collections.singletonMap( "q", String.valueOf( quality ) );
+        }
+        return new MediaType( mediaType.getType(), mediaType.getSubtype(), parameters );
+    }
+
+    public static MediaType negotiate( HttpHeaders headers, MediaType... types ) throws NotAcceptableException {
+        MediaType bestMediaType = null;
+        double bestScore = 0.0;
+        double[] typeScores = new double[types.length];
+        for ( int i = 0; i < types.length; i++ ) {
+            typeScores[i] = Double.parseDouble( types[i].getParameters().getOrDefault( "q", "1.0" ) );
+        }
+        for ( MediaType acceptableMediaType : headers.getAcceptableMediaTypes() ) {
+            double q1;
+            try {
+                q1 = Double.parseDouble( acceptableMediaType.getParameters().getOrDefault( "q", "1.0" ) );
+            } catch ( NumberFormatException e ) {
+                throw new BadRequestException( "Invalid q-value for media type in 'Accept' header." );
+            }
+            for ( int i = 0; i < types.length; i++ ) {
+                MediaType type = types[i];
+                double q2 = typeScores[i];
+                if ( acceptableMediaType.isCompatible( type ) ) {
+                    double score = q1 * q2;
+                    if ( score > bestScore ) {
+                        bestScore = score;
+                        bestMediaType = type;
+                    }
+                }
+            }
+        }
+        if ( bestMediaType == null ) {
+            throw new NotAcceptableException( String.format( "None of the accepted media type are compatible with those produced: %s.",
+                    Stream.of( types ).map( MediaType::toString ).collect( Collectors.joining( ", " ) ) ) );
+        }
+        return bestMediaType;
+    }
 }
diff --git a/gemma-rest/src/main/java/ubic/gemma/rest/util/args/ExpressionAnalysisResultSetArg.java b/gemma-rest/src/main/java/ubic/gemma/rest/util/args/ExpressionAnalysisResultSetArg.java
index c7eb132091..353a1493f4 100644
--- a/gemma-rest/src/main/java/ubic/gemma/rest/util/args/ExpressionAnalysisResultSetArg.java
+++ b/gemma-rest/src/main/java/ubic/gemma/rest/util/args/ExpressionAnalysisResultSetArg.java
@@ -21,10 +21,6 @@ ExpressionAnalysisResultSet getEntity( ExpressionAnalysisResultSetService servic
         return service.loadWithExperimentAnalyzed( getValue() );
     }
 
-    public ExpressionAnalysisResultSet getEntityWithContrastsAndResults( ExpressionAnalysisResultSetService service ) {
-        return service.loadWithResultsAndContrasts( getValue() );
-    }
-
     public static ExpressionAnalysisResultSetArg valueOf( String s ) {
         try {
             return new ExpressionAnalysisResultSetArg( Long.parseLong( s ) );
diff --git a/gemma-rest/src/main/java/ubic/gemma/rest/util/args/ExpressionAnalysisResultSetArgService.java b/gemma-rest/src/main/java/ubic/gemma/rest/util/args/ExpressionAnalysisResultSetArgService.java
index b8171385af..cc191ab4d3 100644
--- a/gemma-rest/src/main/java/ubic/gemma/rest/util/args/ExpressionAnalysisResultSetArgService.java
+++ b/gemma-rest/src/main/java/ubic/gemma/rest/util/args/ExpressionAnalysisResultSetArgService.java
@@ -12,4 +12,16 @@ public class ExpressionAnalysisResultSetArgService extends AbstractEntityArgServ
     public ExpressionAnalysisResultSetArgService( ExpressionAnalysisResultSetService service ) {
         super( service );
     }
+
+    public ExpressionAnalysisResultSet getEntityWithContrastsAndResults( ExpressionAnalysisResultSetArg analysisResultSet ) {
+        return service.loadWithResultsAndContrasts( analysisResultSet.getValue() );
+    }
+
+    public ExpressionAnalysisResultSet getEntityWithContrastsAndResults( ExpressionAnalysisResultSetArg analysisResultSet, int offset, int limit ) {
+        return service.loadWithResultsAndContrasts( analysisResultSet.getValue(), offset, limit );
+    }
+
+    public ExpressionAnalysisResultSet getEntityWithContrastsAndResults( ExpressionAnalysisResultSetArg analysisResultSet, double threshold, int offset, int limit ) {
+        return service.loadWithResultsAndContrasts( analysisResultSet.getValue(), threshold, offset, limit );
+    }
 }
diff --git a/gemma-rest/src/test/java/ubic/gemma/rest/AnalysisResultSetsWebServiceTest.java b/gemma-rest/src/test/java/ubic/gemma/rest/AnalysisResultSetsWebServiceTest.java
index b045cdb492..abe864a00d 100644
--- a/gemma-rest/src/test/java/ubic/gemma/rest/AnalysisResultSetsWebServiceTest.java
+++ b/gemma-rest/src/test/java/ubic/gemma/rest/AnalysisResultSetsWebServiceTest.java
@@ -7,8 +7,6 @@
 import org.junit.Before;
 import org.junit.Test;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.test.context.ActiveProfiles;
-import org.springframework.test.context.web.WebAppConfiguration;
 import ubic.gemma.core.util.test.PersistentDummyObjectHelper;
 import ubic.gemma.model.analysis.expression.diff.*;
 import ubic.gemma.model.common.description.DatabaseEntry;
@@ -22,24 +20,21 @@
 import ubic.gemma.persistence.service.expression.arrayDesign.ArrayDesignService;
 import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentService;
 import ubic.gemma.rest.util.BaseJerseyIntegrationTest;
-import ubic.gemma.rest.util.MalformedArgException;
+import ubic.gemma.rest.util.MediaTypeUtils;
 import ubic.gemma.rest.util.ResponseDataObject;
 import ubic.gemma.rest.util.args.*;
 
 import javax.ws.rs.BadRequestException;
 import javax.ws.rs.NotFoundException;
+import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
-import javax.ws.rs.core.StreamingOutput;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
 import java.io.InputStreamReader;
-import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.List;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.*;
+import static ubic.gemma.rest.util.Assertions.assertThat;
 
 public class AnalysisResultSetsWebServiceTest extends BaseJerseyIntegrationTest {
 
@@ -241,50 +236,55 @@ public void testFindAllWhenDatabaseEntryDoesNotExistThenRaise404NotFound() {
 
     @Test
     public void testFindByIdThenReturn200Success() {
-        ResponseDataObject result = service.getResultSet( ExpressionAnalysisResultSetArg.valueOf( dears.getId().toString() ), false );
-        DifferentialExpressionAnalysisResultSetValueObject dearsVo = ( DifferentialExpressionAnalysisResultSetValueObject ) result.getData();
-        assertEquals( dearsVo.getId(), dears.getId() );
-        assertEquals( dearsVo.getAnalysis().getId(), dea.getId() );
-        assertNotNull( dearsVo.getResults() );
+        assertThat( target( "/resultSets/" + dears.getId() ).request().get() )
+                .hasStatus( Response.Status.OK )
+                .hasMediaType( MediaType.APPLICATION_JSON_TYPE )
+                .hasEncoding( "gzip" )
+                .entity()
+                .hasFieldOrPropertyWithValue( "data.id", dears.getId().intValue() )
+                .hasFieldOrPropertyWithValue( "data.analysis.id", dea.getId().intValue() )
+                .extracting( "data.results" )
+                .isNotNull();
     }
 
     @Test
     public void testFindByIdWhenExcludeResultsThenReturn200Success() {
-        ResponseDataObject result = service.getResultSet( ExpressionAnalysisResultSetArg.valueOf( dears.getId().toString() ), true );
-        DifferentialExpressionAnalysisResultSetValueObject dearsVo = ( DifferentialExpressionAnalysisResultSetValueObject ) result.getData();
-        assertEquals( dearsVo.getId(), dears.getId() );
-        assertEquals( dearsVo.getAnalysis().getId(), dea.getId() );
-        assertNull( dearsVo.getResults() );
+        assertThat( target( "/resultSets/" + dears.getId() ).queryParam( "excludeResults", true ).request().get() )
+                .hasStatus( Response.Status.OK )
+                .hasMediaType( MediaType.APPLICATION_JSON_TYPE )
+                .hasEncoding( "gzip" )
+                .entityAsString()
+                .doesNotContain( "results" );
     }
 
     @Test
     public void testFindByIdWhenInvalidIdentifierThenThrowMalformedArgException() {
-        assertThrows( MalformedArgException.class, () -> service.getResultSet( ExpressionAnalysisResultSetArg.valueOf( "alksdok102" ), false ) );
+        assertThat( target( "/resultSets/alksdok102" ).request().get() )
+                .hasStatus( Response.Status.BAD_REQUEST );
     }
 
     @Test
     public void testFindByIdWhenResultSetDoesNotExistsThenReturn404NotFoundError() {
-        long id = 123129L;
-        NotFoundException e = assertThrows( NotFoundException.class, () -> service.getResultSet( ExpressionAnalysisResultSetArg.valueOf( String.valueOf( id ) ), false ) );
-        assertEquals( e.getResponse().getStatus(), Response.Status.NOT_FOUND.getStatusCode() );
+        assertThat( target( "/resultSets/" + 123129L ).request().get() )
+                .hasStatus( Response.Status.NOT_FOUND );
     }
 
     @Test
-    public void testFindByIdToTsv() throws IOException {
-        StreamingOutput result = service.getResultSetAsTsv( ExpressionAnalysisResultSetArg.valueOf( dears.getId().toString() ) );
-        ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
-        result.write( byteArrayOutputStream );
-        byteArrayOutputStream.toString( StandardCharsets.UTF_8.name() );
-        // FIXME: I could not find the equivalent of withFirstRecordAsHeader() in the builder API
-        CSVParser reader = CSVFormat.Builder.create( CSVFormat.TDF.withFirstRecordAsHeader() )
-                .setSkipHeaderRecord( false )
-                .setCommentMarker( '#' ).build()
-                .parse( new InputStreamReader( new ByteArrayInputStream( byteArrayOutputStream.toByteArray() ) ) );
-        assertEquals( reader.getHeaderNames(), Arrays.asList( "id", "probe_id", "probe_name", "gene_id", "gene_name", "gene_ncbi_id", "gene_official_symbol", "gene_official_name", "pvalue", "corrected_pvalue", "rank" ) );
-        CSVRecord record = reader.iterator().next();
-        assertEquals( record.get( "pvalue" ), "1.0" );
-        assertEquals( record.get( "corrected_pvalue" ), "0.0001" );
-        // rank is null, it should appear as an empty string
-        assertEquals( record.get( "rank" ), "" );
+    public void testFindByIdToTsv() {
+        assertThat( target( "/resultSets/" + dears.getId() ).request( MediaTypeUtils.TEXT_TAB_SEPARATED_VALUES_UTF8 ).get() )
+                .entityAsStream()
+                .satisfies( is -> {
+                    // FIXME: I could not find the equivalent of withFirstRecordAsHeader() in the builder API
+                    CSVParser reader = CSVFormat.Builder.create( CSVFormat.TDF.withFirstRecordAsHeader() )
+                            .setSkipHeaderRecord( false )
+                            .setCommentMarker( '#' ).build()
+                            .parse( new InputStreamReader( is ) );
+                    assertEquals( reader.getHeaderNames(), Arrays.asList( "id", "probe_id", "probe_name", "gene_id", "gene_name", "gene_ncbi_id", "gene_official_symbol", "gene_official_name", "pvalue", "corrected_pvalue", "rank" ) );
+                    CSVRecord record = reader.iterator().next();
+                    assertEquals( record.get( "pvalue" ), "1.0" );
+                    assertEquals( record.get( "corrected_pvalue" ), "0.0001" );
+                    // rank is null, it should appear as an empty string
+                    assertEquals( record.get( "rank" ), "" );
+                } );
     }
 }
\ No newline at end of file
diff --git a/gemma-rest/src/test/java/ubic/gemma/rest/util/ResponseAssert.java b/gemma-rest/src/test/java/ubic/gemma/rest/util/ResponseAssert.java
index 59af4b7261..f168026d8f 100644
--- a/gemma-rest/src/test/java/ubic/gemma/rest/util/ResponseAssert.java
+++ b/gemma-rest/src/test/java/ubic/gemma/rest/util/ResponseAssert.java
@@ -1,6 +1,5 @@
 package ubic.gemma.rest.util;
 
-import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.assertj.core.api.*;
 import org.assertj.core.internal.Maps;
@@ -9,10 +8,7 @@
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.NewCookie;
 import javax.ws.rs.core.Response;
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
 import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -35,20 +31,14 @@ public ResponseAssert( Response actual ) {
                         .sorted( Map.Entry.comparingByKey() )
                         .map( e -> e.getKey() + ": " + String.join( ", ", e.getValue() ) )
                         .collect( Collectors.joining( "\n" ) ),
-                formatEntity( actual.getEntity() ) );
-    }
-
-    private String formatEntity( Object entity ) {
-        if ( entity instanceof ByteArrayInputStream ) {
-            try {
-                return IOUtils.toString( ( InputStream ) entity, StandardCharsets.UTF_8 );
-            } catch ( IOException e ) {
-                throw new RuntimeException( e );
-            } finally {
-                ( ( ByteArrayInputStream ) entity ).reset();
-            }
+                formatEntity( actual ) );
+    }
+
+    private String formatEntity( Response response ) {
+        if ( response.bufferEntity() ) {
+            return response.readEntity( String.class );
         } else {
-            return entity.toString();
+            return response.getEntity().toString();
         }
     }
 

From 71758f4b25eec01f54955c29768d951e69a74d6b Mon Sep 17 00:00:00 2001
From: Guillaume Poirier-Morency 
Date: Tue, 14 May 2024 17:00:43 -0700
Subject: [PATCH 49/61] Few more Assertions.assertThat import fixes

---
 gemma-rest/src/test/java/ubic/gemma/rest/TaxaWebServiceTest.java | 1 +
 .../gemma/rest/analytics/ga4/GoogleAnalytics4ProviderTest.java   | 1 +
 2 files changed, 2 insertions(+)

diff --git a/gemma-rest/src/test/java/ubic/gemma/rest/TaxaWebServiceTest.java b/gemma-rest/src/test/java/ubic/gemma/rest/TaxaWebServiceTest.java
index f0b72b804b..18187c9f99 100644
--- a/gemma-rest/src/test/java/ubic/gemma/rest/TaxaWebServiceTest.java
+++ b/gemma-rest/src/test/java/ubic/gemma/rest/TaxaWebServiceTest.java
@@ -20,6 +20,7 @@
 import java.util.Collections;
 import java.util.List;
 
+import static org.assertj.core.api.Assertions.assertThat;
 import static ubic.gemma.rest.util.Assertions.assertThat;
 
 public class TaxaWebServiceTest extends BaseJerseyIntegrationTest {
diff --git a/gemma-rest/src/test/java/ubic/gemma/rest/analytics/ga4/GoogleAnalytics4ProviderTest.java b/gemma-rest/src/test/java/ubic/gemma/rest/analytics/ga4/GoogleAnalytics4ProviderTest.java
index 868ab6c844..72b190eb1f 100644
--- a/gemma-rest/src/test/java/ubic/gemma/rest/analytics/ga4/GoogleAnalytics4ProviderTest.java
+++ b/gemma-rest/src/test/java/ubic/gemma/rest/analytics/ga4/GoogleAnalytics4ProviderTest.java
@@ -20,6 +20,7 @@
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static ubic.gemma.rest.util.Assertions.assertThat;
 

From 260383772529ebf4a92ce231b871238bdeb6f9cc Mon Sep 17 00:00:00 2001
From: Paul Pavlidis 
Date: Tue, 14 May 2024 17:22:16 -0700
Subject: [PATCH 50/61] Provide methods to make ontology loading lighter, when
 desired.

It makes some difference
---
 .../ExperimentalDesignImporterImpl.java       |  2 +-
 .../gemma/core/ontology/OntologyUtils.java    | 47 +++++++++++++++++++
 2 files changed, 48 insertions(+), 1 deletion(-)

diff --git a/gemma-core/src/main/java/ubic/gemma/core/loader/expression/simple/ExperimentalDesignImporterImpl.java b/gemma-core/src/main/java/ubic/gemma/core/loader/expression/simple/ExperimentalDesignImporterImpl.java
index 47634236f2..d3556c2993 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/loader/expression/simple/ExperimentalDesignImporterImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/loader/expression/simple/ExperimentalDesignImporterImpl.java
@@ -153,7 +153,7 @@ private void addExperimentalFactorsToExperimentalDesign( ExperimentalDesign expe
 
         // make sure that EFO is initialized
         try {
-            OntologyUtils.ensureInitialized( efoService );
+            OntologyUtils.ensureInitializedLite( efoService );
         } catch ( InterruptedException e ) {
             Thread.currentThread().interrupt();
             throw new RuntimeException( e );
diff --git a/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyUtils.java b/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyUtils.java
index 4417bbce7b..8e16a055a7 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyUtils.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyUtils.java
@@ -20,6 +20,7 @@ public class OntologyUtils {
      * possible interrupt
      */
     public static void ensureInitialized( OntologyService service ) throws InterruptedException {
+
         if ( service.isOntologyLoaded() )
             return;
         if ( service.isInitializationThreadAlive() ) {
@@ -30,4 +31,50 @@ public static void ensureInitialized( OntologyService service ) throws Interrupt
             service.initialize( true, false );
         }
     }
+
+    /**
+     * Ensure that a given ontology is initialized, force-loading it via {@link OntologyService#initialize(boolean, boolean)}, but setting the language level to LITE,
+     * Inferencing to NONE, processImports to false, and enable search to false, if the ontology isn't already loaded.
+     * @param service
+     * @throws InterruptedException
+     */
+    public static void ensureInitializedLite( OntologyService service ) throws InterruptedException {
+        if ( service.isOntologyLoaded() )
+            return;
+        if ( service.isInitializationThreadAlive() ) {
+            log.info( String.format( "Waiting for %s to load...", service ) );
+            service.waitForInitializationThread();
+        } else {
+            ensureInitialized( service, OntologyService.InferenceMode.NONE, OntologyService.LanguageLevel.LITE, false, false );
+        }
+    }
+
+    /**
+     * Ensure that a given ontology is initialized, force-loading it via {@link OntologyService#initialize(boolean, boolean)},
+     * but first setting how we load it. However, those parameters are ignored if the ontology is already loaded or in progress.
+     * @param service
+     * @param mode
+     * @param level
+     * @param searchEnabled
+     * @param processImports
+     * @throws InterruptedException  in case the ontology initialization thread is started, we will wait which implies a
+     *      possible interrupt
+     */
+    public static void ensureInitialized( OntologyService service, OntologyService.InferenceMode mode, OntologyService.LanguageLevel level, Boolean searchEnabled, Boolean processImports ) throws InterruptedException {
+
+        if ( service.isOntologyLoaded() )
+            return;
+        if ( service.isInitializationThreadAlive() ) {
+            log.info( String.format( "Waiting for %s to load...", service ) );
+            service.waitForInitializationThread();
+        } else {
+            service.setInferenceMode( mode );
+            service.setSearchEnabled( searchEnabled );
+            service.setLanguageLevel( level );
+            service.setProcessImports( processImports );
+            log.info( String.format( "Force-loading %s ", service ) );
+
+            service.initialize( true, false );
+        }
+    }
 }

From a33218a75a4d083dca556b8a6d1680cca7594aba Mon Sep 17 00:00:00 2001
From: Paul Pavlidis 
Date: Wed, 15 May 2024 09:20:35 -0700
Subject: [PATCH 51/61] fix stupid bug where outlier samples were removed
 entirely from processed data during batch correction

---
 ...nExperimentBatchCorrectionServiceImpl.java | 48 ++++++++++++++++++-
 ...nExperimentBatchCorrectionServiceTest.java | 21 ++++++++
 2 files changed, 68 insertions(+), 1 deletion(-)

diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/ExpressionExperimentBatchCorrectionServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/ExpressionExperimentBatchCorrectionServiceImpl.java
index ab64d055a8..ee8db94f74 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/ExpressionExperimentBatchCorrectionServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/ExpressionExperimentBatchCorrectionServiceImpl.java
@@ -32,6 +32,7 @@
 import ubic.gemma.model.common.quantitationtype.QuantitationType;
 import ubic.gemma.model.common.quantitationtype.ScaleType;
 import ubic.gemma.model.expression.bioAssay.BioAssay;
+import ubic.gemma.model.expression.bioAssayData.BioAssayDimension;
 import ubic.gemma.model.expression.bioAssayData.ProcessedExpressionDataVector;
 import ubic.gemma.model.expression.biomaterial.BioMaterial;
 import ubic.gemma.model.expression.designElement.CompositeSequence;
@@ -159,8 +160,53 @@ public ExpressionDataDoubleMatrix comBat( ExpressionDataDoubleMatrix originalDat
 
         ObjectMatrix design = this.getDesign( ee, finalMatrix );
 
-        return this.doComBat( ee, finalMatrix, design );
+        ExpressionDataDoubleMatrix correctedMatrix = this.doComBat( ee, finalMatrix, design );
 
+        return restoreOutliers( originalDataMatrix, correctedMatrix );
+
+    }
+
+    /**
+     * Restore the outliers by basically overwriting the original matrix with the corrected values, leaving outlier samples as they were.
+     * This is a lot easier than starting over with a new matrix.
+     * @param originalDataMatrix
+     * @param correctedMatrix
+     * @return the originalDataMatrix with the correctedvalues now plugged in, or, if no outliers were present, the correctedMatrix because why not.s
+     */
+    private ExpressionDataDoubleMatrix restoreOutliers( ExpressionDataDoubleMatrix originalDataMatrix, ExpressionDataDoubleMatrix correctedMatrix ) {
+        if ( originalDataMatrix.getBestBioAssayDimension().getBioAssays().size() == correctedMatrix.columns() ) {
+            return correctedMatrix;
+        }
+
+        Set outlierColumns = new HashSet<>();
+        for ( int j = 0; j < originalDataMatrix.columns(); j++ ) {
+            if ( originalDataMatrix.getBioAssaysForColumn( j ).iterator().next().getIsOutlier() ) {
+                outlierColumns.add( j );
+            }
+        }
+
+        if ( outlierColumns.isEmpty() ) {
+            throw new IllegalStateException( "Was expecting some outliers to be present since the corrected matrix is smaller than the original matrix" );
+        }
+
+        log.info( "Restoring " + outlierColumns.size() + " outlier columns" );
+
+        /*
+        Iterate over the rows and columns of the original matrix and copy the values from the corrected matrix.
+        If the column is an outlier in the original matrix, just skip it.
+         */
+        for ( int i = 0; i < originalDataMatrix.rows(); i++ ) {
+            int skip = 0;
+            for ( int j = 0; j < originalDataMatrix.columns(); j++ ) {
+                if ( outlierColumns.contains( j ) ) {
+                    skip++;
+                    continue; // leave it alone; normally this will be an NaN.
+                }
+                originalDataMatrix.set( i, j, correctedMatrix.get( i, j - skip ) );
+            }
+        }
+
+        return originalDataMatrix;
     }
 
     /**
diff --git a/gemma-core/src/test/java/ubic/gemma/core/analysis/preprocess/batcheffects/ExpressionExperimentBatchCorrectionServiceTest.java b/gemma-core/src/test/java/ubic/gemma/core/analysis/preprocess/batcheffects/ExpressionExperimentBatchCorrectionServiceTest.java
index 71b0caffb7..fde2af5a55 100644
--- a/gemma-core/src/test/java/ubic/gemma/core/analysis/preprocess/batcheffects/ExpressionExperimentBatchCorrectionServiceTest.java
+++ b/gemma-core/src/test/java/ubic/gemma/core/analysis/preprocess/batcheffects/ExpressionExperimentBatchCorrectionServiceTest.java
@@ -24,6 +24,7 @@
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.springframework.beans.factory.annotation.Autowired;
+import ubic.basecode.ontology.providers.ExperimentalFactorOntologyService;
 import ubic.gemma.core.datastructure.matrix.ExpressionDataDoubleMatrix;
 import ubic.gemma.core.loader.expression.geo.AbstractGeoServiceTest;
 import ubic.gemma.core.loader.expression.geo.GeoDomainObjectGeneratorLocal;
@@ -31,7 +32,9 @@
 import ubic.gemma.core.loader.expression.simple.ExperimentalDesignImporter;
 import ubic.gemma.core.loader.util.AlreadyExistsInSystemException;
 import ubic.gemma.core.util.test.category.SlowTest;
+import ubic.gemma.model.expression.bioAssay.BioAssay;
 import ubic.gemma.model.expression.experiment.ExpressionExperiment;
+import ubic.gemma.persistence.service.expression.bioAssay.BioAssayService;
 import ubic.gemma.persistence.service.expression.bioAssayData.ProcessedExpressionDataVectorService;
 import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentService;
 
@@ -39,6 +42,7 @@
 import java.util.Collection;
 import java.util.List;
 
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 
 /**
@@ -58,6 +62,9 @@ public class ExpressionExperimentBatchCorrectionServiceTest extends AbstractGeoS
     @Autowired
     private GeoService geoService;
 
+    @Autowired
+    private BioAssayService bioAssayService;
+
     @Autowired
     private ProcessedExpressionDataVectorService processedExpressionDataVectorService;
 
@@ -86,9 +93,21 @@ public void testComBatOnEE() throws Exception {
             newee = ( ExpressionExperiment ) ( ( List ) e.getData() ).iterator().next();
         }
 
+
         assertNotNull( newee );
         newee = expressionExperimentService.thawLite( newee );
+        // add an outlier to excerise batch correction dealing with that.
+        for ( BioAssay ba : newee.getBioAssays() ) {
+            if ( ba.getName().equals( "070314_ETOH-20" ) ) {
+                ba.setIsOutlier( true );
+                bioAssayService.update( ba ); // save it jusut in case ...
+                break;
+            }
+        }
+        newee = expressionExperimentService.thawLite( newee );
+
         processedExpressionDataVectorService.computeProcessedExpressionData( newee );
+
         try ( InputStream deis = this.getClass()
                 .getResourceAsStream( "/data/loader/expression/geo/gse18162Short/design.txt" ) ) {
             experimentalDesignImporter.importDesign( newee, deis );
@@ -96,6 +115,8 @@ public void testComBatOnEE() throws Exception {
         ExpressionDataDoubleMatrix comBat = correctionService.comBat( newee );
         assertNotNull( comBat );
 
+        assertEquals( newee.getBioAssays().size(), comBat.columns() );
+
     }
 
     private void cleanup() {

From 469321acfc580675484b043bc7365311ec369826 Mon Sep 17 00:00:00 2001
From: Paul Pavlidis 
Date: Wed, 15 May 2024 11:38:13 -0700
Subject: [PATCH 52/61] handle case where rank information is missing

see https://github.com/PavlidisLab/GemmaCuration/issues/516
---
 .../ProcessedExpressionDataVectorDaoImpl.java | 34 +++++++------------
 1 file changed, 13 insertions(+), 21 deletions(-)

diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/bioAssayData/ProcessedExpressionDataVectorDaoImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/bioAssayData/ProcessedExpressionDataVectorDaoImpl.java
index a193d76a34..e36f84495c 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/bioAssayData/ProcessedExpressionDataVectorDaoImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/bioAssayData/ProcessedExpressionDataVectorDaoImpl.java
@@ -1053,32 +1053,24 @@ private Collection getProcessedVectors( Expressio
             // cannot fix this here, because we're read-only.
         }
 
-        /*
-         * To help ensure we get a good random set of items, we can do several queries with different random offsets.
-         */
-        //    int numSegments = 2;
-        //  int segmentSize = ( int ) Math.ceil( limit / numSegments );
-        int segmentSize = limit;
-//        if ( limit < numSegments ) {
-//            segmentSize = limit;
-//        }
-
         Query q = this.getSessionFactory().getCurrentSession()
                 .createQuery( " from ProcessedExpressionDataVector dedv "
                         + "where dedv.expressionExperiment = :ee and dedv.rankByMean > 0.5 order by RAND()" ); // order by rand() works?
         q.setParameter( "ee", ee );
-        q.setMaxResults( segmentSize );
+        q.setMaxResults( limit );
 
-        int k = 0;
-        //  while ( result.size() < limit ) {
-        //   int firstResult = new Random().nextInt( availableVectorCount - segmentSize );
-        //  q.setFirstResult( firstResult );
         List list = q.list();
-        //   log.info( list.size() + " retrieved this time firstResult=" + 0 );
+
+        if ( list.isEmpty() ) { // maybe ranks are not set for some reason. 
+            q = this.getSessionFactory().getCurrentSession()
+                    .createQuery( " from ProcessedExpressionDataVector dedv "
+                            + "where dedv.expressionExperiment = :ee order by RAND()" ); // order by rand() works?
+            q.setParameter( "ee", ee );
+            q.setMaxResults( limit );
+            list = q.list();
+        }
+
         result.addAll( list );
-        //   if (result.isEmpty()) break;
-        //  k++;
-        //   }
 
         if ( result.size() > limit ) {
             result = result.stream().limit( limit ).collect( Collectors.toSet() );
@@ -1086,10 +1078,10 @@ private Collection getProcessedVectors( Expressio
 
         if ( timer.getTime() > 1000 )
             AbstractDao.log
-                    .info( "Fetch " + result.size() + " vectors from " + ee.getShortName() + ": " + timer.getTime() + "ms, " + k + " queries were run." );
+                    .info( "Fetch " + result.size() + " vectors from " + ee.getShortName() + ": " + timer.getTime() + "ms "   );
 
         if ( result.isEmpty() ) {
-            AbstractDao.log.warn( "Experiment does not have any processed data vectors" );
+            AbstractDao.log.warn( "Experiment does not have any processed data vectors to display? " + ee );
             return result;
         }
 

From c465b7337199dbb34bd1cd0fadb0b42e28c65f43 Mon Sep 17 00:00:00 2001
From: Paul Pavlidis 
Date: Wed, 15 May 2024 11:59:39 -0700
Subject: [PATCH 53/61] minor

---
 .../bioAssayData/ProcessedExpressionDataVectorDaoImpl.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/bioAssayData/ProcessedExpressionDataVectorDaoImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/bioAssayData/ProcessedExpressionDataVectorDaoImpl.java
index e36f84495c..6f35560787 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/bioAssayData/ProcessedExpressionDataVectorDaoImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/bioAssayData/ProcessedExpressionDataVectorDaoImpl.java
@@ -1061,10 +1061,10 @@ private Collection getProcessedVectors( Expressio
 
         List list = q.list();
 
-        if ( list.isEmpty() ) { // maybe ranks are not set for some reason. 
+        if ( list.isEmpty() ) { // maybe ranks are not set for some reason; can happen e.g. GeneSpring mangled data.
             q = this.getSessionFactory().getCurrentSession()
                     .createQuery( " from ProcessedExpressionDataVector dedv "
-                            + "where dedv.expressionExperiment = :ee order by RAND()" ); // order by rand() works?
+                            + "where dedv.expressionExperiment = :ee order by RAND()" );
             q.setParameter( "ee", ee );
             q.setMaxResults( limit );
             list = q.list();

From 12b4b53ee8b5326444f684df54af22852a72e247 Mon Sep 17 00:00:00 2001
From: Paul Pavlidis 
Date: Wed, 15 May 2024 12:32:17 -0700
Subject: [PATCH 54/61] somehow I didn't quite finish
 https://github.com/PavlidisLab/Gemma/issues/1070

---
 .../api/visualization/VisualizationWidget.js  | 88 ++++++++++---------
 1 file changed, 45 insertions(+), 43 deletions(-)

diff --git a/gemma-web/src/main/webapp/scripts/api/visualization/VisualizationWidget.js b/gemma-web/src/main/webapp/scripts/api/visualization/VisualizationWidget.js
index 5be676f80e..2f5d107560 100755
--- a/gemma-web/src/main/webapp/scripts/api/visualization/VisualizationWidget.js
+++ b/gemma-web/src/main/webapp/scripts/api/visualization/VisualizationWidget.js
@@ -47,13 +47,13 @@ Gemma.DataVectorThumbnailsView = Ext.extend( Ext.DataView, {
    name : "vectorDV",
 
    singleSelect : true,
-   showPValues: true,
+   showPValues : true,
 
    itemSelector : 'div.vizWrap',
 
    /**
     * The data get from the server is not compatible with flotr out-of-the box. A little transformation is needed.
-    * 
+    *
     * @param data for one record.
     * @memberOf Gemma.DataVectorThumbnailsView
     */
@@ -64,7 +64,7 @@ Gemma.DataVectorThumbnailsView = Ext.extend( Ext.DataView, {
    /**
     * Gets the selected node's record; or, if no node is selected, returns the first record; or null if there are no
     * nodes.
-    * 
+    *
     * @return {record}
     */
    getSelectedOrFirst : function() {
@@ -105,7 +105,7 @@ Gemma.prepareProfiles = function( data, showPValues ) {
    var preparedData = [];
    var geneExpressionProfile = data.profiles;
 
-   for (var i = 0; i < geneExpressionProfile.length; i++) {
+   for ( var i = 0; i < geneExpressionProfile.length; i++ ) {
       var profile = geneExpressionProfile[i].profile;
 
       var probeId = geneExpressionProfile[i].probe.id;
@@ -133,7 +133,7 @@ Gemma.prepareProfiles = function( data, showPValues ) {
       if ( genes !== undefined && genes.length > 0 ) {
          var k, gene, link, geneName;
 
-         for (k = 0; k < genes.length; k++) {
+         for ( k = 0; k < genes.length; k++ ) {
             gene = genes[k];
             geneName = genes[k].officialName;
             link = ''
@@ -158,7 +158,7 @@ Gemma.prepareProfiles = function( data, showPValues ) {
        * Turn a flat vector into an array of points (that's what flotr needs)
        */
       var points = [];
-      for (var j = 0; j < profile.length; j++) {
+      for ( var j = 0; j < profile.length; j++ ) {
          var point = [ j, profile[j] ];
          points.push( point );
       }
@@ -251,7 +251,7 @@ Gemma.ProfileTemplate = Ext.extend( Ext.XTemplate, {
 
       Gemma.ProfileTemplate.superclass.overwrite.call( this, el, values, ret );
 
-      for (var i = 0; i < values.length; i++) {
+      for ( var i = 0; i < values.length; i++ ) {
          var randomnumber = Math.floor( Math.random() * 101 );
          var record = values[i];
          var shortName = record.eevo.id; // can be a subset, which has no shortName.
@@ -284,13 +284,13 @@ Gemma.HeatmapTemplate = Ext.extend( Ext.XTemplate, {
       forceFit : true,
       maxBoxHeight : 3,
       allowTargetSizeAdjust : true
-   // make the rows smaller in the thumbnails
+      // make the rows smaller in the thumbnails
    },
 
    overwrite : function( el, values, ret ) {
       Gemma.HeatmapTemplate.superclass.overwrite.call( this, el, values, ret );
 
-      for (var i = 0; i < values.length; i++) {
+      for ( var i = 0; i < values.length; i++ ) {
          var randomnumber = Math.floor( Math.random() * 101 );
          var record = values[i];
          var shortName = record.eevo.id;
@@ -329,18 +329,18 @@ Gemma.getProfileThumbnailTemplate = function( heatmap, havePvalues, smooth, cmpI
       var tmpl = new Gemma.HeatmapTemplate(
          '',
          '
 {shortName}:  {[Ext.util.Format.ellipsis( values.name, Gemma.MAX_EE_NAME_LENGTH)]}    '
-            + pvalueString + '
', '' );
+         + cmpID
+         + '" style="cursor:pointer;float:left;padding: 10px"> {shortName}:  {[Ext.util.Format.ellipsis( values.name, Gemma.MAX_EE_NAME_LENGTH)]}    '
+         + pvalueString + '', '' );
       tmpl.cmpID = cmpID;
       return tmpl;
    } else {
       var tmpl = new Gemma.ProfileTemplate(
          '',
          '
  {shortName}:  {[Ext.util.Format.ellipsis( values.name, Gemma.MAX_EE_NAME_LENGTH)]}     '
-            + pvalueString + '
', '', {
+         + cmpID
+         + '" style="cursor:pointer;float:left;padding: 10px">  {shortName}:  {[Ext.util.Format.ellipsis( values.name, Gemma.MAX_EE_NAME_LENGTH)]}     '
+         + pvalueString + '', '', {
             smooth : smooth
          } );
       tmpl.cmpID = cmpID;
@@ -424,9 +424,11 @@ Gemma.VisualizationZoomPanel = Ext
             }
 
             if ( eevo ) {
+               var eeInfoTitle = "";
 
-               var eeInfoTitle = ""
                   + eevo.shortName
                   + " ("
@@ -471,7 +473,7 @@ Gemma.VisualizationZoomPanel = Ext
                      return k[0] + "" + Ext.util.Format.ellipsis( k[1], Gemma.MAX_THUMBNAILLABEL_LENGTH_CHAR );
                   },
                   position : "sw" // best to be west, if we're expanded...applies
-               // to linecharts.
+                  // to linecharts.
                },
                conditionLegend : false,
                label : true
@@ -522,7 +524,7 @@ Gemma.VisualizationZoomPanel = Ext
                      var conditionLabels = record.get( "factorValuesToNames" );
                      var conditionLabelKey = record.get( "factorNames" );
 
-                     for (var i = 0; i < profiles.length; i++) {
+                     for ( var i = 0; i < profiles.length; i++ ) {
 
                         if ( profiles[i].labelID == probeId ) {
                            if ( profiles[i].selected ) {
@@ -570,7 +572,7 @@ Gemma.VisualizationWithThumbsWindow = Ext.extend( Ext.Window, {
          delete config.title;
       }
       var panelConfigParam = {
-          havePvalues: true
+         havePvalues : true
       };
 
       // add extra config params to panel
@@ -591,7 +593,7 @@ Gemma.VisualizationWithThumbsWindow = Ext.extend( Ext.Window, {
    },
 
    /**
-    * 
+    *
     */
    initComponent : function() {
 
@@ -704,9 +706,9 @@ Gemma.VisualizationWithThumbsPanel = Ext.extend( Ext.Panel, {
    getReturnedGeneCount : function( records ) {
       var returnedGeneIds = {};
       var returnedGeneCount = 0;
-      for (var i = 0; i < records.length; i++) {
-         for (var j = 0; j < records[i].get( 'profiles' ).length; j++) {
-            for (var k = 0; k < records[i].get( 'profiles' )[j].genes.length; k++) {
+      for ( var i = 0; i < records.length; i++ ) {
+         for ( var j = 0; j < records[i].get( 'profiles' ).length; j++ ) {
+            for ( var k = 0; k < records[i].get( 'profiles' )[j].genes.length; k++ ) {
                if ( returnedGeneIds[records[i].get( 'profiles' )[j].genes[k].id] === undefined ) {
                   returnedGeneIds[records[i].get( 'profiles' )[j].genes[k].id] = true;
                   returnedGeneCount++;
@@ -769,7 +771,7 @@ Gemma.VisualizationWithThumbsPanel = Ext.extend( Ext.Panel, {
 
       // force a refresh of the thumbnails.
       var template = Gemma.getProfileThumbnailTemplate( this.heatmapMode, this.havePvalues, /* this.smoothLineGraphs */
-      false, Ext.id() );
+         false, Ext.id() );
       this.dv.setTemplate( template );
 
       // force a refresh of the zoom.
@@ -795,7 +797,7 @@ Gemma.VisualizationWithThumbsPanel = Ext.extend( Ext.Panel, {
 
       // force a refresh of the thumbnails.
       var template = Gemma.getProfileThumbnailTemplate( this.heatmapMode, this.havePvalues, /* this.smoothLineGraphs */
-      false, Ext.id() );
+         false, Ext.id() );
       this.dv.setTemplate( template );
 
       // force a refresh of the zoom.
@@ -812,7 +814,7 @@ Gemma.VisualizationWithThumbsPanel = Ext.extend( Ext.Panel, {
 
    updateTemplate : function() {
       var template = Gemma.getProfileThumbnailTemplate( this.heatmapMode, this.havePvalues, /* this.smoothLineGraphs */
-      false, this.id );
+         false, this.id );
       this.dv.setTemplate( template ); // causes update of thumbnails.
    },
 
@@ -950,7 +952,7 @@ Gemma.VisualizationWithThumbsPanel = Ext.extend( Ext.Panel, {
 
             this.removeAll();
             var factorCount = 0;
-            for (factorCategory in conditionLabelKey) {
+            for ( factorCategory in conditionLabelKey ) {
                factorCount++;
                break;
             }
@@ -1034,7 +1036,7 @@ Gemma.VisualizationWithThumbsPanel = Ext.extend( Ext.Panel, {
                ref : 'toggleViewBtn',
                disabled : true,
                handler : this.switchView.createDelegate( this )
-            }]
+            } ]
          } )
       } );
 
@@ -1141,7 +1143,7 @@ Gemma.VisualizationDifferentialWindow = Ext.extend( Gemma.VisualizationWithThumb
 
 /**
  * Specialization for coexpression display
- * 
+ *
  * @class Gemma.CoexpressionVisualizationWindow
  * @extends Gemma.VisualizationWithThumbsWindow
  */
@@ -1156,7 +1158,7 @@ Gemma.CoexpressionVisualizationWindow = Ext.extend( Gemma.VisualizationWithThumb
 
 /**
  * Represents a VisualizationValueObject.
- * 
+ *
  * @extends Ext.data.Store
  */
 Gemma.VisualizationStore = function( config ) {
@@ -1203,7 +1205,7 @@ Gemma.VisualizationStore = function( config ) {
 };
 
 /**
- * 
+ *
  * @class Gemma.VisualizationStore
  * @extends Ext.data.Store
  */
@@ -1218,7 +1220,7 @@ Ext.extend( Gemma.VisualizationStore, Ext.data.Store, {} );
  * Sort in this order: 1. Query probes that show coexpression or sig. diff ex., ordered by pvalue 2. Target probes that
  * show coexp or sig diff ex. 3. Query probes that do not show coexpression or sig. diff ex. (faded) 4. Target probes
  * that do not show coexp or sig diff ex.
- * 
+ *
  * @param {}
  *           a
  * @param {}
@@ -1298,9 +1300,9 @@ var FactorValueLegend = (function() {
          Ext.DomHelper.append( target, {
             id : id,
             tag : 'div'
-         // ,
-         // width: legendWidth,
-         // height: legendHeight
+            // ,
+            // width: legendWidth,
+            // height: legendHeight
          } );
          var legendDiv = Ext.get( id );
 
@@ -1315,13 +1317,13 @@ var FactorValueLegend = (function() {
          // var ctxDummy = this.el.dom.getContext("2d");
          // CanvasTextFunctions.enable(ctxDummy);
          // ctxDummy.font = fontSize + "px sans-serif";
-         for ( var factorCategory in conditionLabelKey) {
+         for ( var factorCategory in conditionLabelKey ) {
             factorCount++;
             // compute the room needed for the labels.
             if ( ctx.measureText( factorCategory ).width > maxColumnWidth ) {
                maxColumnWidth = ctx.measureText( factorCategory ).width;
             }
-            for ( var factorValue in conditionLabelKey[factorCategory]) {
+            for ( var factorValue in conditionLabelKey[factorCategory] ) {
                factorValueCount++;
                // compute the room needed for the labels.
                var dim = ctx.measureText( factorValue );
@@ -1347,7 +1349,7 @@ var FactorValueLegend = (function() {
          ctx.translate( 10, 20 );
          x = 0;
          y = 0;
-         for ( var factorCategory in conditionLabelKey) {
+         for ( var factorCategory in conditionLabelKey ) {
             facCat = Ext.util.Format.ellipsis( factorCategory, FACTOR_VALUE_LABEL_MAX_CHAR );
             var maxLabelWidthInCategory = 0;
             dim = ctx.measureText( facCat );
@@ -1357,7 +1359,7 @@ var FactorValueLegend = (function() {
             }
             ctx.fillText( facCat, x, y );
             y += PER_CONDITION_LABEL_HEIGHT + 2;
-            for ( var factorValue in conditionLabelKey[factorCategory]) {
+            for ( var factorValue in conditionLabelKey[factorCategory] ) {
                var facVal = Ext.util.Format.ellipsis( factorValue, FACTOR_VALUE_LABEL_MAX_CHAR );
                colour = conditionLabelKey[factorCategory][factorValue];
                ctx.fillStyle = colour;
@@ -1382,11 +1384,11 @@ var FactorValueLegend = (function() {
 
       /**
        * Function: (private) constructCanvas
-       * 
+       *
        * Initializes a canvas. When the browser is IE, we make use of excanvas.
-       * 
+       *
        * Parameters: none
-       * 
+       *
        * Returns: ctx
        */
       function constructCanvas( div, canvasWidth, canvasHeight ) {

From 33535d579c7c1be0c8d88566a6c051efcf78dfd5 Mon Sep 17 00:00:00 2001
From: Guillaume Poirier-Morency 
Date: Wed, 15 May 2024 17:10:12 -0700
Subject: [PATCH 55/61] Separate build and package stages

Packaging involves generating javadocs JARs which is time-consuming and
pointless if the quick unit tests fail.
---
 .jenkins/Jenkinsfile | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/.jenkins/Jenkinsfile b/.jenkins/Jenkinsfile
index cf66ad7e89..091651641f 100644
--- a/.jenkins/Jenkinsfile
+++ b/.jenkins/Jenkinsfile
@@ -61,11 +61,10 @@ pipeline {
         stage('Build') {
             steps {
                 setBuildStatus "Build", "Build #${env.BUILD_NUMBER} has started...", 'PENDING'
-                sh 'mvn -B package -DskipTests'
+                sh 'mvn -B compile'
             }
             post {
                 success {
-                    archiveArtifacts artifacts: '**/target/*.jar, **/target/*.war', fingerprint: true
                     setBuildStatus "Build", "Build #${env.BUILD_NUMBER} succeeded.", 'SUCCESS'
                 }
                 unsuccessful {
@@ -90,6 +89,21 @@ pipeline {
                 }
             }
         }
+        stage('Package') {
+            steps {
+                setBuildStatus "Package", "Package for build #${env.BUILD_NUMBER} have started...", 'PENDING'
+                sh 'mvn -B package -DskipTests'
+            }
+            post {
+                success {
+                    archiveArtifacts artifacts: '**/target/*.jar, **/target/*.war', fingerprint: true
+                    setBuildStatus "Package", "Package for build #${env.BUILD_NUMBER} have passed.", 'SUCCESS'
+                }
+                unsuccessful {
+                    setBuildStatus "Package", "Package for build #${env.BUILD_NUMBER} failed.", 'FAILURE'
+                }
+            }
+        }
         stage('Run integration tests and perform deployment in parallel') {
             when {
                 anyOf {

From 1a5bb07d61b07fdda36b7ebbdc8a9bd458490401 Mon Sep 17 00:00:00 2001
From: Guillaume Poirier-Morency 
Date: Thu, 16 May 2024 11:30:06 -0700
Subject: [PATCH 56/61] Add null defaults for BuildInfo injected values

---
 .../java/ubic/gemma/core/util/BuildInfo.java  |  7 ++--
 .../gemma/core/util/MissingBuildInfoTest.java | 38 +++++++++++++++++++
 2 files changed, 42 insertions(+), 3 deletions(-)
 create mode 100644 gemma-core/src/test/java/ubic/gemma/core/util/MissingBuildInfoTest.java

diff --git a/gemma-core/src/main/java/ubic/gemma/core/util/BuildInfo.java b/gemma-core/src/main/java/ubic/gemma/core/util/BuildInfo.java
index 3faf3023ae..b586fe436f 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/util/BuildInfo.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/util/BuildInfo.java
@@ -1,6 +1,7 @@
 package ubic.gemma.core.util;
 
 import lombok.extern.apachecommons.CommonsLog;
+import org.apache.commons.lang.StringUtils;
 import org.springframework.beans.factory.InitializingBean;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Configuration;
@@ -40,11 +41,11 @@ public static BuildInfo fromClasspath() {
         }
     }
 
-    @Value("${gemma.version}")
+    @Value("${gemma.version:#{null}}")
     private String version;
-    @Value("${gemma.build.timestamp}")
+    @Value("${gemma.build.timestamp:#{null}}")
     private String timestampAsString;
-    @Value("${gemma.build.gitHash}")
+    @Value("${gemma.build.gitHash:#{null}}")
     private String gitHash;
 
     private Date timestamp;
diff --git a/gemma-core/src/test/java/ubic/gemma/core/util/MissingBuildInfoTest.java b/gemma-core/src/test/java/ubic/gemma/core/util/MissingBuildInfoTest.java
new file mode 100644
index 0000000000..1edc10855c
--- /dev/null
+++ b/gemma-core/src/test/java/ubic/gemma/core/util/MissingBuildInfoTest.java
@@ -0,0 +1,38 @@
+package ubic.gemma.core.util;
+
+import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Import;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.junit4.AbstractJUnit4SpringContextTests;
+import ubic.gemma.core.util.test.TestPropertyPlaceholderConfigurer;
+import ubic.gemma.persistence.util.TestComponent;
+
+import static org.junit.Assert.assertNull;
+
+@ContextConfiguration
+public class MissingBuildInfoTest extends AbstractJUnit4SpringContextTests {
+
+    @Import(BuildInfo.class)
+    @Configuration
+    @TestComponent
+    static class BuildInfoContextConfiguration {
+
+        @Bean
+        public static TestPropertyPlaceholderConfigurer testPropertyPlaceholderConfigurer() {
+            return new TestPropertyPlaceholderConfigurer();
+        }
+    }
+
+    @Autowired
+    private BuildInfo buildInfo;
+
+    @Test
+    public void test() {
+        assertNull( buildInfo.getVersion() );
+        assertNull( buildInfo.getTimestamp() );
+        assertNull( buildInfo.getGitHash() );
+    }
+}

From be2a931be6d82d72d08dc7138f63e12d532a97a9 Mon Sep 17 00:00:00 2001
From: Guillaume Poirier-Morency 
Date: Thu, 16 May 2024 11:56:58 -0700
Subject: [PATCH 57/61] Update baseCode to 1.1.23

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 2767275438..ff4f8c2129 100644
--- a/pom.xml
+++ b/pom.xml
@@ -140,7 +140,7 @@
 		
 			baseCode
 			baseCode
-			1.1.23-SNAPSHOT
+			1.1.23
 		
 
 		

From de3a3df34419a92d43de84525bbd41413d6aa495 Mon Sep 17 00:00:00 2001
From: Guillaume Poirier-Morency 
Date: Thu, 16 May 2024 11:59:14 -0700
Subject: [PATCH 58/61] Update dependencies

---
 gemma-core/pom.xml |  2 +-
 pom.xml            | 11 ++++++-----
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/gemma-core/pom.xml b/gemma-core/pom.xml
index 6323dc48f0..a45b12a6f7 100644
--- a/gemma-core/pom.xml
+++ b/gemma-core/pom.xml
@@ -279,7 +279,7 @@
         
             org.apache.commons
             commons-csv
-            1.10.0
+            1.11.0
         
         
             org.apache.commons
diff --git a/pom.xml b/pom.xml
index ff4f8c2129..2c0a72df46 100644
--- a/pom.xml
+++ b/pom.xml
@@ -185,7 +185,7 @@
 		
 			org.aspectj
 			aspectjweaver
-			1.9.22
+			1.9.22.1
 		
 		
 			org.springframework
@@ -310,6 +310,7 @@
 		
 			commons-logging
 			commons-logging
+			1.3.2
 		
 		
 			commons-logging
@@ -708,12 +709,12 @@
 		3.2.18.RELEASE
 		3.2.10.RELEASE
 		2.25.1
-		2.17.0
-		2.2.21
+		2.17.1
+		2.2.22
 		3.9
 		3.6.2
-		1.39.0
-		1.12.5
+		1.39.2
+		1.13.0
 		2.1.4
 		8.4.0
 		

From 991515f63b36aafae0759f154e21ac81fbc616ec Mon Sep 17 00:00:00 2001
From: Guillaume Poirier-Morency 
Date: Thu, 16 May 2024 12:11:14 -0700
Subject: [PATCH 59/61] rest: Update changelogs

---
 gemma-rest/src/main/resources/restapidocs/CHANGELOG.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/gemma-rest/src/main/resources/restapidocs/CHANGELOG.md b/gemma-rest/src/main/resources/restapidocs/CHANGELOG.md
index e5bd861fe5..754f450b51 100644
--- a/gemma-rest/src/main/resources/restapidocs/CHANGELOG.md
+++ b/gemma-rest/src/main/resources/restapidocs/CHANGELOG.md
@@ -3,8 +3,11 @@
 ### Update 2.7.5
 
 - fix a bug in `getTaxonDatasets` sorting parameter, it was indicating `Taxon` instead of `ExpressionExperiment`
-- disambiguate all endpoints that expect a gene identifier with a taxon argument
+- disambiguate all endpoints that expect a gene identifier with a taxon argument, the previous endpoints still exist but
+  will now raise `400 Bad Request` when an ambiguous identifier is supplied instead of returning an arbitrary result
 - add endpoints to retrieve all genes with pagination
+- merge `getResultSets` and `getResultSetsAsTsv` endpoints in the OpenAPI specification
+- add support for offset/limit and threshold arguments for retrieving DE results and retaining most significant probes
 
 ### Update 2.7.4
 

From 3cae4e7aac7b7d7ae603c97e66fb9910e619157a Mon Sep 17 00:00:00 2001
From: Guillaume Poirier-Morency 
Date: Thu, 16 May 2024 12:35:51 -0700
Subject: [PATCH 60/61] Fix double load in UserGroupDaoImpl

---
 .../authentication/UserServiceImpl.java       | 79 ++++++++++---------
 .../security/authentication/package-info.java |  7 ++
 .../common/auditAndSecurity/UserGroupDao.java |  5 --
 .../auditAndSecurity/UserGroupDaoImpl.java    | 73 +++++------------
 .../authentication/UserServiceImplTest.java   | 63 +++++++++++++--
 5 files changed, 122 insertions(+), 105 deletions(-)
 create mode 100644 gemma-core/src/main/java/ubic/gemma/core/security/authentication/package-info.java

diff --git a/gemma-core/src/main/java/ubic/gemma/core/security/authentication/UserServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/security/authentication/UserServiceImpl.java
index 1ad65f06c7..fd52d1ea02 100755
--- a/gemma-core/src/main/java/ubic/gemma/core/security/authentication/UserServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/security/authentication/UserServiceImpl.java
@@ -20,12 +20,14 @@
 import gemma.gsec.acl.domain.AclService;
 import gemma.gsec.authentication.UserExistsException;
 import gemma.gsec.util.SecurityUtil;
+import org.apache.commons.lang.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.dao.DataIntegrityViolationException;
 import org.springframework.dao.InvalidDataAccessResourceUsageException;
 import org.springframework.security.access.AccessDeniedException;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
+import ubic.gemma.model.common.auditAndSecurity.GroupAuthority;
 import ubic.gemma.model.common.auditAndSecurity.User;
 import ubic.gemma.model.common.auditAndSecurity.UserGroup;
 import ubic.gemma.persistence.service.common.auditAndSecurity.UserDao;
@@ -34,18 +36,19 @@
 import java.util.ArrayList;
 import java.util.Collection;
 
+import static java.util.Objects.requireNonNull;
+
 /**
  * @author pavlidis
  */
-@SuppressWarnings("CollectionAddAllCanBeReplacedWithConstructor") // Not possible due to type safety
 @Service
 public class UserServiceImpl implements UserService {
 
     @Autowired
-    UserDao userDao;
+    private UserDao userDao;
 
     @Autowired
-    UserGroupDao userGroupDao;
+    private UserGroupDao userGroupDao;
 
     @Autowired
     private AclService aclService;
@@ -56,26 +59,35 @@ public class UserServiceImpl implements UserService {
     @Override
     @Transactional
     public void addGroupAuthority( gemma.gsec.model.UserGroup group, String authority ) {
-        this.userGroupDao.addAuthority( ( ubic.gemma.model.common.auditAndSecurity.UserGroup ) group, authority );
+        group = requireNonNull( userGroupDao.load( group.getId() ) );
+        for ( gemma.gsec.model.GroupAuthority ga : group.getAuthorities() ) {
+            if ( ga.getAuthority().equals( authority ) ) {
+                return;
+            }
+        }
+        GroupAuthority ga = GroupAuthority.Factory.newInstance();
+        ga.setAuthority( authority );
+        group.getAuthorities().add( ga );
+        update( group );
     }
 
     @Override
     @Transactional
     public void addUserToGroup( gemma.gsec.model.UserGroup group, gemma.gsec.model.User user ) {
+        group = requireNonNull( userGroupDao.load( group.getId() ) );
+        user = requireNonNull( userDao.load( user.getId() ) );
         // add user to list of members
         group.getGroupMembers().add( user );
-        this.userGroupDao.update( ( ubic.gemma.model.common.auditAndSecurity.UserGroup ) group );
     }
 
     @Override
     @Transactional
     public User create( final gemma.gsec.model.User user ) throws UserExistsException {
-
-        if ( user.getUserName() == null ) {
-            throw new IllegalArgumentException( "UserName cannot be null" );
+        if ( StringUtils.isBlank( user.getUserName() ) ) {
+            throw new IllegalArgumentException( "Username cannot be blank" );
         }
 
-        if ( this.userDao.findByUserName( user.getUserName() ) != null ) {
+        if ( this.findByUserName( user.getUserName() ) != null ) {
             throw new UserExistsException( "User '" + user.getUserName() + "' already exists!" );
         }
 
@@ -84,38 +96,34 @@ public User create( final gemma.gsec.model.User user ) throws UserExistsExceptio
         }
 
         try {
-            return this.userDao.create( ( ubic.gemma.model.common.auditAndSecurity.User ) user );
+            return this.userDao.create( ( User ) user );
         } catch ( DataIntegrityViolationException | InvalidDataAccessResourceUsageException e ) {
             throw new UserExistsException( "User '" + user.getUserName() + "' already exists!" );
         }
-
     }
 
     @Override
     @Transactional
     public UserGroup create( gemma.gsec.model.UserGroup group ) {
-        return this.userGroupDao.create( ( ubic.gemma.model.common.auditAndSecurity.UserGroup ) group );
+        return this.userGroupDao.create( ( UserGroup ) group );
     }
 
     @Override
     @Transactional
     public void delete( gemma.gsec.model.User user ) {
-        for ( UserGroup group : this.userDao.loadGroups( ( ubic.gemma.model.common.auditAndSecurity.User ) user ) ) {
+        user = requireNonNull( userDao.load( user.getId() ), "No user with ID: " + user.getId() );
+        for ( UserGroup group : this.userDao.loadGroups( ( User ) user ) ) {
             group.getGroupMembers().remove( user );
-            this.userGroupDao.update( ( ubic.gemma.model.common.auditAndSecurity.UserGroup ) group );
         }
-
-        this.userDao.remove( ( ubic.gemma.model.common.auditAndSecurity.User ) user );
+        this.userDao.remove( ( User ) user );
     }
 
     @Override
     @Transactional
     public void delete( gemma.gsec.model.UserGroup group ) {
-        String groupName = group.getName();
+        group = requireNonNull( userGroupDao.load( group.getId() ), "No group with that name: " + group.getName() );
 
-        if ( !this.groupExists( groupName ) ) {
-            throw new IllegalArgumentException( "No group with that name: " + groupName );
-        }
+        String groupName = group.getName();
 
         /*
          * make sure this isn't one of the special groups
@@ -169,17 +177,13 @@ public boolean groupExists( String name ) {
     @Override
     @Transactional(readOnly = true)
     public Collection findGroupsForUser( gemma.gsec.model.User user ) {
-        Collection ret = new ArrayList<>();
-        ret.addAll( this.userGroupDao.findGroupsForUser( ( ubic.gemma.model.common.auditAndSecurity.User ) user ) );
-        return ret;
+        return new ArrayList<>( this.userGroupDao.findGroupsForUser( ( User ) user ) );
     }
 
     @Override
     @Transactional(readOnly = true)
     public Collection listAvailableGroups() {
-        Collection ret = new ArrayList<>();
-        ret.addAll( this.userGroupDao.loadAll() );
-        return ret;
+        return new ArrayList<>( this.userGroupDao.loadAll() );
     }
 
     @Override
@@ -191,28 +195,27 @@ public User load( final Long id ) {
     @Override
     @Transactional(readOnly = true)
     public Collection loadAll() {
-        Collection ret = new ArrayList<>();
-        ret.addAll( this.userDao.loadAll() );
-        return ret;
+        return new ArrayList<>( this.userDao.loadAll() );
     }
 
     @Override
     @Transactional(readOnly = true)
     public Collection loadGroupAuthorities( gemma.gsec.model.User user ) {
-        Collection ret = new ArrayList<>();
-        ret.addAll( this.userDao.loadGroupAuthorities( ( ubic.gemma.model.common.auditAndSecurity.User ) user ) );
-        return ret;
+        return new ArrayList<>( this.userDao.loadGroupAuthorities( ( User ) user ) );
     }
 
     @Override
+    @Transactional
     public void removeGroupAuthority( gemma.gsec.model.UserGroup group, String authority ) {
-        this.userGroupDao.removeAuthority( ( ubic.gemma.model.common.auditAndSecurity.UserGroup ) group, authority );
+        group = requireNonNull( userGroupDao.load( group.getId() ) );
+        group.getAuthorities().removeIf( ga -> ga.getAuthority().equals( authority ) );
     }
 
     @Override
     @Transactional
     public void removeUserFromGroup( gemma.gsec.model.User user, gemma.gsec.model.UserGroup group ) {
-        group.getGroupMembers().remove( user );
+        group = requireNonNull( userGroupDao.load( group.getId() ) );
+        user = requireNonNull( userDao.load( user.getId() ) );
 
         String userName = user.getUserName();
         String groupName = group.getName();
@@ -225,7 +228,8 @@ public void removeUserFromGroup( gemma.gsec.model.User user, gemma.gsec.model.Us
         if ( AuthorityConstants.USER_GROUP_NAME.equals( groupName ) ) {
             throw new IllegalArgumentException( "You cannot remove users from the USER group!" );
         }
-        this.userGroupDao.update( ( ubic.gemma.model.common.auditAndSecurity.UserGroup ) group );
+
+        group.getGroupMembers().remove( user );
 
         /*
          * TODO: if the group is empty, should we remove it? Not if it is GROUP_USER or ADMIN, but perhaps otherwise.
@@ -235,13 +239,12 @@ public void removeUserFromGroup( gemma.gsec.model.User user, gemma.gsec.model.Us
     @Override
     @Transactional
     public void update( final gemma.gsec.model.User user ) {
-        this.userDao.update( ( ubic.gemma.model.common.auditAndSecurity.User ) user );
+        this.userDao.update( ( User ) user );
     }
 
     @Override
     @Transactional
     public void update( gemma.gsec.model.UserGroup group ) {
-        this.userGroupDao.update( ( ubic.gemma.model.common.auditAndSecurity.UserGroup ) group );
+        this.userGroupDao.update( ( UserGroup ) group );
     }
-
 }
diff --git a/gemma-core/src/main/java/ubic/gemma/core/security/authentication/package-info.java b/gemma-core/src/main/java/ubic/gemma/core/security/authentication/package-info.java
new file mode 100644
index 0000000000..9865bfb2a4
--- /dev/null
+++ b/gemma-core/src/main/java/ubic/gemma/core/security/authentication/package-info.java
@@ -0,0 +1,7 @@
+/**
+ * This package contains implementation of gsec {@link gemma.gsec.authentication.UserService} and {@link gemma.gsec.authentication.UserManager}.
+ */
+@ParametersAreNonnullByDefault
+package ubic.gemma.core.security.authentication;
+
+import javax.annotation.ParametersAreNonnullByDefault;
\ No newline at end of file
diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/common/auditAndSecurity/UserGroupDao.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/common/auditAndSecurity/UserGroupDao.java
index 20e44e9161..b2632dba8c 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/common/auditAndSecurity/UserGroupDao.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/common/auditAndSecurity/UserGroupDao.java
@@ -29,12 +29,7 @@
  */
 public interface UserGroupDao extends BaseDao {
 
-    void addAuthority( UserGroup group, String authority );
-
     UserGroup findByName( java.lang.String name );
 
     Collection findGroupsForUser( User user );
-
-    void removeAuthority( UserGroup group, String authority );
-
 }
diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/common/auditAndSecurity/UserGroupDaoImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/common/auditAndSecurity/UserGroupDaoImpl.java
index b3c17c8f9b..2096f75ed4 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/common/auditAndSecurity/UserGroupDaoImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/common/auditAndSecurity/UserGroupDaoImpl.java
@@ -19,17 +19,16 @@
 package ubic.gemma.persistence.service.common.auditAndSecurity;
 
 import gemma.gsec.AuthorityConstants;
+import org.apache.commons.lang3.ArrayUtils;
 import org.hibernate.SessionFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Repository;
-import ubic.gemma.model.common.auditAndSecurity.GroupAuthority;
+import org.springframework.util.Assert;
 import ubic.gemma.model.common.auditAndSecurity.User;
 import ubic.gemma.model.common.auditAndSecurity.UserGroup;
 import ubic.gemma.persistence.service.AbstractDao;
 
 import java.util.Collection;
-import java.util.Iterator;
-import java.util.Objects;
 
 /**
  * @see ubic.gemma.model.common.auditAndSecurity.UserGroup
@@ -37,22 +36,24 @@
 @Repository
 public class UserGroupDaoImpl extends AbstractDao implements UserGroupDao {
 
+    private static final String[] PROTECTED_GROUP_NAMES = {
+            AuthorityConstants.USER_GROUP_NAME,
+            AuthorityConstants.ADMIN_GROUP_NAME,
+            AuthorityConstants.AGENT_GROUP_NAME
+    };
+
     @Autowired
     public UserGroupDaoImpl( SessionFactory sessionFactory ) {
         super( UserGroup.class, sessionFactory );
     }
 
     @Override
-    public void addAuthority( UserGroup group, String authority ) {
-        for ( gemma.gsec.model.GroupAuthority ga : group.getAuthorities() ) {
-            if ( ga.getAuthority().equals( authority ) ) {
-                return;
-            }
+    public UserGroup find( UserGroup entity ) {
+        if ( entity.getId() != null ) {
+            return super.find( entity );
+        } else {
+            return this.findByName( entity.getName() );
         }
-        GroupAuthority ga = GroupAuthority.Factory.newInstance();
-        ga.setAuthority( authority );
-        group.getAuthorities().add( ga );
-        super.update( group );
     }
 
     @Override
@@ -70,60 +71,24 @@ public Collection findGroupsForUser( User user ) {
                 .setParameter( "user", user ).list();
     }
 
-    @Override
-    public void removeAuthority( UserGroup group, String authority ) {
-        group.getAuthorities().removeIf( ga -> ga.getAuthority().equals( authority ) );
-        this.update( group );
-    }
-
     @Override
     public UserGroup create( final UserGroup userGroup ) {
-        if ( userGroup == null ) {
-            throw new IllegalArgumentException( "UserGroup.create - 'userGroup' can not be null" );
-        }
-        if ( userGroup.getName().equals( AuthorityConstants.USER_GROUP_NAME ) || userGroup.getName()
-                .equals( AuthorityConstants.ADMIN_GROUP_NAME ) || userGroup.getName()
-                .equals( AuthorityConstants.AGENT_GROUP_NAME ) ) {
-            throw new IllegalArgumentException( "Cannot create group with that name: " + userGroup.getName() );
-        }
+        Assert.isTrue( !ArrayUtils.contains( PROTECTED_GROUP_NAMES, userGroup.getName() ),
+                "Cannot create group with name: " + userGroup.getName() );
         return super.create( userGroup );
     }
 
     @Override
     public void remove( UserGroup userGroup ) {
-        // FIXME: this should not be necessary, but we have cases where the group are obtained from a different Hibernate
-        //  session
-        userGroup = Objects.requireNonNull( this.load( userGroup.getId() ),
-                String.format( "No UserGroup with ID %d.", userGroup.getId() ) );
-        // this check is done higher up as well...
-        if ( userGroup.getName().equals( AuthorityConstants.USER_GROUP_NAME ) || userGroup.getName()
-                .equals( AuthorityConstants.ADMIN_GROUP_NAME ) || userGroup.getName()
-                .equals( AuthorityConstants.AGENT_GROUP_NAME ) ) {
-            throw new IllegalArgumentException( "Cannot remove group: " + userGroup );
-        }
+        Assert.isTrue( !ArrayUtils.contains( PROTECTED_GROUP_NAMES, userGroup.getName() ),
+                "Cannot remove group with name: " + userGroup.getName() );
         super.remove( userGroup );
     }
 
     @Override
     public void update( UserGroup userGroup ) {
-        UserGroup groupToUpdate = Objects.requireNonNull( this.load( userGroup.getId() ),
-                String.format( "No UserGroup with ID %d.", userGroup.getId() ) );
-        String name = groupToUpdate.getName();
-        if ( !name.equals( userGroup.getName() ) && ( name.equals( AuthorityConstants.USER_GROUP_NAME ) || name
-                .equals( AuthorityConstants.ADMIN_GROUP_NAME ) || name
-                .equals( AuthorityConstants.AGENT_GROUP_NAME ) ) ) {
-            throw new IllegalArgumentException( "Cannot change name of group: " + groupToUpdate.getName() );
-        }
+        Assert.isTrue( !ArrayUtils.contains( PROTECTED_GROUP_NAMES, userGroup.getName() ),
+                "Cannot update group with name: " + userGroup.getName() );
         super.update( userGroup );
     }
-
-    @Override
-    public UserGroup find( UserGroup entity ) {
-        if ( entity.getId() != null ) {
-            return this.load( entity.getId() );
-        } else {
-            return this.findByName( entity.getName() );
-        }
-    }
-
 }
\ No newline at end of file
diff --git a/gemma-core/src/test/java/ubic/gemma/core/security/authentication/UserServiceImplTest.java b/gemma-core/src/test/java/ubic/gemma/core/security/authentication/UserServiceImplTest.java
index 85644f152f..21b39a1dc8 100755
--- a/gemma-core/src/test/java/ubic/gemma/core/security/authentication/UserServiceImplTest.java
+++ b/gemma-core/src/test/java/ubic/gemma/core/security/authentication/UserServiceImplTest.java
@@ -18,12 +18,21 @@
  */
 package ubic.gemma.core.security.authentication;
 
+import gemma.gsec.SecurityService;
+import gemma.gsec.acl.domain.AclService;
+import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.junit4.AbstractJUnit4SpringContextTests;
 import ubic.gemma.model.common.auditAndSecurity.User;
 import ubic.gemma.model.common.auditAndSecurity.UserGroup;
 import ubic.gemma.persistence.service.common.auditAndSecurity.UserDao;
 import ubic.gemma.persistence.service.common.auditAndSecurity.UserGroupDao;
+import ubic.gemma.persistence.util.TestComponent;
 
 import java.util.Collection;
 import java.util.HashSet;
@@ -33,18 +42,52 @@
 /**
  * @author pavlidis
  */
-public class UserServiceImplTest {
-    private final UserServiceImpl userService = new UserServiceImpl();
-    private final User testUser = User.Factory.newInstance();
+@ContextConfiguration
+public class UserServiceImplTest extends AbstractJUnit4SpringContextTests {
+
+    @Configuration
+    @TestComponent
+    static class UserServiceImplTestContextConfiguration {
+
+        @Bean
+        public UserService userService() {
+            return new UserServiceImpl();
+        }
+
+        @Bean
+        public UserDao userDao() {
+            return mock();
+        }
+
+        @Bean
+        public UserGroupDao userGroupDao() {
+            return mock();
+        }
+
+        @Bean
+        public AclService aclService() {
+            return mock();
+        }
+
+        @Bean
+        public SecurityService securityService() {
+            return mock();
+        }
+    }
+
+    @Autowired
+    private UserService userService;
+
+    @Autowired
     private UserDao userDaoMock;
+
+    private final User testUser = User.Factory.newInstance();
+
     private Collection userGroups;
 
     @Before
     public void setUp() {
-        userDaoMock = mock( UserDao.class );
-        userService.userDao = userDaoMock;
-
-        userService.userGroupDao = mock( UserGroupDao.class );
+        testUser.setId( 1L );
         testUser.setEmail( "foo@bar" );
         testUser.setName( "Foo" );
         testUser.setLastName( "Bar" );
@@ -57,7 +100,11 @@ public void setUp() {
         group.getGroupMembers().add( testUser );
         userGroups = new HashSet<>();
         userGroups.add( group );
+    }
 
+    @After
+    public void resetMocks() {
+        reset( userDaoMock );
     }
 
     @Test
@@ -77,9 +124,9 @@ public void testHandleSaveUser() throws Exception {
 
     @Test
     public void testHandleRemoveUser() {
+        when( userDaoMock.load( testUser.getId() ) ).thenReturn( testUser );
         when( userDaoMock.loadGroups( testUser ) ).thenReturn( userGroups );
         userService.delete( testUser );
         verify( userDaoMock ).remove( testUser );
     }
-
 }

From 58403aaa5c18b30d9e78ae2b9e1ec18061bb3187 Mon Sep 17 00:00:00 2001
From: Guillaume Poirier-Morency 
Date: Thu, 16 May 2024 13:38:46 -0700
Subject: [PATCH 61/61] Update for next development version

---
 gemma-cli/pom.xml            | 2 +-
 gemma-core/pom.xml           | 2 +-
 gemma-groovy-support/pom.xml | 2 +-
 gemma-rest/pom.xml           | 2 +-
 gemma-web/pom.xml            | 2 +-
 pom.xml                      | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/gemma-cli/pom.xml b/gemma-cli/pom.xml
index 8bd34c428a..2932061b74 100644
--- a/gemma-cli/pom.xml
+++ b/gemma-cli/pom.xml
@@ -3,7 +3,7 @@
     
         gemma
         gemma
-        1.31.6-SNAPSHOT
+        1.31.6
     
     4.0.0
     gemma-cli
diff --git a/gemma-core/pom.xml b/gemma-core/pom.xml
index a45b12a6f7..69000e26f1 100644
--- a/gemma-core/pom.xml
+++ b/gemma-core/pom.xml
@@ -3,7 +3,7 @@
     
         gemma
         gemma
-        1.31.6-SNAPSHOT
+        1.31.6
     
     4.0.0
     gemma-core
diff --git a/gemma-groovy-support/pom.xml b/gemma-groovy-support/pom.xml
index 34ffe5d10a..9430a60bff 100644
--- a/gemma-groovy-support/pom.xml
+++ b/gemma-groovy-support/pom.xml
@@ -6,7 +6,7 @@
     
         gemma
         gemma
-        1.31.6-SNAPSHOT
+        1.31.6
     
 
     gemma-groovy-support
diff --git a/gemma-rest/pom.xml b/gemma-rest/pom.xml
index 154c417244..a4f95cf1eb 100644
--- a/gemma-rest/pom.xml
+++ b/gemma-rest/pom.xml
@@ -5,7 +5,7 @@
     
         gemma
         gemma
-        1.31.6-SNAPSHOT
+        1.31.6
     
     4.0.0
 
diff --git a/gemma-web/pom.xml b/gemma-web/pom.xml
index b1c181e211..3e4ee3afe5 100644
--- a/gemma-web/pom.xml
+++ b/gemma-web/pom.xml
@@ -3,7 +3,7 @@
 	
 		gemma
 		gemma
-		1.31.6-SNAPSHOT
+		1.31.6
 	
 	4.0.0
 	gemma-web
diff --git a/pom.xml b/pom.xml
index 2c0a72df46..87637e84e8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -5,7 +5,7 @@
 	Gemma
 	gemma
 	gemma
-	1.31.6-SNAPSHOT
+	1.31.6
 	2005
 	The Gemma Project for meta-analysis of genomics data
 	https://gemma.msl.ubc.ca