From 3b304785a060d476a48fd6bd60bc818b3b36fc5d Mon Sep 17 00:00:00 2001 From: Guillaume Poirier-Morency Date: Wed, 30 Mar 2022 11:03:08 -0700 Subject: [PATCH 01/28] Update versions for hotfix --- gemma-core/pom.xml | 2 +- gemma-web/pom.xml | 2 +- pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/gemma-core/pom.xml b/gemma-core/pom.xml index fc3852e724..731e29480f 100644 --- a/gemma-core/pom.xml +++ b/gemma-core/pom.xml @@ -3,7 +3,7 @@ gemma gemma - 1.27.10 + 1.27.11 4.0.0 gemma-core diff --git a/gemma-web/pom.xml b/gemma-web/pom.xml index bd7732407e..23777ac3a9 100644 --- a/gemma-web/pom.xml +++ b/gemma-web/pom.xml @@ -3,7 +3,7 @@ gemma gemma - 1.27.10 + 1.27.11 4.0.0 gemma-web diff --git a/pom.xml b/pom.xml index 0ec4d92e0c..3b63a5a06d 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ Gemma gemma gemma - 1.27.10 + 1.27.11 2005 The Gemma Project for meta-analysis of genomics data https://gemma.msl.ubc.ca From 411d2465fcb7669271f0db1773b6fec375e3f755 Mon Sep 17 00:00:00 2001 From: Guillaume Poirier-Morency Date: Thu, 9 Dec 2021 14:32:06 -0800 Subject: [PATCH 02/28] Migrate to Apache Commons Collection 4 --- .../links/AbstractMatrixRowPairAnalysis.java | 2 +- .../expression/diff/LinearModelAnalyzer.java | 4 ++-- .../ArrayDesignAnnotationServiceImpl.java | 4 ++-- .../GoldenPathSequenceAnalysis.java | 2 +- .../AffyPowerToolsProbesetSummarize.java | 2 +- .../security/audit/AuditableUtilImpl.java | 18 ++++------------- .../pca/PrincipalComponentAnalysis.java | 15 +++----------- ...DifferentialExpressionAnalysisDaoImpl.java | 2 +- .../auditAndSecurity/AuditEventDaoImpl.java | 20 ++++--------------- .../gemma/persistence/util/CommonQueries.java | 2 +- .../ExpressionExperimentSetController.java | 9 ++++----- .../genome/gene/GeneSetController.java | 8 ++++---- pom.xml | 6 +++--- 13 files changed, 31 insertions(+), 63 deletions(-) diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/expression/coexpression/links/AbstractMatrixRowPairAnalysis.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/expression/coexpression/links/AbstractMatrixRowPairAnalysis.java index 71e5624f97..3f487fe84f 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/analysis/expression/coexpression/links/AbstractMatrixRowPairAnalysis.java +++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/expression/coexpression/links/AbstractMatrixRowPairAnalysis.java @@ -21,7 +21,7 @@ import cern.colt.bitvector.BitMatrix; import cern.colt.list.DoubleArrayList; import cern.colt.list.ObjectArrayList; -import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.time.StopWatch; import org.apache.commons.logging.Log; diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/expression/diff/LinearModelAnalyzer.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/expression/diff/LinearModelAnalyzer.java index 894cc8643a..b3807f4394 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/analysis/expression/diff/LinearModelAnalyzer.java +++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/expression/diff/LinearModelAnalyzer.java @@ -20,8 +20,8 @@ import cern.colt.matrix.DoubleMatrix1D; import cern.colt.matrix.impl.DenseDoubleMatrix1D; -import org.apache.commons.collections.Transformer; -import org.apache.commons.collections.TransformerUtils; +import org.apache.commons.collections4.Transformer; +import org.apache.commons.collections4.TransformerUtils; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.StopWatch; diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ArrayDesignAnnotationServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ArrayDesignAnnotationServiceImpl.java index c69080095d..b3ad2cce8d 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ArrayDesignAnnotationServiceImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ArrayDesignAnnotationServiceImpl.java @@ -18,8 +18,8 @@ */ package ubic.gemma.core.analysis.service; -import org.apache.commons.collections.Transformer; -import org.apache.commons.collections.iterators.TransformIterator; +import org.apache.commons.collections4.Transformer; +import org.apache.commons.collections4.iterators.TransformIterator; import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; diff --git a/gemma-core/src/main/java/ubic/gemma/core/externalDb/GoldenPathSequenceAnalysis.java b/gemma-core/src/main/java/ubic/gemma/core/externalDb/GoldenPathSequenceAnalysis.java index e12f1751c4..d4e694df65 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/externalDb/GoldenPathSequenceAnalysis.java +++ b/gemma-core/src/main/java/ubic/gemma/core/externalDb/GoldenPathSequenceAnalysis.java @@ -18,7 +18,7 @@ */ package ubic.gemma.core.externalDb; -import org.apache.commons.collections.map.LRUMap; +import org.apache.commons.collections4.map.LRUMap; import org.apache.commons.lang3.StringUtils; import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.ResultSetExtractor; diff --git a/gemma-core/src/main/java/ubic/gemma/core/loader/expression/AffyPowerToolsProbesetSummarize.java b/gemma-core/src/main/java/ubic/gemma/core/loader/expression/AffyPowerToolsProbesetSummarize.java index 6efb4cc4e6..473bf58b9a 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/loader/expression/AffyPowerToolsProbesetSummarize.java +++ b/gemma-core/src/main/java/ubic/gemma/core/loader/expression/AffyPowerToolsProbesetSummarize.java @@ -31,7 +31,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.configuration2.ex.ConfigurationException; import org.apache.commons.configuration2.PropertiesConfiguration; import org.apache.commons.lang3.RandomStringUtils; diff --git a/gemma-core/src/main/java/ubic/gemma/core/security/audit/AuditableUtilImpl.java b/gemma-core/src/main/java/ubic/gemma/core/security/audit/AuditableUtilImpl.java index 7ac8fb92ba..6f6d2f2e5d 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/security/audit/AuditableUtilImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/core/security/audit/AuditableUtilImpl.java @@ -18,8 +18,8 @@ */ package ubic.gemma.core.security.audit; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.Predicate; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.Predicate; import org.springframework.stereotype.Component; import ubic.gemma.model.expression.arrayDesign.ArrayDesignValueObject; import ubic.gemma.model.expression.experiment.ExpressionExperimentValueObject; @@ -40,12 +40,7 @@ public void removeTroubledArrayDesigns( Collection value return; } - CollectionUtils.filter( valueObjects, new Predicate() { - @Override - public boolean evaluate( Object vo ) { - return !( ( ArrayDesignValueObject ) vo ).getTroubled(); - } - } ); + CollectionUtils.filter( valueObjects, vo -> !vo.getTroubled() ); } @Override @@ -54,12 +49,7 @@ public void removeTroubledEes( Collection eevos return; } - CollectionUtils.filter( eevos, new Predicate() { - @Override - public boolean evaluate( Object e ) { - return !( ( ExpressionExperimentValueObject ) e ).getTroubled(); - } - } ); + CollectionUtils.filter( eevos, vo -> !vo.getTroubled() ); } } diff --git a/gemma-core/src/main/java/ubic/gemma/model/analysis/expression/pca/PrincipalComponentAnalysis.java b/gemma-core/src/main/java/ubic/gemma/model/analysis/expression/pca/PrincipalComponentAnalysis.java index 1ba858d7e3..620cc3e17b 100644 --- a/gemma-core/src/main/java/ubic/gemma/model/analysis/expression/pca/PrincipalComponentAnalysis.java +++ b/gemma-core/src/main/java/ubic/gemma/model/analysis/expression/pca/PrincipalComponentAnalysis.java @@ -18,8 +18,7 @@ */ package ubic.gemma.model.analysis.expression.pca; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.Predicate; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ArrayUtils; import ubic.basecode.io.ByteArrayConverter; import ubic.gemma.model.analysis.SingleExperimentAnalysis; @@ -27,10 +26,7 @@ import ubic.gemma.model.expression.bioAssayData.BioAssayDimension; import javax.persistence.Transient; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; +import java.util.*; public class PrincipalComponentAnalysis extends SingleExperimentAnalysis { @@ -130,12 +126,7 @@ public List getEigenvectorArrays() throws IllegalArgumentException { result.set( index, dA ); } - CollectionUtils.filter( result, new Predicate() { - @Override - public boolean evaluate( Object object ) { - return object != null; - } - } ); + CollectionUtils.filter( result, Objects::nonNull ); return result; } diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionAnalysisDaoImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionAnalysisDaoImpl.java index fde26da51c..5f579d8a45 100644 --- a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionAnalysisDaoImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionAnalysisDaoImpl.java @@ -18,7 +18,7 @@ */ package ubic.gemma.persistence.service.analysis.expression.diff; -import org.apache.commons.collections.ListUtils; +import org.apache.commons.collections4.ListUtils; import org.apache.commons.lang3.time.StopWatch; import org.hibernate.*; import org.springframework.beans.factory.annotation.Autowired; diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/common/auditAndSecurity/AuditEventDaoImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/common/auditAndSecurity/AuditEventDaoImpl.java index 21df1ccd45..8a15e4d2c9 100644 --- a/gemma-core/src/main/java/ubic/gemma/persistence/service/common/auditAndSecurity/AuditEventDaoImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/common/auditAndSecurity/AuditEventDaoImpl.java @@ -18,8 +18,8 @@ */ package ubic.gemma.persistence.service.common.auditAndSecurity; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.Predicate; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.Predicate; import org.apache.commons.lang3.time.StopWatch; import org.hibernate.Hibernate; import org.hibernate.Query; @@ -193,13 +193,7 @@ public void retainHavingEvent( final Collection a, final Map events = this.getLastEvent( a, type ); - CollectionUtils.filter( a, new Predicate() { - @Override - public boolean evaluate( Object arg0 ) { - //noinspection SuspiciousMethodCalls // this is perfectly fine since we are passing this directly into the filter - return events.containsKey( arg0 ); - } - } ); + CollectionUtils.filter( a, events::containsKey ); } @@ -211,13 +205,7 @@ public void retainLackingEvent( final Collection a, final Map events = this.getLastEvent( a, type ); AbstractDao.log.info( "Phase I: " + timer.getTime() + "ms" ); - CollectionUtils.filter( a, new Predicate() { - @Override - public boolean evaluate( Object arg0 ) { - //noinspection SuspiciousMethodCalls // this is perfectly fine since we are passing this directly into the filter - return !events.containsKey( arg0 ); - } - } ); + CollectionUtils.filter( a, ( Predicate ) arg0 -> !events.containsKey( arg0 ) ); } diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/util/CommonQueries.java b/gemma-core/src/main/java/ubic/gemma/persistence/util/CommonQueries.java index b2a8156f1d..4c983dd45e 100644 --- a/gemma-core/src/main/java/ubic/gemma/persistence/util/CommonQueries.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/util/CommonQueries.java @@ -18,7 +18,7 @@ */ package ubic.gemma.persistence.util; -import org.apache.commons.collections.ListUtils; +import org.apache.commons.collections4.ListUtils; import org.apache.commons.lang3.time.StopWatch; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; diff --git a/gemma-web/src/main/java/ubic/gemma/web/controller/expression/experiment/ExpressionExperimentSetController.java b/gemma-web/src/main/java/ubic/gemma/web/controller/expression/experiment/ExpressionExperimentSetController.java index e03b0f498b..b459ae76bd 100644 --- a/gemma-web/src/main/java/ubic/gemma/web/controller/expression/experiment/ExpressionExperimentSetController.java +++ b/gemma-web/src/main/java/ubic/gemma/web/controller/expression/experiment/ExpressionExperimentSetController.java @@ -18,8 +18,8 @@ */ package ubic.gemma.web.controller.expression.experiment; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.Predicate; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.Predicate; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.StopWatch; import org.springframework.beans.factory.annotation.Autowired; @@ -179,12 +179,11 @@ public Collection getExperimentsInSet( L .getExperimentValueObjectsInSet( groupId ); if ( limit != null && limit > 0 && limit < experimentInSet.size() ) { - //noinspection unchecked - return CollectionUtils.select( experimentInSet, new Predicate() { + return CollectionUtils.select( experimentInSet, new Predicate() { int i = 0; @Override - public boolean evaluate( Object object ) { + public boolean evaluate( ExpressionExperimentDetailsValueObject object ) { return i++ < limit; } } ); diff --git a/gemma-web/src/main/java/ubic/gemma/web/controller/genome/gene/GeneSetController.java b/gemma-web/src/main/java/ubic/gemma/web/controller/genome/gene/GeneSetController.java index 3ccdfda678..1f734c94ea 100755 --- a/gemma-web/src/main/java/ubic/gemma/web/controller/genome/gene/GeneSetController.java +++ b/gemma-web/src/main/java/ubic/gemma/web/controller/genome/gene/GeneSetController.java @@ -19,8 +19,8 @@ package ubic.gemma.web.controller.genome.gene; import gemma.gsec.SecurityService; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.Predicate; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.Predicate; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Controller; @@ -205,11 +205,11 @@ public Collection getGenesInGroup( Long groupId, final Integer Collection genesInGroup = geneSetService.getGenesInGroup( new GeneSetValueObject( groupId ) ); if ( limit != null && limit > 0 && limit < genesInGroup.size() ) { - return CollectionUtils.select( genesInGroup, new Predicate() { + return CollectionUtils.select( genesInGroup, new Predicate() { int i = 0; @Override - public boolean evaluate( Object object ) { + public boolean evaluate( GeneValueObject object ) { return i++ < limit; } } ); diff --git a/pom.xml b/pom.xml index 3b63a5a06d..8bdf60c81f 100644 --- a/pom.xml +++ b/pom.xml @@ -344,9 +344,9 @@ 1.9.4 - commons-collections - commons-collections - 3.2.2 + org.apache.commons + commons-collections4 + 4.4 org.apache.commons From 6f9e34ee2f9337e79823ec3b22ccb09ac166dd59 Mon Sep 17 00:00:00 2001 From: Guillaume Poirier-Morency Date: Fri, 4 Feb 2022 12:49:51 -0800 Subject: [PATCH 03/28] Address vulnerability in commons-collections <3.2.2 --- pom.xml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pom.xml b/pom.xml index 8bdf60c81f..cedf122591 100644 --- a/pom.xml +++ b/pom.xml @@ -348,6 +348,14 @@ commons-collections4 4.4 + + + commons-collections + commons-collections + 3.2.2 + runtime + org.apache.commons commons-configuration2 From a938940bd26cf682c06c5c84bc21a2fa141c6edb Mon Sep 17 00:00:00 2001 From: Guillaume Poirier-Morency Date: Sun, 6 Feb 2022 10:03:15 -0800 Subject: [PATCH 04/28] Require Maven >=3.1.1 Ideally we would target >=3.0.5 to match our build environment, but that's not possible given the plugins that we use. --- gemma-core/pom.xml | 6 +++--- gemma-web/pom.xml | 2 +- pom.xml | 20 ++++++++++++++++++++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/gemma-core/pom.xml b/gemma-core/pom.xml index 731e29480f..c82e1f0fe8 100644 --- a/gemma-core/pom.xml +++ b/gemma-core/pom.xml @@ -14,7 +14,7 @@ org.apache.maven.plugins maven-antrun-plugin - 1.7 + 3.0.0 @@ -120,7 +120,7 @@ org.apache.maven.plugins maven-antrun-plugin - [1.7,) + [3.0.0,) run @@ -150,7 +150,7 @@ org.apache.maven.plugins maven-dependency-plugin - 3.2.0 + 3.1.2 process-resources diff --git a/gemma-web/pom.xml b/gemma-web/pom.xml index 23777ac3a9..cc31f6bc9b 100644 --- a/gemma-web/pom.xml +++ b/gemma-web/pom.xml @@ -22,7 +22,7 @@ com.github.klieber phantomjs-maven-plugin - 0.4 + 0.7 test-compile diff --git a/pom.xml b/pom.xml index cedf122591..1b941af5b1 100644 --- a/pom.xml +++ b/pom.xml @@ -478,6 +478,26 @@ + + org.apache.maven.plugins + maven-enforcer-plugin + 3.0.0 + + + enforce-maven + + enforce + + + + + [3.1.1,) + + + + + + com.amashchenko.maven.plugin gitflow-maven-plugin From 88dbddeaced0500a6cbce97e7a1b633113c862cc Mon Sep 17 00:00:00 2001 From: Guillaume Poirier-Morency Date: Sun, 6 Feb 2022 15:26:57 -0800 Subject: [PATCH 05/28] Use colored output for CLI Highlight the status, PID and invoked method similar to what Spring Boot does. Remove redundant [Gemma %d] prefix since it's pointless as the end-user is certainly aware the CLI is running. Add the PID to the output since it's useful to identify the process. --- gemma-core/src/main/config/log4j.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gemma-core/src/main/config/log4j.properties b/gemma-core/src/main/config/log4j.properties index 7111d89578..58a51f24e1 100644 --- a/gemma-core/src/main/config/log4j.properties +++ b/gemma-core/src/main/config/log4j.properties @@ -10,7 +10,7 @@ log4j.appender.stderr=org.apache.log4j.ConsoleAppender log4j.appender.stderr.target=System.err log4j.appender.stderr.layout=org.apache.log4j.PatternLayout -log4j.appender.stderr.layout.ConversionPattern=[Gemma %d] %p [%t] %C.%M(%L) | %m%n +log4j.appender.stderr.layout.ConversionPattern=%d %highlight{%p} %style{%pid}{magenta} [%t] %style{%C.%M(%L)}{cyan} | %m%n # Log to a file. Note that if File is a relative path, the output file goes wherever the application JVM was started from. # Define gemma.log.dir as a parameter to your JAVA_OPTS and make sure this is passed to java when you start it. From 169c1f70634a4de53a02889faeb64d9a9a66be96 Mon Sep 17 00:00:00 2001 From: Guillaume Poirier-Morency Date: Mon, 7 Feb 2022 09:59:45 -0800 Subject: [PATCH 06/28] Update jstl and taglibs to 1.2 and 1.2.5 respectively --- gemma-web/pom.xml | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/gemma-web/pom.xml b/gemma-web/pom.xml index cc31f6bc9b..4a0b1ff23d 100644 --- a/gemma-web/pom.xml +++ b/gemma-web/pom.xml @@ -352,19 +352,13 @@ javax.servlet jstl - 1.1.2 + 1.2 runtime - taglibs - standard - 1.1.2 - runtime - - - taglibs - request - 1.0.1 + org.apache.taglibs + taglibs-standard-impl + 1.2.5 runtime From b5be7bbac1491a6b14bc954f460fa864ad8941a4 Mon Sep 17 00:00:00 2001 From: Guillaume Poirier-Morency Date: Mon, 7 Feb 2022 10:00:07 -0800 Subject: [PATCH 07/28] Add a few notes and fix typos in pom.xml --- gemma-core/pom.xml | 6 +++--- pom.xml | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/gemma-core/pom.xml b/gemma-core/pom.xml index c82e1f0fe8..6a411f0576 100644 --- a/gemma-core/pom.xml +++ b/gemma-core/pom.xml @@ -264,7 +264,7 @@ 1.7 - + org.apache.lucene lucene-core @@ -276,7 +276,7 @@ ${lucene.version} - + org.apache.jena jena-core @@ -375,7 +375,7 @@ 1.7.4 - + com.googlecode.javaewah JavaEWAH diff --git a/pom.xml b/pom.xml index 1b941af5b1..d1d3cc2a91 100644 --- a/pom.xml +++ b/pom.xml @@ -379,7 +379,7 @@ ${jackson.version} - + xml-apis xml-apis @@ -391,6 +391,7 @@ xercesImpl 2.10.0 + xalan xalan From 43b223199ad2b8104b2a4b6f260ded8712de9fbe Mon Sep 17 00:00:00 2001 From: Guillaume Poirier-Morency Date: Thu, 10 Feb 2022 14:58:42 -0800 Subject: [PATCH 08/28] Add section to pom.xml --- pom.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pom.xml b/pom.xml index d1d3cc2a91..1563984da9 100644 --- a/pom.xml +++ b/pom.xml @@ -18,6 +18,10 @@ gemma-web pom + + scm:git:https://github.com/PavlidisLab/Gemma.git + https://github.com/PavlidisLab/Gemma + GitHub https://github.com/PavlidisLab/Gemma/issues From 9a91f602ec170a61a3b786487aa0d2e4da818235 Mon Sep 17 00:00:00 2001 From: Guillaume Poirier-Morency Date: Sun, 13 Feb 2022 13:12:00 -0800 Subject: [PATCH 09/28] pom: Move schema export task under its own execution As defined, the target is inherited and then shadowed in the version-file execution. Add schema-export as execution . Remove ${maven.dependency.classpath} mention as it was removed in maven-antrun-plugin 3.0.0. Instead, add hibernate-core to the plugin dependencies. This limit the scope of the classpath the plugin has access to. Add ${project.basedir} prefix to the *.hbm.xml file path resolution. --- gemma-core/pom.xml | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/gemma-core/pom.xml b/gemma-core/pom.xml index 6a411f0576..0fb1d789d6 100644 --- a/gemma-core/pom.xml +++ b/gemma-core/pom.xml @@ -15,21 +15,6 @@ org.apache.maven.plugins maven-antrun-plugin 3.0.0 - - - - - - - - - - - - - version-file @@ -46,12 +31,31 @@ + schema-export generate-resources run + + + + + + + + + + + + org.hibernate + hibernate-core + ${hibernate.version} + + org.apache.maven.plugins From e10a1b9d4c8c8c4b06d092f762ade88de12e5fb2 Mon Sep 17 00:00:00 2001 From: Guillaume Poirier-Morency Date: Sun, 13 Feb 2022 16:00:12 -0800 Subject: [PATCH 10/28] Remove gemma-model from SQL resources unpacking --- gemma-core/pom.xml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/gemma-core/pom.xml b/gemma-core/pom.xml index 0fb1d789d6..98b34d9f55 100644 --- a/gemma-core/pom.xml +++ b/gemma-core/pom.xml @@ -171,14 +171,6 @@ ${project.build.directory}/schema **/*.sql - - gemma - gemma-model - 1.10.1 - true - ${project.build.directory}/schema - **/*.sql - From cd74fce87f3430b081cbee59948effb00218e123 Mon Sep 17 00:00:00 2001 From: Guillaume Poirier-Morency Date: Fri, 25 Feb 2022 11:51:56 -0800 Subject: [PATCH 11/28] Mark operations in GeoService as transactional This is a likely cause of the AccessDeniedException as it appear to be caused by an inconsistent ACL state with left-overs from a removed EE entity. --- .../core/loader/expression/geo/service/GeoServiceImpl.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gemma-core/src/main/java/ubic/gemma/core/loader/expression/geo/service/GeoServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/loader/expression/geo/service/GeoServiceImpl.java index d3bd37a158..854e79a7ea 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/loader/expression/geo/service/GeoServiceImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/core/loader/expression/geo/service/GeoServiceImpl.java @@ -21,6 +21,7 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; import ubic.gemma.core.analysis.report.ArrayDesignReportService; import ubic.gemma.core.analysis.report.ExpressionExperimentReportService; import ubic.gemma.core.loader.entrez.pubmed.PubMedXMLFetcher; @@ -76,6 +77,7 @@ public GeoServiceImpl( ArrayDesignReportService arrayDesignReportService, BioAss } @Override + @Transactional public ArrayDesign addElements( ArrayDesign targetPlatform ) { if ( !targetPlatform.getCompositeSequences().isEmpty() ) { @@ -125,6 +127,7 @@ public ArrayDesign addElements( ArrayDesign targetPlatform ) { } @Override + @Transactional public Collection fetchAndLoad( String geoAccession, boolean loadPlatformOnly, boolean doSampleMatching, boolean splitByPlatform ) { return this.fetchAndLoad( geoAccession, loadPlatformOnly, doSampleMatching, splitByPlatform, true, true ); @@ -140,6 +143,7 @@ public Collection fetchAndLoad( String geoAccession, boolean loadPlatformOnly * */ @Override + @Transactional public Collection fetchAndLoad( String geoAccession, boolean loadPlatformOnly, boolean doSampleMatching, boolean splitByPlatform, boolean allowSuperSeriesImport, boolean allowSubSeriesImport ) { From a021006614975529fe55a2b0cffeb6c314ac2f9c Mon Sep 17 00:00:00 2001 From: Guillaume Poirier-Morency Date: Mon, 7 Mar 2022 09:06:18 -0800 Subject: [PATCH 12/28] Comment out CoexpressionQueryQueue usages (fix #309) Remove the @Repository annotation to prevent Spring from initializing the Hibernate DAO support. --- .../CoexpressionQueryQueueImpl.java | 5 +- .../coexpression/CoexpressionServiceImpl.java | 87 +++++++++---------- 2 files changed, 42 insertions(+), 50 deletions(-) diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/association/coexpression/CoexpressionQueryQueueImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/association/coexpression/CoexpressionQueryQueueImpl.java index 8189bbe6f7..98d064ee31 100644 --- a/gemma-core/src/main/java/ubic/gemma/persistence/service/association/coexpression/CoexpressionQueryQueueImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/association/coexpression/CoexpressionQueryQueueImpl.java @@ -23,11 +23,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Lazy; import org.springframework.orm.hibernate3.support.HibernateDaoSupport; import org.springframework.security.core.context.SecurityContext; import org.springframework.security.core.context.SecurityContextHolder; -import org.springframework.stereotype.Repository; import ubic.gemma.model.genome.Gene; import ubic.gemma.persistence.service.genome.GeneDao; @@ -38,8 +36,7 @@ /** * @author Paul */ -@Repository -@Lazy +// @Repository class CoexpressionQueryQueueImpl extends HibernateDaoSupport implements CoexpressionQueryQueue { private static final int QUEUE_SIZE = 1000; diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/association/coexpression/CoexpressionServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/association/coexpression/CoexpressionServiceImpl.java index e9fe4e49b0..32555832d0 100644 --- a/gemma-core/src/main/java/ubic/gemma/persistence/service/association/coexpression/CoexpressionServiceImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/association/coexpression/CoexpressionServiceImpl.java @@ -52,11 +52,11 @@ public class CoexpressionServiceImpl implements CoexpressionService { @Autowired private CoexpressionDao coexpressionDao; - @Autowired - private CoexpressionQueryQueue coexpressionQueryQueue; + // @Autowired + // private CoexpressionQueryQueue coexpressionQueryQueue; - @Autowired - private CoexpressionCache coexpressionCache; + // @Autowired + // private CoexpressionCache coexpressionCache; @Autowired private ExpressionExperimentDao experimentDao; @@ -82,12 +82,12 @@ public void createOrUpdate( BioAssaySet bioAssaySet, List genes = new HashSet<>(); - for ( NonPersistentNonOrderedCoexpLink link : links ) { - genes.add( link.getFirstGene() ); - genes.add( link.getSecondGene() ); - } - this.coexpressionQueryQueue.removeFromQueue( genes ); + // Collection genes = new HashSet<>(); + // for ( NonPersistentNonOrderedCoexpLink link : links ) { + // genes.add( link.getFirstGene() ); + // genes.add( link.getSecondGene() ); + // } + // this.coexpressionQueryQueue.removeFromQueue( genes ); } @Override @@ -102,9 +102,9 @@ public List findCoexpressionRelationships( Gene gene, C List results = this.coexpressionDao .findCoexpressionRelationships( gene, bas, maxResults, quick ); - if ( quick || maxResults > 0 ) { - this.coexpressionQueryQueue.addToFullQueryQueue( gene ); - } + // if ( quick || maxResults > 0 ) { + // this.coexpressionQueryQueue.addToFullQueryQueue( gene ); + // } return results; } @@ -133,7 +133,7 @@ public Map> findCoexpressionRelationships( T // since we require these links occur in all the given data sets, we assume we should cache (if not there // already) - don't bother checking 'quick' and 'maxResults'. - this.possiblyAddToCacheQueue( results ); + // this.possiblyAddToCacheQueue( results ); return results; } @@ -141,25 +141,20 @@ public Map> findCoexpressionRelationships( T @Override public Map> findCoexpressionRelationships( Taxon t, Collection genes, Collection bas, int stringency, int maxResults, boolean quick ) { - Map> results = this.coexpressionDao + // if ( stringency > CoexpressionCache.CACHE_QUERY_STRINGENCY || quick || maxResults > 0 ) { + // this.possiblyAddToCacheQueue( results ); + // } + return this.coexpressionDao .findCoexpressionRelationships( t, genes, bas, stringency, maxResults, quick ); - - if ( stringency > CoexpressionCache.CACHE_QUERY_STRINGENCY || quick || maxResults > 0 ) { - this.possiblyAddToCacheQueue( results ); - } - - return results; } @Override public Map> findInterCoexpressionRelationships( Taxon t, Collection genes, Collection bas, int stringency, boolean quick ) { - Map> results = this.coexpressionDao - .findInterCoexpressionRelationships( t, genes, bas, stringency, quick ); - // these are always candidates for queuing since the constraint on genes is done at the query level. - this.possiblyAddToCacheQueue( results ); - return results; + // this.possiblyAddToCacheQueue( results ); + return this.coexpressionDao + .findInterCoexpressionRelationships( t, genes, bas, stringency, quick ); } @Override @@ -264,26 +259,26 @@ private Map> computeRelativeRanks( TreeMap> links ) { - - if ( !coexpressionCache.isEnabled() ) - return; - - Set toQueue = new HashSet<>(); - for ( Long id : links.keySet() ) { - for ( CoexpressionValueObject link : links.get( id ) ) { - if ( link.isFromCache() ) { - continue; - } - toQueue.add( link.getQueryGeneId() ); - } - } - if ( !toQueue.isEmpty() ) { - CoexpressionServiceImpl.log.info( "Queuing " + toQueue.size() + " genes for coexpression cache warm" ); - coexpressionQueryQueue.addToFullQueryQueue( toQueue ); - } - - } + // private void possiblyAddToCacheQueue( Map> links ) { + + // if ( !coexpressionCache.isEnabled() ) + // return; + + // Set toQueue = new HashSet<>(); + // for ( Long id : links.keySet() ) { + // for ( CoexpressionValueObject link : links.get( id ) ) { + // if ( link.isFromCache() ) { + // continue; + // } + // toQueue.add( link.getQueryGeneId() ); + // } + // } + // if ( !toQueue.isEmpty() ) { + // CoexpressionServiceImpl.log.info( "Queuing " + toQueue.size() + " genes for coexpression cache warm" ); + // coexpressionQueryQueue.addToFullQueryQueue( toQueue ); + // } + + // } private GeneCoexpressionNodeDegreeValueObject updateNodeDegree( Gene gene ) { GeneCoexpressionNodeDegree nd = this.geneCoexpressionNodeDegreeDao.findOrCreate( gene ); From 173624a1e043e7095deea699ee4d6ec64e28d957 Mon Sep 17 00:00:00 2001 From: Guillaume Poirier-Morency Date: Mon, 7 Mar 2022 09:51:08 -0800 Subject: [PATCH 13/28] Use a TaskExecutor to avoid direct thread creation This will prevent an unreasonable number of threads being created by Gemma. Create a default pool of at most 16 threads with an unlimited task queue size. The only remaining threads are for Homologene loading and a few test cases, but these cannot be readily replaced. --- .../apps/NCBIGene2GOAssociationLoaderCLI.java | 4 +++- .../gemma/core/apps/NcbiGeneLoaderCLI.java | 3 ++- .../NCBIGene2GOAssociationLoader.java | 18 ++++++++++-------- .../ArrayDesignProbeMapperServiceImpl.java | 11 ++++++----- .../genome/gene/ncbi/NcbiGeneConverter.java | 13 +++++++++---- .../ncbi/NcbiGeneDomainObjectGenerator.java | 15 ++++++++------- .../genome/gene/ncbi/NcbiGeneLoader.java | 18 ++++++++++-------- .../providers/GeneOntologyServiceImpl.java | 8 +++++--- .../CoexpressionQueryQueueImpl.java | 9 +++++---- .../gemma/applicationContext-serviceBeans.xml | 4 ++++ ...CompositeSequenceGeneMapperServiceTest.java | 6 +++++- .../NCBIGene2GOAssociationParserTest.java | 6 +++++- .../genome/gene/ncbi/NCBIGeneLoadingTest.java | 5 ++++- 13 files changed, 76 insertions(+), 44 deletions(-) diff --git a/gemma-core/src/main/java/ubic/gemma/core/apps/NCBIGene2GOAssociationLoaderCLI.java b/gemma-core/src/main/java/ubic/gemma/core/apps/NCBIGene2GOAssociationLoaderCLI.java index 7c7950879f..83e239969a 100755 --- a/gemma-core/src/main/java/ubic/gemma/core/apps/NCBIGene2GOAssociationLoaderCLI.java +++ b/gemma-core/src/main/java/ubic/gemma/core/apps/NCBIGene2GOAssociationLoaderCLI.java @@ -21,6 +21,7 @@ import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; +import org.springframework.core.task.TaskExecutor; import ubic.gemma.core.apps.GemmaCLI.CommandGroup; import ubic.gemma.core.loader.association.NCBIGene2GOAssociationLoader; import ubic.gemma.core.loader.association.NCBIGene2GOAssociationParser; @@ -64,8 +65,9 @@ protected void buildOptions( Options options ) { @Override protected void doWork() throws Exception { TaxonService taxonService = this.getBean( TaxonService.class ); + TaskExecutor taskExecutor = this.getBean( TaskExecutor.class ); - NCBIGene2GOAssociationLoader gene2GOAssLoader = new NCBIGene2GOAssociationLoader(); + NCBIGene2GOAssociationLoader gene2GOAssLoader = new NCBIGene2GOAssociationLoader( taskExecutor ); gene2GOAssLoader.setPersisterHelper( this.getPersisterHelper() ); diff --git a/gemma-core/src/main/java/ubic/gemma/core/apps/NcbiGeneLoaderCLI.java b/gemma-core/src/main/java/ubic/gemma/core/apps/NcbiGeneLoaderCLI.java index 4002572857..badf0e8892 100755 --- a/gemma-core/src/main/java/ubic/gemma/core/apps/NcbiGeneLoaderCLI.java +++ b/gemma-core/src/main/java/ubic/gemma/core/apps/NcbiGeneLoaderCLI.java @@ -22,6 +22,7 @@ import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.lang3.StringUtils; +import org.springframework.core.task.TaskExecutor; import ubic.gemma.core.apps.GemmaCLI.CommandGroup; import ubic.gemma.core.loader.genome.gene.ncbi.NcbiGeneLoader; import ubic.gemma.core.util.AbstractCLI; @@ -98,7 +99,7 @@ protected boolean requireLogin() { @Override protected void doWork() throws Exception { - loader = new NcbiGeneLoader(); + loader = new NcbiGeneLoader( this.getBean( TaskExecutor.class ) ); TaxonService taxonService = this.getBean( TaxonService.class ); loader.setTaxonService( taxonService ); loader.setPersisterHelper( this.getPersisterHelper() ); diff --git a/gemma-core/src/main/java/ubic/gemma/core/loader/association/NCBIGene2GOAssociationLoader.java b/gemma-core/src/main/java/ubic/gemma/core/loader/association/NCBIGene2GOAssociationLoader.java index cd82336f5c..532ee78ed2 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/loader/association/NCBIGene2GOAssociationLoader.java +++ b/gemma-core/src/main/java/ubic/gemma/core/loader/association/NCBIGene2GOAssociationLoader.java @@ -20,6 +20,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.springframework.core.task.TaskExecutor; import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContext; import org.springframework.security.core.context.SecurityContextHolder; @@ -50,6 +51,11 @@ public class NCBIGene2GOAssociationLoader { private Persister persisterHelper; private NCBIGene2GOAssociationParser parser = null; private int count; + private final TaskExecutor taskExecutor; + + public NCBIGene2GOAssociationLoader( TaskExecutor taskExecutor ) { + this.taskExecutor = taskExecutor; + } public int getCount() { return count; @@ -75,7 +81,7 @@ public void load( final InputStream inputStream ) { final SecurityContext context = SecurityContextHolder.getContext(); final Authentication authentication = context.getAuthentication(); - Thread loadThread = new Thread( new Runnable() { + taskExecutor.execute( new Runnable() { @Override public void run() { NCBIGene2GOAssociationLoader.log.info( "Starting loading" ); @@ -84,9 +90,7 @@ public void run() { } } ); - loadThread.start(); - - Thread parseThread = new Thread( new Runnable() { + taskExecutor.execute( new Runnable() { @Override public void run() { try { @@ -103,8 +107,6 @@ public void run() { } } ); - parseThread.start(); - while ( !this.isProducerDone() || !this.isConsumerDone() ) { try { Thread.sleep( 1000 ); @@ -116,8 +118,8 @@ public void run() { public void load( LocalFile ncbiFile ) { - try (InputStream inputStream = FileTools - .getInputStreamFromPlainOrCompressedFile( ncbiFile.asFile().getAbsolutePath() )) { + try ( InputStream inputStream = FileTools + .getInputStreamFromPlainOrCompressedFile( ncbiFile.asFile().getAbsolutePath() ) ) { this.load( inputStream ); } catch ( IOException e ) { diff --git a/gemma-core/src/main/java/ubic/gemma/core/loader/expression/arrayDesign/ArrayDesignProbeMapperServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/loader/expression/arrayDesign/ArrayDesignProbeMapperServiceImpl.java index 0c72aaeb4a..fd3ced74c1 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/loader/expression/arrayDesign/ArrayDesignProbeMapperServiceImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/core/loader/expression/arrayDesign/ArrayDesignProbeMapperServiceImpl.java @@ -22,6 +22,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.task.TaskExecutor; import org.springframework.security.core.context.SecurityContext; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.stereotype.Component; @@ -91,6 +92,7 @@ public class ArrayDesignProbeMapperServiceImpl implements ArrayDesignProbeMapper private final GeneService geneService; private final Persister persisterHelper; private final ProbeMapper probeMapper; + private final TaskExecutor taskExecutor; @Autowired public ArrayDesignProbeMapperServiceImpl( AnnotationAssociationService annotationAssociationService, @@ -98,7 +100,7 @@ public ArrayDesignProbeMapperServiceImpl( AnnotationAssociationService annotatio ArrayDesignReportService arrayDesignReportService, ArrayDesignService arrayDesignService, ProbeMapper probeMapper, BioSequenceService bioSequenceService, BlatResultService blatResultService, CompositeSequenceService compositeSequenceService, ExpressionDataFileService expressionDataFileService, - GeneProductService geneProductService, GeneService geneService, Persister persisterHelper ) { + GeneProductService geneProductService, GeneService geneService, Persister persisterHelper, TaskExecutor taskExecutor ) { this.annotationAssociationService = annotationAssociationService; this.arrayDesignAnnotationService = arrayDesignAnnotationService; this.arrayDesignReportService = arrayDesignReportService; @@ -111,6 +113,7 @@ public ArrayDesignProbeMapperServiceImpl( AnnotationAssociationService annotatio this.geneProductService = geneProductService; this.geneService = geneService; this.persisterHelper = persisterHelper; + this.taskExecutor = taskExecutor; } @Override @@ -516,16 +519,14 @@ private void load( final BlockingQueue queue, final AtomicBoolean generato final SecurityContext context = SecurityContextHolder.getContext(); assert context != null; - Thread loadThread = new Thread( new Runnable() { + this.taskExecutor.execute( new Runnable() { @Override public void run() { SecurityContextHolder.setContext( context ); ArrayDesignProbeMapperServiceImpl.this.doLoad( queue, generatorDone, loaderDone, persist ); } - }, "PersistBlatAssociations" ); - - loadThread.start(); + }); } diff --git a/gemma-core/src/main/java/ubic/gemma/core/loader/genome/gene/ncbi/NcbiGeneConverter.java b/gemma-core/src/main/java/ubic/gemma/core/loader/genome/gene/ncbi/NcbiGeneConverter.java index 1ad14496fd..66fe6c37e8 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/loader/genome/gene/ncbi/NcbiGeneConverter.java +++ b/gemma-core/src/main/java/ubic/gemma/core/loader/genome/gene/ncbi/NcbiGeneConverter.java @@ -22,6 +22,7 @@ import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.springframework.core.task.TaskExecutor; import ubic.gemma.core.loader.genome.gene.ncbi.model.NCBIGene2Accession; import ubic.gemma.core.loader.genome.gene.ncbi.model.NCBIGeneInfo; import ubic.gemma.core.loader.util.converter.Converter; @@ -63,9 +64,15 @@ public class NcbiGeneConverter implements Converter { NcbiGeneConverter.ensembl.setName( "Ensembl" ); } + private final TaskExecutor taskExecutor; + AtomicBoolean producerDone = new AtomicBoolean( false ); AtomicBoolean sourceDone = new AtomicBoolean( false ); + public NcbiGeneConverter( TaskExecutor taskExecutor ) { + this.taskExecutor = taskExecutor; + } + /** * @return the genBank */ @@ -239,7 +246,7 @@ public void convert( final BlockingQueue geneInfoQueue, final Bloc // start up thread to convert a member of geneInfoQueue to a gene/geneproduct/databaseentry // then push the gene onto the geneQueue for loading - Thread convertThread = new Thread( new Runnable() { + this.taskExecutor.execute( new Runnable() { @Override @SuppressWarnings("synthetic-access") public void run() { @@ -268,9 +275,7 @@ public void run() { } producerDone.set( true ); } - }, "Converter" ); - - convertThread.start(); + }); } public boolean isProducerDone() { diff --git a/gemma-core/src/main/java/ubic/gemma/core/loader/genome/gene/ncbi/NcbiGeneDomainObjectGenerator.java b/gemma-core/src/main/java/ubic/gemma/core/loader/genome/gene/ncbi/NcbiGeneDomainObjectGenerator.java index 7ca42e1173..24d397eedf 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/loader/genome/gene/ncbi/NcbiGeneDomainObjectGenerator.java +++ b/gemma-core/src/main/java/ubic/gemma/core/loader/genome/gene/ncbi/NcbiGeneDomainObjectGenerator.java @@ -20,6 +20,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.springframework.core.task.TaskExecutor; import ubic.basecode.util.FileTools; import ubic.gemma.core.loader.genome.gene.ncbi.model.NCBIGeneInfo; import ubic.gemma.core.loader.genome.gene.ncbi.model.NcbiGeneHistory; @@ -58,8 +59,10 @@ public class NcbiGeneDomainObjectGenerator { // whether to fetch files from ncbi or use existing ones private boolean doDownload = true; private Integer startingNcbiId = null; + private final TaskExecutor taskExecutor; - public NcbiGeneDomainObjectGenerator( Collection taxa ) { + public NcbiGeneDomainObjectGenerator( Collection taxa, TaskExecutor taskExecutor ) { + this.taskExecutor = taskExecutor; if ( taxa != null ) { this.supportedTaxa = new HashMap<>(); @@ -191,8 +194,8 @@ private void processLocalFiles( final LocalFile geneInfoFile, final LocalFile ge // NcbiGeneDomainObjectGenerator.log.debug( "Parsing GeneInfo =" + geneInfoFile.asFile().getAbsolutePath() ); - try (InputStream is = FileTools - .getInputStreamFromPlainOrCompressedFile( geneInfoFile.asFile().getAbsolutePath() )) { + try ( InputStream is = FileTools + .getInputStreamFromPlainOrCompressedFile( geneInfoFile.asFile().getAbsolutePath() ) ) { infoParser.parse( is ); } } catch ( IOException e ) { @@ -247,7 +250,7 @@ private void processLocalFiles( final LocalFile geneInfoFile, final LocalFile ge // all accessions for the gene are done. // 1b) Create a Collection, and push into BlockingQueue - Thread parseThread = new Thread( new Runnable() { + this.taskExecutor.execute( new Runnable() { @Override public void run() { try { @@ -261,9 +264,7 @@ public void run() { NcbiGeneDomainObjectGenerator.log.debug( "Domain object generator done" ); producerDone.set( true ); } - }, "gene2accession parser" ); - - parseThread.start(); + } ); // 1c) As elements get added to BlockingQueue, NCBIGeneConverter // consumes diff --git a/gemma-core/src/main/java/ubic/gemma/core/loader/genome/gene/ncbi/NcbiGeneLoader.java b/gemma-core/src/main/java/ubic/gemma/core/loader/genome/gene/ncbi/NcbiGeneLoader.java index 2fa6ef6c31..7b96076678 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/loader/genome/gene/ncbi/NcbiGeneLoader.java +++ b/gemma-core/src/main/java/ubic/gemma/core/loader/genome/gene/ncbi/NcbiGeneLoader.java @@ -22,6 +22,7 @@ import org.apache.commons.lang3.time.StopWatch; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.springframework.core.task.TaskExecutor; import org.springframework.security.core.context.SecurityContext; import org.springframework.security.core.context.SecurityContextHolder; import ubic.gemma.model.genome.Gene; @@ -54,15 +55,17 @@ public class NcbiGeneLoader { // whether to fetch files from ncbi or use existing ones private boolean doDownload = true; private Integer startingNcbiId = null; + private final TaskExecutor taskExecutor; - public NcbiGeneLoader() { + public NcbiGeneLoader( TaskExecutor taskExecutor ) { + this.taskExecutor = taskExecutor; generatorDone = new AtomicBoolean( false ); converterDone = new AtomicBoolean( false ); loaderDone = new AtomicBoolean( false ); } - public NcbiGeneLoader( Persister persisterHelper ) { - this(); + public NcbiGeneLoader( TaskExecutor taskExecutor, Persister persisterHelper ) { + this( taskExecutor ); this.setPersisterHelper( persisterHelper ); } @@ -216,15 +219,14 @@ private void load( final BlockingQueue geneQueue ) { final SecurityContext context = SecurityContextHolder.getContext(); assert context != null; - Thread loadThread = new Thread( new Runnable() { + taskExecutor.execute( new Runnable() { @Override public void run() { SecurityContextHolder.setContext( context ); NcbiGeneLoader.this.doLoad( geneQueue ); } - }, "Loading" ); - loadThread.start(); + } ); while ( !generatorDone.get() || !converterDone.get() || !loaderDone.get() ) { try { @@ -244,12 +246,12 @@ private void load( String geneInfoFile, String gene2AccFile, String geneHistoryF this.converterDone.set( false ); this.loaderDone.set( false ); - NcbiGeneDomainObjectGenerator sdog = new NcbiGeneDomainObjectGenerator( supportedTaxa ); + NcbiGeneDomainObjectGenerator sdog = new NcbiGeneDomainObjectGenerator( supportedTaxa, taskExecutor ); sdog.setDoDownload( doDownload ); sdog.setProducerDoneFlag( generatorDone ); sdog.setStartingNcbiId( startingNcbiId ); - NcbiGeneConverter converter = new NcbiGeneConverter(); + NcbiGeneConverter converter = new NcbiGeneConverter( taskExecutor ); converter.setSourceDoneFlag( generatorDone ); converter.setProducerDoneFlag( converterDone ); diff --git a/gemma-core/src/main/java/ubic/gemma/core/ontology/providers/GeneOntologyServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/ontology/providers/GeneOntologyServiceImpl.java index 729e0cfaae..16e1bf9eaf 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/ontology/providers/GeneOntologyServiceImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/core/ontology/providers/GeneOntologyServiceImpl.java @@ -29,6 +29,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.jena.larq.IndexLARQ; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.task.TaskExecutor; import org.springframework.stereotype.Component; import ubic.basecode.ontology.OntologyLoader; import ubic.basecode.ontology.model.AnnotationProperty; @@ -133,6 +134,9 @@ private static String toUri( String goId ) { private OntModel model; + @Autowired + private TaskExecutor taskExecutor; + /** * Cache of go term -> parent terms */ @@ -816,7 +820,7 @@ private synchronized void initializeGeneOntology() { if ( GeneOntologyServiceImpl.running.get() ) return; - Thread loadThread = new Thread( new Runnable() { + this.taskExecutor.execute( new Runnable() { @Override public void run() { GeneOntologyServiceImpl.running.set( true ); @@ -845,8 +849,6 @@ public void run() { } ); - loadThread.start(); - } private void logIds( String prefix, Collection terms ) { diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/association/coexpression/CoexpressionQueryQueueImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/association/coexpression/CoexpressionQueryQueueImpl.java index 98d064ee31..aaadbf643a 100644 --- a/gemma-core/src/main/java/ubic/gemma/persistence/service/association/coexpression/CoexpressionQueryQueueImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/association/coexpression/CoexpressionQueryQueueImpl.java @@ -23,6 +23,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.task.TaskExecutor; import org.springframework.orm.hibernate3.support.HibernateDaoSupport; import org.springframework.security.core.context.SecurityContext; import org.springframework.security.core.context.SecurityContextHolder; @@ -47,6 +48,8 @@ class CoexpressionQueryQueueImpl extends HibernateDaoSupport implements Coexpres private CoexpressionDao coexpressionDao; @Autowired private GeneDao geneDao; + @Autowired + private TaskExecutor taskExecutor; @Autowired public CoexpressionQueryQueueImpl( SessionFactory sessionFactory ) { @@ -88,7 +91,7 @@ protected void initDao() throws Exception { super.initDao(); final SecurityContext context = SecurityContextHolder.getContext(); - Thread loadThread = new Thread( new Runnable() { + taskExecutor.execute(new Runnable() { private final int MAX_WARNINGS = 5; @@ -128,9 +131,7 @@ public void run() { } } - }, "Fetching coexpression for recently used genes" ); - loadThread.setDaemon( true ); - loadThread.start(); + }); } private void queryForCache( QueuedGene gene ) { diff --git a/gemma-core/src/main/resources/ubic/gemma/applicationContext-serviceBeans.xml b/gemma-core/src/main/resources/ubic/gemma/applicationContext-serviceBeans.xml index 4d44902c4a..5a881ef61d 100644 --- a/gemma-core/src/main/resources/ubic/gemma/applicationContext-serviceBeans.xml +++ b/gemma-core/src/main/resources/ubic/gemma/applicationContext-serviceBeans.xml @@ -11,6 +11,10 @@ + + + + diff --git a/gemma-core/src/test/java/ubic/gemma/core/analysis/service/CompositeSequenceGeneMapperServiceTest.java b/gemma-core/src/test/java/ubic/gemma/core/analysis/service/CompositeSequenceGeneMapperServiceTest.java index 475e17dafe..fba00da7e3 100644 --- a/gemma-core/src/test/java/ubic/gemma/core/analysis/service/CompositeSequenceGeneMapperServiceTest.java +++ b/gemma-core/src/test/java/ubic/gemma/core/analysis/service/CompositeSequenceGeneMapperServiceTest.java @@ -23,6 +23,7 @@ import org.junit.Test; import org.junit.experimental.categories.Category; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.task.TaskExecutor; import ubic.basecode.util.FileTools; import ubic.gemma.core.apps.Blat; import ubic.gemma.core.apps.ShellDelegatingBlat; @@ -87,6 +88,9 @@ public class CompositeSequenceGeneMapperServiceTest extends AbstractGeoServiceTe @Autowired private ArrayDesignSequenceProcessingService sequenceProcessingService; + @Autowired + private TaskExecutor taskExecutor; + @After public void cleanup() { @@ -206,7 +210,7 @@ private void loadData() throws Exception { } private void loadGeneData() throws Exception { - NcbiGeneLoader loader = new NcbiGeneLoader(); + NcbiGeneLoader loader = new NcbiGeneLoader( taskExecutor ); loader.setTaxonService( taxonService ); loader.setPersisterHelper( this.persisterHelper ); diff --git a/gemma-core/src/test/java/ubic/gemma/core/loader/association/NCBIGene2GOAssociationParserTest.java b/gemma-core/src/test/java/ubic/gemma/core/loader/association/NCBIGene2GOAssociationParserTest.java index a49d4fc8fd..4df6de4b4e 100644 --- a/gemma-core/src/test/java/ubic/gemma/core/loader/association/NCBIGene2GOAssociationParserTest.java +++ b/gemma-core/src/test/java/ubic/gemma/core/loader/association/NCBIGene2GOAssociationParserTest.java @@ -22,6 +22,7 @@ import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.task.TaskExecutor; import ubic.gemma.core.util.test.BaseSpringContextTest; import ubic.gemma.persistence.service.association.Gene2GOAssociationService; @@ -42,13 +43,16 @@ public class NCBIGene2GOAssociationParserTest extends BaseSpringContextTest { @Autowired private Gene2GOAssociationService gene2GOAssociationService; + @Autowired + private TaskExecutor taskExecutor; + /* * Configure parser and loader. Injecting the parser and loader with their dependencies. */ @Before public void setup() { gene2GOAssociationService.removeAll(); - gene2GOAssLoader = new NCBIGene2GOAssociationLoader(); + gene2GOAssLoader = new NCBIGene2GOAssociationLoader( taskExecutor ); gene2GOAssLoader.setParser( new NCBIGene2GOAssociationParser( taxonService.loadAll() ) ); gene2GOAssLoader.setPersisterHelper( this.persisterHelper ); } diff --git a/gemma-core/src/test/java/ubic/gemma/core/loader/genome/gene/ncbi/NCBIGeneLoadingTest.java b/gemma-core/src/test/java/ubic/gemma/core/loader/genome/gene/ncbi/NCBIGeneLoadingTest.java index c2551775cd..efbd254a29 100644 --- a/gemma-core/src/test/java/ubic/gemma/core/loader/genome/gene/ncbi/NCBIGeneLoadingTest.java +++ b/gemma-core/src/test/java/ubic/gemma/core/loader/genome/gene/ncbi/NCBIGeneLoadingTest.java @@ -22,6 +22,7 @@ import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.task.TaskExecutor; import ubic.basecode.util.FileTools; import ubic.gemma.core.genome.gene.service.GeneService; import ubic.gemma.core.genome.gene.service.GeneSetService; @@ -51,6 +52,8 @@ public class NCBIGeneLoadingTest extends BaseSpringContextTest { private GeneSetService geneSetService; @Autowired private GeneProductService geneProductService; + @Autowired + private TaskExecutor taskExecutor; @Before public void setup() { @@ -64,7 +67,7 @@ public void tearDown() { @Test public void testGeneLoader() throws Exception { - NcbiGeneLoader loader = new NcbiGeneLoader( persisterHelper ); + NcbiGeneLoader loader = new NcbiGeneLoader( taskExecutor, persisterHelper ); loader.setTaxonService( taxonService ); String geneInfoTestFile = "/data/loader/genome/gene/gene_info.human.sample"; From 4648c725b270142b7efb1737008e5535dedc4435 Mon Sep 17 00:00:00 2001 From: Guillaume Poirier-Morency Date: Mon, 7 Mar 2022 11:19:52 -0800 Subject: [PATCH 14/28] Reuse the thread pool for running linear models --- .../analysis/expression/diff/LinearModelAnalyzer.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/expression/diff/LinearModelAnalyzer.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/expression/diff/LinearModelAnalyzer.java index b3807f4394..a3c9861c4d 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/analysis/expression/diff/LinearModelAnalyzer.java +++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/expression/diff/LinearModelAnalyzer.java @@ -27,7 +27,10 @@ import org.apache.commons.lang3.time.StopWatch; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Scope; +import org.springframework.core.task.AsyncTaskExecutor; import org.springframework.stereotype.Component; import ubic.basecode.dataStructure.matrix.DenseDoubleMatrix; import ubic.basecode.dataStructure.matrix.DoubleMatrix; @@ -140,6 +143,9 @@ private static BioAssayDimension createBADMap( List columnsToUse ) return reorderedDim; } + @Autowired + private AsyncTaskExecutor taskExecutor; + /** * Determine if any factor should be treated as the intercept term. */ @@ -1432,9 +1438,8 @@ private Map runAnalysis( final DoubleMatrix runAnalysisFuture( final DesignMatrix designMatrix, final DoubleMatrix data, final Map rawResults, final DoubleMatrix1D librarySize, final DifferentialExpressionAnalysisConfig config ) { - ExecutorService service = Executors.newSingleThreadExecutor(); - Future f = service.submit( new Runnable() { + Future f = taskExecutor.submit( new Runnable() { @Override public void run() { StopWatch timer = new StopWatch(); @@ -1476,7 +1481,6 @@ public void run() { } } ); - service.shutdown(); return f; } } From 667f67f75c748914e54704f1bebb8f556d7d086b Mon Sep 17 00:00:00 2001 From: Guillaume Poirier-Morency Date: Mon, 7 Mar 2022 14:28:58 -0800 Subject: [PATCH 15/28] Make the local task executor configurable --- gemma-core/src/main/resources/default.properties | 4 ++++ .../resources/ubic/gemma/applicationContext-serviceBeans.xml | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/gemma-core/src/main/resources/default.properties b/gemma-core/src/main/resources/default.properties index 6a6461cc92..f3f531a31a 100755 --- a/gemma-core/src/main/resources/default.properties +++ b/gemma-core/src/main/resources/default.properties @@ -212,6 +212,10 @@ gemma.coexpressionSearch.maxGenesForQueryGenesOnly=500 #controls the maximum number of genes that can be used for the large coexpression 'my genes only' query, that is, no 'complete the graph' operations. gemma.coexpressionSearch.maxGenesPerCoexLargeQuery=1000 ############################################################ +# Configuration for the local tasks' executor (based on Spring TaskExecutor and AsyncTaskExecutor) +gemma.localTasks.corePoolSize=1 +gemma.localTasks.maxPoolSize=16 +############################################################ # REMOTE TASKS (JMS) CONFIGURATION gemma.jms.brokerURL=localhost:61616 gemma.test.jms.brokerURL=localhost:61617 diff --git a/gemma-core/src/main/resources/ubic/gemma/applicationContext-serviceBeans.xml b/gemma-core/src/main/resources/ubic/gemma/applicationContext-serviceBeans.xml index 5a881ef61d..815b213c5f 100644 --- a/gemma-core/src/main/resources/ubic/gemma/applicationContext-serviceBeans.xml +++ b/gemma-core/src/main/resources/ubic/gemma/applicationContext-serviceBeans.xml @@ -11,8 +11,10 @@ + - + + From 48ff253515d8cd8040bed89afc2f7478f32d7c05 Mon Sep 17 00:00:00 2001 From: Guillaume Poirier-Morency Date: Tue, 8 Mar 2022 09:48:34 -0800 Subject: [PATCH 16/28] Use Future instead of AtomicBoolean to assess completion with NCBI gene and Gene2GO loading --- .../apps/NCBIGene2GOAssociationLoaderCLI.java | 3 +- .../gemma/core/apps/NcbiGeneLoaderCLI.java | 3 +- .../NCBIGene2GOAssociationLoader.java | 57 +++++-------- .../genome/gene/ncbi/NcbiGeneConverter.java | 38 +++------ .../ncbi/NcbiGeneDomainObjectGenerator.java | 33 +++----- .../genome/gene/ncbi/NcbiGeneLoader.java | 80 ++++++------------- ...ompositeSequenceGeneMapperServiceTest.java | 4 +- .../NCBIGene2GOAssociationParserTest.java | 9 +-- .../genome/gene/ncbi/NCBIGeneLoadingTest.java | 24 +++--- 9 files changed, 86 insertions(+), 165 deletions(-) diff --git a/gemma-core/src/main/java/ubic/gemma/core/apps/NCBIGene2GOAssociationLoaderCLI.java b/gemma-core/src/main/java/ubic/gemma/core/apps/NCBIGene2GOAssociationLoaderCLI.java index 83e239969a..d3ebd23d95 100755 --- a/gemma-core/src/main/java/ubic/gemma/core/apps/NCBIGene2GOAssociationLoaderCLI.java +++ b/gemma-core/src/main/java/ubic/gemma/core/apps/NCBIGene2GOAssociationLoaderCLI.java @@ -21,6 +21,7 @@ import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; +import org.springframework.core.task.AsyncTaskExecutor; import org.springframework.core.task.TaskExecutor; import ubic.gemma.core.apps.GemmaCLI.CommandGroup; import ubic.gemma.core.loader.association.NCBIGene2GOAssociationLoader; @@ -65,7 +66,7 @@ protected void buildOptions( Options options ) { @Override protected void doWork() throws Exception { TaxonService taxonService = this.getBean( TaxonService.class ); - TaskExecutor taskExecutor = this.getBean( TaskExecutor.class ); + AsyncTaskExecutor taskExecutor = this.getBean( AsyncTaskExecutor.class ); NCBIGene2GOAssociationLoader gene2GOAssLoader = new NCBIGene2GOAssociationLoader( taskExecutor ); diff --git a/gemma-core/src/main/java/ubic/gemma/core/apps/NcbiGeneLoaderCLI.java b/gemma-core/src/main/java/ubic/gemma/core/apps/NcbiGeneLoaderCLI.java index badf0e8892..0a755515f3 100755 --- a/gemma-core/src/main/java/ubic/gemma/core/apps/NcbiGeneLoaderCLI.java +++ b/gemma-core/src/main/java/ubic/gemma/core/apps/NcbiGeneLoaderCLI.java @@ -22,6 +22,7 @@ import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.lang3.StringUtils; +import org.springframework.core.task.AsyncTaskExecutor; import org.springframework.core.task.TaskExecutor; import ubic.gemma.core.apps.GemmaCLI.CommandGroup; import ubic.gemma.core.loader.genome.gene.ncbi.NcbiGeneLoader; @@ -99,7 +100,7 @@ protected boolean requireLogin() { @Override protected void doWork() throws Exception { - loader = new NcbiGeneLoader( this.getBean( TaskExecutor.class ) ); + loader = new NcbiGeneLoader( this.getBean( AsyncTaskExecutor.class ) ); TaxonService taxonService = this.getBean( TaxonService.class ); loader.setTaxonService( taxonService ); loader.setPersisterHelper( this.getPersisterHelper() ); diff --git a/gemma-core/src/main/java/ubic/gemma/core/loader/association/NCBIGene2GOAssociationLoader.java b/gemma-core/src/main/java/ubic/gemma/core/loader/association/NCBIGene2GOAssociationLoader.java index 532ee78ed2..b1cc7c70e8 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/loader/association/NCBIGene2GOAssociationLoader.java +++ b/gemma-core/src/main/java/ubic/gemma/core/loader/association/NCBIGene2GOAssociationLoader.java @@ -20,7 +20,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.springframework.core.task.TaskExecutor; +import org.springframework.core.task.AsyncTaskExecutor; import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContext; import org.springframework.security.core.context.SecurityContextHolder; @@ -35,7 +35,8 @@ import java.util.Collection; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; -import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; /** * @author keshav @@ -46,14 +47,12 @@ public class NCBIGene2GOAssociationLoader { private static final Log log = LogFactory.getLog( NCBIGene2GOAssociationLoader.class ); private static final int QUEUE_SIZE = 60000; private static final int BATCH_SIZE = 12000; - private final AtomicBoolean producerDone = new AtomicBoolean( false ); - private final AtomicBoolean consumerDone = new AtomicBoolean( false ); private Persister persisterHelper; private NCBIGene2GOAssociationParser parser = null; private int count; - private final TaskExecutor taskExecutor; + private final AsyncTaskExecutor taskExecutor; - public NCBIGene2GOAssociationLoader( TaskExecutor taskExecutor ) { + public NCBIGene2GOAssociationLoader( AsyncTaskExecutor taskExecutor ) { this.taskExecutor = taskExecutor; } @@ -65,32 +64,13 @@ private void setCount( int count ) { this.count = count; } - @SuppressWarnings({ "unused", "WeakerAccess" }) // Possible external use - public boolean isConsumerDone() { - return consumerDone.get(); - } - - @SuppressWarnings({ "unused", "WeakerAccess" }) // Possible external use - public boolean isProducerDone() { - return producerDone.get(); - } - public void load( final InputStream inputStream ) { final BlockingQueue queue = new ArrayBlockingQueue<>( NCBIGene2GOAssociationLoader.QUEUE_SIZE ); final SecurityContext context = SecurityContextHolder.getContext(); final Authentication authentication = context.getAuthentication(); - taskExecutor.execute( new Runnable() { - @Override - public void run() { - NCBIGene2GOAssociationLoader.log.info( "Starting loading" ); - SecurityContextHolder.setContext( context ); - NCBIGene2GOAssociationLoader.this.load( queue ); - } - } ); - - taskExecutor.execute( new Runnable() { + Future producerFuture = taskExecutor.submit( new Runnable() { @Override public void run() { try { @@ -103,16 +83,22 @@ public void run() { throw new RuntimeException( e ); } NCBIGene2GOAssociationLoader.log.info( "Done parsing" ); - producerDone.set( true ); } } ); - while ( !this.isProducerDone() || !this.isConsumerDone() ) { - try { - Thread.sleep( 1000 ); - } catch ( InterruptedException e ) { - e.printStackTrace(); + Future consumerFuture = taskExecutor.submit( new Runnable() { + @Override + public void run() { + NCBIGene2GOAssociationLoader.log.info( "Starting loading" ); + SecurityContextHolder.setContext( context ); + NCBIGene2GOAssociationLoader.this.load( queue, producerFuture ); } + } ); + + try { + consumerFuture.get(); + } catch ( InterruptedException | ExecutionException e ) { + e.printStackTrace(); } } @@ -137,7 +123,7 @@ public void setPersisterHelper( Persister persisterHelper ) { this.persisterHelper = persisterHelper; } - private void load( BlockingQueue queue ) { + private void load( BlockingQueue queue, Future producerFuture ) { NCBIGene2GOAssociationLoader.log.debug( "Entering 'load' " ); @@ -147,7 +133,7 @@ private void load( BlockingQueue queue ) { Collection itemsToPersist = new ArrayList<>(); try { - while ( !( producerDone.get() && queue.isEmpty() ) ) { + while ( !( producerFuture.isDone() && queue.isEmpty() ) ) { Gene2GOAssociation associations = queue.poll(); if ( associations == null ) { @@ -175,7 +161,6 @@ private void load( BlockingQueue queue ) { } } catch ( Exception e ) { - consumerDone.set( true ); NCBIGene2GOAssociationLoader.log.fatal( e, e ); throw new RuntimeException( e ); } @@ -184,7 +169,5 @@ private void load( BlockingQueue queue ) { persisterHelper.persist( itemsToPersist ); NCBIGene2GOAssociationLoader.log.info( "Finished, loaded total of " + count + " GO associations" ); - consumerDone.set( true ); - } } diff --git a/gemma-core/src/main/java/ubic/gemma/core/loader/genome/gene/ncbi/NcbiGeneConverter.java b/gemma-core/src/main/java/ubic/gemma/core/loader/genome/gene/ncbi/NcbiGeneConverter.java index 66fe6c37e8..e4716580be 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/loader/genome/gene/ncbi/NcbiGeneConverter.java +++ b/gemma-core/src/main/java/ubic/gemma/core/loader/genome/gene/ncbi/NcbiGeneConverter.java @@ -22,7 +22,7 @@ import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.springframework.core.task.TaskExecutor; +import org.springframework.core.task.AsyncTaskExecutor; import ubic.gemma.core.loader.genome.gene.ncbi.model.NCBIGene2Accession; import ubic.gemma.core.loader.genome.gene.ncbi.model.NCBIGeneInfo; import ubic.gemma.core.loader.util.converter.Converter; @@ -41,7 +41,7 @@ import java.util.Collection; import java.util.HashSet; import java.util.concurrent.BlockingQueue; -import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.Future; /** * Convert NCBIGene2Accession objects into Gemma Gene objects with associated GeneProducts. Genes without products are @@ -64,12 +64,9 @@ public class NcbiGeneConverter implements Converter { NcbiGeneConverter.ensembl.setName( "Ensembl" ); } - private final TaskExecutor taskExecutor; + private final AsyncTaskExecutor taskExecutor; - AtomicBoolean producerDone = new AtomicBoolean( false ); - AtomicBoolean sourceDone = new AtomicBoolean( false ); - - public NcbiGeneConverter( TaskExecutor taskExecutor ) { + public NcbiGeneConverter( AsyncTaskExecutor taskExecutor ) { this.taskExecutor = taskExecutor; } @@ -239,18 +236,20 @@ public Gene convert( NcbiGeneData data ) { return gene; } - /* + /** * Threaded conversion of domain objects to Gemma objects. + * + * @return a future that completes when the conversion is done, that is when the gene info queue is empty and the + * source is done as well */ - public void convert( final BlockingQueue geneInfoQueue, final BlockingQueue geneQueue ) { + public Future convert( final BlockingQueue geneInfoQueue, final BlockingQueue geneQueue, Future sourceFuture ) { // start up thread to convert a member of geneInfoQueue to a gene/geneproduct/databaseentry // then push the gene onto the geneQueue for loading - this.taskExecutor.execute( new Runnable() { + return this.taskExecutor.submit( new Runnable() { @Override - @SuppressWarnings("synthetic-access") public void run() { - while ( !( sourceDone.get() && geneInfoQueue.isEmpty() ) ) { + while ( !( sourceFuture.isDone() && geneInfoQueue.isEmpty() ) ) { try { NcbiGeneData data = geneInfoQueue.poll(); if ( data == null ) { @@ -273,21 +272,8 @@ public void run() { break; } } - producerDone.set( true ); } - }); - } - - public boolean isProducerDone() { - return this.producerDone.get(); - } - - public void setProducerDoneFlag( AtomicBoolean flag ) { - this.producerDone = flag; - } - - public void setSourceDoneFlag( AtomicBoolean flag ) { - this.sourceDone = flag; + } ); } private PhysicalLocation getPhysicalLocation( NCBIGene2Accession acc, Gene gene ) { diff --git a/gemma-core/src/main/java/ubic/gemma/core/loader/genome/gene/ncbi/NcbiGeneDomainObjectGenerator.java b/gemma-core/src/main/java/ubic/gemma/core/loader/genome/gene/ncbi/NcbiGeneDomainObjectGenerator.java index 24d397eedf..e613dd2141 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/loader/genome/gene/ncbi/NcbiGeneDomainObjectGenerator.java +++ b/gemma-core/src/main/java/ubic/gemma/core/loader/genome/gene/ncbi/NcbiGeneDomainObjectGenerator.java @@ -20,7 +20,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.springframework.core.task.TaskExecutor; +import org.springframework.core.task.AsyncTaskExecutor; import ubic.basecode.util.FileTools; import ubic.gemma.core.loader.genome.gene.ncbi.model.NCBIGeneInfo; import ubic.gemma.core.loader.genome.gene.ncbi.model.NcbiGeneHistory; @@ -36,7 +36,7 @@ import java.util.HashSet; import java.util.Map; import java.util.concurrent.BlockingQueue; -import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.Future; /** * Combines information from the gene2accession and gene_info files from NCBI Gene. @@ -51,7 +51,6 @@ public class NcbiGeneDomainObjectGenerator { private static final String GENE2ACCESSION_FILE = "gene2accession"; private static final String GENEHISTORY_FILE = "gene_history"; private static final String GENEENSEMBL_FILE = "gene2ensembl"; - private AtomicBoolean producerDone = new AtomicBoolean( false ); private Map supportedTaxa = null; private Collection supportedTaxaWithNCBIGenes = null; private boolean filter = true; @@ -59,9 +58,9 @@ public class NcbiGeneDomainObjectGenerator { // whether to fetch files from ncbi or use existing ones private boolean doDownload = true; private Integer startingNcbiId = null; - private final TaskExecutor taskExecutor; + private final AsyncTaskExecutor taskExecutor; - public NcbiGeneDomainObjectGenerator( Collection taxa, TaskExecutor taskExecutor ) { + public NcbiGeneDomainObjectGenerator( Collection taxa, AsyncTaskExecutor taskExecutor ) { this.taskExecutor = taskExecutor; if ( taxa != null ) { @@ -90,10 +89,7 @@ public void setDoDownload( boolean doDownload ) { this.doDownload = doDownload; } - /** - * @param queue queue - */ - public void generate( final BlockingQueue queue ) { + public Future generate( final BlockingQueue queue ) { NcbiGeneDomainObjectGenerator.log.info( "Fetching..." ); NCBIGeneFileFetcher fetcher = new NCBIGeneFileFetcher(); @@ -104,10 +100,10 @@ public void generate( final BlockingQueue queue ) { LocalFile geneHistoryFile = fetcher.fetch( NcbiGeneDomainObjectGenerator.GENEHISTORY_FILE ).iterator().next(); LocalFile geneEnsemblFile = fetcher.fetch( NcbiGeneDomainObjectGenerator.GENEENSEMBL_FILE ).iterator().next(); - this.processLocalFiles( geneInfoFile, gene2AccessionFile, geneHistoryFile, geneEnsemblFile, queue ); + return this.processLocalFiles( geneInfoFile, gene2AccessionFile, geneHistoryFile, geneEnsemblFile, queue ); } - public void generateLocal( String geneInfoFilePath, String gene2AccesionFilePath, String geneHistoryFilePath, + public Future generateLocal( String geneInfoFilePath, String gene2AccesionFilePath, String geneHistoryFilePath, String geneEnsemblFilePath, BlockingQueue queue ) { assert gene2AccesionFilePath != null; @@ -136,21 +132,13 @@ public void generateLocal( String geneInfoFilePath, String gene2AccesionFilePath geneEnsemblFile.setLocalURL( geneEnsemblUrl ); } - this.processLocalFiles( geneInfoFile, gene2AccessionFile, geneHistoryFile, geneEnsemblFile, queue ); + return this.processLocalFiles( geneInfoFile, gene2AccessionFile, geneHistoryFile, geneEnsemblFile, queue ); } catch ( IOException e ) { throw new RuntimeException( e ); } } - public boolean isProducerDone() { - return producerDone.get(); - } - - public void setProducerDoneFlag( AtomicBoolean flag ) { - this.producerDone = flag; - } - /** * Those taxa that are supported by GEMMA and have genes in NCBI. * @@ -164,7 +152,7 @@ public void setStartingNcbiId( Integer startingNcbiId ) { this.startingNcbiId = startingNcbiId; } - private void processLocalFiles( final LocalFile geneInfoFile, final LocalFile gene2AccessionFile, + private Future processLocalFiles( final LocalFile geneInfoFile, final LocalFile gene2AccessionFile, LocalFile geneHistoryFile, LocalFile geneEnsemblFile, final BlockingQueue geneDataQueue ) { final NcbiGeneInfoParser infoParser = new NcbiGeneInfoParser(); @@ -250,7 +238,7 @@ private void processLocalFiles( final LocalFile geneInfoFile, final LocalFile ge // all accessions for the gene are done. // 1b) Create a Collection, and push into BlockingQueue - this.taskExecutor.execute( new Runnable() { + return this.taskExecutor.submit( new Runnable() { @Override public void run() { try { @@ -262,7 +250,6 @@ public void run() { throw new RuntimeException( e ); } NcbiGeneDomainObjectGenerator.log.debug( "Domain object generator done" ); - producerDone.set( true ); } } ); diff --git a/gemma-core/src/main/java/ubic/gemma/core/loader/genome/gene/ncbi/NcbiGeneLoader.java b/gemma-core/src/main/java/ubic/gemma/core/loader/genome/gene/ncbi/NcbiGeneLoader.java index 7b96076678..b7a00cf51c 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/loader/genome/gene/ncbi/NcbiGeneLoader.java +++ b/gemma-core/src/main/java/ubic/gemma/core/loader/genome/gene/ncbi/NcbiGeneLoader.java @@ -22,7 +22,7 @@ import org.apache.commons.lang3.time.StopWatch; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.springframework.core.task.TaskExecutor; +import org.springframework.core.task.AsyncTaskExecutor; import org.springframework.security.core.context.SecurityContext; import org.springframework.security.core.context.SecurityContextHolder; import ubic.gemma.model.genome.Gene; @@ -34,7 +34,7 @@ import java.util.HashSet; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; -import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.Future; /** * Load or update information about genes from the NCBI Gene database. @@ -45,9 +45,6 @@ public class NcbiGeneLoader { private static final int QUEUE_SIZE = 1000; private static final Log log = LogFactory.getLog( NcbiGeneConverter.class.getName() ); - private final AtomicBoolean generatorDone; - private final AtomicBoolean converterDone; - private final AtomicBoolean loaderDone; private Persister persisterHelper; private int loadedGeneCount = 0; private TaxonService taxonService; @@ -55,16 +52,13 @@ public class NcbiGeneLoader { // whether to fetch files from ncbi or use existing ones private boolean doDownload = true; private Integer startingNcbiId = null; - private final TaskExecutor taskExecutor; + private final AsyncTaskExecutor taskExecutor; - public NcbiGeneLoader( TaskExecutor taskExecutor ) { + public NcbiGeneLoader( AsyncTaskExecutor taskExecutor ) { this.taskExecutor = taskExecutor; - generatorDone = new AtomicBoolean( false ); - converterDone = new AtomicBoolean( false ); - loaderDone = new AtomicBoolean( false ); } - public NcbiGeneLoader( TaskExecutor taskExecutor, Persister persisterHelper ) { + public NcbiGeneLoader( AsyncTaskExecutor taskExecutor, Persister persisterHelper ) { this( taskExecutor ); this.setPersisterHelper( persisterHelper ); } @@ -76,11 +70,6 @@ public int getLoadedGeneCount() { return loadedGeneCount; } - @SuppressWarnings("BooleanMethodIsAlwaysInverted") // Better semantics - public boolean isLoaderDone() { - return loaderDone.get(); - } - /** * download the gene_info and gene2accession files, then call load * @@ -108,13 +97,13 @@ public void load( String geneInfoFile, String gene2AccFile, String geneHistoryFi } - public void load( String geneInfoFile, String gene2AccFile, String geneHistoryFile, String geneEnsemblFile, + public Future load( String geneInfoFile, String gene2AccFile, String geneHistoryFile, String geneEnsemblFile, Taxon t ) { Collection taxaToUse = new HashSet<>(); taxaToUse.add( t ); - this.load( geneInfoFile, gene2AccFile, geneHistoryFile, geneEnsemblFile, taxaToUse ); + return this.load( geneInfoFile, gene2AccFile, geneHistoryFile, geneEnsemblFile, taxaToUse ); } @@ -181,10 +170,10 @@ public void setStartingNcbiId( Integer startNcbiid ) { this.startingNcbiId = startNcbiid; } - void doLoad( final BlockingQueue geneQueue ) { + private void doLoad( final BlockingQueue geneQueue, Future converterFuture ) { StopWatch timer = new StopWatch(); timer.start(); - while ( !( converterDone.get() && geneQueue.isEmpty() ) ) { + while ( !( converterFuture.isDone() && geneQueue.isEmpty() ) ) { Gene gene = null; try { // the converted genes. @@ -204,79 +193,56 @@ void doLoad( final BlockingQueue geneQueue ) { } catch ( Exception e ) { NcbiGeneLoader.log.error( "Error while loading gene: " + gene + ": " + e.getMessage(), e ); - loaderDone.set( true ); throw new RuntimeException( e ); } } NcbiGeneLoader.log.info( "Loaded " + loadedGeneCount + " genes. " ); - loaderDone.set( true ); } /** * @param geneQueue a blocking queue of genes to be loaded into the database loads genes into the database */ - private void load( final BlockingQueue geneQueue ) { + private Future load( final BlockingQueue geneQueue, Future converterFuture ) { final SecurityContext context = SecurityContextHolder.getContext(); assert context != null; - taskExecutor.execute( new Runnable() { - @Override - public void run() { - SecurityContextHolder.setContext( context ); - NcbiGeneLoader.this.doLoad( geneQueue ); - } - + return taskExecutor.submit( () -> { + SecurityContextHolder.setContext( context ); + NcbiGeneLoader.this.doLoad( geneQueue, converterFuture ); } ); - - while ( !generatorDone.get() || !converterDone.get() || !loaderDone.get() ) { - try { - Thread.sleep( 1000 ); - } catch ( InterruptedException e ) { - e.printStackTrace(); - } - } } - private void load( String geneInfoFile, String gene2AccFile, String geneHistoryFile, String geneEnsemblFile, + private Future load( String geneInfoFile, String gene2AccFile, String geneHistoryFile, String geneEnsemblFile, Collection supportedTaxa ) { - /* - * In case this is reused. - */ - this.generatorDone.set( false ); - this.converterDone.set( false ); - this.loaderDone.set( false ); - NcbiGeneDomainObjectGenerator sdog = new NcbiGeneDomainObjectGenerator( supportedTaxa, taskExecutor ); sdog.setDoDownload( doDownload ); - sdog.setProducerDoneFlag( generatorDone ); sdog.setStartingNcbiId( startingNcbiId ); NcbiGeneConverter converter = new NcbiGeneConverter( taskExecutor ); - converter.setSourceDoneFlag( generatorDone ); - converter.setProducerDoneFlag( converterDone ); // create queue for GeneInfo objects final BlockingQueue geneInfoQueue = new ArrayBlockingQueue<>( NcbiGeneLoader.QUEUE_SIZE ); final BlockingQueue geneQueue = new ArrayBlockingQueue<>( NcbiGeneLoader.QUEUE_SIZE ); // Threaded producer - loading files into queue as GeneInfo objects + Future producerFuture; if ( StringUtils.isEmpty( geneInfoFile ) || StringUtils.isEmpty( geneInfoFile ) ) { - sdog.generate( geneInfoQueue ); + producerFuture = sdog.generate( geneInfoQueue ); } else { - sdog.generateLocal( geneInfoFile, gene2AccFile, geneHistoryFile, geneEnsemblFile, geneInfoQueue ); + producerFuture = sdog.generateLocal( geneInfoFile, gene2AccFile, geneHistoryFile, geneEnsemblFile, geneInfoQueue ); } + // update taxon table to indicate that now there are genes loaded for that taxa. + // all or nothing so that if fails for some taxa then no taxa will be updated. + this.updateTaxaWithGenesUsable( sdog.getSupportedTaxaWithNCBIGenes() ); + // Threaded consumer/producer - consumes GeneInfo objects and generates // Gene/GeneProduct/DatabaseEntry entries - converter.convert( geneInfoQueue, geneQueue ); + Future converterFuture = converter.convert( geneInfoQueue, geneQueue, producerFuture ); // Threaded consumer. Consumes Gene objects and persists them into // the database - this.load( geneQueue ); - - // update taxon table to indicate that now there are genes loaded for that taxa. - // all or nothing so that if fails for some taxa then no taxa will be updated. - this.updateTaxaWithGenesUsable( sdog.getSupportedTaxaWithNCBIGenes() ); + return this.load( geneQueue, converterFuture ); } } diff --git a/gemma-core/src/test/java/ubic/gemma/core/analysis/service/CompositeSequenceGeneMapperServiceTest.java b/gemma-core/src/test/java/ubic/gemma/core/analysis/service/CompositeSequenceGeneMapperServiceTest.java index fba00da7e3..4f2eebcf25 100644 --- a/gemma-core/src/test/java/ubic/gemma/core/analysis/service/CompositeSequenceGeneMapperServiceTest.java +++ b/gemma-core/src/test/java/ubic/gemma/core/analysis/service/CompositeSequenceGeneMapperServiceTest.java @@ -23,7 +23,7 @@ import org.junit.Test; import org.junit.experimental.categories.Category; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.task.TaskExecutor; +import org.springframework.core.task.AsyncTaskExecutor; import ubic.basecode.util.FileTools; import ubic.gemma.core.apps.Blat; import ubic.gemma.core.apps.ShellDelegatingBlat; @@ -89,7 +89,7 @@ public class CompositeSequenceGeneMapperServiceTest extends AbstractGeoServiceTe private ArrayDesignSequenceProcessingService sequenceProcessingService; @Autowired - private TaskExecutor taskExecutor; + private AsyncTaskExecutor taskExecutor; @After public void cleanup() { diff --git a/gemma-core/src/test/java/ubic/gemma/core/loader/association/NCBIGene2GOAssociationParserTest.java b/gemma-core/src/test/java/ubic/gemma/core/loader/association/NCBIGene2GOAssociationParserTest.java index 4df6de4b4e..7da16da898 100644 --- a/gemma-core/src/test/java/ubic/gemma/core/loader/association/NCBIGene2GOAssociationParserTest.java +++ b/gemma-core/src/test/java/ubic/gemma/core/loader/association/NCBIGene2GOAssociationParserTest.java @@ -21,8 +21,7 @@ import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; - -import org.springframework.core.task.TaskExecutor; +import org.springframework.core.task.AsyncTaskExecutor; import ubic.gemma.core.util.test.BaseSpringContextTest; import ubic.gemma.persistence.service.association.Gene2GOAssociationService; @@ -44,7 +43,7 @@ public class NCBIGene2GOAssociationParserTest extends BaseSpringContextTest { private Gene2GOAssociationService gene2GOAssociationService; @Autowired - private TaskExecutor taskExecutor; + private AsyncTaskExecutor taskExecutor; /* * Configure parser and loader. Injecting the parser and loader with their dependencies. @@ -64,8 +63,8 @@ public void setup() { @Test public void testParseAndLoad() throws Exception { - try (InputStream is = this.getClass().getResourceAsStream( "/data/loader/association/gene2go.gz" ); - InputStream gZipIs = new GZIPInputStream( is )) { + try ( InputStream is = this.getClass().getResourceAsStream( "/data/loader/association/gene2go.gz" ); + InputStream gZipIs = new GZIPInputStream( is ) ) { gene2GOAssLoader.load( gZipIs ); gZipIs.close(); diff --git a/gemma-core/src/test/java/ubic/gemma/core/loader/genome/gene/ncbi/NCBIGeneLoadingTest.java b/gemma-core/src/test/java/ubic/gemma/core/loader/genome/gene/ncbi/NCBIGeneLoadingTest.java index efbd254a29..979ce75315 100644 --- a/gemma-core/src/test/java/ubic/gemma/core/loader/genome/gene/ncbi/NCBIGeneLoadingTest.java +++ b/gemma-core/src/test/java/ubic/gemma/core/loader/genome/gene/ncbi/NCBIGeneLoadingTest.java @@ -1,8 +1,8 @@ /* * The Gemma project - * + * * Copyright (c) 2006 University of British Columbia - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -22,7 +22,7 @@ import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.task.TaskExecutor; +import org.springframework.core.task.AsyncTaskExecutor; import ubic.basecode.util.FileTools; import ubic.gemma.core.genome.gene.service.GeneService; import ubic.gemma.core.genome.gene.service.GeneSetService; @@ -36,6 +36,7 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.concurrent.Future; import static org.junit.Assert.*; @@ -53,7 +54,7 @@ public class NCBIGeneLoadingTest extends BaseSpringContextTest { @Autowired private GeneProductService geneProductService; @Autowired - private TaskExecutor taskExecutor; + private AsyncTaskExecutor taskExecutor; @Before public void setup() { @@ -80,13 +81,11 @@ public void testGeneLoader() throws Exception { assertNotNull( ta ); - loader.load( FileTools.resourceToPath( geneInfoTestFile ), FileTools.resourceToPath( gene2AccTestFile ), + Future loaderFuture = loader.load( FileTools.resourceToPath( geneInfoTestFile ), FileTools.resourceToPath( gene2AccTestFile ), FileTools.resourceToPath( geneHistoryFile ), null, ta ); // wait until the loader is done. - while ( !loader.isLoaderDone() ) { - Thread.sleep( 100 ); - } + loaderFuture.get(); // loader is done. // check if it loaded elements to the database @@ -134,12 +133,11 @@ public void testGeneLoader() throws Exception { String updatedHistory = "/data/loader/genome/gene/gene_history.human.changed.sample"; String geneEnsemblFile = "/data/loader/genome/gene/gene2ensembl.human.sample"; - loader.load( FileTools.resourceToPath( geneInfoTestFile ), FileTools.resourceToPath( gene2AccTestFile ), + loaderFuture = loader.load( FileTools.resourceToPath( geneInfoTestFile ), FileTools.resourceToPath( gene2AccTestFile ), FileTools.resourceToPath( updatedHistory ), FileTools.resourceToPath( geneEnsemblFile ), ta ); // wait until the loader is done. - while ( !loader.isLoaderDone() ) { - Thread.sleep( 100 ); - } + loaderFuture.get(); + Collection updatedTestGene = geneService.findByOfficialSymbol( "TEAD1" ); assertEquals( 1, updatedTestGene.size() ); g = updatedTestGene.iterator().next(); @@ -148,7 +146,7 @@ public void testGeneLoader() throws Exception { g = geneService.findByNCBIId( 1 ); assertEquals( "ENSG00000121410", g.getEnsemblId() ); - + // test remove... geneProductService.remove( products ); From 3edb7370071e4e1e09fc195d014350216f4ad860 Mon Sep 17 00:00:00 2001 From: Guillaume Poirier-Morency Date: Tue, 8 Mar 2022 15:49:00 -0800 Subject: [PATCH 17/28] Remove Ehcache showdown listener, it's already handled by Spring --- gemma-web/src/main/webapp/WEB-INF/web.xml | 3 --- 1 file changed, 3 deletions(-) diff --git a/gemma-web/src/main/webapp/WEB-INF/web.xml b/gemma-web/src/main/webapp/WEB-INF/web.xml index b1882fde4c..2dc65c0e96 100644 --- a/gemma-web/src/main/webapp/WEB-INF/web.xml +++ b/gemma-web/src/main/webapp/WEB-INF/web.xml @@ -91,9 +91,6 @@ org.springframework.security.web.session.HttpSessionEventPublisher - - net.sf.ehcache.constructs.web.ShutdownListener - From 2e76a10acc31f0bc070ff6f58d0188b5546df114 Mon Sep 17 00:00:00 2001 From: Guillaume Poirier-Morency Date: Tue, 8 Mar 2022 15:54:25 -0800 Subject: [PATCH 18/28] Eagerly load gemma-rest, gemma-ws first and gemma-web after JAWR is ready --- gemma-web/src/main/webapp/WEB-INF/web.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gemma-web/src/main/webapp/WEB-INF/web.xml b/gemma-web/src/main/webapp/WEB-INF/web.xml index 2dc65c0e96..afb71ef5c9 100644 --- a/gemma-web/src/main/webapp/WEB-INF/web.xml +++ b/gemma-web/src/main/webapp/WEB-INF/web.xml @@ -96,6 +96,8 @@ gemma org.springframework.web.servlet.DispatcherServlet + + 2 @@ -107,6 +109,7 @@ transformWsdlLocations false + 0 @@ -121,6 +124,7 @@ openApi.configuration.location /WEB-INF/classes/openapi-configuration.yaml + 0 From 95ad63336f886f5a38bdde8ec4519a1e65114821 Mon Sep 17 00:00:00 2001 From: Guillaume Poirier-Morency Date: Tue, 8 Mar 2022 16:12:12 -0800 Subject: [PATCH 19/28] Fix all deprecated API usages --- .../gemma/persistence/persister/GenomePersister.java | 4 ++-- .../common/auditAndSecurity/AuditEventDaoImpl.java | 2 +- .../common/description/CharacteristicDaoImpl.java | 2 +- .../service/common/measurement/UnitDaoImpl.java | 2 +- .../service/expression/bioAssay/BioAssayDaoImpl.java | 12 +++++++----- .../ProcessedExpressionDataVectorDaoImpl.java | 2 +- .../designElement/CompositeSequenceDaoImpl.java | 4 ++-- .../expression/experiment/FactorValueDaoImpl.java | 2 +- .../persistence/service/genome/GeneDaoImpl.java | 4 ++-- .../genome/biosequence/BioSequenceDaoImpl.java | 2 +- .../service/genome/gene/GeneProductDaoImpl.java | 6 +++--- gemma-web/src/main/webapp/WEB-INF/web.xml | 2 +- 12 files changed, 23 insertions(+), 21 deletions(-) diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/persister/GenomePersister.java b/gemma-core/src/main/java/ubic/gemma/persistence/persister/GenomePersister.java index c11b2a5842..267aca47d4 100644 --- a/gemma-core/src/main/java/ubic/gemma/persistence/persister/GenomePersister.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/persister/GenomePersister.java @@ -614,7 +614,7 @@ private BioSequence persistOrUpdateBioSequence( BioSequence bioSequence ) { } // I don't fully understand what's going on here, but if we don't do this we fail to synchronize changes. - this.getSession().evict( existingBioSequence ); + this.getSessionFactory().getCurrentSession().evict( existingBioSequence ); bioSequenceDao.update( existingBioSequence ); // also tried merge, without the update, doesn't work. return existingBioSequence; @@ -647,7 +647,7 @@ private Gene persistOrUpdateGene( Gene gene ) { * This allows stale data to exist in this Session, but flushing prematurely causes constraint violations. * Probably we should fix this some other way. */ - this.getSession().setFlushMode( FlushMode.COMMIT ); + this.getSessionFactory().getCurrentSession().setFlushMode( FlushMode.COMMIT ); return this.updateGene( existingGene, gene ); diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/common/auditAndSecurity/AuditEventDaoImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/common/auditAndSecurity/AuditEventDaoImpl.java index 8a15e4d2c9..a3d11242e4 100644 --- a/gemma-core/src/main/java/ubic/gemma/persistence/service/common/auditAndSecurity/AuditEventDaoImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/common/auditAndSecurity/AuditEventDaoImpl.java @@ -282,7 +282,7 @@ private void tryAddAllToResult( Collection result, String queryString //noinspection unchecked result.addAll( queryObject.list() ); } catch ( org.hibernate.HibernateException ex ) { - throw super.convertHibernateAccessException( ex ); + throw getHibernateTemplate().convertHibernateAccessException( ex ); } } diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/common/description/CharacteristicDaoImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/common/description/CharacteristicDaoImpl.java index 2195498ebf..3ab94c8ca3 100644 --- a/gemma-core/src/main/java/ubic/gemma/persistence/service/common/description/CharacteristicDaoImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/common/description/CharacteristicDaoImpl.java @@ -78,7 +78,7 @@ public List browse( Integer start, Integer limit, String orderFi public Collection findByCategory( String query ) { //noinspection unchecked - return this.getSession() + return this.getSessionFactory().getCurrentSession() .createQuery( "select distinct char from Characteristic as char where char.category like :search" ) .setParameter( "search", query + "%" ).list(); } diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/common/measurement/UnitDaoImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/common/measurement/UnitDaoImpl.java index d627ee384a..83db4e6fa3 100644 --- a/gemma-core/src/main/java/ubic/gemma/persistence/service/common/measurement/UnitDaoImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/common/measurement/UnitDaoImpl.java @@ -63,7 +63,7 @@ public Unit find( Unit unit ) { } return ( Unit ) result; } catch ( org.hibernate.HibernateException ex ) { - throw super.convertHibernateAccessException( ex ); + throw getHibernateTemplate().convertHibernateAccessException( ex ); } } } \ No newline at end of file diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/bioAssay/BioAssayDaoImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/bioAssay/BioAssayDaoImpl.java index 2d2ccda17f..8b905d92be 100644 --- a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/bioAssay/BioAssayDaoImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/bioAssay/BioAssayDaoImpl.java @@ -23,6 +23,7 @@ import org.hibernate.Hibernate; import org.hibernate.LockOptions; import org.hibernate.SessionFactory; +import org.hibernate.classic.Session; import org.hibernate.jdbc.Work; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Repository; @@ -94,7 +95,7 @@ public BioAssay find( BioAssay bioAssay ) { } return ( BioAssay ) result; } catch ( org.hibernate.HibernateException ex ) { - throw super.convertHibernateAccessException( ex ); + throw getHibernateTemplate().convertHibernateAccessException( ex ); } } @@ -136,16 +137,17 @@ public void thaw( final BioAssay bioAssay ) { this.getSessionFactory().getCurrentSession().doWork( new Work() { @Override public void execute( Connection connection ) { - BioAssayDaoImpl.this.getSession().buildLockRequest( LockOptions.NONE ).lock( bioAssay ); + Session session = getSessionFactory().getCurrentSession(); + session.buildLockRequest( LockOptions.NONE ).lock( bioAssay ); Hibernate.initialize( bioAssay.getArrayDesignUsed() ); Hibernate.initialize( bioAssay.getOriginalPlatform() ); BioMaterial bm = bioAssay.getSampleUsed(); - BioAssayDaoImpl.this.getSession().buildLockRequest( LockOptions.NONE ).lock( bm ); + session.buildLockRequest( LockOptions.NONE ).lock( bm ); Hibernate.initialize( bm ); Hibernate.initialize( bm.getBioAssaysUsedIn() ); Hibernate.initialize( bm.getFactorValues() ); - BioAssayDaoImpl.this.getSession().evict( bm ); - BioAssayDaoImpl.this.getSession().evict( bioAssay ); + session.evict( bm ); + session.evict( bioAssay ); } } ); } catch ( Throwable th ) { diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/bioAssayData/ProcessedExpressionDataVectorDaoImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/bioAssayData/ProcessedExpressionDataVectorDaoImpl.java index 1331cc439f..b57616dcf4 100644 --- a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/bioAssayData/ProcessedExpressionDataVectorDaoImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/bioAssayData/ProcessedExpressionDataVectorDaoImpl.java @@ -502,7 +502,7 @@ public Collection ensureLog2Scale( new ExpressionDataDoubleMatrix( rawPreferredDataVectors ) ); preferredDataVectors.addAll( matrix.toRawDataVectors() ); preferredMaskedDataQuantitationType.setScale( ScaleType.LOG2 ); - this.getSession().update( preferredMaskedDataQuantitationType ); + this.getSessionFactory().getCurrentSession().update( preferredMaskedDataQuantitationType ); } else { preferredDataVectors.addAll( rawPreferredDataVectors ); } diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/designElement/CompositeSequenceDaoImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/designElement/CompositeSequenceDaoImpl.java index d9d3e7f227..1b5f65c29e 100644 --- a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/designElement/CompositeSequenceDaoImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/designElement/CompositeSequenceDaoImpl.java @@ -155,7 +155,7 @@ public Collection findByBioSequence( BioSequence bioSequence compositeSequences = queryObject.list(); } catch ( org.hibernate.HibernateException ex ) { - throw super.convertHibernateAccessException( ex ); + throw getHibernateTemplate().convertHibernateAccessException( ex ); } return compositeSequences; } @@ -173,7 +173,7 @@ public Collection findByBioSequenceName( String name ) { compositeSequences = queryObject.list(); } catch ( org.hibernate.HibernateException ex ) { - throw super.convertHibernateAccessException( ex ); + throw getHibernateTemplate().convertHibernateAccessException( ex ); } return compositeSequences; } diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/FactorValueDaoImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/FactorValueDaoImpl.java index 4214255540..92c60b494a 100644 --- a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/FactorValueDaoImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/FactorValueDaoImpl.java @@ -131,7 +131,7 @@ public FactorValue find( FactorValue factorValue ) { } return ( FactorValue ) result; } catch ( org.hibernate.HibernateException ex ) { - throw super.convertHibernateAccessException( ex ); + throw getHibernateTemplate().convertHibernateAccessException( ex ); } } diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/genome/GeneDaoImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/genome/GeneDaoImpl.java index a6a4067fc9..ec470d9a2f 100644 --- a/gemma-core/src/main/java/ubic/gemma/persistence/service/genome/GeneDaoImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/genome/GeneDaoImpl.java @@ -242,7 +242,7 @@ public Collection getCompositeSequences( Gene gene, ArrayDesi compSeq = queryObject.list(); } catch ( org.hibernate.HibernateException ex ) { - throw super.convertHibernateAccessException( ex ); + throw getHibernateTemplate().convertHibernateAccessException( ex ); } return compSeq; } @@ -295,7 +295,7 @@ public int getPlatformCountById( Long id ) { "select count(distinct cs.arrayDesign) from Gene as gene inner join gene.products gp, BioSequence2GeneProduct" + " as bs2gp, CompositeSequence as cs where gp=bs2gp.geneProduct " + " and cs.biologicalCharacteristic=bs2gp.bioSequence " + " and gene.id = :id "; - List r = this.getSession().createQuery( queryString ).setParameter( "id", id ).list(); + List r = this.getSessionFactory().getCurrentSession().createQuery( queryString ).setParameter( "id", id ).list(); return ( ( Long ) r.iterator().next() ).intValue(); } diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/genome/biosequence/BioSequenceDaoImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/genome/biosequence/BioSequenceDaoImpl.java index ad1a0be232..98ad368e02 100644 --- a/gemma-core/src/main/java/ubic/gemma/persistence/service/genome/biosequence/BioSequenceDaoImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/genome/biosequence/BioSequenceDaoImpl.java @@ -149,7 +149,7 @@ public Collection getGenesByName( String search ) { .setString( "search", search ).list(); } catch ( org.hibernate.HibernateException ex ) { - throw super.convertHibernateAccessException( ex ); + throw getHibernateTemplate().convertHibernateAccessException( ex ); } } diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/genome/gene/GeneProductDaoImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/genome/gene/GeneProductDaoImpl.java index 4b04526cb3..803d49ed54 100644 --- a/gemma-core/src/main/java/ubic/gemma/persistence/service/genome/gene/GeneProductDaoImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/genome/gene/GeneProductDaoImpl.java @@ -73,7 +73,7 @@ public Collection getGenesByName( String search ) { .setString( "search", search ).list(); } catch ( org.hibernate.HibernateException ex ) { - throw super.convertHibernateAccessException( ex ); + throw getHibernateTemplate().convertHibernateAccessException( ex ); } } @@ -86,7 +86,7 @@ public Collection getGenesByNcbiId( String search ) { .setString( "search", search ).list(); } catch ( org.hibernate.HibernateException ex ) { - throw super.convertHibernateAccessException( ex ); + throw getHibernateTemplate().convertHibernateAccessException( ex ); } } @@ -198,7 +198,7 @@ public GeneProduct find( GeneProduct geneProduct ) { AbstractDao.log.debug( "Found: " + result ); return ( GeneProduct ) result; } catch ( org.hibernate.HibernateException ex ) { - throw super.convertHibernateAccessException( ex ); + throw getHibernateTemplate().convertHibernateAccessException( ex ); } } diff --git a/gemma-web/src/main/webapp/WEB-INF/web.xml b/gemma-web/src/main/webapp/WEB-INF/web.xml index afb71ef5c9..735a51fc3f 100644 --- a/gemma-web/src/main/webapp/WEB-INF/web.xml +++ b/gemma-web/src/main/webapp/WEB-INF/web.xml @@ -57,7 +57,7 @@ sitemesh - com.opensymphony.module.sitemesh.filter.PageFilter + com.opensymphony.sitemesh.webapp.SiteMeshFilter From ec298d29fa943b9ccf1d6a0fa133f512f29bf654 Mon Sep 17 00:00:00 2001 From: Guillaume Poirier-Morency Date: Wed, 16 Mar 2022 09:30:05 -0700 Subject: [PATCH 20/28] Add removeAll() to remove all entities in a single transaction --- .../java/ubic/gemma/persistence/service/AbstractDao.java | 5 +++++ .../ubic/gemma/persistence/service/AbstractService.java | 6 ++++++ .../main/java/ubic/gemma/persistence/service/BaseDao.java | 5 +++++ .../java/ubic/gemma/persistence/service/BaseService.java | 5 +++++ 4 files changed, 21 insertions(+) diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/AbstractDao.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/AbstractDao.java index a8d21591f0..17b5050ab0 100644 --- a/gemma-core/src/main/java/ubic/gemma/persistence/service/AbstractDao.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/AbstractDao.java @@ -135,6 +135,11 @@ public void remove( T entity ) { this.getSessionFactory().getCurrentSession().delete( entity ); } + @Override + public void removeAll() { + this.remove( this.loadAll() ); + } + @Override @Transactional public void update( Collection entities ) { diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/AbstractService.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/AbstractService.java index 7c1d5c46cf..8709b57b36 100644 --- a/gemma-core/src/main/java/ubic/gemma/persistence/service/AbstractService.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/AbstractService.java @@ -90,6 +90,12 @@ public void remove( O entity ) { mainDao.remove( entity ); } + @Override + @Transactional + public void removeAll() { + mainDao.removeAll(); + } + @Override @Transactional public void update( Collection entities ) { diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/BaseDao.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/BaseDao.java index b9cfd4f4f2..a71bd74382 100644 --- a/gemma-core/src/main/java/ubic/gemma/persistence/service/BaseDao.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/BaseDao.java @@ -93,6 +93,11 @@ public interface BaseDao { */ void remove( T entity ); + /** + * Remove all entities from persistent storage. + */ + void removeAll(); + /** * @param entities Update the entities. Not supported if the entities are immutable. */ diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/BaseService.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/BaseService.java index 9979fa54d2..1c92a83703 100644 --- a/gemma-core/src/main/java/ubic/gemma/persistence/service/BaseService.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/BaseService.java @@ -91,6 +91,11 @@ public interface BaseService { */ void remove( O entity ); + /** + * Remove all entities from the persistent storage. + */ + void removeAll(); + /** * Updates all entities in the given collection in the persistent storage. * From 1607806b50e8800f01c8a25f45383a8f8b62157a Mon Sep 17 00:00:00 2001 From: Guillaume Poirier-Morency Date: Wed, 16 Mar 2022 09:32:57 -0700 Subject: [PATCH 21/28] Perform removal of multiple entities in a batch fashion --- .../java/ubic/gemma/persistence/service/AbstractDao.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/AbstractDao.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/AbstractDao.java index 17b5050ab0..a24e31b707 100644 --- a/gemma-core/src/main/java/ubic/gemma/persistence/service/AbstractDao.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/AbstractDao.java @@ -118,8 +118,13 @@ public Integer countAll() { @Override public void remove( Collection entities ) { + int i = 0; for ( T e : entities ) { this.remove( e ); + if ( ++i % BATCH_SIZE == 0 ) { + this.getSessionFactory().getCurrentSession().flush(); + this.getSessionFactory().getCurrentSession().clear(); + } } } From e0797e93524e31e4ffd61e9dc49c9f8b22fce051 Mon Sep 17 00:00:00 2001 From: Guillaume Poirier-Morency Date: Wed, 16 Mar 2022 09:33:44 -0700 Subject: [PATCH 22/28] Use removeAll() to clear fixtures in DatasetsRestTest --- .../java/ubic/gemma/web/services/rest/DatasetsRestTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gemma-web/src/test/java/ubic/gemma/web/services/rest/DatasetsRestTest.java b/gemma-web/src/test/java/ubic/gemma/web/services/rest/DatasetsRestTest.java index e9a5f7f83d..435f18ade2 100644 --- a/gemma-web/src/test/java/ubic/gemma/web/services/rest/DatasetsRestTest.java +++ b/gemma-web/src/test/java/ubic/gemma/web/services/rest/DatasetsRestTest.java @@ -41,7 +41,7 @@ public class DatasetsRestTest extends BaseSpringWebTest { @Before public void setUp() { // FIXME: this should not be necessary, but other tests are not cleaning up their fixtures - expressionExperimentService.remove( expressionExperimentService.loadAll() ); + expressionExperimentService.removeAll(); for ( int i = 0; i < 10; i++ ) { ees.add( this.getNewTestPersistentCompleteExpressionExperiment() ); } From 4f25a17176bbc82c812886797354cdef121d3073 Mon Sep 17 00:00:00 2001 From: Guillaume Poirier-Morency Date: Wed, 16 Mar 2022 12:27:41 -0700 Subject: [PATCH 23/28] Fix unnecessary raw and processed data vectors removal Because of the cascading when calling ExpressionExperiment.getXDataVectors().clear(), this is not necessary and actually result in an error when the session is flushed as of 41081d2cdb7fad3ca3d911887dc3e9d91a8ac101. --- .../experiment/ExpressionExperimentServiceImpl.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentServiceImpl.java index c10ccf88a2..0cc49d05a0 100755 --- a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentServiceImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentServiceImpl.java @@ -930,10 +930,8 @@ public ExpressionExperiment replaceRawVectors( ExpressionExperiment ee, for ( RawExpressionDataVector oldV : eeToUpdate.getRawExpressionDataVectors() ) { qtsToRemove.add( oldV.getQuantitationType() ); } - rawExpressionDataVectorDao.remove( eeToUpdate.getRawExpressionDataVectors() ); // should not be necessary - processedVectorService.remove( eeToUpdate.getProcessedExpressionDataVectors() ); // should not be necessary - eeToUpdate.getProcessedExpressionDataVectors().clear(); // this should be sufficient - eeToUpdate.getRawExpressionDataVectors().clear(); // should be sufficient + eeToUpdate.getProcessedExpressionDataVectors().clear(); + eeToUpdate.getRawExpressionDataVectors().clear(); // These QTs might still be getting used by the replaced vectors. for ( RawExpressionDataVector newVec : newVectors ) { From c6bea91fca1cbe45b2397c0f5a740e806e18bb29 Mon Sep 17 00:00:00 2001 From: Guillaume Poirier-Morency Date: Wed, 16 Mar 2022 12:46:04 -0700 Subject: [PATCH 24/28] Add warnings for removing raw and processed data vectors --- .../ProcessedExpressionDataVectorService.java | 6 ++++++ .../bioAssayData/RawExpressionDataVectorService.java | 12 +++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/bioAssayData/ProcessedExpressionDataVectorService.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/bioAssayData/ProcessedExpressionDataVectorService.java index 9f0f7ca9e6..ba4645a53e 100644 --- a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/bioAssayData/ProcessedExpressionDataVectorService.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/bioAssayData/ProcessedExpressionDataVectorService.java @@ -170,7 +170,13 @@ Map>> getRanksB @Secured({ "GROUP_USER", "ACL_SECURABLE_EDIT" }) void removeProcessedDataVectors( final ExpressionExperiment expressionExperiment ); + /** + * @deprecated never use this method, you can use {@link #removeProcessedDataVectors(ExpressionExperiment)} + * instead, or clear {@link ExpressionExperiment#getProcessedExpressionDataVectors()} directly. The relationship is + * actually managed by Hibernate. + */ @Override + @Deprecated void remove( Collection processedExpressionDataVectors ); List getDiffExVectors( Long resultSetId, Double threshold, int maxNumberOfResults ); diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/bioAssayData/RawExpressionDataVectorService.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/bioAssayData/RawExpressionDataVectorService.java index 495e85ba2c..e7c9be832b 100644 --- a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/bioAssayData/RawExpressionDataVectorService.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/bioAssayData/RawExpressionDataVectorService.java @@ -20,6 +20,7 @@ import org.springframework.security.access.annotation.Secured; import ubic.gemma.model.expression.bioAssayData.RawExpressionDataVector; +import ubic.gemma.model.expression.experiment.ExpressionExperiment; import java.util.Collection; @@ -28,17 +29,26 @@ */ public interface RawExpressionDataVectorService extends DesignElementDataVectorService { - @Override @Secured({ "GROUP_ADMIN" }) RawExpressionDataVector load( Long id ); + /** + * @deprecated never use this method, instead clear {@link ExpressionExperiment#getProcessedExpressionDataVectors()} + * directly. The relationship is actually managed by Hibernate. + */ @Override @Secured({ "GROUP_ADMIN" }) + @Deprecated void remove( Collection vectors ); + /** + * @deprecated never use this method, instead clear {@link ExpressionExperiment#getProcessedExpressionDataVectors()} + * directly. The relationship is actually managed by Hibernate. + */ @Override @Secured({ "GROUP_ADMIN" }) + @Deprecated void remove( RawExpressionDataVector designElementDataVector ); @Override From ebd6f012afa21b6c061bedfc6ca1f54e93dc685e Mon Sep 17 00:00:00 2001 From: Paul Pavlidis Date: Thu, 17 Mar 2022 11:38:52 -0700 Subject: [PATCH 25/28] fix help --- .../java/ubic/gemma/core/apps/ExperimentalDesignImportCli.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gemma-core/src/main/java/ubic/gemma/core/apps/ExperimentalDesignImportCli.java b/gemma-core/src/main/java/ubic/gemma/core/apps/ExperimentalDesignImportCli.java index 0df52a0870..2e98cb2f66 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/apps/ExperimentalDesignImportCli.java +++ b/gemma-core/src/main/java/ubic/gemma/core/apps/ExperimentalDesignImportCli.java @@ -57,8 +57,7 @@ protected void buildOptions( Options options ) { Option expOption = Option.builder( "e" ).required().hasArg().argName( "Expression experiment name" ) .desc( - "Expression experiment short name. Most tools recognize comma-delimited values given on the command line, " - + "and if this option is omitted, the tool will be applied to all expression experiments." ) + "Expression experiment short name" ) .longOpt( "experiment" ).build(); options.addOption( expOption ); From ddc296151381d8410116a286b2afae642c745fb9 Mon Sep 17 00:00:00 2001 From: Guillaume Poirier-Morency Date: Wed, 30 Mar 2022 10:51:30 -0700 Subject: [PATCH 26/28] Fix inefficient query when generating daily datasets and platforms report This is related to #234 and very likely fix it. However, there could be other queries that result in very intensive CPU/memory usage. --- ...DifferentialExpressionAnalysisDaoImpl.java | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionAnalysisDaoImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionAnalysisDaoImpl.java index 5f579d8a45..de9696c057 100644 --- a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionAnalysisDaoImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionAnalysisDaoImpl.java @@ -388,15 +388,22 @@ public Map> getAnalysesByE */ //noinspection unchecked Collection hits = this.getSessionFactory().getCurrentSession().createQuery( - "select distinct a from DifferentialExpressionAnalysis a join fetch a.experimentAnalyzed e join" - + " fetch a.resultSets rs join fetch rs.hitListSizes where e.id in (:eeids)" ) - .setParameterList( "eeids", expressionExperimentIds ).setFirstResult( offset ) - .setMaxResults( limit > 0 ? limit : -1 ).list(); - - /* - * FIXME the above query yields a warning - * "firstResult/maxResults specified with collection fetch; applying in memory!" - */ + "select distinct a from DifferentialExpressionAnalysis a " + + "join fetch a.experimentAnalyzed e " + + "where e.id in (:eeIds)" ) + .setParameterList( "eeIds", expressionExperimentIds ) + .setFirstResult( offset ) + .setMaxResults( limit > 0 ? limit : -1 ) + .list(); + + // initialize result sets and hit list sizes + // this is necessary because the DEA VO constructor will ignore uninitialized associations + for ( DifferentialExpressionAnalysis hit : hits ) { + Hibernate.initialize( hit.getResultSets() ); + for ( ExpressionAnalysisResultSet rs : hit.getResultSets() ) { + Hibernate.initialize( rs.getHitListSizes() ); + } + } Map> ee2fv = new HashMap<>(); List fvs; From a803ca3fbf56ae55bb518f7844ec9893872efbfd Mon Sep 17 00:00:00 2001 From: Guillaume Poirier-Morency Date: Wed, 30 Mar 2022 11:36:04 -0700 Subject: [PATCH 27/28] Update Log4j to 2.17.2 This fixes the Slack appender. It couldn't be cherry-picked from the development branch because Log4j version is managed in the pavlab-starter-parent package. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1563984da9..fa5553d090 100644 --- a/pom.xml +++ b/pom.xml @@ -670,7 +670,7 @@ 8.5.72 8.0.28 3.6.2 - 2.17.1 + 2.17.2 1.7.32 1.18.0 From b48cbad60f6c9db16caed93bbeae878fa136f4cb Mon Sep 17 00:00:00 2001 From: Guillaume Poirier-Morency Date: Wed, 24 Nov 2021 15:21:21 -0800 Subject: [PATCH 28/28] Remove Lso suffix in Gemma CLI logging pattern --- gemma-core/src/main/config/log4j.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gemma-core/src/main/config/log4j.properties b/gemma-core/src/main/config/log4j.properties index 58a51f24e1..fe4c8f6ddf 100644 --- a/gemma-core/src/main/config/log4j.properties +++ b/gemma-core/src/main/config/log4j.properties @@ -19,7 +19,7 @@ log4j.appender.file.File=${gemma.log.dir}/gemma.log log4j.appender.file.MaxFileSize=10000KB log4j.appender.file.MaxBackupIndex=10 log4j.appender.file.layout=org.apache.log4j.PatternLayout -log4j.appender.file.layout.ConversionPattern=[Gemma - %t (%d)] %p %C.%M(%L) | %m%nLso +log4j.appender.file.layout.ConversionPattern=[Gemma - %t (%d)] %p %C.%M(%L) | %m%n # Log file for WARNING and higher level errors log4j.appender.warningFile=org.apache.log4j.DailyRollingFileAppender