MAX_EMPTY_CHUNKS_IN_A_ROW ) {
- log.info("Have already skipped " + numSkippedChunks + " chunks, still no records: bailing");
+ if ( numSkippedChunks > MAX_EMPTY_CHUNKS_IN_A_ROW ) {
+ log.info( "Have already skipped " + numSkippedChunks + " chunks, still no records: bailing" );
break;
}
start += NCBI_CHUNK_SIZE;
@@ -368,11 +368,11 @@ protected void doWork() throws Exception {
+ "\t" + geoRecord.getSubSeriesOf()
+ "\t" + geoRecord.getPubMedIds()
+ "\t" + geoRecord.getTitle()
- + "\t" + geoRecord.getSummary()
+ + "\t" + clean(geoRecord.getSummary())
+ "\t" + geoRecord.getMeshHeadings()
- + "\t" + geoRecord.getSampleDetails()
- + "\t" + geoRecord.getLibraryStrategy()
- + "\t" + geoRecord.getOverallDesign() + "\n" );
+ + "\t" + clean(geoRecord.getSampleDetails())
+ + "\t" + clean(geoRecord.getLibraryStrategy())
+ + "\t" + clean(geoRecord.getOverallDesign()) + "\n" );
seen.add( geoRecord.getGeoAccession() );
@@ -387,4 +387,14 @@ protected void doWork() throws Exception {
}
+
+ /**
+ * Replace any 'tab' characters with spaces.
+ * @param s
+ * @return
+ */
+ String clean( String s ) {
+ return s.replace( '\t', ' ' );
+ }
+
}
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/LoadExpressionDataCli.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/LoadExpressionDataCli.java
index 404cd50674..447560724b 100644
--- a/gemma-cli/src/main/java/ubic/gemma/core/apps/LoadExpressionDataCli.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/LoadExpressionDataCli.java
@@ -75,6 +75,7 @@ public class LoadExpressionDataCli extends AbstractAuthenticatedCLI {
private boolean splitByPlatform = false;
private boolean suppressPostProcessing = false;
private boolean updateOnly = false;
+ private String softFile = null;
@Override
public CommandGroup getCommandGroup() {
@@ -124,6 +125,8 @@ protected void buildOptions( Options options ) {
options.addOption( Option.builder( "nopost" ).desc( "Suppress postprocessing steps" ).build() );
+ options.addOption( Option.builder( "softfile" ).desc( "Load directly from soft.gz file; use with single accession via -e" ).hasArg().argName( "soft file path" ).build() );
+
/*
* add 'allowsub/super' series option;
*/
@@ -141,6 +144,14 @@ protected void doWork() throws Exception {
"You must specific either a file or accessions on the command line" );
}
+ if ( softFile != null ) {
+ Collection ees = geoService.loadFromSoftFile( accessions, softFile, platformOnly, doMatching, splitByPlatform );
+ for ( Object object : ees ) {
+ addSuccessObject( object );
+ }
+ return;
+ }
+
if ( accessions != null ) {
AbstractCLI.log.info( "Got accession(s) from command line " + accessions );
String[] accsToRun = StringUtils.split( accessions, ',' );
@@ -228,6 +239,13 @@ protected void processOptions( CommandLine commandLine ) {
this.suppressPostProcessing = commandLine.hasOption( "nopost" );
+ if ( commandLine.hasOption( "softfile" ) ) {
+ this.softFile = commandLine.getOptionValue( "softfile" );
+ if ( accessions == null || accessions.split( "," ).length > 1 ) {
+ throw new IllegalArgumentException( "You must specify exactly one accession to load from a SOFT file" );
+ }
+
+ }
}
private void processAccession( String accession ) {
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/loader/association/phenotype/LoadEvidenceForClassifier.java b/gemma-cli/src/main/java/ubic/gemma/core/loader/association/phenotype/LoadEvidenceForClassifier.java
index f1bc663678..61835560d6 100644
--- a/gemma-cli/src/main/java/ubic/gemma/core/loader/association/phenotype/LoadEvidenceForClassifier.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/loader/association/phenotype/LoadEvidenceForClassifier.java
@@ -25,6 +25,7 @@
import ubic.gemma.model.common.description.MedicalSubjectHeading;
import ubic.gemma.persistence.util.Settings;
+import javax.annotation.Nullable;
import java.io.*;
import java.util.HashSet;
import java.util.Set;
@@ -59,6 +60,12 @@ public String getCommandName() {
return "loadEvidenceForClassifier";
}
+ @Nullable
+ @Override
+ public String getShortDesc() {
+ return null;
+ }
+
@Override
protected void buildOptions( Options options ) {
}
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/util/AbstractAuthenticatedCLI.java b/gemma-cli/src/main/java/ubic/gemma/core/util/AbstractAuthenticatedCLI.java
index 2caf470887..273528c431 100644
--- a/gemma-cli/src/main/java/ubic/gemma/core/util/AbstractAuthenticatedCLI.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/util/AbstractAuthenticatedCLI.java
@@ -66,11 +66,6 @@ public abstract class AbstractAuthenticatedCLI extends AbstractCLI {
@Autowired
private ManualAuthenticationService manAuthentication;
- @Override
- public String getShortDesc() {
- return "No description provided";
- }
-
/**
* Indicate if the command requires authentication.
*
diff --git a/gemma-cli/src/test/java/ubic/gemma/core/apps/NCBIGene2GOAssociationLoaderCLITest.java b/gemma-cli/src/test/java/ubic/gemma/core/apps/NCBIGene2GOAssociationLoaderCLITest.java
index 3afdd45593..fcbf23dc57 100644
--- a/gemma-cli/src/test/java/ubic/gemma/core/apps/NCBIGene2GOAssociationLoaderCLITest.java
+++ b/gemma-cli/src/test/java/ubic/gemma/core/apps/NCBIGene2GOAssociationLoaderCLITest.java
@@ -1,6 +1,7 @@
package ubic.gemma.core.apps;
import gemma.gsec.authentication.ManualAuthenticationService;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.springframework.beans.factory.annotation.Autowired;
@@ -24,7 +25,6 @@
import static org.mockito.Mockito.*;
import static ubic.gemma.core.util.test.Assumptions.assumeThatResourceIsAvailable;
-@Category(SlowTest.class)
@ContextConfiguration
@TestExecutionListeners(WithSecurityContextTestExecutionListener.class)
public class NCBIGene2GOAssociationLoaderCLITest extends BaseCliTest {
@@ -79,6 +79,8 @@ public PersisterHelper persisterHelper() {
private ExternalDatabaseService externalDatabaseService;
@Test
+ @Ignore("This test is too slow, see https://github.com/PavlidisLab/Gemma/issues/1056 for details")
+ @Category(SlowTest.class)
@WithMockUser(authorities = { "GROUP_ADMIN" })
public void test() throws Exception {
assumeThatResourceIsAvailable( "ftp://ftp.ncbi.nih.gov/gene/DATA/gene2go.gz" );
diff --git a/gemma-cli/src/test/java/ubic/gemma/core/util/test/BaseCliTest.java b/gemma-cli/src/test/java/ubic/gemma/core/util/test/BaseCliTest.java
index 4413c44cd1..6da0450626 100644
--- a/gemma-cli/src/test/java/ubic/gemma/core/util/test/BaseCliTest.java
+++ b/gemma-cli/src/test/java/ubic/gemma/core/util/test/BaseCliTest.java
@@ -2,12 +2,12 @@
import org.springframework.test.context.ActiveProfiles;
import org.springframework.test.context.junit4.AbstractJUnit4SpringContextTests;
-import ubic.gemma.persistence.util.SpringProfiles;
+import ubic.gemma.persistence.util.EnvironmentProfiles;
/**
* Minimal setup
*/
-@ActiveProfiles({ "cli", SpringProfiles.TEST })
+@ActiveProfiles({ "cli", EnvironmentProfiles.TEST })
public abstract class BaseCliTest extends AbstractJUnit4SpringContextTests {
}
diff --git a/gemma-core/pom.xml b/gemma-core/pom.xml
index 29d94a385f..31a8933710 100644
--- a/gemma-core/pom.xml
+++ b/gemma-core/pom.xml
@@ -3,7 +3,7 @@
gemma
gemma
- 1.31.1
+ 1.31.2
4.0.0
gemma-core
diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/expression/diff/LinearModelAnalyzer.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/expression/diff/LinearModelAnalyzer.java
index 7ccf21a933..bb24b7c786 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/analysis/expression/diff/LinearModelAnalyzer.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/expression/diff/LinearModelAnalyzer.java
@@ -27,6 +27,7 @@
import org.apache.commons.lang3.time.StopWatch;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.DisposableBean;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
import ubic.basecode.dataStructure.matrix.DenseDoubleMatrix;
@@ -71,7 +72,7 @@
*/
@Component
@Scope(value = "prototype")
-public class LinearModelAnalyzer extends AbstractDifferentialExpressionAnalyzer {
+public class LinearModelAnalyzer extends AbstractDifferentialExpressionAnalyzer implements DisposableBean {
/**
* Preset levels for which we will store the HitListSizes.
@@ -123,7 +124,7 @@ public static DoubleMatrix makeDataMatrix( ObjectMatrix columnsToUse ) {
+ public static BioAssayDimension createBADMap( List columnsToUse ) {
/*
* Indices of the biomaterials in the original matrix.
*/
@@ -143,6 +144,18 @@ private static BioAssayDimension createBADMap( List columnsToUse )
return reorderedDim;
}
+ /**
+ * Executor used for performing analyses in the background while the current thread is reporting progress.
+ *
+ * This bean is using the prototype scope, so a single-thread executor is suitable to prevent concurrent analyses.
+ */
+ private final ExecutorService executorService = Executors.newSingleThreadExecutor();
+
+ @Override
+ public void destroy() throws Exception {
+ executorService.shutdown();
+ }
+
/**
* Determine if any factor should be treated as the intercept term.
*/
@@ -383,10 +396,6 @@ public Collection run( ExpressionExperiment expr
throw new RuntimeException( String.format( "There are no processed EVs for %s.", expressionExperiment ) );
}
- /*
- * FIXME remove flagged outlier samples ... at some point; so we don't carry NaNs around unnecessarily.
- */
-
return this.run( expressionExperiment, dmatrix, config );
}
@@ -1097,7 +1106,7 @@ private DifferentialExpressionAnalysis makeAnalysisEntity( BioAssaySet bioAssayS
expressionAnalysis.setDescription( "Linear model with " + config.getFactorsToInclude().size() + " factors"
+ ( interceptFactor == null ? "" : " with intercept treated as factor" )
+ ( interactionFactorLists.isEmpty() ? "" : " with interaction" )
- + ( subsetFactorValue == null ? "" : "Using subset " + bioAssaySet + " subset value= " + subsetFactorValue ) );
+ + ( subsetFactorValue == null ? "" : " Using subset " + bioAssaySet + " subset value= " + subsetFactorValue ) );
expressionAnalysis.setSubsetFactorValue( subsetFactorValue );
Set resultSets = this
@@ -1367,7 +1376,7 @@ private Map runAnalysis( final DoubleMatrix> f = Executors.newSingleThreadExecutor().submit( () -> {
+ Future> f = executorService.submit( () -> {
StopWatch timer = new StopWatch();
timer.start();
LeastSquaresFit fit;
diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/MeanVarianceServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/MeanVarianceServiceImpl.java
index 68017a9183..da5e4e9780 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/MeanVarianceServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/MeanVarianceServiceImpl.java
@@ -69,8 +69,8 @@ public MeanVarianceRelation create( ExpressionExperiment ee, boolean forceRecomp
}
ExpressionDataDoubleMatrix intensities = expressionDataMatrixService.getProcessedExpressionDataMatrix( ee );
- if ( intensities == null ) {
- throw new IllegalStateException( "Could not locate intensity matrix for " + ee.getShortName() );
+ if ( intensities == null || intensities.rows() == 0 ) {
+ throw new IllegalStateException( "Could not locate intensity matrix, or it was empty, for " + ee.getShortName() );
}
QuantitationType qt = expressionExperimentService.getPreferredQuantitationType( ee );
@@ -83,6 +83,11 @@ public MeanVarianceRelation create( ExpressionExperiment ee, boolean forceRecomp
log.warn( "Problem log transforming data. Check that the appropriate log scale is used. Mean-variance will be computed as is." );
}
+ if ( intensities.rows() < 3 ) {
+ // this data is pretty much useless altogether, so an exception is the right way to go.
+ throw new IllegalStateException( "Not enough data left after filtering to proceed (" + intensities.rows() + " rows for " + ee.getShortName() + ")" );
+ }
+
mvr = expressionExperimentService.updateMeanVarianceRelation( ee, calculateMeanVariance( intensities ) );
auditTrailService.addUpdateEvent( ee, MeanVarianceUpdateEvent.class, "Mean-variance has been updated." );
diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/OutlierDetectionServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/OutlierDetectionServiceImpl.java
index 1fb0724c8e..a1a31870ae 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/OutlierDetectionServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/OutlierDetectionServiceImpl.java
@@ -54,7 +54,7 @@ public Collection getOutlierDetails( ExpressionExperiment ee ) {
public Collection identifyOutliersByMedianCorrelation( ExpressionExperiment ee ) {
DoubleMatrix cormat = sampleCoexpressionAnalysisService.loadBestMatrix( ee );
if ( cormat == null ) {
- cormat = sampleCoexpressionAnalysisService.computeIfNecessary( ee );
+ cormat = sampleCoexpressionAnalysisService.compute( ee, sampleCoexpressionAnalysisService.prepare( ee ) );
}
if ( cormat.rows() == 0 ) {
OutlierDetectionServiceImpl.log.warn( "Correlation matrix is empty, cannot check for outliers" );
diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/PreprocessorServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/PreprocessorServiceImpl.java
index e36367a1f5..9429f437d9 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/PreprocessorServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/PreprocessorServiceImpl.java
@@ -248,7 +248,7 @@ private void processForPca( ExpressionExperiment ee ) throws SVDRelatedPreproces
*/
private void processForSampleCorrelation( ExpressionExperiment ee ) throws SampleCoexpressionRelatedPreprocessingException {
try {
- sampleCoexpressionAnalysisService.compute( ee );
+ sampleCoexpressionAnalysisService.compute( ee, sampleCoexpressionAnalysisService.prepare( ee ) );
} catch ( RuntimeException e ) {
auditTrailService.addUpdateEvent( ee, FailedSampleCorrelationAnalysisEvent.class, null, e );
throw new SampleCoexpressionRelatedPreprocessingException( ee, e );
diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/TwoChannelMissingValuesImpl.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/TwoChannelMissingValuesImpl.java
index 297874fd6a..cab485b9e4 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/TwoChannelMissingValuesImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/TwoChannelMissingValuesImpl.java
@@ -108,6 +108,16 @@ public Collection computeMissingValues( ExpressionExper
ee = expressionExperimentService.thawLite( ee );
Collection usefulQuantitationTypes = ExpressionDataMatrixBuilder
.getUsefulQuantitationTypes( ee );
+
+ // check that we don't already have a missing value QT
+ for ( QuantitationType qt : usefulQuantitationTypes ) {
+ if ( StandardQuantitationType.PRESENTABSENT.equals(qt.getType()) ) {
+ log.warn( "This experiment already has a missing value quantitation type, no action will be taken and empty collection returned" );
+ return new HashSet<>();
+ }
+ }
+
+
StopWatch timer = new StopWatch();
timer.start();
TwoChannelMissingValuesImpl.log.info( "Loading vectors ..." );
@@ -239,7 +249,7 @@ private Collection computeMissingValues( ExpressionExpe
TwoChannelMissingValuesImpl.log.info( "Finished: " + count + " vectors examined for missing values" );
log.info( "Persisting " + results.size() + " vectors ... " );
- // saving twice is needed to get the QT filled in properly.
+ // saving twice is needed to get the QT filled in properly. ??Why??
source = expressionExperimentService.save( source );
source.getRawExpressionDataVectors().addAll( results );
source = expressionExperimentService.save( source );
diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchConfoundUtils.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchConfoundUtils.java
index 4c8568933b..618a8dbe83 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchConfoundUtils.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchConfoundUtils.java
@@ -175,6 +175,12 @@ private static Collection factorBatchConfoundTest( BioAssaySet ee
factorValueIndexes.put( fv.getId(), index++ );
}
+
+ if ( factorValueIndexes.size() < 2 ) {
+ // This can happen if we're looking at a subset by this factor
+ continue; // to the next factor
+ }
+
Map factorValueMembership = new HashMap<>();
for ( Long bmId : bmToFv.keySet() ) {
diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/ComBat.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/ComBat.java
index d592160a86..97cbb6ed2c 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/ComBat.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/ComBat.java
@@ -71,7 +71,6 @@ public class ComBat {
private final ObjectMatrix sampleInfo;
private final DoubleMatrix data;
-
private boolean hasMissing = false;
private int numSamples;
private int numBatches;
@@ -104,19 +103,26 @@ public class ComBat {
*/
private DoubleMatrix2D x;
- public ComBat( DoubleMatrix data, ObjectMatrix sampleInfo ) {
- if ( data.columns() < 4 ) {
- throw new IllegalArgumentException( "Cannot run ComBat with fewer than 4 samples" );
- }
+ /**
+ * Constructor that can be used just for testing correctability (data is not provided) - FIXME refactor so it's not a constructor.
+ * @param sampleInfo
+ * @throws ComBatException
+ */
+ public ComBat( ObjectMatrix sampleInfo ) throws ComBatException {
+ this.sampleInfo = sampleInfo;
+ solver = new Algebra();
+ computeDesignMatrix();
+ this.data = null;
+ }
+
+ public ComBat( DoubleMatrix data, ObjectMatrix sampleInfo ) throws ComBatException {
this.data = data;
this.sampleInfo = sampleInfo;
solver = new Algebra();
y = new DenseDoubleMatrix2D( data.asArray() );
this.initPartA();
- // DesignMatrix dm = new DesignMatrix( sampleInfo );
- // this.x = dm.getMatrix();
x = this.computeDesignMatrix();
}
@@ -133,8 +139,7 @@ public DoubleMatrix2D getDesignMatrix() {
*/
public void plot( String filePrefix ) {
- if ( this.gammaHat == null )
- throw new IllegalArgumentException( "You must call 'run' first" );
+ if ( this.gammaHat == null ) throw new IllegalArgumentException( "You must call 'run' first" );
/*
* View the distribution of gammaHat, which we assume will have a normal distribution
@@ -206,6 +211,10 @@ public DoubleMatrix2D run() throws ComBatException {
*/
public DoubleMatrix2D run( boolean parametric ) throws ComBatException {
+ if ( data.columns() < 4 ) {
+ throw new IllegalArgumentException( "Cannot run ComBat with fewer than 4 samples" );
+ }
+
StopWatch timer = new StopWatch();
timer.start();
@@ -228,11 +237,9 @@ public DoubleMatrix2D run( boolean parametric ) throws ComBatException {
gammaBar = new DoubleArrayList();
t2 = new DoubleArrayList();
for ( int batchIndex = 0; batchIndex < gammaHat.rows(); batchIndex++ ) {
- double mean = DescriptiveWithMissing
- .mean( new DoubleArrayList( gammaHat.viewRow( batchIndex ).toArray() ) );
+ double mean = DescriptiveWithMissing.mean( new DoubleArrayList( gammaHat.viewRow( batchIndex ).toArray() ) );
gammaBar.add( mean );
- t2.add( DescriptiveWithMissing
- .sampleVariance( new DoubleArrayList( gammaHat.viewRow( batchIndex ).toArray() ), mean ) );
+ t2.add( DescriptiveWithMissing.sampleVariance( new DoubleArrayList( gammaHat.viewRow( batchIndex ).toArray() ), mean ) );
}
// assertEquals( -0.092144, gammaBar.get( 0 ), 0.001 );
@@ -276,9 +283,9 @@ public DoubleMatrix2D run( boolean parametric ) throws ComBatException {
/**
* Special standardization: partial regression of covariates
*
- * @param b b
- * @param A A
- * @return double matrix 2d
+ * @param b the data matrix
+ * @param A the design matrix
+ * @return double matrix 2d representing the data with the covariates regressed out and standardized
*/
DoubleMatrix2D standardize( DoubleMatrix2D b, DoubleMatrix2D A ) {
@@ -311,8 +318,7 @@ DoubleMatrix2D standardize( DoubleMatrix2D b, DoubleMatrix2D A ) {
}
varpooled = var;
} else {
- varpooled = y.copy().assign( solver.transpose( solver.mult( x, beta ) ), Functions.minus )
- .assign( Functions.pow( 2 ) );
+ varpooled = y.copy().assign( solver.transpose( solver.mult( x, beta ) ), Functions.minus ).assign( Functions.pow( 2 ) );
DoubleMatrix2D scale = new DenseDoubleMatrix2D( numSamples, 1 );
scale.assign( 1.0 / numSamples );
varpooled = solver.mult( varpooled, scale );
@@ -355,9 +361,7 @@ private void writePlot( OutputStream os, XYSeries empirical, XYSeries theory ) {
XYSeriesCollection xySeriesCollection = new XYSeriesCollection();
xySeriesCollection.addSeries( empirical );
xySeriesCollection.addSeries( theory );
- JFreeChart chart = ChartFactory
- .createXYLineChart( "", "Magnitude", "Density", xySeriesCollection, PlotOrientation.VERTICAL, false,
- false, false );
+ JFreeChart chart = ChartFactory.createXYLineChart( "", "Magnitude", "Density", xySeriesCollection, PlotOrientation.VERTICAL, false, false, false );
chart.getXYPlot().setRangeGridlinesVisible( false );
chart.getXYPlot().setDomainGridlinesVisible( false );
XYItemRenderer renderer = chart.getXYPlot().getRenderer();
@@ -386,9 +390,7 @@ private void runParametric( final DoubleMatrix2D sdata, DoubleMatrix2D gammastar
DoubleMatrix1D[] batchResults;
- batchResults = this.itSol( batchData, gammaHat.viewRow( batchIndex ), deltaHat.viewRow( batchIndex ),
- gammaBar.get( batchIndex ), t2.get( batchIndex ), aPrior.get( batchIndex ),
- bPrior.get( batchIndex ) );
+ batchResults = this.itSol( batchData, gammaHat.viewRow( batchIndex ), deltaHat.viewRow( batchIndex ), gammaBar.get( batchIndex ), t2.get( batchIndex ), aPrior.get( batchIndex ), bPrior.get( batchIndex ) );
for ( int j = 0; j < batchResults[0].size(); j++ ) {
double v = batchResults[0].get( j );
@@ -430,8 +432,7 @@ public void run() {
for ( int k = firstBatch; k < lastBatch; k++ ) {
String batchId = batchIds[k];
DoubleMatrix2D batchData = ComBat.this.getBatchData( sdata, batchId );
- DoubleMatrix1D[] batchResults = ComBat.this
- .nonParametricFit( batchData, gammaHat.viewRow( k ), deltaHat.viewRow( k ) );
+ DoubleMatrix1D[] batchResults = ComBat.this.nonParametricFit( batchData, gammaHat.viewRow( k ), deltaHat.viewRow( k ) );
results.put( batchId, batchResults );
}
}
@@ -529,19 +530,36 @@ private DoubleMatrix2D computeDesignMatrix() {
* Find the batch
*/
DesignMatrix d = null;
- int batchFactorColumnIndex = sampleInfo.getColIndexByName( ComBat.BATCH_COLUMN_NAME );
- Object[] batchFactor = sampleInfo.getColumn( batchFactorColumnIndex );
+ int batchFactorColumnIndex = this.sampleInfo.getColIndexByName( ComBat.BATCH_COLUMN_NAME );
+ Object[] batchFactor = this.sampleInfo.getColumn( batchFactorColumnIndex );
if ( batchFactor != null ) {
d = new DesignMatrix( batchFactor, 1, ComBat.BATCH_COLUMN_NAME );
}
if ( d == null ) {
throw new IllegalStateException( "No batch factor was found" );
}
- ObjectMatrix sampleInfoWithoutBatchFactor = this
- .getSampleInfoWithoutBatchFactor( batchFactorColumnIndex );
+ ObjectMatrix sampleInfoWithoutBatchFactor = this.getSampleInfoWithoutBatchFactor( batchFactorColumnIndex );
+
+ // Check if the experimental design is going to be usable.
+ DesignMatrix nonBatchFactorDesign = new DesignMatrix( sampleInfoWithoutBatchFactor );
+ int nonbatchrank = solver.rank( nonBatchFactorDesign.getDoubleMatrix() );
+ if ( nonbatchrank < nonBatchFactorDesign.getDoubleMatrix().columns() ) {
+ /*
+ FIXME If this is because of duplicate columns, we _could_ try to fix it.
+ */
+ throw new ComBatException( "Non-batch factor part of the model matrix is not of full rank (Rank " + nonbatchrank + " < " + nonBatchFactorDesign.getDoubleMatrix().columns() + " columns); batch correction cannot proceed" );
+ }
d.add( sampleInfoWithoutBatchFactor );
design = d.getDoubleMatrix();
+
+ // guard against problems.
+ int ranka = solver.rank( design );
+ if ( ranka < design.columns() ) {
+ // if we get here, it probably means there is a confound between the batches and the other factors.
+ throw new ComBatException( "Model matrix is not of full rank (Rank " + ranka + " < " + design.columns() + " columns); batch correction cannot proceed" );
+ }
+
return design;
}
@@ -590,16 +608,14 @@ private DoubleMatrix2D getBatchDesign( String batchId ) {
}
private ObjectMatrix getSampleInfoWithoutBatchFactor( int batchFactorColumnIndex ) {
- ObjectMatrix sampleInfoWithoutBatchFactor = new ObjectMatrixImpl<>( sampleInfo.rows(),
- sampleInfo.columns() - 1 );
+ ObjectMatrix sampleInfoWithoutBatchFactor = new ObjectMatrixImpl<>( sampleInfo.rows(), sampleInfo.columns() - 1 );
boolean warned = false;
int r = 0;
for ( int i = 0; i < sampleInfo.rows(); i++ ) {
int c = 0;
for ( int j = 0; j < sampleInfo.columns(); j++ ) {
- if ( j == batchFactorColumnIndex )
- continue;
+ if ( j == batchFactorColumnIndex ) continue;
if ( i == 0 ) {
sampleInfoWithoutBatchFactor.addColumnName( sampleInfo.getColName( j ) );
}
@@ -655,8 +671,7 @@ private void initPartA() {
*/
for ( String batchId : batches.keySet() ) {
if ( batches.get( batchId ).size() < 2 ) {
- throw new IllegalArgumentException(
- "Batch correction not possible with less than 2 samples in any batch. Consider combining batches." );
+ throw new IllegalArgumentException( "Batch correction not possible with less than 2 samples in any batch. Consider combining batches." );
}
}
@@ -664,8 +679,7 @@ private void initPartA() {
numProbes = y.rows();
}
- private DoubleMatrix1D[] itSol( DoubleMatrix2D matrix, DoubleMatrix1D gHat, DoubleMatrix1D dHat, double gbar,
- double t2b, double a, double b ) throws ComBatException {
+ private DoubleMatrix1D[] itSol( DoubleMatrix2D matrix, DoubleMatrix1D gHat, DoubleMatrix1D dHat, double gbar, double t2b, double a, double b ) throws ComBatException {
DoubleMatrix1D n = this.rowNonMissingCounts( matrix );
DoubleMatrix1D gold = gHat;
@@ -681,11 +695,9 @@ private DoubleMatrix1D[] itSol( DoubleMatrix2D matrix, DoubleMatrix1D gHat, Doub
DoubleMatrix1D sum2 = this.stepSum( matrix, gnew );
DoubleMatrix1D dnew = this.postVar( sum2, n, a, b );
- DoubleMatrix1D gnewtmp = gnew.copy().assign( gold, Functions.minus ).assign( Functions.abs )
- .assign( gold, Functions.div );
+ DoubleMatrix1D gnewtmp = gnew.copy().assign( gold, Functions.minus ).assign( Functions.abs ).assign( gold, Functions.div );
- DoubleMatrix1D dnewtmp = dnew.copy().assign( dold, Functions.minus ).assign( Functions.abs )
- .assign( dold, Functions.div );
+ DoubleMatrix1D dnewtmp = dnew.copy().assign( dold, Functions.minus ).assign( Functions.abs ).assign( dold, Functions.div );
double gnewmax;
double dnewmax;
if ( hasMissing ) {
@@ -706,9 +718,7 @@ private DoubleMatrix1D[] itSol( DoubleMatrix2D matrix, DoubleMatrix1D gHat, Doub
* For certain data sets, we just flail around; for example if there are only two samples. This is a
* bailout for exceptional circumstances.
*/
- throw new ComBatException(
- "Failed to converge within " + MAXITERS + " iterations, last delta was " + String
- .format( "%.2g", change ) );
+ throw new ComBatException( "Failed to converge within " + MAXITERS + " iterations, last delta was " + String.format( "%.2g", change ) );
}
}
@@ -740,8 +750,7 @@ private DoubleMatrix1D[] nonParametricFit( DoubleMatrix2D matrix, DoubleMatrix1D
double sumdLH = 0.0;
for ( int j = 0; j < matrix.rows(); j++ ) {
- if ( j == i )
- continue;
+ if ( j == i ) continue;
double g = gHat.getQuick( j );
double d = dHat.getQuick( j );
@@ -756,8 +765,7 @@ private DoubleMatrix1D[] nonParametricFit( DoubleMatrix2D matrix, DoubleMatrix1D
double LH = ( 1.0 / Math.pow( twopi * d, no2 ) ) * Math.exp( -sum2 / ( 2 * d ) );
- if ( Double.isNaN( LH ) )
- continue;
+ if ( Double.isNaN( LH ) ) continue;
double gLH = g * LH;
double dLH = d * LH;
@@ -778,12 +786,10 @@ private DoubleMatrix1D[] nonParametricFit( DoubleMatrix2D matrix, DoubleMatrix1D
return new DoubleMatrix1D[] { gstar, dstar };
}
- private DoubleMatrix1D postMean( DoubleMatrix1D ghat, double gbar, DoubleMatrix1D n, DoubleMatrix1D dstar,
- double t2b ) {
+ private DoubleMatrix1D postMean( DoubleMatrix1D ghat, double gbar, DoubleMatrix1D n, DoubleMatrix1D dstar, double t2b ) {
DoubleMatrix1D result = new DenseDoubleMatrix1D( ghat.size() );
for ( int i = 0; i < ghat.size(); i++ ) {
- result.set( i, ( t2b * n.get( i ) * ghat.get( i ) + dstar.get( i ) * gbar ) / ( t2b * n.get( i ) + dstar
- .get( i ) ) );
+ result.set( i, ( t2b * n.get( i ) * ghat.get( i ) + dstar.get( i ) * gbar ) / ( t2b * n.get( i ) + dstar.get( i ) ) );
}
return result;
}
@@ -807,8 +813,7 @@ private DoubleMatrix2D rawAdjust( DoubleMatrix2D sdata, DoubleMatrix2D gammastar
DoubleMatrix2D Xbb = this.getBatchDesign( batchId );
- DoubleMatrix2D adjustedBatch = batchData.copy()
- .assign( solver.transpose( solver.mult( Xbb, gammastar ) ), Functions.minus );
+ DoubleMatrix2D adjustedBatch = batchData.copy().assign( solver.transpose( solver.mult( Xbb, gammastar ) ), Functions.minus );
DoubleMatrix1D deltaStarRow = deltastar.viewRow( batchNum );
deltaStarRow.assign( Functions.sqrt );
@@ -849,8 +854,7 @@ private DoubleMatrix2D restoreScale( DoubleMatrix2D adjustedData ) {
private DoubleMatrix1D rowNonMissingCounts( DoubleMatrix2D matrix ) {
DoubleMatrix1D result = new DenseDoubleMatrix1D( matrix.rows() );
for ( int i = 0; i < matrix.rows(); i++ ) {
- result.set( i, DescriptiveWithMissing
- .sizeWithoutMissingValues( new DoubleArrayList( matrix.viewRow( i ).toArray() ) ) );
+ result.set( i, DescriptiveWithMissing.sizeWithoutMissingValues( new DoubleArrayList( matrix.viewRow( i ).toArray() ) ) );
}
return result;
}
@@ -871,8 +875,7 @@ private DoubleMatrix1D stepSum( DoubleMatrix2D matrix, DoubleMatrix1D gnew ) {
* subtract column gnew from each column of data; square; then sum over each row.
*/
- DoubleMatrix2D deltas = matrix.copy().assign( ( s.mult( s.transpose( g ), a ) ), Functions.minus )
- .assign( Functions.square );
+ DoubleMatrix2D deltas = matrix.copy().assign( ( s.mult( s.transpose( g ), a ) ), Functions.minus ).assign( Functions.square );
DoubleMatrix1D sumsq = new DenseDoubleMatrix1D( deltas.rows() );
sumsq.assign( 0.0 );
diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/ExpressionExperimentBatchCorrectionServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/ExpressionExperimentBatchCorrectionServiceImpl.java
index 9f9948ba07..ecbd85d2c4 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/ExpressionExperimentBatchCorrectionServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/ExpressionExperimentBatchCorrectionServiceImpl.java
@@ -24,7 +24,9 @@
import ubic.basecode.dataStructure.matrix.ObjectMatrix;
import ubic.basecode.dataStructure.matrix.ObjectMatrixImpl;
import ubic.basecode.math.MatrixStats;
+import ubic.basecode.math.linearmodels.DesignMatrix;
import ubic.basecode.util.FileTools;
+import ubic.gemma.core.analysis.expression.diff.LinearModelAnalyzer;
import ubic.gemma.core.analysis.util.ExperimentalDesignUtils;
import ubic.gemma.core.datastructure.matrix.ExpressionDataDoubleMatrix;
import ubic.gemma.model.common.description.Characteristic;
@@ -34,6 +36,7 @@
import ubic.gemma.model.expression.bioAssayData.ProcessedExpressionDataVector;
import ubic.gemma.model.expression.biomaterial.BioMaterial;
import ubic.gemma.model.expression.designElement.CompositeSequence;
+import ubic.gemma.model.expression.experiment.BatchEffectType;
import ubic.gemma.model.expression.experiment.ExperimentalFactor;
import ubic.gemma.model.expression.experiment.ExpressionExperiment;
import ubic.gemma.model.expression.experiment.FactorValue;
@@ -65,20 +68,19 @@ public class ExpressionExperimentBatchCorrectionServiceImpl implements Expressio
@Override
public boolean checkCorrectability( ExpressionExperiment ee ) {
-// for ( QuantitationType qt : expressionExperimentService.getQuantitationTypes( ee ) ) {
-// if ( qt.getIsBatchCorrected() ) { // this avoids batch correcting when we don't need to
-// ExpressionExperimentBatchCorrectionServiceImpl.log
-// .info( "Experiment already has a batch-corrected quantitation type: " + ee + ": " + qt );
-// return false;
-// }
-// }
-
ExperimentalFactor batch = this.getBatchFactor( ee );
if ( batch == null ) {
ExpressionExperimentBatchCorrectionServiceImpl.log.info( "No batch factor found: " + ee );
return false;
}
+ BatchEffectType bet = expressionExperimentService.getBatchEffect( ee );
+ if ( BatchEffectType.NO_BATCH_EFFECT_SUCCESS.equals( bet ) || BatchEffectType.SINGLE_BATCH_SUCCESS.equals( bet ) ) {
+ ExpressionExperimentBatchCorrectionServiceImpl.log.info( "Experiment does not require batch correction as " +
+ "batch effect is negligible or it's a single batch: " + ee );
+ return false;
+ }
+
if ( expressionExperimentService.getArrayDesignsUsed( ee ).size() > 1 ) {
log.info( String.format( "%s cannot be batch-corrected: multiplatform; you must switch/merge first.", ee ) );
return false;
@@ -111,10 +113,6 @@ public boolean checkCorrectability( ExpressionExperiment ee ) {
}
}
}
- /*
- * consider merging batches. - we already do this when we create the batch factor, so in general batches should
- * always have at least 2 samples
- */
for ( Long batchId : batches.keySet() ) {
if ( batches.get( batchId ) < 2 ) {
ExpressionExperimentBatchCorrectionServiceImpl.log
@@ -124,6 +122,17 @@ public boolean checkCorrectability( ExpressionExperiment ee ) {
}
}
+ /*
+ Get the experimental design matrix we would use for batch correction. If this is not full rank we can't proceed.
+ */
+ ObjectMatrix design = this.getDesign( ee );
+ ObjectMatrix designU = this.convertFactorValuesToStrings( design );
+ try {
+ new ComBat( designU ); // without data, just to check
+ } catch ( ComBatException c ) { // probably because it's not full rank.
+ log.info( c.getMessage() );
+ return false;
+ }
return true;
}
@@ -144,10 +153,41 @@ public ExpressionDataDoubleMatrix comBat( ExpressionDataDoubleMatrix originalDat
return null;
}
- ObjectMatrix design = this.getDesign( ee, originalDataMatrix );
+ ExpressionDataDoubleMatrix finalMatrix = removeOutliers( originalDataMatrix, ee );
+
+ ObjectMatrix design = this.getDesign( ee, finalMatrix );
+
+ return this.doComBat( ee, finalMatrix, design );
+
+ }
+
+ /**
+ * Remove outlier samples from the data matrix, based on outliers that were flagged in the experiment (not just candidate outliers)
+ * @param originalDataMatrix
+ * @param ee
+ * @return the original matrix, or if outliers were present, a new matrix with the outliers removed
+ */
+ public static ExpressionDataDoubleMatrix removeOutliers( ExpressionDataDoubleMatrix originalDataMatrix, ExpressionExperiment ee ) {
+
+ List columnsToKeep = new ArrayList<>();
+ for ( BioAssay ba : ee.getBioAssays() ) {
+ if ( !ba.getIsOutlier() ) {
+
+ /*
+ Find the index of this bioassay in the originalDataMatrix
+ */
+ columnsToKeep.add( ba.getSampleUsed() );
- return this.doComBat( ee, originalDataMatrix, design );
+ } else {
+ log.info( "Dropping outlier sample: " + ba + " from batch correction" );
+ }
+ }
+ ExpressionDataDoubleMatrix finalMatrix = originalDataMatrix;
+ if ( columnsToKeep.size() < originalDataMatrix.columns() ) {
+ finalMatrix = new ExpressionDataDoubleMatrix( originalDataMatrix, columnsToKeep, LinearModelAnalyzer.createBADMap( columnsToKeep ) );
+ }
+ return finalMatrix;
}
@Override
@@ -212,7 +252,8 @@ private ObjectMatrix convertFactorValuesToStrings(
return designU;
}
- private ExpressionDataDoubleMatrix doComBat( ExpressionExperiment ee, ExpressionDataDoubleMatrix originalDataMatrix,
+ private ExpressionDataDoubleMatrix doComBat( ExpressionExperiment ee, ExpressionDataDoubleMatrix
+ originalDataMatrix,
ObjectMatrix design ) {
ObjectMatrix designU = this.convertFactorValuesToStrings( design );
DoubleMatrix matrix = originalDataMatrix.getMatrix();
@@ -239,7 +280,8 @@ private ExpressionDataDoubleMatrix doComBat( ExpressionExperiment ee, Expression
try {
results = comBat.run( true ); // false: NONPARAMETRIC
} catch ( ComBatException e ) {
- throw new IllegalStateException( "ComBat failed likely due to confounds between batching and other factors: " + e.getMessage(), e );
+ log.error( e.getMessage() );
+ return null;
}
// note these plots always reflect the parametric setup.
@@ -274,13 +316,18 @@ private ExpressionDataDoubleMatrix doComBat( ExpressionExperiment ee, Expression
return correctedExpressionDataMatrix;
}
+
+ private ObjectMatrix getDesign( ExpressionExperiment ee ) {
+ return this.getDesign( ee, null );
+ }
+
/**
* Extract sample information, format into something ComBat can use.
*
* Certain factors are removed at this stage, notably "DE_Exclude/Include" factors.
*
* @param ee
- * @param mat
+ * @param mat only used to get sample ordering?
* @return design matrix
*/
private ObjectMatrix getDesign( ExpressionExperiment ee,
@@ -291,8 +338,13 @@ private ObjectMatrix getDesign( Express
/* remove experimental factors that are for DE_Exclude */
List retainedFactors = experimentalFactors.stream().filter( ef -> retainForBatchCorrection( ef ) ).collect( Collectors.toList() );
- List orderedSamples = ExperimentalDesignUtils.getOrderedSamples( mat, retainedFactors );
-
+ List orderedSamples = new ArrayList<>();
+ if ( mat == null ) {
+ // we're using this in a mode where we don't care about the order.
+ orderedSamples = ee.getBioAssays().stream().map( BioAssay::getSampleUsed ).collect( Collectors.toList() );
+ } else {
+ orderedSamples = ExperimentalDesignUtils.getOrderedSamples( mat, retainedFactors );
+ }
return ExperimentalDesignUtils.sampleInfoMatrix( retainedFactors, orderedSamples,
ExperimentalDesignUtils.getBaselineConditions( orderedSamples, retainedFactors ) );
}
@@ -303,7 +355,7 @@ private ObjectMatrix getDesign( Express
* @return true if the factor should be used in the model for batch correction
*/
private boolean retainForBatchCorrection( ExperimentalFactor ef ) {
- if ( ef.getCategory().getCategoryUri() != null && ef.getCategory().getCategoryUri().equals( COLLECTION_OF_MATERIAL_URI ) ) {
+ if ( ef.getCategory() != null && COLLECTION_OF_MATERIAL_URI.equals( ef.getCategory().getCategoryUri() ) ) {
for ( FactorValue fv : ef.getFactorValues() ) {
for ( Characteristic c : fv.getCharacteristics() ) {
if ( c.getValueUri() != null && ( c.getValueUri().equals( DE_EXCLUDE_URI ) || c.getValueUri().equals( DE_INCLUDE_URI ) ) ) {
@@ -313,6 +365,7 @@ private boolean retainForBatchCorrection( ExperimentalFactor ef ) {
}
}
}
+ log.info( "Retaining factor " + ef.getName() + " for batch correction model" );
return true;
}
@@ -346,7 +399,8 @@ private QuantitationType makeNewQuantitationType( QuantitationType oldQt ) {
*
* @return updated designU
*/
- private ObjectMatrix orderMatrix( DoubleMatrix matrix,
+ private ObjectMatrix orderMatrix
+ ( DoubleMatrix matrix,
ObjectMatrix designU ) {
ObjectMatrix result = new ObjectMatrixImpl<>( designU.rows(), designU.columns() );
diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/filter/ExpressionExperimentFilter.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/filter/ExpressionExperimentFilter.java
index a9e4a2f32c..115e1307b9 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/filter/ExpressionExperimentFilter.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/filter/ExpressionExperimentFilter.java
@@ -108,6 +108,8 @@ public static ExpressionDataDoubleMatrix tooFewDistinctValues( ExpressionDataDou
}
/**
+ * @param matrix the data matrix
+ * @param threshold fraction of values that must be distinct. Thus if set to 0.5, a vector of 10 values must have at 5 distinct values.
* @param tolerance differences smaller than this are counted as "the same value".
*/
public static ExpressionDataDoubleMatrix tooFewDistinctValues( ExpressionDataDoubleMatrix matrix,
diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/svd/SVDServiceHelperImpl.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/svd/SVDServiceHelperImpl.java
index b81392d5b7..86a706b52b 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/svd/SVDServiceHelperImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/svd/SVDServiceHelperImpl.java
@@ -37,10 +37,7 @@
import ubic.gemma.model.expression.bioAssayData.ProcessedExpressionDataVector;
import ubic.gemma.model.expression.biomaterial.BioMaterial;
import ubic.gemma.model.expression.designElement.CompositeSequence;
-import ubic.gemma.model.expression.experiment.ExperimentalFactor;
-import ubic.gemma.model.expression.experiment.ExpressionExperiment;
-import ubic.gemma.model.expression.experiment.ExpressionExperimentValueObject;
-import ubic.gemma.model.expression.experiment.FactorValue;
+import ubic.gemma.model.expression.experiment.*;
import ubic.gemma.persistence.service.analysis.expression.pca.PrincipalComponentAnalysisService;
import ubic.gemma.persistence.service.common.auditAndSecurity.AuditTrailService;
import ubic.gemma.persistence.service.expression.bioAssayData.ProcessedExpressionDataVectorService;
@@ -100,28 +97,34 @@ public static void populateBMFMap( Map> bi
for ( FactorValue fv : bm.getFactorValues() ) {
ExperimentalFactor experimentalFactor = fv.getExperimentalFactor();
double valueToStore;
- if ( fv.getMeasurement() != null ) {
- switch ( fv.getMeasurement().getRepresentation() ) {
- case INT:
- valueToStore = Integer.parseInt( fv.getMeasurement().getValue() );
- break;
- case LONG:
- valueToStore = Long.parseLong( fv.getMeasurement().getValue() );
- break;
- case DOUBLE:
- valueToStore = Double.parseDouble( fv.getMeasurement().getValue() );
- break;
- default:
- // non-numerical measurement can be treated as categorical
- valueToStore = fv.getId().doubleValue();
+ if ( experimentalFactor.getType().equals( FactorType.CONTINUOUS ) ) {
+ if ( fv.getMeasurement() != null && fv.getMeasurement().getValue() != null ) { // continuous
+ try {
+ switch ( fv.getMeasurement().getRepresentation() ) {
+ case INT:
+ valueToStore = Integer.parseInt( fv.getMeasurement().getValue() );
+ break;
+ case LONG:
+ valueToStore = Long.parseLong( fv.getMeasurement().getValue() );
+ break;
+ case DOUBLE:
+ valueToStore = Double.parseDouble( fv.getMeasurement().getValue() );
+ break;
+ default:
+ // non-numerical measurement can be treated as categorical
+ valueToStore = fv.getId().doubleValue();
+ }
+ } catch ( NumberFormatException e ) {
+ valueToStore = Double.NaN; // due to a missing value
+ }
+ } else {
+ valueToStore =Double.NaN;
}
} else {
- /*
- * This is a hack so we don't need special datastructures for
- * continuous and categorical. We're storing the ID but as a double.
- */
+ // for categorical factors, we use the ids as dummy values.
valueToStore = fv.getId().doubleValue();
}
+
bioMaterialFactorMap
.computeIfAbsent( experimentalFactor, k -> new HashMap<>() )
.put( bm.getId(), valueToStore );
@@ -506,7 +509,7 @@ private void analyzeComponent( SVDValueObject svo, int componentNumber, DoubleMa
svo.setPCFactorCorrelationPval( componentNumber, ef,
CorrelationStats.spearmanPvalue( factorCorrelation, eigenGeneWithoutMissing.size() ) );
} else {
- // one-way ANOVA on ranks.
+ // one-way ANOVA on ranks. This test is pretty underpowered.
double kwPVal = KruskalWallis.test( eigenGeneWithoutMissing, groupings );
svo.setPCFactorCorrelationPval( componentNumber, ef, kwPVal );
@@ -516,19 +519,20 @@ private void analyzeComponent( SVDValueObject svo, int componentNumber, DoubleMa
double corrPvalue = CorrelationStats
.spearmanPvalue( factorCorrelation, eigenGeneWithoutMissing.size() );
assert Math.abs( factorCorrelation ) < 1.0 + 1e-2; // sanity.
+
/*
- * Avoid storing a pvalue, as it's hard to compare. If the regular linear correlation is strong,
+ * If the regular linear correlation is strong,
* then we should just use that -- basically, it means the order we have the groups happens to be a
- * good one. Of course we could just store pvalues, but that's not easy to use either.
+ * good one.
*/
if ( corrPvalue <= kwPVal ) {
svo.setPCFactorCorrelation( componentNumber, ef, factorCorrelation );
+ svo.setPCFactorCorrelationPval( componentNumber, ef, corrPvalue);
} else {
// hack. A bit like turning pvalues into prob it
double approxCorr = CorrelationStats
.correlationForPvalue( kwPVal, eigenGeneWithoutMissing.size() );
svo.setPCFactorCorrelation( componentNumber, ef, approxCorr );
-
}
}
diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/report/ArrayDesignReportService.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/report/ArrayDesignReportService.java
index 639a904037..e563397c84 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/analysis/report/ArrayDesignReportService.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/report/ArrayDesignReportService.java
@@ -66,4 +66,6 @@ public interface ArrayDesignReportService {
String getLastGeneMappingEvent( Long id );
+ String getCreateDate( Long id );
+
}
\ No newline at end of file
diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/report/ArrayDesignReportServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/report/ArrayDesignReportServiceImpl.java
index 046184861a..61b3d626e8 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/analysis/report/ArrayDesignReportServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/report/ArrayDesignReportServiceImpl.java
@@ -87,6 +87,7 @@ public void generateAllArrayDesignReport() {
adVo.setNumProbesToGenes( Long.toString( numCsGenes ) );
adVo.setNumGenes( Long.toString( numGenes ) );
adVo.setDateCached( timestamp );
+
// remove file first
File f = new File( ArrayDesignReportServiceImpl.HOME_DIR + File.separatorChar
+ ArrayDesignReportServiceImpl.ARRAY_DESIGN_REPORT_DIR + File.separatorChar
@@ -97,10 +98,10 @@ public void generateAllArrayDesignReport() {
return;
}
}
- try (FileOutputStream fos = new FileOutputStream( ArrayDesignReportServiceImpl.HOME_DIR + File.separatorChar
+ try ( FileOutputStream fos = new FileOutputStream( ArrayDesignReportServiceImpl.HOME_DIR + File.separatorChar
+ ArrayDesignReportServiceImpl.ARRAY_DESIGN_REPORT_DIR + File.separatorChar
+ ArrayDesignReportServiceImpl.ARRAY_DESIGN_SUMMARY );
- ObjectOutputStream oos = new ObjectOutputStream( fos )) {
+ ObjectOutputStream oos = new ObjectOutputStream( fos ) ) {
oos.writeObject( adVo );
} catch ( Throwable e ) {
// cannot write to file. Just fail gracefully.
@@ -172,8 +173,8 @@ public void generateArrayDesignReport( ArrayDesignValueObject adVo ) {
}
}
- try (FileOutputStream fos = new FileOutputStream( reportFileName );
- ObjectOutputStream oos = new ObjectOutputStream( fos )) {
+ try ( FileOutputStream fos = new FileOutputStream( reportFileName );
+ ObjectOutputStream oos = new ObjectOutputStream( fos ) ) {
oos.writeObject( adVo );
} catch ( Throwable e ) {
ArrayDesignReportServiceImpl.log.error( "Cannot write to file: " + reportFileName, e );
@@ -207,7 +208,7 @@ public ArrayDesignValueObject getSummaryObject( Long id ) {
+ File.separatorChar + ArrayDesignReportServiceImpl.ARRAY_DESIGN_REPORT_FILE_NAME_PREFIX + "."
+ id );
if ( f.exists() ) {
- try (FileInputStream fis = new FileInputStream( f ); ObjectInputStream ois = new ObjectInputStream( fis )) {
+ try ( FileInputStream fis = new FileInputStream( f ); ObjectInputStream ois = new ObjectInputStream( fis ) ) {
adVo = ( ArrayDesignValueObject ) ois.readObject();
@@ -230,7 +231,7 @@ public ArrayDesignValueObject getSummaryObject() {
+ ArrayDesignReportServiceImpl.ARRAY_DESIGN_REPORT_DIR + File.separatorChar
+ ArrayDesignReportServiceImpl.ARRAY_DESIGN_SUMMARY );
if ( f.exists() ) {
- try (FileInputStream fis = new FileInputStream( f ); ObjectInputStream ois = new ObjectInputStream( fis )) {
+ try ( FileInputStream fis = new FileInputStream( f ); ObjectInputStream ois = new ObjectInputStream( fis ) ) {
adVo = ( ArrayDesignValueObject ) ois.readObject();
} catch ( Throwable e ) {
return null;
@@ -293,6 +294,7 @@ public void fillEventInformation( Collection adVos ) {
Map sequenceUpdateEvents = events.get( ArrayDesignSequenceUpdateEvent.class );
Map sequenceAnalysisEvents = events.get( ArrayDesignSequenceAnalysisEvent.class );
Map repeatAnalysisEvents = events.get( ArrayDesignRepeatAnalysisEvent.class );
+ Map creationEvents = auditEventService.getCreateEvents( arrayDesigns );
for ( ArrayDesignValueObject adVo : adVos ) {
@@ -326,6 +328,12 @@ public void fillEventInformation( Collection adVos ) {
}
}
+ if ( creationEvents.containsKey( ad ) ) {
+ AuditEvent event = creationEvents.get( ad );
+ if ( event != null ) {
+ adVo.setCreateDate( event.getDate() );
+ }
+ }
}
watch.stop();
@@ -404,7 +412,18 @@ public String getLastRepeatMaskEvent( Long id ) {
@Override
public String getLastGeneMappingEvent( Long id ) {
return this.getLastEvent( id, ArrayDesignGeneMappingEvent.class );
+ }
+
+ @Override
+ public String getCreateDate( Long id ) {
+ ArrayDesign ad = arrayDesignService.load( id );
+
+ if ( ad == null )
+ return "";
+ List events = auditEventService.getEvents( ad );
+ AuditEvent lastEvent = events.get( 0 );
+ return DateFormatUtils.format( lastEvent.getDate(), "yyyy.MMM.dd hh:mm aa" );
}
private String getLastEvent( Long id, Class eventType ) {
diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/report/DatabaseViewGeneratorImpl.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/report/DatabaseViewGeneratorImpl.java
index f98c5769f5..14d022234c 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/analysis/report/DatabaseViewGeneratorImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/report/DatabaseViewGeneratorImpl.java
@@ -144,7 +144,7 @@ private void generateDatasetTissueView( Integer limit, Collection> blatResultCoun
String targetStarts = ( String ) row[17];
Long queryId = ( ( BigInteger ) row[18] ).longValue();
- int hash = 1;
- int prime = 31;
- hash = prime * hash + chromId.hashCode();
- hash = prime * hash + targetStart.hashCode();
- hash = prime * hash + targetEnd.hashCode();
- hash = prime * hash + targetStarts.hashCode();
- hash = prime * hash + queryId.hashCode();
+ int hash = Objects.hash( chromId, targetStart, targetEnd, targetStarts, queryId );
EntityUtils.populateMapSet( blatResultCount, csId, hash );
diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataFileService.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataFileService.java
index b585f60853..434a54dbe7 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataFileService.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataFileService.java
@@ -25,17 +25,21 @@
import ubic.gemma.model.expression.experiment.ExpressionExperiment;
import ubic.gemma.persistence.util.Settings;
+import javax.annotation.Nullable;
+import javax.annotation.ParametersAreNonnullByDefault;
import java.io.File;
import java.io.IOException;
import java.io.Writer;
import java.util.Collection;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
/**
* @author paul
*/
@SuppressWarnings("unused") // Possible external use
+@ParametersAreNonnullByDefault
public interface ExpressionDataFileService extends TsvFileService {
String DATA_ARCHIVE_FILE_SUFFIX = ".zip";
@@ -56,7 +60,7 @@ void analysisResultSetsToString( Collection results
List analysisResultSetToString( ExpressionAnalysisResultSet ears,
Map geneAnnotations, StringBuilder buf, Map probe2String,
- List sortedFirstColumnOfResults );
+ @Nullable List sortedFirstColumnOfResults );
/**
* Delete any existing coexpression, data, or differential expression data files.
@@ -119,12 +123,10 @@ List analysisResultSetToString( Expression
* @param compress compress?
* @param fileName file name
* @param filtered fitlered?
- * @return file
+ * @return file, or empty if the experiment has no processed expression data
* @throws IOException when there are IO problems
*/
- @SuppressWarnings("UnusedReturnValue")
- // Possible external use
- File writeDataFile( ExpressionExperiment ee, boolean filtered, String fileName, boolean compress )
+ Optional writeProcessedExpressionDataFile( ExpressionExperiment ee, boolean filtered, String fileName, boolean compress )
throws IOException, FilteringException;
/**
@@ -134,7 +136,7 @@ File writeDataFile( ExpressionExperiment ee, boolean filtered, String fileName,
* To write to a string, consider using {@link java.io.StringWriter}.
*
* @param ee the expression experiment
- * @param qt a quantitation type to use, otherwise the preferred one is picked
+ * @param qt a quantitation type to use
* @param writer the destination for the raw expression data
* @throws IOException if operations with the writer fails
*/
@@ -150,7 +152,7 @@ File writeDataFile( ExpressionExperiment ee, boolean filtered, String fileName,
* @param writer the destination for the raw expression data
* @throws IOException if operations with the writer fails
*/
- void writeProcessedExpressionData( ExpressionExperiment ee, QuantitationType qt, Writer writer ) throws IOException;
+ void writeProcessedExpressionData( ExpressionExperiment ee, Writer writer ) throws IOException;
/**
* Write or located the coexpression data file for a given experiment
@@ -159,7 +161,7 @@ File writeDataFile( ExpressionExperiment ee, boolean filtered, String fileName,
* @param forceWrite whether to force write
* @return file
*/
- File writeOrLocateCoexpressionDataFile( ExpressionExperiment ee, boolean forceWrite );
+ Optional writeOrLocateCoexpressionDataFile( ExpressionExperiment ee, boolean forceWrite );
/**
* Locate or create a data file containing the 'preferred and masked' expression data matrix, with filtering for low
@@ -170,19 +172,19 @@ File writeDataFile( ExpressionExperiment ee, boolean filtered, String fileName,
* @param filtered filtered
* @param forceWrite force re-write even if file already exists and is up to date.
* @param ee the experiment
- * @return file
+ * @return file, or empty if the experiment has no processed vectors
*/
- File writeOrLocateDataFile( ExpressionExperiment ee, boolean forceWrite, boolean filtered ) throws FilteringException;
+ Optional writeOrLocateProcessedDataFile( ExpressionExperiment ee, boolean forceWrite, boolean filtered ) throws FilteringException;
/**
* Locate or create a new data file for the given quantitation type. The output will include gene information if it
* can be located from its own file.
*
* @param forceWrite To not return the existing file, but create it anew.
- * @param type the quantitaion type
+ * @param type the quantitation type
* @return file
*/
- File writeOrLocateDataFile( QuantitationType type, boolean forceWrite );
+ File writeOrLocateRawExpressionDataFile( ExpressionExperiment ee, QuantitationType type, boolean forceWrite );
/**
* Locate or create an experimental design file for a given experiment.
@@ -205,14 +207,14 @@ File writeDataFile( ExpressionExperiment ee, boolean filtered, String fileName,
Collection writeOrLocateDiffExpressionDataFiles( ExpressionExperiment ee, boolean forceWrite );
/**
- * @param filtered if the data should be filtered.
- * @param ee the experiment
- * @param forceWrite whether to force write
- * @return file
+ * @see #writeOrLocateProcessedDataFile(ExpressionExperiment, boolean, boolean)
*/
- File writeOrLocateJSONDataFile( ExpressionExperiment ee, boolean forceWrite, boolean filtered ) throws FilteringException;
+ Optional writeOrLocateJSONProcessedExpressionDataFile( ExpressionExperiment ee, boolean forceWrite, boolean filtered ) throws FilteringException;
- File writeOrLocateJSONDataFile( QuantitationType type, boolean forceWrite );
+ /**
+ * @see #writeOrLocateRawExpressionDataFile(ExpressionExperiment, QuantitationType, boolean)
+ */
+ File writeOrLocateJSONRawExpressionDataFile( ExpressionExperiment ee, QuantitationType type, boolean forceWrite );
void deleteDiffExArchiveFile( DifferentialExpressionAnalysis analysis );
@@ -229,5 +231,5 @@ File writeDataFile( ExpressionExperiment ee, boolean filtered, String fileName,
* @throws IOException when there was a problem during write
*/
void writeDiffExArchiveFile( BioAssaySet ee, DifferentialExpressionAnalysis analysis,
- DifferentialExpressionAnalysisConfig config ) throws IOException;
+ @Nullable DifferentialExpressionAnalysisConfig config ) throws IOException;
}
\ No newline at end of file
diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataFileServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataFileServiceImpl.java
index 2c37ab4911..9c1fde5c44 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataFileServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataFileServiceImpl.java
@@ -58,6 +58,8 @@
import ubic.gemma.persistence.util.EntityUtils;
import ubic.gemma.persistence.util.Settings;
+import javax.annotation.Nullable;
+import javax.annotation.ParametersAreNonnullByDefault;
import java.io.*;
import java.nio.file.Paths;
import java.util.*;
@@ -74,6 +76,7 @@
* @author paul
*/
@Component
+@ParametersAreNonnullByDefault
public class ExpressionDataFileServiceImpl extends AbstractFileService implements ExpressionDataFileService {
private static final Log log = LogFactory.getLog( ArrayDesignAnnotationServiceImpl.class.getName() );
@@ -142,12 +145,11 @@ public void analysisResultSetsToString( Collection
@Override
public List analysisResultSetToString( ExpressionAnalysisResultSet ears,
Map geneAnnotations, StringBuilder buf, Map probe2String,
- List sortedFirstColumnOfResults ) {
+ @Nullable List sortedFirstColumnOfResults ) {
if ( sortedFirstColumnOfResults == null ) { // Sort P values in ears (because 1st column)
sortedFirstColumnOfResults = new ArrayList<>( ears.getResults() );
- Collections.sort( sortedFirstColumnOfResults,
- DifferentialExpressionAnalysisResultComparator.Factory.newInstance() );
+ sortedFirstColumnOfResults.sort( DifferentialExpressionAnalysisResultComparator.Factory.newInstance() );
}
// Generate a description of the factors involved "factor1_factor2", trying to be R-friendly
@@ -234,7 +236,7 @@ public void deleteDiffExArchiveFile( DifferentialExpressionAnalysis analysis ) {
@Override
public File getDiffExpressionAnalysisArchiveFile( Long analysisId, boolean forceCreate ) {
- DifferentialExpressionAnalysis analysis = this.differentialExpressionAnalysisService.load( analysisId );
+ DifferentialExpressionAnalysis analysis = this.differentialExpressionAnalysisService.loadOrFail( analysisId );
return getDiffExpressionAnalysisArchiveFile( analysis, forceCreate );
}
@@ -351,7 +353,7 @@ private File getNewestFile( File file ) {
}
@Override
- public File writeDataFile( ExpressionExperiment ee, boolean filtered, String fileName, boolean compress )
+ public Optional writeProcessedExpressionDataFile( ExpressionExperiment ee, boolean filtered, String fileName, boolean compress )
throws IOException, FilteringException {
File f = new File( fileName );
return this.writeDataFile( ee, filtered, f, compress );
@@ -359,10 +361,13 @@ public File writeDataFile( ExpressionExperiment ee, boolean filtered, String fil
@Override
public void writeDiffExArchiveFile( BioAssaySet experimentAnalyzed, DifferentialExpressionAnalysis analysis,
- DifferentialExpressionAnalysisConfig config ) throws IOException {
+ @Nullable DifferentialExpressionAnalysisConfig config ) throws IOException {
Collection arrayDesigns = this.expressionExperimentService
.getArrayDesignsUsed( experimentAnalyzed );
Map geneAnnotations = this.getGeneAnnotationsAsStrings( arrayDesigns );
+ if ( analysis.getExperimentAnalyzed().getId() == null ) {// this can happen when using -nodb
+ analysis.getExperimentAnalyzed().setId( experimentAnalyzed.getId() );
+ }
String filename = this.getDiffExArchiveFileName( analysis );
File f = this.getOutputFile( filename );
@@ -419,12 +424,15 @@ public void writeRawExpressionData( ExpressionExperiment ee, QuantitationType qt
@Override
@Transactional(readOnly = true)
- public void writeProcessedExpressionData( ExpressionExperiment ee, QuantitationType qt, Writer writer ) throws IOException {
+ public void writeProcessedExpressionData( ExpressionExperiment ee, Writer writer ) throws IOException {
ee = expressionExperimentService.find( ee );
if ( ee == null ) {
throw new IllegalArgumentException( "ExpressionExperiment has been removed." );
}
- ExpressionDataDoubleMatrix matrix = expressionDataMatrixService.getProcessedExpressionDataMatrix( ee, qt );
+ ExpressionDataDoubleMatrix matrix = expressionDataMatrixService.getProcessedExpressionDataMatrix( ee );
+ if ( matrix == null ) {
+ throw new IllegalArgumentException( "ExpressionExperiment has no processed data vectors." );
+ }
Set ads = matrix.getDesignElements().stream()
.map( CompositeSequence::getArrayDesign )
.collect( Collectors.toSet() );
@@ -432,7 +440,7 @@ public void writeProcessedExpressionData( ExpressionExperiment ee, QuantitationT
}
@Override
- public File writeOrLocateCoexpressionDataFile( ExpressionExperiment ee, boolean forceWrite ) {
+ public Optional writeOrLocateCoexpressionDataFile( ExpressionExperiment ee, boolean forceWrite ) {
ee = expressionExperimentService.thawLite( ee );
@@ -440,11 +448,14 @@ public File writeOrLocateCoexpressionDataFile( ExpressionExperiment ee, boolean
File f = this.getOutputFile( this.getCoexpressionDataFilename( ee ) );
if ( !forceWrite && f.canRead() ) {
ExpressionDataFileServiceImpl.log.info( f + " exists, not regenerating" );
- return f;
+ return Optional.of( f );
}
- this.writeCoexpressionData( f, ee );
- return f;
+ if ( this.writeCoexpressionData( f, ee ) ) {
+ return Optional.of( f );
+ } else {
+ return Optional.empty();
+ }
} catch ( IOException e ) {
throw new RuntimeException( e );
}
@@ -452,13 +463,13 @@ public File writeOrLocateCoexpressionDataFile( ExpressionExperiment ee, boolean
}
@Override
- public File writeOrLocateDataFile( ExpressionExperiment ee, boolean forceWrite, boolean filtered ) throws FilteringException {
+ public Optional writeOrLocateProcessedDataFile( ExpressionExperiment ee, boolean forceWrite, boolean filtered ) throws FilteringException {
try {
File f = this.getOutputFile( ee, filtered );
Date check = expressionExperimentService.getLastArrayDesignUpdate( ee );
if ( this.checkFileOkToReturn( forceWrite, f, check ) ) {
- return f;
+ return Optional.of( f );
}
return this.writeDataFile( ee, filtered, f, true );
@@ -469,7 +480,7 @@ public File writeOrLocateDataFile( ExpressionExperiment ee, boolean forceWrite,
}
@Override
- public File writeOrLocateDataFile( QuantitationType type, boolean forceWrite ) {
+ public File writeOrLocateRawExpressionDataFile( ExpressionExperiment ee, QuantitationType type, boolean forceWrite ) {
try {
File f = this.getOutputFile( type );
@@ -485,7 +496,7 @@ public File writeOrLocateDataFile( QuantitationType type, boolean forceWrite ) {
Collection arrayDesigns = this.getArrayDesigns( vectors );
Map geneAnnotations = this.getGeneAnnotationsAsStringsByProbe( arrayDesigns );
- if ( vectors.size() == 0 ) {
+ if ( vectors.isEmpty() ) {
ExpressionDataFileServiceImpl.log.warn( "No vectors for " + type );
return null;
}
@@ -533,27 +544,29 @@ public Collection writeOrLocateDiffExpressionDataFiles( ExpressionExperime
}
@Override
- public File writeOrLocateJSONDataFile( ExpressionExperiment ee, boolean forceWrite, boolean filtered ) throws FilteringException {
+ public Optional writeOrLocateJSONProcessedExpressionDataFile( ExpressionExperiment ee, boolean forceWrite, boolean filtered ) throws FilteringException {
try {
File f = this.getOutputFile( ee, filtered );
if ( !forceWrite && f.canRead() ) {
ExpressionDataFileServiceImpl.log.info( f + " exists, not regenerating" );
- return f;
+ return Optional.of( f );
}
ExpressionDataFileServiceImpl.log.info( "Creating new JSON expression data file: " + f.getName() );
ExpressionDataDoubleMatrix matrix = this.getDataMatrix( ee, filtered );
-
+ if ( matrix == null ) {
+ return Optional.empty();
+ }
this.writeJson( f, matrix );
- return f;
+ return Optional.of( f );
} catch ( IOException e ) {
throw new RuntimeException( e );
}
}
@Override
- public File writeOrLocateJSONDataFile( QuantitationType type, boolean forceWrite ) {
+ public File writeOrLocateJSONRawExpressionDataFile( ExpressionExperiment ee, QuantitationType type, boolean forceWrite ) {
try {
File f = this.getJSONOutputFile( type );
@@ -566,7 +579,7 @@ public File writeOrLocateJSONDataFile( QuantitationType type, boolean forceWrite
Collection vectors = rawAndProcessedExpressionDataVectorService.findAndThaw( type );
- if ( vectors.size() == 0 ) {
+ if ( vectors.isEmpty() ) {
ExpressionDataFileServiceImpl.log.warn( "No vectors for " + type );
return null;
}
@@ -611,13 +624,27 @@ private String analysisResultSetWithContrastsToString( ExpressionAnalysisResultS
for ( DifferentialExpressionAnalysisResult dear : resultSet.getResults() ) {
StringBuilder rowBuffer = new StringBuilder();
- if ( geneAnnotations.isEmpty() ) {
- rowBuffer.append( dear.getProbe().getName() );
- } else {
- this.addGeneAnnotationsToLine( rowBuffer, dear, geneAnnotations );
+// if ( geneAnnotations.isEmpty() ) {
+// rowBuffer.append( dear.getProbe().getName() );
+// } else {
+ this.addGeneAnnotationsToLine( rowBuffer, dear, geneAnnotations );
+ // }
+
+ /*
+ If there are no results for the DEAR then we wouldn't expect contrasts, so we just leave a blank.
+ */
+ if ( dear.getPvalue() == null ) {
+ String contrastData = "\t\t";
+ rowBuffer.append( contrastData );
+ buf.append( rowBuffer ).append( '\n' );
+ continue;
}
- assert dear.getContrasts().size() == 1;
+
+ if ( dear.getContrasts().size() != 1 ) {
+ //
+ throw new IllegalStateException( "Expected exactly one contrast for continuous factor" );
+ }
ContrastResult contrast = dear.getContrasts().iterator().next();
@@ -642,7 +669,9 @@ private String analysisResultSetWithContrastsToString( ExpressionAnalysisResultS
Collection usedFactorValueIds = new HashSet<>();
for ( DifferentialExpressionAnalysisResult dear : resultSet.getResults() ) {
for ( ContrastResult contrast : dear.getContrasts() ) {
- usedFactorValueIds.add( contrast.getFactorValue().getId() );
+ if ( contrast.getFactorValue() != null ) {
+ usedFactorValueIds.add( contrast.getFactorValue().getId() );
+ }
}
break; // only have to look at one.
}
@@ -690,7 +719,7 @@ private String analysisResultSetWithContrastsToString( ExpressionAnalysisResultS
rowBuffer.append( s );
}
- buf.append( rowBuffer.toString() ).append( '\n' );
+ buf.append( rowBuffer ).append( '\n' );
} // resultSet.getResults() loop
}
@@ -733,7 +762,7 @@ private boolean checkFileOkToReturn( boolean forceWrite, File f, Date check ) {
* @param analysis (might not be persistent)
*/
private String convertDiffExpressionAnalysisData( DifferentialExpressionAnalysis analysis,
- Map geneAnnotations, DifferentialExpressionAnalysisConfig config ) {
+ Map geneAnnotations, @Nullable DifferentialExpressionAnalysisConfig config ) {
if ( analysis.getId() != null )
analysis = differentialExpressionAnalysisService.thawFully( analysis );
Collection results = analysis.getResultSets();
@@ -756,7 +785,7 @@ private String convertDiffExpressionAnalysisData( DifferentialExpressionAnalysis
* eneAnnotations
*/
private String convertDiffExpressionResultSetData( ExpressionAnalysisResultSet resultSet,
- Map geneAnnotations, DifferentialExpressionAnalysisConfig config ) {
+ Map geneAnnotations, @Nullable DifferentialExpressionAnalysisConfig config ) {
// Write header.
// Write contrasts data.
return this.makeDiffExpressionResultSetFileHeader( resultSet, geneAnnotations, config ) + this
@@ -789,6 +818,7 @@ private String getDataFileName( ExpressionExperiment ee, String filteredAdd, Str
return ee.getId() + "_" + FileTools.cleanForFileName( ee.getShortName() ) + "_expmat" + filteredAdd + suffix;
}
+ @Nullable
private ExpressionDataDoubleMatrix getDataMatrix( ExpressionExperiment ee, boolean filtered ) throws FilteringException {
ee = expressionExperimentService.thawLite( ee );
ExpressionDataDoubleMatrix matrix;
@@ -801,7 +831,7 @@ private ExpressionDataDoubleMatrix getDataMatrix( ExpressionExperiment ee, boole
matrix = expressionDataMatrixService.getProcessedExpressionDataMatrix( ee );
}
if ( matrix == null ) {
- throw new RuntimeException( String.format( "%s has no processed expression vectors.", ee ) );
+ log.warn( String.format( "%s has no processed expression vectors.", ee ) );
}
return matrix;
}
@@ -860,12 +890,12 @@ private File getDiffExpressionAnalysisArchiveFile( DifferentialExpressionAnalysi
return f;
}
- private String getFactorValueString( FactorValue fv ) {
+ private String getFactorValueString( @Nullable FactorValue fv ) {
String result;
if ( fv == null )
return "null";
- if ( fv.getCharacteristics() != null && fv.getCharacteristics().size() > 0 ) {
+ if ( fv.getCharacteristics() != null && !fv.getCharacteristics().isEmpty() ) {
StringBuilder fvString = new StringBuilder();
for ( Characteristic c : fv.getCharacteristics() ) {
fvString.append( c.getValue() ).append( "_" );
@@ -879,7 +909,7 @@ private String getFactorValueString( FactorValue fv ) {
return "no_data";
// R-friendly, but no need to add "X" to the beginning since this is a suffix.
- return result.replaceAll( "[\\W]+", "." );
+ return result.replaceAll( "\\W+", "." );
}
/**
@@ -956,7 +986,7 @@ private String getOutputFilename( QuantitationType type ) {
private String makeDiffExpressionFileHeader( DifferentialExpressionAnalysis analysis,
Collection resultSets, Map geneAnnotations,
- DifferentialExpressionAnalysisConfig config ) {
+ @Nullable DifferentialExpressionAnalysisConfig config ) {
if ( analysis.getId() != null ) // It might not be a persistent analysis: using -nodb
analysis = differentialExpressionAnalysisService.thaw( analysis );
@@ -983,7 +1013,7 @@ private String makeDiffExpressionFileHeader( DifferentialExpressionAnalysis anal
}
if ( config != null ) {
- buf.append( config.toString() );
+ buf.append( config );
} else if ( analysis.getProtocol() != null && StringUtils
.isNotBlank( analysis.getProtocol().getDescription() ) ) {
buf.append( analysis.getProtocol().getDescription() );
@@ -1027,7 +1057,7 @@ private String makeDiffExpressionFileHeader( DifferentialExpressionAnalysis anal
}
private String makeDiffExpressionResultSetFileHeader( ExpressionAnalysisResultSet resultSet,
- Map geneAnnotations, DifferentialExpressionAnalysisConfig config ) {
+ Map geneAnnotations, @Nullable DifferentialExpressionAnalysisConfig config ) {
StringBuilder buf = new StringBuilder();
BioAssaySet bas = resultSet.getAnalysis().getExperimentAnalyzed();
@@ -1100,8 +1130,7 @@ private String makeDiffExpressionResultSetFileHeader( ExpressionAnalysisResultSe
/**
* Loads the probe to probe coexpression link information for a given expression experiment and writes it to disk.
*/
- private void writeCoexpressionData( File file, ExpressionExperiment ee ) throws IOException {
-
+ private boolean writeCoexpressionData( File file, ExpressionExperiment ee ) throws IOException {
Taxon tax = expressionExperimentService.getTaxon( ee );
assert tax != null;
@@ -1109,7 +1138,7 @@ private void writeCoexpressionData( File file, ExpressionExperiment ee ) throws
if ( geneLinks.isEmpty() ) {
log.warn( "No coexpression links for this experiment, file will not be created: " + ee );
- return;
+ return false;
}
ExpressionDataFileServiceImpl.log.info( "Creating new coexpression data file: " + file.getAbsolutePath() );
@@ -1139,20 +1168,23 @@ private void writeCoexpressionData( File file, ExpressionExperiment ee ) throws
writer.write( buf.toString() );
}
+ return true;
}
/**
* @param compress if true, file will be output in GZIP format.
*/
- private File writeDataFile( ExpressionExperiment ee, boolean filtered, File f, boolean compress )
+ private Optional writeDataFile( ExpressionExperiment ee, boolean filtered, File f, boolean compress )
throws IOException, FilteringException {
ExpressionDataFileServiceImpl.log.info( "Creating new expression data file: " + f.getName() );
ExpressionDataDoubleMatrix matrix = this.getDataMatrix( ee, filtered );
-
+ if ( matrix == null ) {
+ return Optional.empty();
+ }
Collection arrayDesigns = expressionExperimentService.getArrayDesignsUsed( ee );
Map geneAnnotations = this.getGeneAnnotationsAsStringsByProbe( arrayDesigns );
this.writeMatrix( f, geneAnnotations, matrix, compress );
- return f;
+ return Optional.of( f );
}
/**
@@ -1220,10 +1252,6 @@ private void writeVectors( File file, Collection vector
@Override
@Transactional(readOnly = true)
public void writeTsv( ExpressionExperiment entity, Writer writer ) throws IOException {
- QuantitationType qt = expressionExperimentService.getMaskedPreferredQuantitationType( entity );
- if ( qt == null ) {
- throw new IllegalArgumentException( String.format( "%s lacks a preferred masked quantitation type.", entity ) );
- }
- writeProcessedExpressionData( entity, qt, writer );
+ writeProcessedExpressionData( entity, writer );
}
}
diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataMatrixService.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataMatrixService.java
index 87010fb6fb..a60415c9aa 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataMatrixService.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataMatrixService.java
@@ -14,15 +14,12 @@
*/
package ubic.gemma.core.analysis.service;
-import org.springframework.transaction.annotation.Transactional;
import ubic.basecode.dataStructure.matrix.DoubleMatrix;
import ubic.gemma.core.analysis.preprocess.filter.FilterConfig;
import ubic.gemma.core.analysis.preprocess.filter.FilteringException;
-import ubic.gemma.core.analysis.preprocess.filter.NoRowsLeftAfterFilteringException;
import ubic.gemma.core.datastructure.matrix.ExpressionDataDoubleMatrix;
import ubic.gemma.model.common.quantitationtype.QuantitationType;
import ubic.gemma.model.expression.bioAssayData.ProcessedExpressionDataVector;
-import ubic.gemma.model.expression.bioAssayData.RawExpressionDataVector;
import ubic.gemma.model.expression.experiment.ExpressionExperiment;
import ubic.gemma.model.genome.Gene;
import ubic.gemma.persistence.service.expression.bioAssayData.ProcessedExpressionDataVectorDao;
@@ -68,18 +65,10 @@ ExpressionDataDoubleMatrix getFilteredMatrix( String arrayDesignName, FilterConf
@Nullable
ExpressionDataDoubleMatrix getProcessedExpressionDataMatrix( ExpressionExperiment ee );
- /**
- * Obtain a processed expression data matrix for a given quantitation type.
- * @param quantitationType the quantitation type, or null to pick the preferred one
- */
- @Nullable
- ExpressionDataDoubleMatrix getProcessedExpressionDataMatrix( ExpressionExperiment ee, @Nullable QuantitationType quantitationType );
-
/**
* Obtain a raw expression data matrix for a given quantitation type
- * @param quantitationType the quantitation type, or null to pick the preferred one
*/
- ExpressionDataDoubleMatrix getRawExpressionDataMatrix( ExpressionExperiment ee, @Nullable QuantitationType quantitationType );
+ ExpressionDataDoubleMatrix getRawExpressionDataMatrix( ExpressionExperiment ee, QuantitationType quantitationType );
DoubleMatrix getRankMatrix( Collection genes,
Collection ees, ProcessedExpressionDataVectorDao.RankMethod method );
diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataMatrixServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataMatrixServiceImpl.java
index 2ecb16d095..71b62a55bb 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataMatrixServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataMatrixServiceImpl.java
@@ -112,16 +112,6 @@ public ExpressionDataDoubleMatrix getProcessedExpressionDataMatrix( ExpressionEx
return new ExpressionDataDoubleMatrix( dataVectors );
}
- @Override
- @Transactional(readOnly = true)
- public ExpressionDataDoubleMatrix getProcessedExpressionDataMatrix( ExpressionExperiment ee, QuantitationType quantitationType ) {
- Collection vectors = processedExpressionDataVectorService.findByExpressionExperiment( ee, quantitationType );
- if ( vectors.isEmpty() ) {
- return null;
- }
- return new ExpressionDataDoubleMatrix( vectors );
- }
-
@Override
@Transactional(readOnly = true)
public ExpressionDataDoubleMatrix getRawExpressionDataMatrix( ExpressionExperiment ee, QuantitationType quantitationType ) {
diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/service/GeneMultifunctionalityPopulationServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/service/GeneMultifunctionalityPopulationServiceImpl.java
index 6d8fe339d7..bfceeadc58 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/analysis/service/GeneMultifunctionalityPopulationServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/service/GeneMultifunctionalityPopulationServiceImpl.java
@@ -39,7 +39,6 @@
import ubic.gemma.persistence.service.genome.taxon.TaxonService;
import java.util.*;
-import java.util.stream.Collectors;
/**
* Compute gene multifunctionality and store it in the database.
@@ -233,6 +232,11 @@ private Map> fetchGoAnnotations( Collection genes ) {
*/
Set terms = new HashSet<>( annots.size() );
for ( Characteristic t : annots ) {
+ if ( t.getValueUri() == null ) {
+ GeneMultifunctionalityPopulationServiceImpl.log
+ .warn( "Free-text term annotated to " + gene + " : " + t );
+ continue;
+ }
OntologyTerm term = goService.getTerm( t.getValueUri() );
if ( term == null || term.isObsolete() ) {
GeneMultifunctionalityPopulationServiceImpl.log
diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/util/ExperimentalDesignUtils.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/util/ExperimentalDesignUtils.java
index c533fc5e75..cd6e29990d 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/analysis/util/ExperimentalDesignUtils.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/util/ExperimentalDesignUtils.java
@@ -181,7 +181,7 @@ public static boolean isBatch( ExperimentalFactor ef ) {
}
Characteristic category = ef.getCategory();
return ef.getName().equals( ExperimentalDesignUtils.BATCH_FACTOR_NAME ) &&
- ( category != null && category.getCategory().equals( ExperimentalDesignUtils.BATCH_FACTOR_CATEGORY_NAME ) );
+ ( category != null && ExperimentalDesignUtils.BATCH_FACTOR_CATEGORY_NAME.equals( category.getCategory() ) );
}
/**
@@ -306,11 +306,16 @@ private static Object extractFactorValueForSample( Map ontology2CharacteristicValueObject(
Collection ontologyTerms ) {
Set characteristicsVO = new HashSet<>();
for ( OntologyTerm ontologyTerm : ontologyTerms ) {
+ if ( ontologyTerm.getLabel() == null ) {
+ log.warn( "Term with null label: " + ontologyTerm.getUri() + "; it cannot be converted to a CharacteristicValueObject" );
+ continue;
+ }
characteristicsVO.add( new CharacteristicValueObject( ontologyTerm.getLabel().toLowerCase(), ontologyTerm.getUri() ) );
}
return characteristicsVO;
diff --git a/gemma-core/src/main/java/ubic/gemma/core/association/phenotype/PhenotypeAssociationManagerServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/association/phenotype/PhenotypeAssociationManagerServiceImpl.java
index 3bcec1822a..397380b885 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/association/phenotype/PhenotypeAssociationManagerServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/association/phenotype/PhenotypeAssociationManagerServiceImpl.java
@@ -769,6 +769,7 @@ public Collection searchInDatabaseForPhenotype( Strin
}
return ontologyTermsFound.stream()
+ .filter( t -> t.getLabel() != null )
.map( t -> new CharacteristicValueObject( t.getLabel().toLowerCase(), t.getUri() ) )
.limit( maxResults > 0 ? maxResults : Long.MAX_VALUE )
.collect( Collectors.toCollection( TreeSet::new ) );
diff --git a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataDoubleMatrixUtil.java b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataDoubleMatrixUtil.java
index 9a8c2002c2..006fd2415f 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataDoubleMatrixUtil.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataDoubleMatrixUtil.java
@@ -54,8 +54,19 @@
public class ExpressionDataDoubleMatrixUtil {
private static final double LOGARITHM_BASE = 2.0;
- private static final int COLUMNS_LIMIT = 4;
- private static final double VALUES_LIMIT = 0.7;
+
+
+ /**
+ * This threshold is used to determine if a row has too many identical value; a value of N means that the number of distinct values in the
+ * expression vector of length M must be at least N * M.
+ */
+ private static final double MINIMUM_UNIQUE_VALUES_FRACTION_PER_ELEMENT = 0.3;
+
+ /**
+ * We don't apply the "unique values" filter to matrices with fewer columns than this.
+ */
+ private static final int MINIMUM_COLUMNS_TO_APPLY_UNIQUE_VALUES_FILTER = 4;
+
private static final Log log = LogFactory.getLog( ExpressionDataDoubleMatrixUtil.class.getName() );
@@ -85,12 +96,12 @@ public static ExpressionDataDoubleMatrix filterAndLog2Transform( ExpressionDataD
}
r = dmatrix.rows();
- if ( dmatrix.columns() > ExpressionDataDoubleMatrixUtil.COLUMNS_LIMIT ) {
+ if ( dmatrix.columns() > ExpressionDataDoubleMatrixUtil.MINIMUM_COLUMNS_TO_APPLY_UNIQUE_VALUES_FILTER ) {
/* This threshold had been 10^-5, but it's probably too stringent. Also remember
* the data are log transformed the threshold should be transformed as well (it's not that simple),
* but that's a minor effect.
* To somewhat counter the effect of lowering this stringency, increasing the stringency on VALUES_LIMIT may help */
- dmatrix = ExpressionExperimentFilter.tooFewDistinctValues( dmatrix, ExpressionDataDoubleMatrixUtil.VALUES_LIMIT, 0.001 );
+ dmatrix = ExpressionExperimentFilter.tooFewDistinctValues( dmatrix, ExpressionDataDoubleMatrixUtil.MINIMUM_UNIQUE_VALUES_FRACTION_PER_ELEMENT, 0.001 );
if ( dmatrix.rows() < r ) {
ExpressionDataDoubleMatrixUtil.log.info( ( r - dmatrix.rows() ) + " rows removed due to too many identical values" );
}
@@ -186,6 +197,14 @@ public static ExpressionDataDoubleMatrix ensureLog2Scale( ExpressionDataDoubleMa
ExpressionDataDoubleMatrixUtil.log.info( " **** Converting from log10 to log2 **** " );
MatrixStats.convertToLog2( transformedMatrix, 10 );
break;
+ case LOG1P:
+ ExpressionDataDoubleMatrixUtil.log.info( " **** Converting from log1p to log2 **** " );
+ for ( int i = 0; i < transformedMatrix.rows(); i++ ) {
+ for ( int j = 0; j < transformedMatrix.columns(); j++ ) {
+ transformedMatrix.set( i, j, Math.log( Math.expm1( transformedMatrix.get( i, j ) ) ) / Math.log( 2 ) );
+ }
+ }
+ break;
case LINEAR:
ExpressionDataDoubleMatrixUtil.log.info( " **** LOG TRANSFORMING **** " );
MatrixStats.logTransform( transformedMatrix );
@@ -482,8 +501,6 @@ private static boolean isZScore( DoubleMatrix1D vector ) {
return true;
}
// FIXME: use a faster algorithm for the median, there's a O(n) approach
- // sort only if necessary, median expects a sorted input
- v.sort();
return isCloseToZero( DescriptiveWithMissing.median( v ) );
}
diff --git a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataMatrixColumnSort.java b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataMatrixColumnSort.java
index a07bf72e81..c8e5cf4daa 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataMatrixColumnSort.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataMatrixColumnSort.java
@@ -20,14 +20,18 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.commons.math3.analysis.function.Exp;
import ubic.basecode.dataStructure.matrix.DoubleMatrix;
import ubic.gemma.core.analysis.expression.diff.BaselineSelection;
import ubic.gemma.core.analysis.util.ExperimentalDesignUtils;
+import ubic.gemma.model.common.description.Characteristic;
import ubic.gemma.model.expression.bioAssay.BioAssay;
import ubic.gemma.model.expression.biomaterial.BioMaterial;
import ubic.gemma.model.expression.experiment.ExperimentalFactor;
import ubic.gemma.model.expression.experiment.FactorValue;
+import ubic.gemma.model.expression.experiment.Geeq;
+import javax.annotation.Nullable;
import java.util.*;
import java.util.Map.Entry;
@@ -89,13 +93,22 @@ public static Map getBaselineLevels( List getBaselineLevels( List DoubleMatrix orderByExperimentalDesign( DoubleMatrix mat ) {
+ return ExpressionDataMatrixColumnSort.orderByExperimentalDesign( mat, null );
+ }
+
+ public static DoubleMatrix orderByExperimentalDesign( DoubleMatrix mat, @Nullable ExperimentalFactor primaryFactor ) {
List bioAssays = mat.getColNames();
@@ -191,7 +208,7 @@ public static DoubleMatrix orderByExperimentalDesign( DoubleMat
start.add( bioAssay.getSampleUsed() );
bm2ba.put( bioAssay.getSampleUsed(), bioAssay );
}
- List bm = ExpressionDataMatrixColumnSort.orderByExperimentalDesign( start, null );
+ List bm = ExpressionDataMatrixColumnSort.orderByExperimentalDesign( start, null, primaryFactor );
List newBioAssayOrder = new ArrayList<>();
for ( BioMaterial bioMaterial : bm ) {
assert bm2ba.containsKey( bioMaterial );
@@ -200,14 +217,19 @@ public static DoubleMatrix orderByExperimentalDesign( DoubleMat
return mat.subsetColumns( newBioAssayOrder );
}
+
+ public static List orderByExperimentalDesign( ExpressionDataMatrix mat ) {
+ return ExpressionDataMatrixColumnSort.orderByExperimentalDesign( mat, null );
+ }
+
/**
* @param mat matrix
* @return bio materials
*/
- public static List orderByExperimentalDesign( ExpressionDataMatrix mat ) {
+ public static List orderByExperimentalDesign( ExpressionDataMatrix mat, @Nullable ExperimentalFactor primaryFactor ) {
List start = ExpressionDataMatrixColumnSort.getBms( mat );
- List ordered = ExpressionDataMatrixColumnSort.orderByExperimentalDesign( start, null );
+ List ordered = ExpressionDataMatrixColumnSort.orderByExperimentalDesign( start, null, primaryFactor );
assert ordered.size() == start.size() : "Expected " + start.size() + ", got " + ordered.size();
@@ -216,12 +238,23 @@ public static List orderByExperimentalDesign( ExpressionDataMatrix<
}
/**
- * @param factors, can be null
- * @param start start
+ * @param start BioMaterials to sort
* @return bio materials
*/
public static List orderByExperimentalDesign( List start,
Collection factors ) {
+ return ExpressionDataMatrixColumnSort.orderByExperimentalDesign( start, factors, null );
+ }
+
+ /**
+ *
+ * @param start BioMaterials to sort
+ * @param factors, can be null
+ * @param primaryFactor to start with, can be null
+ * @return bio materials
+ */
+ public static List orderByExperimentalDesign( List start,
+ @Nullable Collection factors, @Nullable ExperimentalFactor primaryFactor ) {
if ( start.size() == 1 ) {
return start;
@@ -232,7 +265,7 @@ public static List orderByExperimentalDesign( List sta
}
Collection unsortedFactors;
- if ( factors != null ) {
+ if ( factors != null && !factors.isEmpty() ) {
unsortedFactors = factors;
} else {
unsortedFactors = ExpressionDataMatrixColumnSort.getFactors( start );
@@ -245,7 +278,7 @@ public static List orderByExperimentalDesign( List sta
// sort factors: which one do we want to sort by first
List sortedFactors = ExpressionDataMatrixColumnSort
- .orderFactorsByExperimentalDesign( start, unsortedFactors );
+ .orderFactorsByExperimentalDesign( start, unsortedFactors, primaryFactor );
// sort biomaterials using sorted factors
return ExpressionDataMatrixColumnSort.orderBiomaterialsBySortedFactors( start, sortedFactors );
}
@@ -274,18 +307,34 @@ public int compare( BioMaterial o1, BioMaterial o2 ) {
/**
* @return list of factors, sorted from simplest (fewest number of values from the biomaterials passed in) to least
- * simple. Continuous factors will always be first, and batch factors last.
+ * simple.
*/
private static List orderFactorsByExperimentalDesign( List start,
- Collection factors ) {
+ Collection factors, ExperimentalFactor primaryFactor ) {
- if ( factors == null || factors.isEmpty() ) {
+ if ( ( factors == null || factors.isEmpty() ) && primaryFactor == null ) {
ExpressionDataMatrixColumnSort.log.warn( "No factors supplied for sorting" );
return new LinkedList<>();
}
+ // if we are provided a primary factor, we just work with it.
LinkedList sortedFactors = new LinkedList<>();
+
+ if ( factors == null || factors.isEmpty() && primaryFactor != null ) {
+ sortedFactors.add( primaryFactor );
+ return sortedFactors;
+ }
+
Collection factorsToTake = new HashSet<>( factors );
+
+ if ( primaryFactor != null ) {
+ if ( !factorsToTake.contains( primaryFactor ) ) {
+ throw new IllegalArgumentException( "Primary factor not in the list of factors" );
+ }
+ sortedFactors.add( primaryFactor );
+ factorsToTake.remove( primaryFactor );
+ }
+
while ( !factorsToTake.isEmpty() ) {
ExperimentalFactor simplest = ExpressionDataMatrixColumnSort.chooseSimplestFactor( start, factorsToTake );
if ( simplest == null ) {
@@ -358,7 +407,7 @@ private static ExperimentalFactor chooseSimplestFactor( List bms,
for ( ExperimentalFactor ef : factors ) {
if ( ExperimentalDesignUtils.isContinuous( ef ) ) {
- return ef;
+ // return ef;
}
/*
@@ -484,6 +533,11 @@ private static Collection getFactors( Collection factorValues = bm.getFactorValues();
for ( FactorValue fv : factorValues ) {
+
+ if ( fv.getCharacteristics().stream().map( Characteristic::getValue ).anyMatch( "DE_Exclude"::equalsIgnoreCase ) ) {
+ continue;
+ }
+
if ( !usedFactorValues.containsKey( fv.getExperimentalFactor() ) ) {
usedFactorValues.put( fv.getExperimentalFactor(), new HashSet<>() );
}
@@ -565,11 +619,8 @@ private static List orderByFactor( ExperimentalFactor ef, Map
* Sort biomaterials according to a list of ordered factors.
- *
- *
- * If any factor is continuous, we sort by it and don't do any further sorting.
*
- * Otherwise, for categorical factors, we sort recursively (by levels of the first factor, then
+ * For categorical factors, we sort recursively (by levels of the first factor, then
* within that by levels of the second factor etc.)
*
* Any batch factor is used last (we sort by batch only within the most granular factor's levels)
@@ -610,12 +661,6 @@ private static List orderBiomaterialsBySortedFactors( List ordered = ExpressionDataMatrixColumnSort.orderByFactor( simplest, fv2bms, start );
- // Abort ordering, so we are ordered only by the first continuous factor.
- if ( ExperimentalDesignUtils.isContinuous( simplest ) ) {
- assert ordered != null;
- return ordered;
- }
-
LinkedList factorsStillToDo = new LinkedList<>();
factorsStillToDo.addAll( factors );
factorsStillToDo.remove( simplest );
diff --git a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/MatrixWriter.java b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/MatrixWriter.java
index e0ebe41c8f..d6f858f060 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/MatrixWriter.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/MatrixWriter.java
@@ -44,7 +44,7 @@
public class MatrixWriter extends AbstractFileService> {
public void write( Writer writer, ExpressionDataMatrix matrix,
- Map> geneAnnotations, boolean writeHeader, boolean orderByDesign )
+ @Nullable Map> geneAnnotations, boolean writeHeader, boolean orderByDesign )
throws IOException {
this.write( writer, matrix, geneAnnotations, writeHeader, true, true, orderByDesign );
}
diff --git a/gemma-core/src/main/java/ubic/gemma/core/genome/gene/service/GeneSearchServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/genome/gene/service/GeneSearchServiceImpl.java
index 9850096eef..2d5eef3706 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/genome/gene/service/GeneSearchServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/genome/gene/service/GeneSearchServiceImpl.java
@@ -27,7 +27,6 @@
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.access.AccessDeniedException;
import org.springframework.stereotype.Service;
-import ubic.basecode.ontology.search.OntologySearchException;
import ubic.gemma.core.genome.gene.*;
import ubic.gemma.core.ontology.providers.GeneOntologyService;
import ubic.gemma.core.search.*;
@@ -245,8 +244,10 @@ public Collection searchGenesAndGeneGroups( String qu
taxon = geneSetService.getTaxon( gs );
GeneSetValueObject gsVo = geneSetValueObjectHelper.convertToValueObject( gs );
srDo = new SearchResultDisplayObject( gsVo );
- srDo.setTaxonId( taxon.getId() );
- srDo.setTaxonName( taxon.getCommonName() );
+ if ( taxon != null ) {
+ srDo.setTaxonId( taxon.getId() );
+ srDo.setTaxonName( taxon.getCommonName() );
+ }
geneSets.add( srDo );
}
taxon = null;
@@ -404,9 +405,11 @@ private List> retainGeneSetsOfThisTaxon( Long taxonId, Lis
for ( SearchResult sr : geneSetSearchResults ) {
GeneSet gs = sr.getResultObject();
if ( gs != null ) {
- GeneSetValueObject gsVo = geneSetValueObjectHelper.convertToValueObject( gs );
+ Set geneSetTaxaIds = geneSetService.getTaxa( gs ).stream()
+ .map( Taxon::getId )
+ .collect( Collectors.toSet() );
isSetOwnedByUser.put( gs.getId(), securityService.isOwnedByCurrentUser( gs ) );
- if ( Objects.equals( gsVo.getTaxonId(), taxonId ) ) {
+ if ( geneSetTaxaIds.contains( taxonId ) ) {
taxonCheckedSets.add( sr );
}
}
diff --git a/gemma-core/src/main/java/ubic/gemma/core/genome/gene/service/GeneSetService.java b/gemma-core/src/main/java/ubic/gemma/core/genome/gene/service/GeneSetService.java
index 762dee961b..dd6cb3c64f 100755
--- a/gemma-core/src/main/java/ubic/gemma/core/genome/gene/service/GeneSetService.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/genome/gene/service/GeneSetService.java
@@ -38,6 +38,7 @@
import javax.annotation.ParametersAreNonnullByDefault;
import java.util.Collection;
import java.util.List;
+import java.util.Set;
/**
* Service for managing gene sets
@@ -152,7 +153,7 @@ public interface GeneSetService extends BaseService, BaseVoEnabledServi
* @see GeneSetDao GeneSetDao for security filtering
*/
@Secured({ "IS_AUTHENTICATED_ANONYMOUSLY", "AFTER_ACL_COLLECTION_READ" })
- Collection loadAll( Taxon tax );
+ Collection loadAll( @Nullable Taxon tax );
/**
* Returns the {@link GeneSet}s for the currently logged in {@link User} - i.e, ones for which the current user has
@@ -303,10 +304,16 @@ Collection updateDatabaseEntity(
* all the genes
*
* @param geneSet gene set
- * @return the taxon or null if the gene set param was null
+ * @return a taxon, or null if the gene set has no member
*/
+ @Nullable
Taxon getTaxon( GeneSet geneSet );
+ /**
+ * Obtain all the taxa for the members of a given gene set.
+ */
+ Set getTaxa( GeneSet geneSet );
+
@Override
@Secured({ "GROUP_USER", "ACL_SECURABLE_EDIT" })
void update( GeneSet entity );
diff --git a/gemma-core/src/main/java/ubic/gemma/core/genome/gene/service/GeneSetServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/genome/gene/service/GeneSetServiceImpl.java
index c2566e8a79..fad346956c 100755
--- a/gemma-core/src/main/java/ubic/gemma/core/genome/gene/service/GeneSetServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/genome/gene/service/GeneSetServiceImpl.java
@@ -121,7 +121,7 @@ public Collection findByName( String name, Taxon taxon ) {
@Override
@Transactional(readOnly = true)
- public Collection loadAll( Taxon tax ) {
+ public Collection loadAll( @Nullable Taxon tax ) {
return this.geneSetDao.loadAll( tax );
}
@@ -487,12 +487,13 @@ public TaxonValueObject getTaxonVOforGeneSetVO( SessionBoundGeneSetValueObject g
@Override
@Transactional(readOnly = true)
public Taxon getTaxon( GeneSet geneSet ) {
- if ( geneSet == null )
- return null;
- Taxon tmpTax;
- tmpTax = geneSetDao.getTaxon( geneSet.getId() );
+ return geneSetDao.getTaxon( geneSet );
+ }
- return tmpTax;
+ @Override
+ @Transactional(readOnly = true)
+ public Set getTaxa( GeneSet geneSet ) {
+ return new HashSet<>( geneSetDao.getTaxa( geneSet ) );
}
private void checkGeneList( GeneSet gset, Collection updatedGenelist, Collection genes ) {
diff --git a/gemma-core/src/main/java/ubic/gemma/core/job/executor/webapp/TaskRunningServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/job/executor/webapp/TaskRunningServiceImpl.java
index c865b22f7f..7b630980d1 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/job/executor/webapp/TaskRunningServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/job/executor/webapp/TaskRunningServiceImpl.java
@@ -20,6 +20,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.DisposableBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.concurrent.DelegatingSecurityContextCallable;
import org.springframework.stereotype.Component;
@@ -44,7 +45,7 @@
*/
// Valid, inspection is not parsing the context file for some reason
@Component
-public class TaskRunningServiceImpl implements TaskRunningService {
+public class TaskRunningServiceImpl implements TaskRunningService, DisposableBean {
private static final Log log = LogFactory.getLog( TaskRunningServiceImpl.class );
private final ExecutorService executorService = Executors.newFixedThreadPool( 20 );
private final Map submittedTasks = new ConcurrentHashMap<>();
@@ -54,6 +55,19 @@ public class TaskRunningServiceImpl implements TaskRunningService {
@Autowired
private TaskPostProcessing taskPostProcessing;
+ @Override
+ public void destroy() throws Exception {
+ log.info( "Shutting down TaskRunningService executor..." );
+ executorService.shutdown();
+ if ( !executorService.isTerminated() ) {
+ log.warn( "There are still running tasks, will wait at most 5 minutes before shutting them down." );
+ }
+ if ( !executorService.awaitTermination( 5, TimeUnit.MINUTES ) ) {
+ log.info( "TaskRunningService executor was still running after 5 minutes, interrupting pending tasks..." );
+ executorService.shutdownNow();
+ }
+ }
+
@Override
public SubmittedTask getSubmittedTask( String taskId ) {
return submittedTasks.get( taskId );
diff --git a/gemma-core/src/main/java/ubic/gemma/core/loader/association/phenotype/PhenotypeProcessingUtil.java b/gemma-core/src/main/java/ubic/gemma/core/loader/association/phenotype/PhenotypeProcessingUtil.java
index 21033fe5b1..e586c01f30 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/loader/association/phenotype/PhenotypeProcessingUtil.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/loader/association/phenotype/PhenotypeProcessingUtil.java
@@ -562,7 +562,7 @@ private void parseManualMappingFile() throws IOException {
if ( ontologyTerm != null ) {
- if ( ontologyTerm.getLabel().equalsIgnoreCase( valueStaticFile ) ) {
+ if ( valueStaticFile.equalsIgnoreCase( ontologyTerm.getLabel() ) ) {
if ( manualDescriptionToValuesUriMapping.get( termId ) != null ) {
col = manualDescriptionToValuesUriMapping.get( termId );
@@ -917,8 +917,12 @@ private boolean findUsingManualMappingFile( String meshOrOmimId, String annotato
meshOrOmimId + this.findExtraInfoMeshDescription( meshOrOmimId ) + " PARENT: (" );
for ( OntologyTerm o : onParents ) {
-
- String meshId = this.changeToId( o.getUri() );
+ String termUri = o.getUri();
+ if ( termUri == null ) {
+ log.warn( "Ignoring free-text term " + o );
+ continue;
+ }
+ String meshId = this.changeToId( termUri );
Collection uri = this.findManualMappingTermValueUri( meshId );
if ( uri != null && !uri.isEmpty() ) {
phenotypesUri.addAll( uri );
@@ -1185,8 +1189,13 @@ private Map> meshToDiseaseTerms( Collection> diseaseTerms = new HashMap<>();
for ( OntologyTerm m : meshTerms ) {
+ String termUri = m.getUri();
+ if ( termUri == null ) {
+ log.warn( "Ignoring free-text term: " + m );
+ continue;
+ }
- String meshId = this.changeToId( m.getUri() );
+ String meshId = this.changeToId( termUri );
Collection onDisease = this.findOntologyTermsUriWithDiseaseId( meshId );
diff --git a/gemma-core/src/main/java/ubic/gemma/core/loader/expression/ExpressionExperimentPlatformSwitchService.java b/gemma-core/src/main/java/ubic/gemma/core/loader/expression/ExpressionExperimentPlatformSwitchService.java
index 66773ad296..76c7ae74af 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/loader/expression/ExpressionExperimentPlatformSwitchService.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/loader/expression/ExpressionExperimentPlatformSwitchService.java
@@ -278,6 +278,8 @@ private void cleanupUnused( ExpressionExperiment ee, Collection() );
+ // numberOfSample is updated later when the BAs are populated
if ( series.getSampleCorrespondence().size() == 0 ) {
throw new IllegalArgumentException( "No sample correspondence!" );
@@ -2703,7 +2704,7 @@ private FactorValue findMatchingExperimentalFactorValue( Collection fetchAndLoad( String geoAccession, boolean loadPlatformOnly, boole
void setGeoDomainObjectGenerator( GeoDomainObjectGenerator generator );
+ /**
+ * Load from a SOFT file. This can be used for testing but maybe there are other situations it is useful.
+ *
+ * @param accession e.g GSE1234
+ * @param softFile the full path to the SOFT file. The file name has to be [accession].soft.gz
+ * @return a single experiment.
+ */
+ Collection loadFromSoftFile( String accession, String softFile, boolean loadPlatformOnly, boolean doSampleMatching, boolean splitByPlatform );
}
\ No newline at end of file
diff --git a/gemma-core/src/main/java/ubic/gemma/core/loader/expression/geo/service/GeoServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/loader/expression/geo/service/GeoServiceImpl.java
index 57f19d8912..b99a27d12a 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/loader/expression/geo/service/GeoServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/loader/expression/geo/service/GeoServiceImpl.java
@@ -19,7 +19,6 @@
package ubic.gemma.core.loader.expression.geo.service;
import org.apache.commons.lang3.StringUtils;
-import org.hibernate.cfg.Settings;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@@ -28,10 +27,7 @@
import ubic.gemma.core.analysis.report.ExpressionExperimentReportService;
import ubic.gemma.core.annotation.reference.BibliographicReferenceService;
import ubic.gemma.core.loader.entrez.pubmed.PubMedXMLFetcher;
-import ubic.gemma.core.loader.expression.geo.DatasetCombiner;
-import ubic.gemma.core.loader.expression.geo.GeoConverter;
-import ubic.gemma.core.loader.expression.geo.GeoDomainObjectGenerator;
-import ubic.gemma.core.loader.expression.geo.GeoSampleCorrespondence;
+import ubic.gemma.core.loader.expression.geo.*;
import ubic.gemma.core.loader.expression.geo.model.*;
import ubic.gemma.core.loader.util.AlreadyExistsInSystemException;
import ubic.gemma.model.common.auditAndSecurity.eventType.ExpressionExperimentUpdateFromGEOEvent;
@@ -44,7 +40,6 @@
import ubic.gemma.model.expression.designElement.CompositeSequence;
import ubic.gemma.model.expression.experiment.ExpressionExperiment;
import ubic.gemma.model.genome.biosequence.BioSequence;
-import ubic.gemma.persistence.persister.Persister;
import ubic.gemma.persistence.service.ExpressionExperimentPrePersistService;
import ubic.gemma.persistence.service.common.auditAndSecurity.AuditTrailService;
import ubic.gemma.persistence.service.common.description.CharacteristicService;
@@ -53,8 +48,8 @@
import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentService;
import ubic.gemma.persistence.service.genome.taxon.TaxonService;
import ubic.gemma.persistence.util.ArrayDesignsForExperimentCache;
-import ubic.gemma.persistence.util.SettingsConfig;
+import java.io.File;
import java.util.*;
/**
@@ -67,7 +62,7 @@ public class GeoServiceImpl extends AbstractGeoService {
private static final String GEO_DB_NAME = "GEO";
- @Value( "${geo.minimumSamplesToLoad}")
+ @Value("${geo.minimumSamplesToLoad}")
private int MINIMUM_SAMPLE_COUNT_TO_LOAD;
private final ArrayDesignReportService arrayDesignReportService;
@@ -389,6 +384,15 @@ public void updateFromGEO( String geoAccession ) {
}
+ @Override
+ @Transactional
+ public Collection loadFromSoftFile( String accession, String softFile, boolean loadPlatformOnly, boolean doSampleMatching, boolean splitByPlatform ) {
+ File f = new File( softFile );
+ this.setGeoDomainObjectGenerator(
+ new GeoDomainObjectGeneratorLocal( f.getParent() ) );
+ return fetchAndLoad( accession, loadPlatformOnly, doSampleMatching, splitByPlatform );
+ }
+
private void check( Collection result ) {
for ( ExpressionExperiment expressionExperiment : result ) {
this.check( expressionExperiment );
diff --git a/gemma-core/src/main/java/ubic/gemma/core/loader/expression/simple/ExperimentalDesignImporterImpl.java b/gemma-core/src/main/java/ubic/gemma/core/loader/expression/simple/ExperimentalDesignImporterImpl.java
index 04ede86afb..47634236f2 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/loader/expression/simple/ExperimentalDesignImporterImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/loader/expression/simple/ExperimentalDesignImporterImpl.java
@@ -368,9 +368,7 @@ private void addFactorValuesToExperimentalFactor( ExperimentalFactor experimenta
ExperimentalDesignImporterImpl.log.debug( "Factor is categorical" );
Statement newVc = Statement.Factory.newInstance();
if ( category != null ) {
- String category2 = category.getCategory();
- assert category2 != null;
- newVc.setCategory( category2 );
+ newVc.setCategory( category.getCategory() );
newVc.setCategoryUri( category.getCategoryUri() );
}
newVc.setValue( value ); // don't have a valueUri at this point
@@ -580,12 +578,12 @@ private Map mapBioMaterialsToNamePossibilities( Collection<
private Characteristic termForCategoryLookup( String category, Collection terms ) {
OntologyTerm t = null;
- String lookup = category.replaceAll( "_", " " ).toLowerCase();
+ String lookup = category.replaceAll( "_", " " );
for ( OntologyTerm to : terms ) {
- if ( to.getLabel().equals( category )
- || to.getLabel().toLowerCase().equals( lookup )
+ if ( category.equalsIgnoreCase( to.getLabel() )
+ || lookup.equalsIgnoreCase( to.getLabel() )
// if EFO is loaded, the "sex" term is actually named "biological sex"
- || ( to.getLabel().equals( "biological sex" ) && lookup.equals( "sex" ) ) ) {
+ || ( "biological sex".equalsIgnoreCase( to.getLabel() ) && "sex".equalsIgnoreCase( lookup ) ) ) {
t = to;
break;
}
diff --git a/gemma-core/src/main/java/ubic/gemma/core/ontology/GoMetricImpl.java b/gemma-core/src/main/java/ubic/gemma/core/ontology/GoMetricImpl.java
index ad18c75ab0..2ddbbfadff 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/ontology/GoMetricImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/ontology/GoMetricImpl.java
@@ -19,6 +19,7 @@
package ubic.gemma.core.ontology;
+import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -113,7 +114,7 @@ public Double computeMaxSimilarity( Gene queryGene, Gene targetGene, Map termCountMap, String
int termCount = termCountMap.get( term );
OntologyTerm ont = geneOntologyService.getTerm( term );
+ if ( ont == null ) {
+ GoMetricImpl.log.warn( "No GO term found for: " + term + ", will fallback to zero for the number of children occurrences." );
+ return 0;
+ }
+
Collection children = ont.getChildren( false, partOf );
if ( children.isEmpty() ) {
@@ -397,7 +403,7 @@ private Double checkParents( OntologyTerm ontoM, OntologyTerm ontoC, Map getChildren( Collection terms, boolean di
public Collection getCategoryTerms() {
return categoryTerms.stream()
.map( term -> {
+ String termUri = term.getUri();
+ if ( termUri == null ) {
+ return term; // a free-text category
+ }
if ( experimentalFactorOntologyService.isOntologyLoaded() ) {
- OntologyTerm efoTerm = experimentalFactorOntologyService.getTerm( term.getUri() );
+ OntologyTerm efoTerm = experimentalFactorOntologyService.getTerm( termUri );
if ( efoTerm != null ) {
return efoTerm;
}
@@ -448,7 +452,7 @@ public String getDefinition( String uri ) {
if ( ot != null ) {
for ( AnnotationProperty ann : ot.getAnnotations() ) {
// FIXME: not clear this will work with all ontologies. UBERON, HP, MP, MONDO does it this way.
- if ( ann.getUri().equals( "http://purl.obolibrary.org/obo/IAO_0000115" ) ) {
+ if ( "http://purl.obolibrary.org/obo/IAO_0000115".equals( ann.getUri() ) ) {
return ann.getContents();
}
}
@@ -681,6 +685,10 @@ private Collection findCharacteristicsFromOntology( S
for ( OntologyTerm ontologyTerm : ontologyTerms ) {
// if the ontology term wasnt already found in the database
if ( characteristicFromDatabaseWithValueUri.get( ontologyTerm.getUri() ) == null ) {
+ if ( ontologyTerm.getLabel() == null ) {
+ log.warn( "Term with null label: " + ontologyTerm.getUri() + "; it cannot be converted to a CharacteristicValueObject" );
+ continue;
+ }
CharacteristicValueObject phenotype = new CharacteristicValueObject( ontologyTerm.getLabel().toLowerCase(), ontologyTerm.getUri() );
characteristicsFromOntology.add( phenotype );
}
diff --git a/gemma-core/src/main/java/ubic/gemma/core/ontology/providers/GeneOntologyServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/ontology/providers/GeneOntologyServiceImpl.java
index d36278a874..7c4270e35c 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/ontology/providers/GeneOntologyServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/ontology/providers/GeneOntologyServiceImpl.java
@@ -76,6 +76,9 @@ public static String asRegularGoId( OntologyTerm term ) {
if ( term == null )
return null;
String uri = term.getUri();
+ if ( uri == null ) {
+ return null;
+ }
return GeneOntologyServiceImpl.asRegularGoId( uri );
}
diff --git a/gemma-core/src/main/java/ubic/gemma/core/search/GeneSetSearchImpl.java b/gemma-core/src/main/java/ubic/gemma/core/search/GeneSetSearchImpl.java
index a10a027c94..4a3e8ed4ea 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/search/GeneSetSearchImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/search/GeneSetSearchImpl.java
@@ -32,6 +32,7 @@
import ubic.gemma.core.genome.gene.GeneSetValueObjectHelper;
import ubic.gemma.core.genome.gene.service.GeneSetService;
import ubic.gemma.core.ontology.providers.GeneOntologyService;
+import ubic.gemma.model.common.description.CharacteristicValueObject;
import ubic.gemma.model.genome.Gene;
import ubic.gemma.model.genome.Taxon;
import ubic.gemma.model.genome.TaxonValueObject;
@@ -39,7 +40,6 @@
import ubic.gemma.model.genome.gene.GeneSetMember;
import ubic.gemma.model.genome.gene.GeneSetValueObject;
import ubic.gemma.model.genome.gene.GeneValueObject;
-import ubic.gemma.model.common.description.CharacteristicValueObject;
import ubic.gemma.persistence.service.association.Gene2GOAssociationService;
import ubic.gemma.persistence.service.genome.taxon.TaxonService;
import ubic.gemma.persistence.util.EntityUtils;
diff --git a/gemma-core/src/main/java/ubic/gemma/core/util/BuildInfo.java b/gemma-core/src/main/java/ubic/gemma/core/util/BuildInfo.java
index 79170613f3..3faf3023ae 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/util/BuildInfo.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/util/BuildInfo.java
@@ -1,32 +1,41 @@
package ubic.gemma.core.util;
+import lombok.extern.apachecommons.CommonsLog;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Configuration;
+import org.springframework.core.io.ClassPathResource;
import org.springframework.format.datetime.DateFormatter;
-import ubic.gemma.persistence.util.Settings;
import javax.annotation.Nullable;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
import java.text.DateFormat;
+import java.text.ParseException;
import java.util.Date;
import java.util.Locale;
+import java.util.Properties;
+@CommonsLog
@Configuration
public class BuildInfo implements InitializingBean {
private static final String MAVEN_DATETIME_PATTERN = "yyyy-MM-dd'T'HH:mm:ss'Z'";
/**
- * Retrieves the build info using {@link Settings}
- * @deprecated simply autowire this configuration to access build information at runtime
+ * Retrieve build information directly from the classpath.
*/
- @Deprecated
- public static BuildInfo fromSettings() {
- try {
- return new BuildInfo( Settings.getString( "gemma.version" ),
- Settings.getString( "gemma.build.timestamp" ),
- Settings.getString( "gemma.build.gitHash" ) );
- } catch ( Exception e ) {
+ public static BuildInfo fromClasspath() {
+ Properties props = new Properties();
+ try ( InputStream reader = new ClassPathResource( "/ubic/gemma/version.properties" ).getInputStream() ) {
+ props.load( reader );
+ return new BuildInfo( props.getProperty( "gemma.version" ),
+ props.getProperty( "gemma.build.timestamp" ),
+ props.getProperty( "gemma.build.gitHash" ) );
+ } catch ( FileNotFoundException e ) {
+ return new BuildInfo();
+ } catch ( IOException e ) {
throw new RuntimeException( e );
}
}
@@ -44,7 +53,7 @@ public BuildInfo() {
}
- private BuildInfo( String version, String timestampAsString, String gitHash ) throws Exception {
+ private BuildInfo( String version, String timestampAsString, String gitHash ) {
this.version = version;
this.timestampAsString = timestampAsString;
this.gitHash = gitHash;
@@ -52,10 +61,15 @@ private BuildInfo( String version, String timestampAsString, String gitHash ) th
}
@Override
- public void afterPropertiesSet() throws Exception {
+ public void afterPropertiesSet() {
if ( timestampAsString != null ) {
- timestamp = new DateFormatter( MAVEN_DATETIME_PATTERN )
- .parse( timestampAsString, Locale.getDefault() );
+ try {
+ timestamp = new DateFormatter( MAVEN_DATETIME_PATTERN )
+ .parse( timestampAsString, Locale.getDefault() );
+
+ } catch ( ParseException e ) {
+ log.error( "Failed to parse build timestamp.", e );
+ }
}
}
diff --git a/gemma-core/src/main/java/ubic/gemma/core/util/DummyMailSender.java b/gemma-core/src/main/java/ubic/gemma/core/util/DummyMailSender.java
index a36a358c60..10411dc818 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/util/DummyMailSender.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/util/DummyMailSender.java
@@ -22,7 +22,7 @@
import org.springframework.mail.MailException;
import org.springframework.mail.MailSender;
import org.springframework.mail.SimpleMailMessage;
-import ubic.gemma.persistence.util.SpringProfiles;
+import ubic.gemma.persistence.util.EnvironmentProfiles;
/**
* Mock mail sender for testing.
@@ -38,7 +38,7 @@ public class DummyMailSender implements MailSender, InitializingBean {
@Override
public void afterPropertiesSet() throws Exception {
- if ( environment.acceptsProfiles( SpringProfiles.DEV ) ) {
+ if ( environment.acceptsProfiles( EnvironmentProfiles.DEV ) ) {
log.warn( "Emails will be sent to a dummy mail sender. If this is not intended, activate the 'production' profile by setting -Dspring.profiles.active=production." );
}
}
diff --git a/gemma-core/src/main/java/ubic/gemma/core/visualization/ExperimentalDesignVisualizationService.java b/gemma-core/src/main/java/ubic/gemma/core/visualization/ExperimentalDesignVisualizationService.java
index ed5adaf8de..50a2f8ef0c 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/visualization/ExperimentalDesignVisualizationService.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/visualization/ExperimentalDesignVisualizationService.java
@@ -18,6 +18,7 @@
import ubic.gemma.model.expression.bioAssayData.DoubleVectorValueObject;
import ubic.gemma.model.expression.experiment.ExperimentalFactor;
+import javax.annotation.Nullable;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.Map;
@@ -27,6 +28,18 @@
*/
public interface ExperimentalDesignVisualizationService {
+ /**
+ * Put data vectors in the order you'd want to display the experimental design. This causes the "isReorganized" flag
+ * of the dedVs to be set to true.
+ *
+ * @param dedVs dedVs, already sliced for the subset of samples needed for display (if necessary); will be modified
+ * @param primaryFactor if non-null this factor will be used to order the data, otherwise the first factor will be chosen using built-in heuristics. Set this to ensure that
+ * the data is ordered by the factor you want as in the case of showing DE genes
+ * @return Map of EE ids to "layouts", which are Maps of BioAssays to map of experimental factors to doubles.
+ */
+ Map>> sortVectorDataByDesign(
+ Collection dedVs, @Nullable ExperimentalFactor primaryFactor );
+
/**
* Put data vectors in the order you'd want to display the experimental design. This causes the "isReorganized" flag
* of the dedVs to be set to true.
diff --git a/gemma-core/src/main/java/ubic/gemma/core/visualization/ExperimentalDesignVisualizationServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/visualization/ExperimentalDesignVisualizationServiceImpl.java
index 9c111d9b9d..4bb11e5cca 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/visualization/ExperimentalDesignVisualizationServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/visualization/ExperimentalDesignVisualizationServiceImpl.java
@@ -24,6 +24,8 @@
import org.apache.commons.lang3.time.StopWatch;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.commons.math3.analysis.function.Exp;
+import org.apache.log4j.Layout;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import ubic.basecode.dataStructure.matrix.DoubleMatrix;
@@ -44,6 +46,7 @@
import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentService;
import ubic.gemma.persistence.util.EntityUtils;
+import javax.annotation.Nullable;
import java.awt.*;
import java.io.File;
import java.io.IOException;
@@ -60,10 +63,50 @@
@Component
public class ExperimentalDesignVisualizationServiceImpl implements ExperimentalDesignVisualizationService {
+ class LayoutSelection {
+ private Long experimentId;
+ private Long factorId; // nullable.
+
+ public LayoutSelection( Long experimentId, @Nullable ExperimentalFactor primaryFactor ) {
+ this.experimentId = experimentId;
+
+ if ( primaryFactor != null )
+ this.factorId = primaryFactor.getId();
+ }
+
+ public LayoutSelection( Long experimentId, Long primaryFactorId ) {
+ this.experimentId = experimentId;
+ this.factorId = primaryFactorId;
+ }
+
+ public LayoutSelection( Long experimentId ) {
+ this.experimentId = experimentId;
+ this.factorId = null;
+ }
+
+ public int hashCode() {
+ return Objects.hash( experimentId, factorId );
+ }
+
+ public boolean equals( Object obj ) {
+ if ( this == obj ) {
+ return true;
+ }
+ if ( obj == null ) {
+ return false;
+ }
+ if ( getClass() != obj.getClass() ) {
+ return false;
+ }
+ LayoutSelection other = ( LayoutSelection ) obj;
+ return Objects.equals( experimentId, other.experimentId ) && Objects.equals( factorId, other.factorId );
+ }
+ }
+
/**
* Cache of layouts for experiments, keyed by experiment ID.
*/
- private final Map>> cachedLayouts = new ConcurrentHashMap<>();
+ private final Map>> cachedLayouts = new ConcurrentHashMap<>();
private final Log log = LogFactory.getLog( this.getClass().getName() );
private final ExpressionExperimentService expressionExperimentService;
@@ -75,6 +118,12 @@ public ExperimentalDesignVisualizationServiceImpl( ExpressionExperimentService e
@Override
public Map>> sortVectorDataByDesign(
Collection dedVs ) {
+ return this.sortVectorDataByDesign( dedVs, null );
+ }
+
+ @Override
+ public Map>> sortVectorDataByDesign(
+ Collection dedVs, ExperimentalFactor primaryFactor ) {
//cachedLayouts.clear(); // uncomment FOR DEBUGGING.
@@ -90,9 +139,9 @@ public Map> newOrderingsForBioAssayDimensions = new HashMap<>();
for ( DoubleVectorValueObject vec : dedVs ) {
@@ -105,17 +154,22 @@ public Map> layout = null;
- if ( cachedLayouts.containsKey( vec.getExpressionExperiment().getId() ) ) {
- layout = cachedLayouts.get( vec.getExpressionExperiment().getId() );
- } else if ( vec.getExpressionExperiment().getClass()
+
+ LayoutSelection cacheKey = null;
+
+ if ( vec.getExpressionExperiment().getClass()
.isInstance( ExpressionExperimentSubsetValueObject.class ) ) {
- // subset.
- layout = cachedLayouts.get( ( ( ExpressionExperimentSubsetValueObject ) vec.getExpressionExperiment() )
- .getSourceExperiment() );
+ cacheKey = new LayoutSelection( ( ( ExpressionExperimentSubsetValueObject ) vec.getExpressionExperiment() ).getSourceExperiment(), primaryFactor );
+ } else {
+ cacheKey = new LayoutSelection( vec.getExpressionExperiment().getId(), primaryFactor );
}
+
+ layout = cachedLayouts.get( cacheKey );
+
+
if ( layout == null || layout.isEmpty() ) {
- log.warn( "Did not find cached layout for " + vec.getId() );
+ log.debug( "Did not find cached layout for " + vec.getId() + ( primaryFactor != null ? " PrimaryFactor=" + primaryFactor.getName() : "" ) );
continue;
}
@@ -216,12 +270,11 @@ public Map> extension = this
- .getExperimentalDesignLayout( actualEe, expressionExperimentService.getBioAssayDimensions( actualEe ) );
+ .getExperimentalDesignLayout( actualEe, expressionExperimentService.getBioAssayDimensions( actualEe ), null );
for ( BioAssayValueObject vbaVo : bioAssayDimension.getBioAssays() ) {
assert extension.containsKey( vbaVo );
@@ -313,9 +366,9 @@ private LinkedHashMap> getExperimentalDesignLayout(
ExpressionExperiment e ) {
- if ( cachedLayouts.containsKey( e.getId() ) ) {
- return cachedLayouts.get( e.getId() );
+ if ( cachedLayouts.containsKey( new LayoutSelection( e.getId() ) ) ) {
+ return cachedLayouts.get( new LayoutSelection( e.getId() ) );
}
Collection bds = expressionExperimentService.getBioAssayDimensions( e );
e = this.expressionExperimentService.thawLite( e );
LinkedHashMap> result = this
- .getExperimentalDesignLayout( e, bds );
+ .getExperimentalDesignLayout( e, bds, null );
- cachedLayouts.put( e.getId(), result );
+ cachedLayouts.put( new LayoutSelection( e.getId() ), result );
return result;
}
@@ -357,11 +410,12 @@ private LinkedHashMap> getExperimentalDesignLayout(
- BioAssaySet experiment, Collection bds ) {
+ BioAssaySet experiment, Collection bds, ExperimentalFactor primaryFactor ) {
LinkedHashMap> result = new LinkedHashMap<>();
@@ -375,7 +429,7 @@ private LinkedHashMap bms = ExpressionDataMatrixColumnSort.orderByExperimentalDesign( mat );
+ List bms = ExpressionDataMatrixColumnSort.orderByExperimentalDesign( mat, primaryFactor );
Map fvV = new HashMap<>();
@@ -433,22 +487,22 @@ private LinkedHashMap alreadyFilled = new HashSet<>();
-
if ( expressionExperiment.getRawExpressionDataVectors().isEmpty() ) {
AbstractPersister.log.debug( "Filling in bioassays" );
for ( BioAssay bioAssay : expressionExperiment.getBioAssays() ) {
this.fillInBioAssayAssociations( bioAssay, caches );
- alreadyFilled.add( bioAssay );
}
} else {
AbstractPersister.log.debug( "Filling in bioassays via data vectors" ); // usual case.
+ Set alreadyFilled;
alreadyFilled = this.fillInExpressionExperimentDataVectorAssociations( expressionExperiment, caches );
expressionExperiment.setBioAssays( alreadyFilled );
+ expressionExperiment.setNumberOfSamples( alreadyFilled.size() );
}
}
diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/AbstractQueryFilteringVoEnabledDao.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/AbstractQueryFilteringVoEnabledDao.java
index 2276f3be65..0e2784ffe4 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/AbstractQueryFilteringVoEnabledDao.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/AbstractQueryFilteringVoEnabledDao.java
@@ -15,6 +15,8 @@
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
+import static java.util.Objects.requireNonNull;
+
/**
* Partial implementation of {@link FilteringVoEnabledDao} based on the Hibernate {@link Query} API.
*
@@ -343,7 +345,8 @@ private Slice doLoadValueObjectsWithCache( @Nullable Filters filters, @Nulla
private long doCountWithCache( @Nullable Filters filters, boolean cacheable ) {
StopWatch timer = StopWatch.createStarted();
try {
- return ( Long ) this.getFilteringCountQuery( filters ).setCacheable( cacheable ).uniqueResult();
+ return ( Long ) requireNonNull( this.getFilteringCountQuery( filters ).setCacheable( cacheable ).uniqueResult(),
+ String.format( "Counting query for %s returned null.", elementClass.getName() ) );
} finally {
timer.stop();
if ( timer.getTime( TimeUnit.MILLISECONDS ) > REPORT_SLOW_QUERY_AFTER_MS ) {
diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/TableMaintenanceUtil.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/TableMaintenanceUtil.java
index dfbbaaf2cb..e940364a1d 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/TableMaintenanceUtil.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/TableMaintenanceUtil.java
@@ -20,8 +20,6 @@
import org.springframework.security.access.annotation.Secured;
-import java.nio.file.Path;
-
/**
* @author paul
*/
@@ -67,6 +65,15 @@ public interface TableMaintenanceUtil {
@Secured({ "GROUP_AGENT" })
int updateExpressionExperiment2CharacteristicEntries();
+ /**
+ * Update a specific level of the {@code EXPRESSION_EXPERIMENT2CHARACTERISTIC} table.
+ * @param level the level to update which is either {@link ubic.gemma.model.expression.experiment.ExpressionExperiment},
+ * {@link ubic.gemma.model.expression.biomaterial.BioMaterial} or {@link ubic.gemma.model.expression.experiment.ExperimentalDesign}
+ * @return the number of records that were created or updated
+ */
+ @Secured({ "GROUP_AGENT" })
+ int updateExpressionExperiment2CharacteristicEntries( Class level );
+
/**
* Update the {@code EXPRESSION_EXPERIMENT2_ARRAY_DESIGN} table.
* @return the number of records that were created or updated
@@ -74,9 +81,6 @@ public interface TableMaintenanceUtil {
@Secured({ "GROUP_AGENT" })
int updateExpressionExperiment2ArrayDesignEntries();
- @Secured({ "GROUP_ADMIN" })
- void setGene2CsInfoPath( Path gene2CsInfoPath );
-
// for tests only, to keep from getting emails.
@Secured({ "GROUP_ADMIN" })
void disableEmail();
diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/TableMaintenanceUtilImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/TableMaintenanceUtilImpl.java
index 28328505a5..b8f0fe281a 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/TableMaintenanceUtilImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/TableMaintenanceUtilImpl.java
@@ -26,6 +26,7 @@
import org.apache.commons.logging.LogFactory;
import org.hibernate.SessionFactory;
import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
import org.springframework.mail.SimpleMailMessage;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@@ -43,7 +44,6 @@
import ubic.gemma.persistence.service.common.description.ExternalDatabaseService;
import ubic.gemma.persistence.service.genome.GeneDao;
import ubic.gemma.persistence.util.MailEngine;
-import ubic.gemma.persistence.util.Settings;
import javax.annotation.Nullable;
import java.io.IOException;
@@ -52,7 +52,6 @@
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
-import java.nio.file.Paths;
import java.util.Calendar;
import java.util.Collection;
import java.util.Date;
@@ -90,24 +89,25 @@ public class TableMaintenanceUtilImpl implements TableMaintenanceUtil {
+ "and AOI.OBJECT_ID = I.ID "
+ "and ACE.SID_FK = (select ACLSID.ID from ACLSID where ACLSID.GRANTED_AUTHORITY = 'IS_AUTHENTICATED_ANONYMOUSLY') "
+ "group by AOI.ID), 0)";
- /**
- * Query used to repopulate the EXPRESSION_EXPERIMENT2CHARACTERISTIC table.
- */
- private static final String E2C_QUERY =
- "insert into EXPRESSION_EXPERIMENT2CHARACTERISTIC (ID, NAME, DESCRIPTION, CATEGORY, CATEGORY_URI, `VALUE`, VALUE_URI, ORIGINAL_VALUE, EVIDENCE_CODE, EXPRESSION_EXPERIMENT_FK, ACL_IS_AUTHENTICATED_ANONYMOUSLY_MASK, LEVEL) "
- + "select C.ID, C.NAME, C.DESCRIPTION, C.CATEGORY, C.CATEGORY_URI, C.`VALUE`, C.VALUE_URI, C.ORIGINAL_VALUE, C.EVIDENCE_CODE, I.ID, (" + SELECT_ANONYMOUS_MASK + "), cast(:eeClass as char(256)) "
+
+ private static final String EE2C_EE_QUERY =
+ "select MIN(C.ID), C.NAME, C.DESCRIPTION, C.CATEGORY, C.CATEGORY_URI, C.`VALUE`, C.VALUE_URI, C.ORIGINAL_VALUE, C.EVIDENCE_CODE, I.ID, (" + SELECT_ANONYMOUS_MASK + "), cast(? as char(256)) "
+ "from INVESTIGATION I "
+ "join CHARACTERISTIC C on I.ID = C.INVESTIGATION_FK "
+ "where I.class = 'ExpressionExperiment' "
- + "union "
- + "select C.ID, C.NAME, C.DESCRIPTION, C.CATEGORY, C.CATEGORY_URI, C.`VALUE`, C.VALUE_URI, C.ORIGINAL_VALUE, C.EVIDENCE_CODE, I.ID, (" + SELECT_ANONYMOUS_MASK + "), cast(:bmClass as char(255)) "
+ + "group by I.ID, COALESCE(C.CATEGORY_URI, C.CATEGORY), COALESCE(C.VALUE_URI, C.`VALUE`)";
+
+ private static final String EE2C_BM_QUERY =
+ "select MIN(C.ID), C.NAME, C.DESCRIPTION, C.CATEGORY, C.CATEGORY_URI, C.`VALUE`, C.VALUE_URI, C.ORIGINAL_VALUE, C.EVIDENCE_CODE, I.ID, (" + SELECT_ANONYMOUS_MASK + "), cast(? as char(255)) "
+ "from INVESTIGATION I "
+ "join BIO_ASSAY BA on I.ID = BA.EXPRESSION_EXPERIMENT_FK "
+ "join BIO_MATERIAL BM on BA.SAMPLE_USED_FK = BM.ID "
+ "join CHARACTERISTIC C on BM.ID = C.BIO_MATERIAL_FK "
+ "where I.class = 'ExpressionExperiment' "
- + "union "
- + "select C.ID, C.NAME, C.DESCRIPTION, C.CATEGORY, C.CATEGORY_URI, C.`VALUE`, C.VALUE_URI, C.ORIGINAL_VALUE, C.EVIDENCE_CODE, I.ID, (" + SELECT_ANONYMOUS_MASK + "), cast(:edClass as char(255)) "
+ + "group by I.ID, COALESCE(C.CATEGORY_URI, C.CATEGORY), COALESCE(C.VALUE_URI, C.`VALUE`)";
+
+ private static final String EE2C_ED_QUERY =
+ "select MIN(C.ID), C.NAME, C.DESCRIPTION, C.CATEGORY, C.CATEGORY_URI, C.`VALUE`, C.VALUE_URI, C.ORIGINAL_VALUE, C.EVIDENCE_CODE, I.ID, (" + SELECT_ANONYMOUS_MASK + "), cast(? as char(255)) "
+ "from INVESTIGATION I "
+ "join EXPERIMENTAL_DESIGN on I.EXPERIMENTAL_DESIGN_FK = EXPERIMENTAL_DESIGN.ID "
+ "join EXPERIMENTAL_FACTOR EF on EXPERIMENTAL_DESIGN.ID = EF.EXPERIMENTAL_DESIGN_FK "
@@ -115,21 +115,24 @@ public class TableMaintenanceUtilImpl implements TableMaintenanceUtil {
+ "join CHARACTERISTIC C on FV.ID = C.FACTOR_VALUE_FK "
// remove C.class = 'Statement' once the old-style characteristics are removed (see https://github.com/PavlidisLab/Gemma/issues/929 for details)
+ "where I.class = 'ExpressionExperiment' and C.class = 'Statement' "
- + "on duplicate key update NAME = VALUES(NAME), DESCRIPTION = VALUES(DESCRIPTION), CATEGORY = VALUES(CATEGORY), CATEGORY_URI = VALUES(CATEGORY_URI), `VALUE` = VALUES(`VALUE`), VALUE_URI = VALUES(VALUE_URI), ORIGINAL_VALUE = VALUES(ORIGINAL_VALUE), EVIDENCE_CODE = VALUES(EVIDENCE_CODE), ACL_IS_AUTHENTICATED_ANONYMOUSLY_MASK = VALUES(ACL_IS_AUTHENTICATED_ANONYMOUSLY_MASK)";
+ + "group by I.ID, COALESCE(C.CATEGORY_URI, C.CATEGORY), COALESCE(C.VALUE_URI, C.`VALUE`)";
private static final String EE2AD_QUERY = "insert into EXPRESSION_EXPERIMENT2ARRAY_DESIGN (EXPRESSION_EXPERIMENT_FK, ARRAY_DESIGN_FK, IS_ORIGINAL_PLATFORM, ACL_IS_AUTHENTICATED_ANONYMOUSLY_MASK) "
+ "select I.ID, AD.ID, FALSE, (" + SELECT_ANONYMOUS_MASK + ") from INVESTIGATION I "
+ "join BIO_ASSAY BA on I.ID = BA.EXPRESSION_EXPERIMENT_FK "
+ "join ARRAY_DESIGN AD on BA.ARRAY_DESIGN_USED_FK = AD.ID "
+ + "where I.class = 'ExpressionExperiment' "
+ "group by I.ID, AD.ID "
+ "union "
+ "select I.ID, AD.ID, TRUE, (" + SELECT_ANONYMOUS_MASK + ") from INVESTIGATION I "
+ "join BIO_ASSAY BA on I.ID = BA.EXPRESSION_EXPERIMENT_FK "
+ "join ARRAY_DESIGN AD on BA.ORIGINAL_PLATFORM_FK = AD.ID "
+ + "where I.class = 'ExpressionExperiment' "
+ "group by I.ID, AD.ID "
+ "on duplicate key update ACL_IS_AUTHENTICATED_ANONYMOUSLY_MASK = VALUES(ACL_IS_AUTHENTICATED_ANONYMOUSLY_MASK)";
- private static final Path DEFAULT_GENE2CS_INFO_PATH = Paths.get( Settings.getString( "gemma.appdata.home" ), "DbReports", "gene2cs.info" );
+
private static final Log log = LogFactory.getLog( TableMaintenanceUtil.class.getName() );
+
@Autowired
private AuditEventService auditEventService;
@@ -142,7 +145,12 @@ public class TableMaintenanceUtilImpl implements TableMaintenanceUtil {
@Autowired
private SessionFactory sessionFactory;
- private Path gene2CsInfoPath = DEFAULT_GENE2CS_INFO_PATH;
+ @Value("${gemma.gene2cs.path}")
+ private Path gene2CsInfoPath;
+
+ @Value("${gemma.admin.email}")
+ private String adminEmailAddress;
+
private boolean sendEmail = true;
@Override
@@ -217,16 +225,52 @@ public void updateGene2CsEntries() {
@Timed
public int updateExpressionExperiment2CharacteristicEntries() {
log.info( "Updating the EXPRESSION_EXPERIMENT2CHARACTERISTIC table..." );
- int updated = sessionFactory.getCurrentSession().createSQLQuery( E2C_QUERY )
+ int updated = sessionFactory.getCurrentSession()
+ .createSQLQuery(
+ "insert into EXPRESSION_EXPERIMENT2CHARACTERISTIC (ID, NAME, DESCRIPTION, CATEGORY, CATEGORY_URI, `VALUE`, VALUE_URI, ORIGINAL_VALUE, EVIDENCE_CODE, EXPRESSION_EXPERIMENT_FK, ACL_IS_AUTHENTICATED_ANONYMOUSLY_MASK, LEVEL) "
+ + EE2C_EE_QUERY
+ + " union "
+ + EE2C_BM_QUERY
+ + " union "
+ + EE2C_ED_QUERY + " "
+ + "on duplicate key update NAME = VALUES(NAME), DESCRIPTION = VALUES(DESCRIPTION), CATEGORY = VALUES(CATEGORY), CATEGORY_URI = VALUES(CATEGORY_URI), `VALUE` = VALUES(`VALUE`), VALUE_URI = VALUES(VALUE_URI), ORIGINAL_VALUE = VALUES(ORIGINAL_VALUE), EVIDENCE_CODE = VALUES(EVIDENCE_CODE), ACL_IS_AUTHENTICATED_ANONYMOUSLY_MASK = VALUES(ACL_IS_AUTHENTICATED_ANONYMOUSLY_MASK), LEVEL = VALUES(LEVEL)" )
.addSynchronizedQuerySpace( EE2C_QUERY_SPACE )
- .setParameter( "eeClass", ExpressionExperiment.class )
- .setParameter( "bmClass", BioMaterial.class )
- .setParameter( "edClass", ExperimentalDesign.class )
+ .setParameter( 0, ExpressionExperiment.class )
+ .setParameter( 1, BioMaterial.class )
+ .setParameter( 2, ExperimentalDesign.class )
.executeUpdate();
log.info( String.format( "Done updating the EXPRESSION_EXPERIMENT2CHARACTERISTIC table; %d entries were updated.", updated ) );
return updated;
}
+ @Override
+ @Timed
+ @Transactional
+ public int updateExpressionExperiment2CharacteristicEntries( Class level ) {
+ String query;
+ if ( level.equals( ExpressionExperiment.class ) ) {
+ query = EE2C_EE_QUERY;
+ } else if ( level.equals( BioMaterial.class ) ) {
+ query = EE2C_BM_QUERY;
+ } else if ( level.equals( ExperimentalDesign.class ) ) {
+ query = EE2C_ED_QUERY;
+ } else {
+ throw new IllegalArgumentException( "Level must be one of ExpressionExperiment.class, BioMaterial.class or ExperimentalDesign.class." );
+ }
+ log.info( "Updating the EXPRESSION_EXPERIMENT2CHARACTERISTIC table at level " + level + "..." );
+ int updated = sessionFactory.getCurrentSession()
+ .createSQLQuery(
+ "insert into EXPRESSION_EXPERIMENT2CHARACTERISTIC (ID, NAME, DESCRIPTION, CATEGORY, CATEGORY_URI, `VALUE`, VALUE_URI, ORIGINAL_VALUE, EVIDENCE_CODE, EXPRESSION_EXPERIMENT_FK, ACL_IS_AUTHENTICATED_ANONYMOUSLY_MASK, LEVEL) "
+ + query + " "
+ + "on duplicate key update NAME = VALUES(NAME), DESCRIPTION = VALUES(DESCRIPTION), CATEGORY = VALUES(CATEGORY), CATEGORY_URI = VALUES(CATEGORY_URI), `VALUE` = VALUES(`VALUE`), VALUE_URI = VALUES(VALUE_URI), ORIGINAL_VALUE = VALUES(ORIGINAL_VALUE), EVIDENCE_CODE = VALUES(EVIDENCE_CODE), ACL_IS_AUTHENTICATED_ANONYMOUSLY_MASK = VALUES(ACL_IS_AUTHENTICATED_ANONYMOUSLY_MASK), LEVEL = VALUES(LEVEL)" )
+ .addSynchronizedQuerySpace( EE2C_QUERY_SPACE )
+ .setParameter( 0, level )
+ .executeUpdate();
+ log.info( String.format( "Done updating the EXPRESSION_EXPERIMENT2CHARACTERISTIC table at %s level; %d entries were updated.",
+ level.getSimpleName(), updated ) );
+ return updated;
+ }
+
@Override
@Transactional
public int updateExpressionExperiment2ArrayDesignEntries() {
@@ -238,14 +282,6 @@ public int updateExpressionExperiment2ArrayDesignEntries() {
return updated;
}
- /**
- * For use in tests.
- */
- @Override
- public void setGene2CsInfoPath( Path gene2CsInfoPath ) {
- this.gene2CsInfoPath = gene2CsInfoPath;
- }
-
/**
* For use in tests.
*/
@@ -289,7 +325,6 @@ private void sendEmail( Gene2CsStatus results ) {
if ( !sendEmail )
return;
SimpleMailMessage msg = new SimpleMailMessage();
- String adminEmailAddress = Settings.getAdminEmailAddress();
if ( StringUtils.isBlank( adminEmailAddress ) ) {
TableMaintenanceUtilImpl.log
.warn( "No administrator email address could be found, so gene2cs status email will not be sent." );
diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionAnalysisDao.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionAnalysisDao.java
index 403ab86181..97935314af 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionAnalysisDao.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionAnalysisDao.java
@@ -56,10 +56,15 @@ Collection find( Gene gene, ExpressionAnalysisRe
double threshold );
/**
- * @return analyses associated with the factor, either through the subsetfactor or as factors for resultsets.
+ * Retrieve analyses associated with the factor, either through the subset factor or as factors for result sets.
*/
Collection findByFactor( ExperimentalFactor ef );
+ /**
+ * @see #findByFactor(ExperimentalFactor)
+ */
+ Collection findByFactors( Collection experimentalFactors );
+
Map> findByExperimentIds( Collection investigationIds );
Collection findExperimentsWithAnalyses( Gene gene );
diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionAnalysisDaoImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionAnalysisDaoImpl.java
index a4ee6d20c2..aa49bfdc75 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionAnalysisDaoImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionAnalysisDaoImpl.java
@@ -291,24 +291,57 @@ public Collection find( Gene gene, ExpressionAna
@Override
public Collection findByFactor( ExperimentalFactor ef ) {
+ Set results = new HashSet<>();
// subset factorValues factors.
- @SuppressWarnings("unchecked")
- Collection result = this.getSessionFactory().getCurrentSession()
- .createQuery( "select distinct a from DifferentialExpressionAnalysis a join a.subsetFactorValue ssf"
- + " join ssf.experimentalFactor efa where efa = :ef " )
+ //noinspection unchecked
+ results.addAll( this.getSessionFactory().getCurrentSession()
+ .createQuery( "select distinct a from DifferentialExpressionAnalysis a "
+ + "join a.subsetFactorValue ssf "
+ + "where ssf.experimentalFactor = :ef" )
.setParameter( "ef", ef )
- .list();
+ .list() );
// factors used in the analysis.
//noinspection unchecked
- result.addAll( this.getSessionFactory().getCurrentSession()
- .createQuery( "select distinct a from DifferentialExpressionAnalysis a join a.resultSets rs"
- + " left join rs.baselineGroup bg join rs.experimentalFactors efa where efa = :ef " )
+ results.addAll( this.getSessionFactory().getCurrentSession()
+ .createQuery( "select distinct a from DifferentialExpressionAnalysis a "
+ + "join a.resultSets rs "
+ + "join rs.experimentalFactors efa "
+ + "where efa = :ef" )
.setParameter( "ef", ef )
.list() );
- return result;
+ return results;
+ }
+
+ @Override
+ public Collection findByFactors( Collection experimentalFactors ) {
+ if ( experimentalFactors.isEmpty() ) {
+ return Collections.emptySet();
+ }
+
+ Set results = new HashSet<>();
+ // subset factorValues factors.
+ //noinspection unchecked
+ results.addAll( this.getSessionFactory().getCurrentSession()
+ .createQuery( "select distinct a from DifferentialExpressionAnalysis a "
+ + "join a.subsetFactorValue ssf "
+ + "where ssf.experimentalFactor in :efs" )
+ .setParameterList( "efs", experimentalFactors )
+ .list() );
+
+ // factors used in the analysis
+ //noinspection unchecked
+ results.addAll( this.getSessionFactory().getCurrentSession()
+ .createQuery( "select distinct a from DifferentialExpressionAnalysis a "
+ + "join a.resultSets rs "
+ + "join rs.experimentalFactors efa "
+ + "where efa in :efs" )
+ .setParameterList( "efs", experimentalFactors )
+ .list() );
+
+ return results;
}
@Override
diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionAnalysisService.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionAnalysisService.java
index 63066df668..70f01865e5 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionAnalysisService.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionAnalysisService.java
@@ -133,4 +133,11 @@ Map> getAnalysesByExperiment(
Collection ids, int offset, int limit );
+ /**
+ * Remove analyses using the given factor.
+ * @return the number of analysis removed
+ */
+ int removeForExperimentalFactor( ExperimentalFactor experimentalFactor );
+
+ int removeForExperimentalFactors( Collection experimentalFactors );
}
diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionAnalysisServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionAnalysisServiceImpl.java
index f3b6ab521e..a964eb42e2 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionAnalysisServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionAnalysisServiceImpl.java
@@ -253,6 +253,22 @@ public void removeForExperiment( BioAssaySet ee ) {
this.remove( diffAnalyses );
}
+ @Override
+ @Transactional
+ public int removeForExperimentalFactor( ExperimentalFactor experimentalFactor ) {
+ Collection found = differentialExpressionAnalysisDao.findByFactor( experimentalFactor );
+ this.remove( found );
+ return found.size();
+ }
+
+ @Override
+ @Transactional
+ public int removeForExperimentalFactors( Collection experimentalFactors ) {
+ Collection found = differentialExpressionAnalysisDao.findByFactors( experimentalFactors );
+ this.remove( found );
+ return found.size();
+ }
+
@Override
@Transactional(readOnly = true)
public Collection findByExperiment( BioAssaySet experiment ) {
diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionResultDao.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionResultDao.java
index 409d58ab26..e89cac8dfa 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionResultDao.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionResultDao.java
@@ -19,10 +19,8 @@
package ubic.gemma.persistence.service.analysis.expression.diff;
import ubic.basecode.math.distribution.Histogram;
-import ubic.gemma.model.analysis.expression.diff.ExpressionAnalysisResultSet;
import ubic.gemma.model.analysis.expression.diff.*;
import ubic.gemma.model.expression.experiment.BioAssaySet;
-import ubic.gemma.model.expression.experiment.ExperimentalFactor;
import ubic.gemma.model.expression.experiment.ExpressionExperimentValueObject;
import ubic.gemma.model.genome.Gene;
import ubic.gemma.persistence.service.BaseDao;
@@ -39,93 +37,52 @@ public interface DifferentialExpressionResultDao extends BaseDao> find( Gene gene,
+ Map> findByGeneAndExperimentAnalyzed( Gene gene,
Collection experimentsAnalyzed, double threshold, int limit );
/**
* Given a list of experiments and a threshold value finds all the probes that met the cut off in the given
* experiments
- *
- * @param experimentsAnalyzed ees
- * @param threshold threshold
- * @param limit limit
- * @return map to diff exp VOs
*/
- Map> find(
+ Map> findByExperimentAnalyzed(
Collection experimentsAnalyzed, double threshold, int limit );
/**
- * @param gene gene
- * @return a map of a collection of {@link DifferentialExpressionAnalysisResult}s keyed by {@link BioAssaySet}.
+ * Find differential expression results for a given gene, grouped by experiment.
+ * @return a map of a collection of {@link DifferentialExpressionAnalysisResult}s keyed by {@link BioAssaySet}.
*/
- Map> find( Gene gene );
+ Map> findByGene( Gene gene );
/**
- * @param gene gene
- * @param experimentsAnalyzed ees
- * @return a map of a collection of {@link DifferentialExpressionAnalysisResult}s keyed by
+ * Find differential expression results for a given gene and set of experiments, grouped by experiment.
+ * @return a map of a collection of {@link DifferentialExpressionAnalysisResult}s keyed by
* {@link BioAssaySet}.
*/
- Map> find( Gene gene,
+ Map> findByGeneAndExperimentAnalyzed( Gene gene,
Collection experimentsAnalyzed );
/**
- * @param geneIds gene ids
- * @param resultSets result sets
- * @return map of resultSetId to map of gene to DiffExprGeneSearchResult
+ * @return map of resultSetId to map of gene to DiffExprGeneSearchResult
*/
Map> findDiffExAnalysisResultIdsInResultSets(
Collection resultSets, Collection geneIds );
- List findGeneInResultSets( Gene gene, ExpressionAnalysisResultSet resultSet,
- Collection arrayDesignIds, int limit );
-
List findInResultSet( ExpressionAnalysisResultSet resultSet, Double threshold,
int maxResultsToReturn, int minNumberOfResults );
- /**
- * Given a list of result sets finds the diff expression results that met the given threshold
- *
- * @param resultsAnalyzed resultsAnalyzed
- * @param threshold threshold
- * @param limit limit
- * @return map to diff exp VOs
- */
- Map> findInResultSets(
- Collection resultsAnalyzed, double threshold, int limit );
-
- DifferentialExpressionAnalysis getAnalysis( ExpressionAnalysisResultSet rs );
-
- Collection getExperimentalFactors(
- DifferentialExpressionAnalysisResult differentialExpressionAnalysisResult );
-
- Map> getExperimentalFactors(
- Collection differentialExpressionAnalysisResults );
-
Map loadContrastDetailsForResults( Collection ids );
- void thaw( Collection results );
-
- void thaw( final DifferentialExpressionAnalysisResult result );
-
/**
* Find differential expression for a gene, exceeding a given significance level (using the corrected pvalue field)
*
* @param gene gene
* @param threshold threshold
* @param limit limit
- * @return map to diff exp VOs
+ * @return map to diff exp VOs
*/
Map> find( Gene gene, double threshold,
int limit );
Histogram loadPvalueDistribution( Long resultSetId );
-
}
diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionResultDaoImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionResultDaoImpl.java
index 607e529f43..209562d31f 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionResultDaoImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionResultDaoImpl.java
@@ -21,21 +21,22 @@
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.StopWatch;
import org.hibernate.*;
-import org.hibernate.type.StandardBasicTypes;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Repository;
+import org.springframework.util.Assert;
import ubic.basecode.io.ByteArrayConverter;
import ubic.basecode.math.distribution.Histogram;
import ubic.basecode.util.BatchIterator;
import ubic.basecode.util.SQLUtils;
import ubic.gemma.model.analysis.expression.diff.*;
-import ubic.gemma.model.expression.designElement.CompositeSequence;
-import ubic.gemma.model.expression.experiment.*;
+import ubic.gemma.model.expression.experiment.BioAssaySet;
+import ubic.gemma.model.expression.experiment.ExperimentalFactor;
+import ubic.gemma.model.expression.experiment.ExperimentalFactorValueObject;
+import ubic.gemma.model.expression.experiment.ExpressionExperimentValueObject;
import ubic.gemma.model.genome.Gene;
import ubic.gemma.model.genome.gene.GeneValueObject;
import ubic.gemma.persistence.service.AbstractDao;
import ubic.gemma.persistence.util.CommonQueries;
-import ubic.gemma.persistence.util.EntityUtils;
import ubic.gemma.persistence.util.TaskCancelledException;
import java.math.BigInteger;
@@ -53,61 +54,15 @@ public class DifferentialExpressionResultDaoImpl extends AbstractDao> find( Gene gene,
+ public Map> findByGeneAndExperimentAnalyzed( Gene gene,
Collection experimentsAnalyzed, double threshold, int limit ) {
-
- StopWatch timer = new StopWatch();
- timer.start();
- String qs = DifferentialExpressionResultDaoImpl.fetchResultsByGeneAndExperimentsQuery
- + " and r.correctedPvalue < :threshold";
-
- if ( limit > 0 ) {
- qs += " order by r.correctedPvalue";
- }
-
- Map> results = new HashMap<>();
-
- if ( experimentsAnalyzed.size() == 0 ) {
- return results;
+ if ( experimentsAnalyzed.isEmpty() ) {
+ return Collections.emptyMap();
}
+ StopWatch timer = StopWatch.createStarted();
List qResult = getSessionFactory().getCurrentSession()
- .createQuery( qs )
+ .createQuery( DIFF_EX_RESULTS_BY_GENE_QUERY + " and e.id in (:experimentAnalyzed)"
+ + ( limit > 0 ? " order by r.correctedPvalue" : "" ) )
.setParameter( "gene", gene )
.setParameterList( "experimentsAnalyzed", experimentsAnalyzed )
.setParameter( "threshold", threshold )
@@ -158,142 +102,98 @@ public Map() );
- }
-
- results.get( ee ).add( new DifferentialExpressionValueObject( probeResult ) );
- }
-
- AbstractDao.log
- .warn( "Num experiments with probe analysis results (with limit = " + limit + ") : " + results.size()
- + ". Number of probes returned in total: " + qResult.size() );
+ Map> results = groupDiffExResultVos( qResult );
+ AbstractDao.log.info( String.format( "Num experiments with probe analysis results (with limit = %d) : %d. Number of probes returned in total: %d",
+ limit, results.size(), qResult.size() ) );
timer.stop();
if ( timer.getTime() > 1000 ) {
- AbstractDao.log.info( "Diff ex results: " + timer.getTime() + " ms" );
+ AbstractDao.log.warn( "Diff ex results: " + timer.getTime() + " ms" );
}
return results;
}
@Override
- public Map> find(
+ public Map> findByExperimentAnalyzed(
Collection experiments, double qvalueThreshold, int limit ) {
-
- Map> results = new HashMap<>();
-
- if ( experiments.size() == 0 ) {
- return results;
+ if ( experiments.isEmpty() ) {
+ return Collections.emptyMap();
}
-
- StopWatch timer = new StopWatch();
- timer.start();
-
+ StopWatch timer = StopWatch.createStarted();
List qResult = getSessionFactory().getCurrentSession()
- .createQuery( DifferentialExpressionResultDaoImpl.fetchResultsByExperimentsQuery )
+ .createQuery( "select e, r from DifferentialExpressionAnalysis a "
+ + "join a.experimentAnalyzed e "
+ + "join a.resultSets rs join rs.results r "
+ + "where e.id in (:experimentsAnalyzed) and r.correctedPvalue < :threshold"
+ + ( limit > 0 ? " order by r.correctedPvalue" : "" ) )
.setParameterList( "experimentsAnalyzed", experiments )
.setParameter( "threshold", qvalueThreshold )
.setMaxResults( limit )
.setCacheable( true )
.setCacheRegion( "diffExResult" )
.list();
-
- for ( Object o : qResult ) {
-
- Object[] oa = ( Object[] ) o;
- DifferentialExpressionAnalysisResult probeResult = ( DifferentialExpressionAnalysisResult ) oa[1];
- ExpressionExperimentValueObject eevo = ( ( BioAssaySet ) oa[0] ).createValueObject();
- if ( !results.containsKey( eevo ) ) {
- results.put( eevo, new ArrayList() );
+ try {
+ return groupDiffExResultVos( qResult );
+ } finally {
+ timer.stop();
+ if ( timer.getTime() > 1000 ) {
+ AbstractDao.log.info( "Diff ex results: " + timer.getTime() + " ms" );
}
-
- results.get( eevo ).add( new DifferentialExpressionValueObject( probeResult ) );
- }
-
- timer.stop();
- if ( timer.getTime() > 1000 ) {
- AbstractDao.log.info( "Diff ex results: " + timer.getTime() + " ms" );
}
-
- return results;
}
@Override
- public Map> find( Gene gene ) {
- StopWatch timer = new StopWatch();
- timer.start();
- Map> results = new HashMap<>();
- if ( gene == null )
- return results;
-
- List qResult = getSessionFactory().getCurrentSession().createQuery( DifferentialExpressionResultDaoImpl.fetchResultsByGene )
+ public Map> findByGene( Gene gene ) {
+ Assert.notNull( gene );
+ StopWatch timer = StopWatch.createStarted();
+ List qResult = getSessionFactory().getCurrentSession()
+ .createQuery( DIFF_EX_RESULTS_BY_GENE_QUERY )
.setParameter( "gene", gene )
.setCacheable( true )
.list();
-
- for ( Object o : qResult ) {
-
- Object[] oa = ( Object[] ) o;
- ExpressionExperimentValueObject ee = ( ( BioAssaySet ) oa[0] ).createValueObject();
- DifferentialExpressionValueObject probeResult = new DifferentialExpressionValueObject(
- ( DifferentialExpressionAnalysisResult ) oa[1] );
-
- if ( !results.containsKey( ee ) ) {
- results.put( ee, new ArrayList() );
+ try {
+ return groupDiffExResultVos( qResult );
+ } finally {
+ timer.stop();
+ if ( timer.getTime() > 1000 ) {
+ AbstractDao.log.warn( "Diff ex results: " + timer.getTime() + " ms" );
}
-
- results.get( ee ).add( probeResult );
- }
- timer.stop();
- if ( timer.getTime() > 1000 ) {
- AbstractDao.log.info( "Diff ex results: " + timer.getTime() + " ms" );
}
- return results;
}
@Override
- public Map> find( Gene gene,
+ public Map> findByGeneAndExperimentAnalyzed( Gene gene,
Collection experimentsAnalyzed ) {
-
- Map> results = new HashMap<>();
-
- if ( experimentsAnalyzed.size() == 0 ) {
- return results;
+ Assert.notNull( gene );
+ if ( experimentsAnalyzed.isEmpty() ) {
+ return Collections.emptyMap();
}
-
- StopWatch timer = new StopWatch();
- timer.start();
-
+ StopWatch timer = StopWatch.createStarted();
List qResult = this.getSessionFactory().getCurrentSession()
- .createQuery( DifferentialExpressionResultDaoImpl.fetchResultsByGeneAndExperimentsQuery )
+ .createQuery( DIFF_EX_RESULTS_BY_GENE_QUERY + " and e.id in (:experimentsAnalyzed)" )
.setParameter( "gene", gene )
.setParameterList( "experimentsAnalyzed", experimentsAnalyzed )
.list();
+ try {
+ return groupDiffExResultVos( qResult );
+ } finally {
+ timer.stop();
+ if ( timer.getTime() > 1000 ) {
+ AbstractDao.log.warn( "Diff ex results: " + timer.getTime() + " ms" );
+ }
+ }
+ }
+ private Map> groupDiffExResultVos( List qResult ) {
+ Map> results = new HashMap<>();
for ( Object o : qResult ) {
-
Object[] oa = ( Object[] ) o;
- ExpressionExperimentValueObject ee = new ExpressionExperimentValueObject( ( ExpressionExperiment ) oa[0] );
- DifferentialExpressionValueObject probeResult = new DifferentialExpressionValueObject(
- ( DifferentialExpressionAnalysisResult ) oa[1] );
- probeResult.setExpressionExperiment( ee );
-
- if ( !results.containsKey( ee ) ) {
- results.put( ee, new ArrayList() );
- }
-
- results.get( ee ).add( probeResult );
- }
- timer.stop();
- if ( timer.getTime() > 1000 ) {
- AbstractDao.log.info( "Diff ex results: " + timer.getTime() + " ms" );
+ ExpressionExperimentValueObject ee = ( ( BioAssaySet ) oa[0] ).createValueObject();
+ DifferentialExpressionAnalysisResult dear = ( DifferentialExpressionAnalysisResult ) oa[1];
+ Hibernate.initialize( dear.getProbe() );
+ DifferentialExpressionValueObject probeResult = new DifferentialExpressionValueObject( dear );
+ results.computeIfAbsent( ee, k -> new ArrayList<>() ).add( probeResult );
}
return results;
}
@@ -331,8 +231,11 @@ public Map> findDiffExAnalysisResultId
assert !resultSetsNeeded.isEmpty();
- org.hibernate.SQLQuery queryObject = session.createSQLQuery(
- DifferentialExpressionResultDaoImpl.fetchBatchDifferentialExpressionAnalysisResultsByResultSetsAndGeneQuery );
+ SQLQuery queryObject = session.createSQLQuery(
+ "SELECT dear.PROBE_FK, dear.ID,"
+ + " dear.RESULT_SET_FK, dear.CORRECTED_PVALUE, dear.PVALUE "
+ + " FROM DIFFERENTIAL_EXPRESSION_ANALYSIS_RESULT dear FORCE INDEX (probeResultSets) WHERE dear.RESULT_SET_FK IN (:rs_ids) AND "
+ + " dear.PROBE_FK IN (:probe_ids) " );
/*
* These values have been tweaked to probe for performance issues.
@@ -352,7 +255,7 @@ public Map> findDiffExAnalysisResultId
+ " resultSets; batch size=" + geneBatchSize );
}
- final int numResultSetBatches = ( int ) Math.ceil( resultSetsNeeded.size() / resultSetBatchSize );
+ final int numResultSetBatches = ( int ) Math.ceil( ( double ) resultSetsNeeded.size() / ( double ) resultSetBatchSize );
queryObject.setFlushMode( FlushMode.MANUAL );
@@ -382,7 +285,7 @@ public Map> findDiffExAnalysisResultId
queryObject.setParameterList( "rs_ids", resultSetIdBatch );
int numGeneBatchesDone = 0;
- final int numGeneBatches = ( int ) Math.ceil( cs2GeneIdMap.size() / geneBatchSize );
+ final int numGeneBatches = ( int ) Math.ceil( ( double ) cs2GeneIdMap.size() / ( double ) geneBatchSize );
StopWatch innerQt = new StopWatch();
@@ -478,65 +381,25 @@ public Map> findDiffExAnalysisResultId
return results;
}
- @Override
- public List findGeneInResultSets( Gene gene, ExpressionAnalysisResultSet resultSet,
- Collection arrayDesignIds, int limit ) {
-
- StopWatch timer = new StopWatch();
- timer.start();
-
- List results;
-
- Session session = this.getSessionFactory().getCurrentSession();
- org.hibernate.SQLQuery queryObject = session
- .createSQLQuery( DifferentialExpressionResultDaoImpl.fetchResultsByResultSetAndGeneQuery );
-
- queryObject.setLong( "gene_id", gene.getId() );
- queryObject.setLong( "rs_id", resultSet.getId() );
-
- queryObject.setMaxResults( limit );
-
- queryObject.addScalar( "CORRECTED_PVALUE", StandardBasicTypes.DOUBLE );
- //noinspection unchecked
- results = queryObject.list();
-
- timer.stop();
- if ( AbstractDao.log.isDebugEnabled() )
- AbstractDao.log
- .debug( "Fetching probeResults from resultSet " + resultSet.getId() + " for gene " + gene.getId()
- + "and " + arrayDesignIds.size() + "arrays took : " + timer.getTime() + " ms" );
-
- return results;
- }
-
@Override
public List findInResultSet( ExpressionAnalysisResultSet resultSet,
Double threshold, int limit, int minNumberOfResults ) {
+ Assert.notNull( resultSet, "The result set must not be null." );
+ Assert.isTrue( minNumberOfResults > 0, "Minimum number of results must be greater than zero." );
- if ( minNumberOfResults < 1 ) {
- throw new IllegalArgumentException( "Minimum number of results must be greater than one" );
- }
-
- List results = new ArrayList<>();
-
- if ( resultSet == null ) {
- return results;
- }
-
- results = differentialExpressionResultCache.getTopHits( resultSet );
- if ( results != null && results.size() >= minNumberOfResults ) {
+ // check for top hits in cache
+ List cachedResults = differentialExpressionResultCache.getTopHits( resultSet );
+ if ( cachedResults != null && cachedResults.size() >= minNumberOfResults ) {
AbstractDao.log.info( "Top hits already in cache" );
- return results;
+ return cachedResults;
}
- results = new ArrayList<>();
-
- // get it.
-
+ // retrieve top hits from database
StopWatch timer = new StopWatch();
timer.start();
- String qs = DifferentialExpressionResultDaoImpl.fetchResultsBySingleResultSetQuery
- + " and r.correctedPvalue <= :threshold order by r.correctedPvalue";
+ String qs = "select r from DifferentialExpressionAnalysisResult r "
+ + "where r.resultSet = :resultSet and r.correctedPvalue <= :threshold "
+ + "order by r.correctedPvalue";
List qResult = getSessionFactory().getCurrentSession().createQuery( qs )
.setParameter( "resultSet", resultSet )
@@ -548,140 +411,31 @@ public List findInResultSet( ExpressionAnalys
if ( qResult.size() < minNumberOfResults ) {
// FIXME this is kind of dumb. If we always return the top minimum, why not just always get that?
AbstractDao.log.info( "Too few results met threshold, repeating to just get the top hits" );
- qs = DifferentialExpressionResultDaoImpl.fetchResultsBySingleResultSetQuery + " order by r.correctedPvalue";
+ qs = "select r from DifferentialExpressionAnalysisResult r "
+ + "where r.resultSet = :resultSet "
+ + "order by r.correctedPvalue";
qResult = getSessionFactory().getCurrentSession().createQuery( qs )
.setParameter( "resultSet", resultSet )
.setMaxResults( minNumberOfResults )
.list();
}
+ List results = new ArrayList<>();
for ( Object o : qResult ) {
- DifferentialExpressionValueObject probeResult = new DifferentialExpressionValueObject(
- ( DifferentialExpressionAnalysisResult ) o );
+ DifferentialExpressionAnalysisResult dear = ( DifferentialExpressionAnalysisResult ) o;
+ Hibernate.initialize( dear.getProbe() );
+ DifferentialExpressionValueObject probeResult = new DifferentialExpressionValueObject( dear );
results.add( probeResult );
}
- timer.stop();
- if ( timer.getTime() > 1000 ) {
- AbstractDao.log.info( "Diff ex results: " + timer.getTime() + " ms" );
- }
-
differentialExpressionResultCache.addToTopHitsCache( resultSet, results );
- return results;
- }
-
- /**
- * Given a list of result sets finds the results that met the given threshold
- *
- * @param limit - max number of results to return.
- */
- @Override
- public Map> findInResultSets(
- Collection resultsAnalyzed, double threshold, int limit ) {
-
- Map> results = new HashMap<>();
-
- if ( resultsAnalyzed.size() == 0 ) {
- return results;
- }
-
- // Integer bin = Math.log10(threshold);
-
- StopWatch timer = new StopWatch();
- timer.start();
- String qs = DifferentialExpressionResultDaoImpl.fetchResultsByResultSetQuery
- + " and r.correctedPvalue < :threshold order by r.correctedPvalue";
-
- List qResult = getSessionFactory().getCurrentSession().createQuery( qs )
- .setParameterList( "resultsAnalyzed", resultsAnalyzed )
- .setParameter( "threshold", threshold )
- .setMaxResults( limit )
- .list();
-
- for ( Object o : qResult ) {
-
- Object[] oa = ( Object[] ) o;
- ExpressionAnalysisResultSet ee = ( ExpressionAnalysisResultSet ) oa[0];
- DifferentialExpressionAnalysisResult probeResult = ( DifferentialExpressionAnalysisResult ) oa[1];
-
- if ( !results.containsKey( ee ) ) {
- results.put( ee, new ArrayList() );
- }
-
- results.get( ee ).add( probeResult );
- }
-
timer.stop();
if ( timer.getTime() > 1000 ) {
AbstractDao.log.info( "Diff ex results: " + timer.getTime() + " ms" );
}
- return results;
- }
-
- @Override
- public DifferentialExpressionAnalysis getAnalysis( ExpressionAnalysisResultSet rs ) {
- return ( DifferentialExpressionAnalysis ) this.getSessionFactory().getCurrentSession()
- .createQuery( "select distinct a from DifferentialExpressionAnalysis a join a.resultSets r where r=:r" )
- .setParameter( "r", rs )
- .uniqueResult();
- }
-
- @Override
- public Collection getExperimentalFactors(
- DifferentialExpressionAnalysisResult differentialExpressionAnalysisResult ) {
-
- //noinspection unchecked
- return this.getSessionFactory().getCurrentSession().createQuery( "select ef from ExpressionAnalysisResultSet rs"
- + " inner join rs.results r inner join rs.experimentalFactors ef where r=:differentialExpressionAnalysisResult" )
- .setParameter( "differentialExpressionAnalysisResult", differentialExpressionAnalysisResult ).list();
-
- }
-
- @Override
- public Map> getExperimentalFactors(
- Collection differentialExpressionAnalysisResults ) {
- StopWatch timer = new StopWatch();
- timer.start();
- Map> factorsByResult = new HashMap<>();
- if ( differentialExpressionAnalysisResults.isEmpty() ) {
- return factorsByResult;
- }
-
- //language=HQL
- final String queryString = "select rs.experimentalFactors, r from ExpressionAnalysisResultSet rs"
- + " inner join rs.results r where r in (:differentialExpressionAnalysisResults)";
-
- List qr = this.getSessionFactory().getCurrentSession()
- .createQuery( queryString )
- .setParameterList( "differentialExpressionAnalysisResults", differentialExpressionAnalysisResults )
- .list();
-
- if ( qr == null || qr.isEmpty() )
- return factorsByResult;
-
- for ( Object o : qr ) {
- Object[] ar = ( Object[] ) o;
- ExperimentalFactor f = ( ExperimentalFactor ) ar[0];
- DifferentialExpressionAnalysisResult res = ( DifferentialExpressionAnalysisResult ) ar[1];
-
- assert differentialExpressionAnalysisResults.contains( res );
-
- if ( !factorsByResult.containsKey( res ) ) {
- factorsByResult.put( res, new HashSet() );
- }
-
- factorsByResult.get( res ).add( f );
-
- if ( AbstractDao.log.isDebugEnabled() )
- AbstractDao.log.debug( res );
- }
- timer.stop();
- if ( timer.getTime() > 1000 ) {
- AbstractDao.log.info( "factors by results: " + timer.getTime() + " ms" );
- }
- return factorsByResult;
+ return results;
}
/**
@@ -690,9 +444,6 @@ public Map>
@Override
public Map loadContrastDetailsForResults( Collection ids ) {
//language=SQL
- final String queryString = "SELECT DISTINCT c.ID, c.LOG_FOLD_CHANGE, c.FACTOR_VALUE_FK,"
- + " c.DIFFERENTIAL_EXPRESSION_ANALYSIS_RESULT_FK, c.PVALUE FROM CONTRAST_RESULT c"
- + " WHERE c.DIFFERENTIAL_EXPRESSION_ANALYSIS_RESULT_FK IN (:ids) ";
Map probeResults = new HashMap<>();
@@ -700,7 +451,10 @@ public Map loadContrastDetailsForResults( Collection
return probeResults;
}
- SQLQuery query = this.getSessionFactory().getCurrentSession().createSQLQuery( queryString );
+ SQLQuery query = this.getSessionFactory().getCurrentSession()
+ .createSQLQuery( "SELECT DISTINCT c.ID, c.LOG_FOLD_CHANGE, c.FACTOR_VALUE_FK,"
+ + " c.DIFFERENTIAL_EXPRESSION_ANALYSIS_RESULT_FK, c.PVALUE FROM CONTRAST_RESULT c"
+ + " WHERE c.DIFFERENTIAL_EXPRESSION_ANALYSIS_RESULT_FK IN (:ids) " );
int BATCH_SIZE = 2000; // previously: 500, then 1000. New optimized query is plenty fast.
StopWatch timer = new StopWatch();
@@ -737,36 +491,15 @@ public Map loadContrastDetailsForResults( Collection
if ( timer.getTime() > 2000 ) {
AbstractDao.log.info( "Fetch " + batch.size() + " results with contrasts: " + timer.getTime()
- + "ms; query was\n " + queryString.replace( ":ids", StringUtils.join( batch, "," ) ) );
+ + "ms; query was\n " + ( "SELECT DISTINCT c.ID, c.LOG_FOLD_CHANGE, c.FACTOR_VALUE_FK,"
+ + " c.DIFFERENTIAL_EXPRESSION_ANALYSIS_RESULT_FK, c.PVALUE FROM CONTRAST_RESULT c"
+ + " WHERE c.DIFFERENTIAL_EXPRESSION_ANALYSIS_RESULT_FK IN (:ids) " ).replace( ":ids", StringUtils.join( batch, "," ) ) );
}
}
return probeResults;
}
- @Override
- public void thaw( final Collection results ) {
- for ( DifferentialExpressionAnalysisResult result : results ) {
- Hibernate.initialize( result );
- CompositeSequence cs = result.getProbe();
- Hibernate.initialize( cs );
- Hibernate.initialize( result.getContrasts() );
- }
- }
-
- @Override
- public void thaw( final DifferentialExpressionAnalysisResult result ) {
- Hibernate.initialize( result );
-
- CompositeSequence cs = result.getProbe();
- Hibernate.initialize( cs );
-
- Collection contrasts = result.getContrasts();
- for ( ContrastResult contrast : contrasts ) {
- Hibernate.initialize( contrast.getFactorValue() );
- }
- }
-
@Override
public Map> find( Gene gene,
double threshold, int limit ) {
@@ -775,14 +508,16 @@ public Map 0.0 ) {
sql = sql + " and d.CORRECTED_PVALUE < :threshold ";
}
if ( limit > 0 ) {
- sql = sql + " order by d.PVALUE ASC ";
+ sql = sql + " order by d.PVALUE";
}
SQLQuery query = session.createSQLQuery( sql );
@@ -849,9 +584,10 @@ public Map ees = session.createQuery(
- "select ee, rs from ExpressionAnalysisResultSet rs join fetch rs.experimentalFactors join rs.analysis a join a.experimentAnalyzed ee"
- + " where rs.id in (:rsids)" )
+ List ees = session
+ .createQuery( "select ee, rs from ExpressionAnalysisResultSet rs "
+ + "join fetch rs.experimentalFactors join rs.analysis a join a.experimentAnalyzed ee "
+ + "where rs.id in (:rsids)" )
.setParameterList( "rsids", resultSets ).list();
/*
@@ -862,7 +598,7 @@ public Map() );
+ results.put( evo, new ArrayList<>() );
}
for ( Iterator it = resultsInter.values().iterator(); it.hasNext(); ) {
@@ -948,7 +684,7 @@ private int processResultTuple( Object resultRow, Map() );
+ resultsFromDb.put( resultSetId, new HashMap<>() );
}
assert cs2GeneIdMap.containsKey( probeId );
@@ -1030,11 +766,11 @@ private Map> fillFromCache( Map() );
+ foundInCache.put( r, new HashSet<>() );
for ( DiffExprGeneSearchResult result : cached ) {
if ( !results.containsKey( r ) ) {
- results.put( r, new HashMap() );
+ results.put( r, new HashMap<>() );
}
results.get( r ).put( result.getGeneId(), result );
@@ -1116,9 +852,7 @@ private Map> getProbesForGenesInResultSetBatch( Session s
* resultSet
*/
private void processDiffExResultHit( Map results, Long resultSetId, Long geneId,
- Long resultId, Double correctedPvalue, Double uncorrectedPvalue ) {
-
- assert correctedPvalue != null;
+ Long resultId, double correctedPvalue, double uncorrectedPvalue ) {
DiffExprGeneSearchResult r = results.get( geneId );
if ( r == null ) { // first encounter
diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionResultService.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionResultService.java
index f37d8c58a3..a7122e09cf 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionResultService.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionResultService.java
@@ -115,38 +115,9 @@ Map> fi
Map> findDiffExAnalysisResultIdsInResultSets(
Collection resultSets, Collection geneIds );
- List findGeneInResultSet( Gene gene, ExpressionAnalysisResultSet resultSet, Collection arrayDesignIds,
- int limit );
-
List findInResultSet( ExpressionAnalysisResultSet ar, Double threshold,
int maxResultsToReturn, int minNumberOfResults );
- /**
- * Given a list of result sets finds the diff expression results that met the given threshold
- *
- * @param threshold threshold
- * @param limit limit
- * @return map to diff ex VOs
- */
- @Secured({ "IS_AUTHENTICATED_ANONYMOUSLY", "AFTER_ACL_MAP_READ" })
- Map> findInResultSets(
- Collection resultsAnalyzed, double threshold, int limit );
-
- /**
- * Fetch the analysis associated with a result set.
- *
- * @param rs result set
- * @return diff ex.
- */
- DifferentialExpressionAnalysis getAnalysis( ExpressionAnalysisResultSet rs );
-
- Map> getExperimentalFactors(
- Collection differentialExpressionAnalysisResults );
-
- @Secured({ "IS_AUTHENTICATED_ANONYMOUSLY", "AFTER_ACL_COLLECTION_READ" })
- Collection getExperimentalFactors(
- DifferentialExpressionAnalysisResult differentialExpressionAnalysisResult );
-
/**
* @param ids ids
* @return map of result to contrasts value object.
diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionResultServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionResultServiceImpl.java
index 06d18d1a2c..bbaa875380 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionResultServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionResultServiceImpl.java
@@ -24,7 +24,6 @@
import ubic.basecode.math.distribution.Histogram;
import ubic.gemma.model.analysis.expression.diff.ExpressionAnalysisResultSet;
import ubic.gemma.model.analysis.expression.diff.*;
-import ubic.gemma.model.expression.experiment.ExperimentalFactor;
import ubic.gemma.model.expression.experiment.ExpressionExperimentValueObject;
import ubic.gemma.model.genome.Gene;
import ubic.gemma.persistence.service.AbstractService;
@@ -53,27 +52,27 @@ public DifferentialExpressionResultServiceImpl( DifferentialExpressionResultDao
@Transactional(readOnly = true)
public Map> find(
Collection experimentsAnalyzed, double threshold, int limit ) {
- return this.DERDao.find( experimentsAnalyzed, threshold, limit );
+ return this.DERDao.findByExperimentAnalyzed( experimentsAnalyzed, threshold, limit );
}
@Override
@Transactional(readOnly = true)
public Map> find( Gene gene ) {
- return this.DERDao.find( gene );
+ return this.DERDao.findByGene( gene );
}
@Override
@Transactional(readOnly = true)
public Map> find( Gene gene,
Collection experimentsAnalyzed ) {
- return this.DERDao.find( gene, experimentsAnalyzed );
+ return this.DERDao.findByGeneAndExperimentAnalyzed( gene, experimentsAnalyzed );
}
@Override
@Transactional(readOnly = true)
public Map> find( Gene gene,
Collection experimentsAnalyzed, double threshold, int limit ) {
- return this.DERDao.find( gene, experimentsAnalyzed, threshold, limit );
+ return this.DERDao.findByGeneAndExperimentAnalyzed( gene, experimentsAnalyzed, threshold, limit );
}
@Override
@@ -90,13 +89,6 @@ public Map> findDiffExAnalysisResultId
return this.DERDao.findDiffExAnalysisResultIdsInResultSets( resultSets, geneIds );
}
- @Override
- @Transactional(readOnly = true)
- public List findGeneInResultSet( Gene gene, ExpressionAnalysisResultSet resultSet,
- Collection arrayDesignIds, int limit ) {
- return this.DERDao.findGeneInResultSets( gene, resultSet, arrayDesignIds, limit );
- }
-
@Override
@Transactional(readOnly = true)
public List findInResultSet( ExpressionAnalysisResultSet resultSet,
@@ -104,33 +96,6 @@ public List findInResultSet( ExpressionAnalys
return this.DERDao.findInResultSet( resultSet, threshold, maxResultsToReturn, minNumberOfResults );
}
- @Override
- @Transactional(readOnly = true)
- public Map> findInResultSets(
- Collection resultsAnalyzed, double threshold, int limit ) {
- return this.DERDao.findInResultSets( resultsAnalyzed, threshold, limit );
- }
-
- @Override
- @Transactional(readOnly = true)
- public DifferentialExpressionAnalysis getAnalysis( ExpressionAnalysisResultSet rs ) {
- return this.DERDao.getAnalysis( rs );
- }
-
- @Override
- @Transactional(readOnly = true)
- public Map> getExperimentalFactors(
- Collection differentialExpressionAnalysisResults ) {
- return this.DERDao.getExperimentalFactors( differentialExpressionAnalysisResults );
- }
-
- @Override
- @Transactional(readOnly = true)
- public Collection getExperimentalFactors(
- DifferentialExpressionAnalysisResult differentialExpressionAnalysisResult ) {
- return this.DERDao.getExperimentalFactors( differentialExpressionAnalysisResult );
- }
-
@Override
@Transactional(readOnly = true)
public Map loadContrastDetailsForResults( Collection ids ) {
@@ -142,5 +107,4 @@ public Map loadContrastDetailsForResults( Collection
public Histogram loadPvalueDistribution( Long resultSetId ) {
return this.DERDao.loadPvalueDistribution( resultSetId );
}
-
}
\ No newline at end of file
diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/ExpressionAnalysisResultSetDao.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/ExpressionAnalysisResultSetDao.java
index 75db8cd857..4bd6c795ff 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/ExpressionAnalysisResultSetDao.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/ExpressionAnalysisResultSetDao.java
@@ -20,8 +20,8 @@
import ubic.gemma.model.analysis.expression.diff.DifferentialExpressionAnalysis;
import ubic.gemma.model.analysis.expression.diff.DifferentialExpressionAnalysisResult;
-import ubic.gemma.model.analysis.expression.diff.ExpressionAnalysisResultSet;
import ubic.gemma.model.analysis.expression.diff.DifferentialExpressionAnalysisResultSetValueObject;
+import ubic.gemma.model.analysis.expression.diff.ExpressionAnalysisResultSet;
import ubic.gemma.model.common.Identifiable;
import ubic.gemma.model.common.description.DatabaseEntry;
import ubic.gemma.model.expression.experiment.BioAssaySet;
@@ -56,10 +56,10 @@ public interface ExpressionAnalysisResultSetDao extends AnalysisResultSetDao
* This is much faster than navigating through the probe's alignments, transcripts and then genes as it uses the
* internal GENE2CS table described in {@link TableMaintenanceUtil#updateGene2CsEntries()}.
- *
+ *
* Note: Not all probes have associated genes, so you should use {@link Map#getOrDefault(Object, Object)} with an
* empty collection to handle this case.
*/
@@ -76,5 +76,8 @@ public interface ExpressionAnalysisResultSetDao extends AnalysisResultSetDao findByBioAssaySetInAndDatabaseEntryInLimit( @Nullable Collection bioAssaySets, @Nullable Collection databaseEntries, @Nullable Filters filters, int offset, int limit, @Nullable Sort sort );
+ /**
+ * Initialize the analysis and subset factor vale.
+ */
void thaw( ExpressionAnalysisResultSet ears );
}
diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/ExpressionAnalysisResultSetDaoImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/ExpressionAnalysisResultSetDaoImpl.java
index 82fd05d0db..aea15139e5 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/ExpressionAnalysisResultSetDaoImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/ExpressionAnalysisResultSetDaoImpl.java
@@ -27,10 +27,7 @@
import org.hibernate.type.StandardBasicTypes;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Repository;
-import ubic.gemma.model.analysis.expression.diff.DifferentialExpressionAnalysis;
-import ubic.gemma.model.analysis.expression.diff.DifferentialExpressionAnalysisResultSetValueObject;
-import ubic.gemma.model.analysis.expression.diff.ExpressionAnalysisResultSet;
-import ubic.gemma.model.analysis.expression.diff.PvalueDistribution;
+import ubic.gemma.model.analysis.expression.diff.*;
import ubic.gemma.model.common.description.Characteristic;
import ubic.gemma.model.common.description.DatabaseEntry;
import ubic.gemma.model.common.protocol.Protocol;
@@ -79,13 +76,17 @@ public ExpressionAnalysisResultSet loadWithResultsAndContrasts( Long id ) {
ExpressionAnalysisResultSet ears = ( ExpressionAnalysisResultSet ) getSessionFactory().getCurrentSession()
.createQuery( "select ears from ExpressionAnalysisResultSet ears "
+ "left join fetch ears.results res "
- + "left join fetch res.probe probe "
- + "left join fetch probe.biologicalCharacteristic bc "
- + "left join fetch bc.sequenceDatabaseEntry "
+ "left join fetch res.contrasts "
+ "where ears.id = :rsId" )
.setParameter( "rsId", id )
.uniqueResult();
+ if ( ears != null ) {
+ for ( DifferentialExpressionAnalysisResult r : ears.getResults() ) {
+ // will also initialize the biological characteristics and sequence database entries
+ // this is efficient because of batch loading and second-level caching
+ Hibernate.initialize( r.getProbe() );
+ }
+ }
if ( timer.getTime() > 1000 ) {
log.info( String.format( "Loaded [%s id=%d] with results, probes and contrasts in %d ms.",
elementClass.getName(), id, timer.getTime() ) );
@@ -139,12 +140,20 @@ public void thaw( ExpressionAnalysisResultSet ears ) {
// this drastically reduces the number of columns fetched which would anyway be repeated
Hibernate.initialize( ears.getAnalysis() );
Hibernate.initialize( ears.getAnalysis().getExperimentAnalyzed() );
+
// it is faster to query those separately because there's a large number of rows fetched via the results &
// contrasts and only a handful of factors
Hibernate.initialize( ears.getExperimentalFactors() );
+
// factor values are always eagerly fetched (see ExperimentalFactor.hbm.xml), so we don't need to initialize.
// I still think it's neat to use stream API for that though in case we ever make them lazy:
// resultSet.getExperimentalFactors().stream().forEach( Hibernate::initialize );
+
+ // this needs to be initialized because it does not appear in the experimental factors
+ if ( ears.getAnalysis().getSubsetFactorValue() != null ) {
+ Hibernate.initialize( ears.getAnalysis().getSubsetFactorValue() );
+ Hibernate.initialize( ears.getAnalysis().getSubsetFactorValue().getExperimentalFactor() );
+ }
}
@Override
diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/sampleCoexpression/PreparedCoexMatrices.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/sampleCoexpression/PreparedCoexMatrices.java
new file mode 100644
index 0000000000..2a9ab06bb9
--- /dev/null
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/sampleCoexpression/PreparedCoexMatrices.java
@@ -0,0 +1,35 @@
+/*
+ * The Gemma project
+ *
+ * Copyright (c) 2011 University of British Columbia
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package ubic.gemma.persistence.service.analysis.expression.sampleCoexpression;
+
+import ubic.gemma.model.analysis.expression.coexpression.SampleCoexpressionMatrix;
+
+import javax.annotation.Nullable;
+
+/**
+ * Helper class for sample coexpression analysis.
+ */
+public class PreparedCoexMatrices {
+
+ public PreparedCoexMatrices( @Nullable SampleCoexpressionMatrix matrix, @Nullable SampleCoexpressionMatrix regressedMatrix ) {
+ this.matrix = matrix;
+ this.regressedMatrix = regressedMatrix;
+ }
+
+ @Nullable
+ SampleCoexpressionMatrix matrix;
+ @Nullable
+ SampleCoexpressionMatrix regressedMatrix;
+}
diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/sampleCoexpression/SampleCoexpressionAnalysisService.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/sampleCoexpression/SampleCoexpressionAnalysisService.java
index 08fa9cd586..35bb569ff6 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/sampleCoexpression/SampleCoexpressionAnalysisService.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/sampleCoexpression/SampleCoexpressionAnalysisService.java
@@ -14,8 +14,11 @@
*/
package ubic.gemma.persistence.service.analysis.expression.sampleCoexpression;
+import org.springframework.transaction.annotation.Isolation;
+import org.springframework.transaction.annotation.Transactional;
import ubic.basecode.dataStructure.matrix.DoubleMatrix;
import ubic.gemma.model.analysis.expression.coexpression.SampleCoexpressionAnalysis;
+import ubic.gemma.model.analysis.expression.coexpression.SampleCoexpressionMatrix;
import ubic.gemma.model.expression.bioAssay.BioAssay;
import ubic.gemma.model.expression.experiment.ExpressionExperiment;
@@ -57,6 +60,14 @@ public interface SampleCoexpressionAnalysisService {
@Nullable
DoubleMatrix loadBestMatrix( ExpressionExperiment ee );
+
+ @Transactional(readOnly = true)
+ @Nullable
+ public DoubleMatrix retrieveExisting( ExpressionExperiment ee );
+
+ @Transactional(readOnly = true)
+ PreparedCoexMatrices prepare( ExpressionExperiment ee );
+
/**
* Computes sample correlation matrices for the given experiment. If the experiment already has any, they are
* removed. See {@link SampleCoexpressionAnalysis} for the description of the matrices that are computed.
@@ -64,22 +75,8 @@ public interface SampleCoexpressionAnalysisService {
* @param ee the experiment to create a sample correlation matrix for.
* @return the regressed coexpression matrix if available, otherwise the full
*/
- DoubleMatrix compute( ExpressionExperiment ee );
-
- /**
- * Computes sample correlation matrices for the given experiment if necessary.
- *
- * The computation happens if at least one of the following condition is met:
- *
- * - there is no {@link SampleCoexpressionAnalysis} for the given experiment
- * - the full coexpression matrix is null
- * - the regressed coexpression matrix is null and there is an important factor
- *
- *
- * @see #compute(ExpressionExperiment)
- * @return the regressed coexpression matrix if available, otherwise the full
- */
- DoubleMatrix computeIfNecessary( ExpressionExperiment ee );
+ @Transactional(isolation = Isolation.SERIALIZABLE)
+ DoubleMatrix compute( ExpressionExperiment ee, PreparedCoexMatrices matrices );
boolean hasAnalysis( ExpressionExperiment ee );
@@ -89,4 +86,5 @@ public interface SampleCoexpressionAnalysisService {
* @param ee the experiment to remove the analysis for.
*/
void removeForExperiment( ExpressionExperiment ee );
-}
\ No newline at end of file
+}
+
diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/sampleCoexpression/SampleCoexpressionAnalysisServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/sampleCoexpression/SampleCoexpressionAnalysisServiceImpl.java
index 776e174c40..8cfe992744 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/sampleCoexpression/SampleCoexpressionAnalysisServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/sampleCoexpression/SampleCoexpressionAnalysisServiceImpl.java
@@ -116,44 +116,56 @@ public DoubleMatrix loadBestMatrix( ExpressionExperiment ee
}
@Override
- @Transactional(isolation = Isolation.SERIALIZABLE)
- public DoubleMatrix computeIfNecessary( ExpressionExperiment ee ) {
+ @Transactional(readOnly = true)
+ public DoubleMatrix retrieveExisting( ExpressionExperiment ee ) {
ExpressionExperiment thawedee = this.expressionExperimentService.thawLite( ee );
SampleCoexpressionAnalysis analysis = sampleCoexpressionAnalysisDao.load( thawedee );
if ( analysis == null || analysis.getFullCoexpressionMatrix() == null || this.shouldComputeRegressed( thawedee, analysis ) ) {
SampleCoexpressionAnalysisServiceImpl.log
.info( String.format( SampleCoexpressionAnalysisServiceImpl.MSG_INFO_RUNNING_SCM, thawedee.getId() ) );
- return this.compute( thawedee );
+ return null;
} else {
this.logCormatStatus( analysis, false );
return toDoubleMatrix( analysis.getBestCoexpressionMatrix() );
}
}
+
+ @Override
+ @Transactional(readOnly = true)
+ public PreparedCoexMatrices prepare( ExpressionExperiment ee ) {
+ // Create new analysis
+ Collection vectors = processedExpressionDataVectorService
+ .getProcessedDataVectors( ee );
+ SampleCoexpressionMatrix matrix = this.getMatrix( ee, false, vectors );
+ SampleCoexpressionMatrix regressedMatrix = this.getMatrix( ee, true, vectors );
+ return new PreparedCoexMatrices( matrix, regressedMatrix );
+ }
+
+
/**
* Unfortunately, this method breaks under high contention (see #400,
* so we need to fully lock the database while undergoing using {@link Isolation#SERIALIZABLE} transaction isolation
* level. This annotation also has to be appled to other methods of this class that call compute() directly or
* indirectly.
+ *
+ * PP changed this to do more of the processing in a read-only transaction
*/
@Override
@Transactional(isolation = Isolation.SERIALIZABLE)
- public DoubleMatrix compute( ExpressionExperiment ee ) {
+ public DoubleMatrix compute( ExpressionExperiment ee, PreparedCoexMatrices matrices ) {
+ SampleCoexpressionMatrix matrix = matrices.matrix;
+
+ if ( matrix == null ) {
+ throw new RuntimeException( "Full coexpression matrix could not be computed." );
+ }
ExpressionExperiment thawedee = this.expressionExperimentService.thawLite( ee );
// Remove any old data
this.removeForExperiment( thawedee );
- // Create new analysis
- Collection vectors = processedExpressionDataVectorService
- .getProcessedDataVectors( thawedee );
- SampleCoexpressionMatrix matrix = this.getMatrix( thawedee, false, vectors );
- SampleCoexpressionMatrix regressedMatrix = this.getMatrix( thawedee, true, vectors );
-
- if ( matrix == null ) {
- throw new RuntimeException( "Full coexpression matrix could not be computed." );
- }
+ SampleCoexpressionMatrix regressedMatrix = matrices.regressedMatrix; // this one is optional
// this one is optional
if ( regressedMatrix == null ) {
@@ -408,3 +420,5 @@ private ExpressionDataDoubleMatrix regressionResiduals( ExpressionDataDoubleMatr
return new ExpressionDataDoubleMatrix( dmatrix, f, dmatrix.getQuantitationTypes() );
}
}
+
+
diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/association/coexpression/CoexpressionDaoImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/association/coexpression/CoexpressionDaoImpl.java
index f1ffd82883..08b5579d61 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/association/coexpression/CoexpressionDaoImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/association/coexpression/CoexpressionDaoImpl.java
@@ -2083,11 +2083,12 @@ private void removeGeneCoexpressedWith( Long geneId, Collection removedGen
.createQuery( "from GeneCoexpressedGenes where geneId = :id" ).setParameter( "id", geneId )
.uniqueResult();
// note this might be a no-op.
- for ( Long g : removedGenes ) {
- gcti.removeEntity( g );
- sess.update( gcti );
+ if ( gcti != null ) {
+ for ( Long g : removedGenes ) {
+ gcti.removeEntity( g );
+ sess.update( gcti );
+ }
}
-
}
/**
diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/association/coexpression/CoexpressionQueryQueueImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/association/coexpression/CoexpressionQueryQueueImpl.java
index 8543a56d07..479e5aa262 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/association/coexpression/CoexpressionQueryQueueImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/association/coexpression/CoexpressionQueryQueueImpl.java
@@ -43,7 +43,7 @@ class CoexpressionQueryQueueImpl implements CoexpressionQueryQueue, Initializing
private final CoexpressionDao coexpressionDao;
private final GeneDao geneDao;
- private Future queryCacheFuture;
+ private final ExecutorService queryCacheExecutor = Executors.newSingleThreadExecutor();
public CoexpressionQueryQueueImpl( CoexpressionDao coexpressionDao, GeneDao geneDao ) {
this.coexpressionDao = coexpressionDao;
@@ -79,7 +79,7 @@ public void removeFromQueue( Collection genes ) {
@Override
public void afterPropertiesSet() throws Exception {
- queryCacheFuture = Executors.newSingleThreadExecutor().submit( new DelegatingSecurityContextRunnable( new Runnable() {
+ queryCacheExecutor.submit( new DelegatingSecurityContextRunnable( new Runnable() {
private static final int MAX_WARNINGS = 5;
@@ -109,13 +109,13 @@ public void run() {
}
}
}
-
} ) );
}
@Override
public void destroy() throws Exception {
- queryCacheFuture.cancel( true );
+ // filling the coexpression cache is not really critical, so we can safely interrupt it
+ queryCacheExecutor.shutdownNow();
}
private void queryForCache( Long geneId ) {
diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/common/auditAndSecurity/AuditEventDao.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/common/auditAndSecurity/AuditEventDao.java
index 6a0e59399d..d49b8ea0cc 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/common/auditAndSecurity/AuditEventDao.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/common/auditAndSecurity/AuditEventDao.java
@@ -76,4 +76,6 @@ Map, Map> getLastEventsBy
void retainHavingEvent( Collection a, Class type );
void retainLackingEvent( Collection a, Class type );
+
+ Map getCreateEvents( Collection auditables );
}
diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/common/auditAndSecurity/AuditEventDaoImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/common/auditAndSecurity/AuditEventDaoImpl.java
index 023b8ccfb7..6bbe63666c 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/common/auditAndSecurity/AuditEventDaoImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/common/auditAndSecurity/AuditEventDaoImpl.java
@@ -48,8 +48,8 @@ public class AuditEventDaoImpl extends AbstractDao implements AuditE
* Classes that we track for 'updated since'. This is used for "What's new" functionality.
*/
private static final String[] AUDITABLES_TO_TRACK_FOR_WHATS_NEW = {
- "ubic.gemma.model.expression.arrayDesign.ArrayDesign",
- "ubic.gemma.model.expression.experiment.ExpressionExperiment" };
+ "ubic.gemma.model.expression.arrayDesign.ArrayDesign",
+ "ubic.gemma.model.expression.experiment.ExpressionExperiment" };
@Autowired
public AuditEventDaoImpl( SessionFactory sessionFactory ) {
@@ -68,8 +68,8 @@ public List getEvents( final Auditable auditable ) {
Long id = auditable.getAuditTrail().getId();
//noinspection unchecked
return this.getSessionFactory().getCurrentSession()
- .createQuery( "select e from AuditTrail t join t.events e where t.id = :id order by e.date,e.id " )
- .setParameter( "id", id ).list();
+ .createQuery( "select e from AuditTrail t join t.events e where t.id = :id order by e.date,e.id " )
+ .setParameter( "id", id ).list();
}
@@ -85,13 +85,13 @@ public AuditEvent getLastEvent( Auditable auditable, Class, Map> getLastEventsByType(
- Collection auditables, Collection> types ) {
+ Collection auditables, Collection> types ) {
Map, Map> results = new HashMap<>();
for ( Class ti : types ) {
Map results2 = getLastEvents( auditables, ti, null );
results.put( ti, results2.entrySet().stream()
- .filter( e -> ti.isAssignableFrom( e.getValue().getEventType().getClass() ) )
- .collect( Collectors.toMap( Map.Entry::getKey, Map.Entry::getValue ) ) );
+ .filter( e -> ti.isAssignableFrom( e.getValue().getEventType().getClass() ) )
+ .collect( Collectors.toMap( Map.Entry::getKey, Map.Entry::getValue ) ) );
}
return results;
}
@@ -108,7 +108,7 @@ public Collection getNewSinceDate( Date date ) {
Collection result = new HashSet<>();
for ( String clazz : AuditEventDaoImpl.AUDITABLES_TO_TRACK_FOR_WHATS_NEW ) {
String queryString = "select distinct adb from " + clazz
- + " adb inner join adb.auditTrail atr inner join atr.events as ae where ae.date > :date and ae.action='C'";
+ + " adb inner join adb.auditTrail atr inner join atr.events as ae where ae.date > :date and ae.action='C'";
this.tryAddAllToResult( result, queryString, date );
}
return result;
@@ -126,7 +126,7 @@ public Collection getUpdatedSinceDate( Date date ) {
Collection result = new HashSet<>();
for ( String clazz : AuditEventDaoImpl.AUDITABLES_TO_TRACK_FOR_WHATS_NEW ) {
String queryString = "select distinct adb from " + clazz
- + " adb inner join adb.auditTrail atr inner join atr.events as ae where ae.date > :date and ae.action='U'";
+ + " adb inner join adb.auditTrail atr inner join atr.events as ae where ae.date > :date and ae.action='U'";
this.tryAddAllToResult( result, queryString, date );
}
return result;
@@ -139,7 +139,7 @@ public boolean hasEvent( Auditable a, Class type ) {
@Override
public void retainHavingEvent( final Collection a,
- final Class type ) {
+ final Class type ) {
final Map events = this.getLastEvents( a, type, null );
@@ -149,7 +149,7 @@ public void retainHavingEvent( final Collection a,
@Override
public void retainLackingEvent( final Collection a,
- final Class type ) {
+ final Class type ) {
StopWatch timer = new StopWatch();
timer.start();
final Map events = this.getLastEvents( a, type, null );
@@ -159,6 +159,32 @@ public void retainLackingEvent( final Collection a,
}
+ public Map getCreateEvents( final Collection auditables ) {
+
+ if ( auditables.isEmpty() ) {
+ return Collections.emptyMap();
+ }
+
+ Map result = new HashMap<>( auditables.size() );
+
+ final Map atMap = auditables.stream()
+ .collect( Collectors.toMap( a -> a.getAuditTrail().getId(), Function.identity() ) );
+
+ final String queryString = "select trail.id, ae from AuditTrail trail join trail.events" +
+ " ae where trail.id in :trails and ae.action = 'C'";
+ Query queryObject = this.getSessionFactory().getCurrentSession()
+ .createQuery( queryString )
+ .setParameterList( "trails", atMap.keySet() );
+ List qr = queryObject.list();
+ for ( Object o : qr ) {
+ Object[] ar = ( Object[] ) o;
+ Long t = ( Long ) ar[0];
+ AuditEvent e = ( AuditEvent ) ar[1];
+ result.put( atMap.get( t ), e );
+ }
+ return result;
+ }
+
private Map