Only consider NGS samples (#246)

jenniferboedker · KochTobi · wow-such-code · web-flow · commit 5f924c608ebc · 2022-09-15T10:35:03.000+02:00
Filters openbis projects for projects containing NGS samples and thus being of type DNA or RNA. This remove most samples for which no tracking information is available. Furthermore, full proteomics projects are removed since we cannot provide reliable tracking information here. Also, this PR fixes the number of samples shown for the project status.

Co-authored-by: Tobias Koch &lt;tobias.koch@qbic.uni-tuebingen.de&gt;
Co-authored-by: wow-such-code &lt;andreas.friedrich@uni-tuebingen.de&gt;
diff --git a/sample-tracking-status-overview-app/src/main/groovy/life/qbic/portal/sampletracking/DependencyManager.groovy b/sample-tracking-status-overview-app/src/main/groovy/life/qbic/portal/sampletracking/DependencyManager.groovy
@@ -107,7 +107,7 @@ class DependencyManager {
             "/v2/samples",
             "/status",
             "/v2/projects",
-            credentials)
+            credentials, getNgsSampleRepository())
   }
 
     /**
@@ -174,4 +174,8 @@ class DependencyManager {
   DownloadManifestProvider getDownloadManifestProvider() {
     return sampleTrackingConnector
   }
+
+  NgsSampleRepository getNgsSampleRepository() {
+    return openBisConnector
+  }
 }
diff --git a/sample-tracking-status-overview-app/src/main/groovy/life/qbic/portal/sampletracking/data/NgsSampleRepository.java b/sample-tracking-status-overview-app/src/main/groovy/life/qbic/portal/sampletracking/data/NgsSampleRepository.java
@@ -0,0 +1,18 @@
+package life.qbic.portal.sampletracking.data;
+
+import java.util.List;
+
+/**
+ * <b>Repository for loading NGS samples</b>
+ *
+ * @since 1.1.3
+ */
+public interface NgsSampleRepository {
+
+    /**
+     * Finds all NGS samples for a given project
+     * @param projectCode the code specifying a project
+     * @return a list of sample codes from all NGS samples of that project
+     */
+    public List<String> findNGSSamplesForProject(String projectCode);
+}
diff --git a/sample-tracking-status-overview-app/src/main/groovy/life/qbic/portal/sampletracking/data/OpenBisConnector.java b/sample-tracking-status-overview-app/src/main/groovy/life/qbic/portal/sampletracking/data/OpenBisConnector.java
@@ -1,5 +1,8 @@
 package life.qbic.portal.sampletracking.data;
 
+import static java.util.stream.Collectors.groupingBy;
+import static java.util.stream.Collectors.toList;
+
 import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi;
 import ch.ethz.sis.openbis.generic.asapi.v3.dto.common.search.SearchResult;
 import ch.ethz.sis.openbis.generic.asapi.v3.dto.project.fetchoptions.ProjectFetchOptions;
@@ -8,28 +11,26 @@
 import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.search.SampleSearchCriteria;
 import ch.systemsx.cisd.common.spring.HttpInvokerUtils;
 import java.util.ArrayList;
-import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Optional;
 import java.util.function.Function;
 import java.util.function.Predicate;
-import java.util.stream.Collectors;
 import life.qbic.datamodel.dtos.portal.PortalUser;
 import life.qbic.portal.sampletracking.view.projects.viewmodel.Project;
 import life.qbic.portal.sampletracking.view.samples.viewmodel.Sample;
 
-public class OpenBisConnector implements ProjectRepository, SampleRepository {
+public class OpenBisConnector implements ProjectRepository, SampleRepository, NgsSampleRepository {
 
   private final String sessionToken;
 
   private final IApplicationServerApi api;
 
   private static final int TIMEOUT = 10_000;
-
-  private final List<ch.ethz.sis.openbis.generic.asapi.v3.dto.project.Project> cachedProjects = new ArrayList<>();
-  private final Map<String, List<ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample>> cachedSamples = new HashMap<>();
+  private final Map<String, List<ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample>> sampleCache = new HashMap<>();
+  private final Map<String, ch.ethz.sis.openbis.generic.asapi.v3.dto.project.Project> projectCache = new HashMap<>();
 
 
 
@@ -43,64 +44,72 @@ public OpenBisConnector(Credentials credentials, PortalUser portalUser,
 
   @Override
   public List<Project> findAllProjects() {
-    if (cachedProjects.isEmpty()) {
-      ProjectFetchOptions fetchOptions = new ProjectFetchOptions();
-      fetchOptions.withSpace();
-      SearchResult<ch.ethz.sis.openbis.generic.asapi.v3.dto.project.Project> projectSearchResult =
-          api.searchProjects(sessionToken, new ProjectSearchCriteria(), fetchOptions);
-
-      List<ch.ethz.sis.openbis.generic.asapi.v3.dto.project.Project> projects = projectSearchResult.getObjects().stream()
-          .filter(hasValidProjectCode())
-          .sorted(Comparator.comparing(
-              ch.ethz.sis.openbis.generic.asapi.v3.dto.project.Project::getModificationDate).reversed())
-          .collect(Collectors.toList());
-      cachedProjects.addAll(projects);
+    if (projectCache.isEmpty()) {
+      updateCache();
     }
-    return cachedProjects.stream()
-        .map(it -> new Project(it.getCode(),
-            Optional.ofNullable(it.getDescription()).orElse("")))
-        .collect(Collectors.toList());
+    return projectCache.entrySet().stream()
+        .map(it -> new Project(it.getKey(), Optional.ofNullable(it.getValue().getDescription()).orElse("")))
+        .collect(toList());
+  }
+
+  @Override
+  public List<String> findNGSSamplesForProject(String projectCode){
+
+    return findAllSamplesForProject(projectCode).stream()
+            .map(Sample::code)
+            .collect(toList());
   }
 
   @Override
   public List<Sample> findAllSamplesForProject(String projectCode) {
 
-    List<ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample> samples;
-    if (cachedSamples.containsKey(projectCode)) {
-      samples = cachedSamples.get(projectCode);
-    } else {
-      SampleSearchCriteria sampleSearchCriteria = new SampleSearchCriteria();
-      sampleSearchCriteria.withCode().thatStartsWith(projectCode);
-      sampleSearchCriteria.withType().withCode().thatEquals("Q_TEST_SAMPLE");
-      SampleFetchOptions fetchOptions = new SampleFetchOptions();
-      fetchOptions.withType();
-      fetchOptions.withProperties();
-      SearchResult<ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample> sampleSearchResult = api.searchSamples(
-          sessionToken, sampleSearchCriteria, fetchOptions);
-      samples = sampleSearchResult.getObjects();
-      cachedSamples.put(projectCode, samples);
-    }
-    return samples.stream()
+    List<ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample> cachedProjectSamples = sampleCache
+        .entrySet().stream()
+        .filter(it -> it.getKey().equals(projectCode)).map(Entry::getValue).findAny().orElse(new ArrayList<>());
+    return cachedProjectSamples.stream().map(convertToSample()).collect(toList());
+  }
+
+  private void updateCache() {
+
+    SampleSearchCriteria isNgs = new SampleSearchCriteria();
+    isNgs.withOrOperator();
+    isNgs.withProperty("Q_SAMPLE_TYPE").thatContains("DNA");
+    isNgs.withProperty("Q_SAMPLE_TYPE").thatContains("RNA");
+    // this guarantees that no q-entity is contained as only test samples have Q_SAMPLE_TYPE
+
+    SampleFetchOptions fetchOptions = new SampleFetchOptions();
+    fetchOptions.withType();
+    fetchOptions.withProperties();
+    SearchResult<ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample> sampleSearchResult = api.searchSamples(
+        sessionToken, isNgs, fetchOptions);
+    Map<String, List<ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample>> projectSampleMap = sampleSearchResult.getObjects()
+        .stream()
         .filter(hasValidSampleCode())
-        .filter(notAnEntity())
-        .map(convertToSample())
-        .collect(Collectors.toList());
+        .collect(groupingBy(this::getProject));
+    sampleCache.putAll(projectSampleMap);
+
+    ProjectFetchOptions projectFetchOptions = new ProjectFetchOptions();
+    projectFetchOptions.withSpace();
+    ProjectSearchCriteria projectSearchCriteria = new ProjectSearchCriteria();
+
+    SearchResult<ch.ethz.sis.openbis.generic.asapi.v3.dto.project.Project> projectSearchResult = api.searchProjects(
+        sessionToken, projectSearchCriteria, projectFetchOptions);
+    List<ch.ethz.sis.openbis.generic.asapi.v3.dto.project.Project> projects = projectSearchResult.getObjects()
+        .stream().filter(it -> sampleCache.containsKey(it.getCode())).collect(toList());
+    projects.forEach(it -> projectCache.put(it.getCode(), it));
   }
 
-  private Predicate<? super ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample> hasValidSampleCode() {
-    return sample -> sample.getCode().matches("Q[A-X0-9]{4}[0-9]{3}[A-X0-9]{2}$");
+  private String getProject(
+      ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample sample) {
+    return sample.getCode().substring(0, 5);
   }
 
-  private Predicate<ch.ethz.sis.openbis.generic.asapi.v3.dto.project.Project> hasValidProjectCode() {
-    return project -> project.getCode().matches("^Q[A-X0-9]{4}$");
+  private Predicate<? super ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample> hasValidSampleCode() {
+    return sample -> sample.getCode().matches("Q[A-X0-9]{4}[0-9]{3}[A-X0-9]{2}$");
   }
 
   private static Function<ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample, Sample> convertToSample() {
     return it -> new Sample(it.getCode(), it.getProperty("Q_SECONDARY_NAME"));
   }
 
-  private static Predicate<ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample> notAnEntity() {
-    return it -> !it.getCode().contains("ENTITY");
-  }
-
 }
diff --git a/sample-tracking-status-overview-app/src/main/groovy/life/qbic/portal/sampletracking/data/SampleTrackingConnector.java b/sample-tracking-status-overview-app/src/main/groovy/life/qbic/portal/sampletracking/data/SampleTrackingConnector.java
@@ -50,15 +50,17 @@ public class SampleTrackingConnector implements ProjectStatusProvider, SampleSta
   private final String serviceAddress;
   private final String serviceUser;
   private final String userPass;
+  private final NgsSampleRepository ngsSampleRepository;
 
   public SampleTrackingConnector(String serviceUrlBase, String samplesSuffix, String statusSuffix,
-      String projectsSuffix, Credentials credentials) {
+                                 String projectsSuffix, Credentials credentials, NgsSampleRepository ngsSampleRepository) {
     this.samplesSuffix = samplesSuffix;
     this.statusSuffix = statusSuffix;
     this.projectsSuffix = projectsSuffix;
     this.serviceAddress = Objects.requireNonNull(serviceUrlBase);
     this.serviceUser = Objects.requireNonNull(credentials.getUser());
     this.userPass = Objects.requireNonNull(credentials.getPassword());
+    this.ngsSampleRepository = ngsSampleRepository;
   }
 
   @Override
@@ -113,14 +115,14 @@ public Instant statusValidSince() {
   public Optional<ProjectStatus> getForProject(String projectCode) {
     Optional<ProjectStatus> cachedStatusForProject = getCachedStatusForProject(projectCode);
     return cachedStatusForProject.isPresent() ? cachedStatusForProject
-        : askServiceForProject(projectCode).map(ProjectSatusMapper::toProjectStatus);
+        : askServiceForProject(projectCode).map(ProjectStatusMapper::toProjectStatus);
   }
 
   private Optional<ProjectStatus> getCachedStatusForProject(String projectCode) {
     synchronized (cachedProjects) {
       if (cachedProjects.containsKey(projectCode)) {
         Project project = cachedProjects.get(projectCode);
-        return Optional.of(ProjectSatusMapper.toProjectStatus(project));
+        return Optional.of(ProjectStatusMapper.toProjectStatus(project));
       }
     }
     return Optional.empty();
@@ -158,6 +160,10 @@ private Optional<Project> askServiceForProject(String projectCode) {
     try {
       List<Sample> samples = objectMapper.readValue(projectJson, new TypeReference<ArrayList<Sample>>() {
       });
+
+      List<String> ngsSamples = ngsSampleRepository.findNGSSamplesForProject(projectCode);
+      samples = samples.stream().filter(sample -> ngsSamples.contains(sample.sampleCode)).collect(Collectors.toList());
+
       Project project = new Project();
       project.addAll(samples);
       synchronized (cachedProjects) {
@@ -259,7 +265,7 @@ private Optional<Sample> askServiceForSample(String sampleCode) {
     }
   }
 
-  static private class ProjectSatusMapper {
+  static private class ProjectStatusMapper {
 
     static ProjectStatus toProjectStatus(Project project) {
       int totalCount = project.size();
diff --git a/sample-tracking-status-overview-app/src/main/groovy/life/qbic/portal/sampletracking/view/projects/ProjectStatusComponent.java b/sample-tracking-status-overview-app/src/main/groovy/life/qbic/portal/sampletracking/view/projects/ProjectStatusComponent.java
@@ -16,8 +16,6 @@
 
 public class ProjectStatusComponent extends Composite {
 
-  public static final int COLUMN_WIDTH = 100;
-
   private final ExecutorService executorService;
   private final ProjectStatusProvider trackingStatusProvider;
   private final Label receivedCountLabel;
@@ -90,12 +88,9 @@ public ProjectStatusComponent(Project project, ExecutorService executorService,
     panelContent.setMargin(false); // determined by CSS
     panelContent.setSpacing(false); // determined by CSS
     panelContent.addComponent(spinner);
-//    panelContent.setComponentAlignment(spinner, Alignment.TOP_CENTER);
 
     panelContent.addComponent(statusLayout);
-//    panelContent.setComponentAlignment(statusLayout, Alignment.TOP_CENTER);
     panelContent.addComponent(errorMessage);
-//    panelContent.setComponentAlignment(errorMessage, Alignment.TOP_CENTER);
 
     setCompositionRoot(panelContent);
     panelContent.setWidthUndefined();
diff --git a/sample-tracking-status-overview-app/src/main/groovy/life/qbic/portal/sampletracking/view/projects/ProjectView.java b/sample-tracking-status-overview-app/src/main/groovy/life/qbic/portal/sampletracking/view/projects/ProjectView.java
@@ -137,6 +137,7 @@ private void selectProject(Project project) {
       removeFileDownloaders(downloadButton);
     }
     if (project.projectStatus().totalCount() < 1) {
+      samplesButton.setEnabled(false);
       return;
     }
     samplesButton.setEnabled(true);

Original file line number	Diff line number	Diff line change
`@@ -107,7 +107,7 @@ class DependencyManager {`
`107`	`107`	`"/v2/samples",`
`108`	`108`	`"/status",`
`109`	`109`	`"/v2/projects",`
`110`		`- credentials)`
	`110`	`+ credentials, getNgsSampleRepository())`
`111`	`111`	`}`
`112`	`112`
`113`	`113`	`/**`
`@@ -174,4 +174,8 @@ class DependencyManager {`
`174`	`174`	`DownloadManifestProvider getDownloadManifestProvider() {`
`175`	`175`	`return sampleTrackingConnector`
`176`	`176`	`}`
	`177`	`+`
	`178`	`+ NgsSampleRepository getNgsSampleRepository() {`
	`179`	`+ return openBisConnector`
	`180`	`+ }`
`177`	`181`	`}`
Original file line number	Diff line number	Diff line change
`@@ -137,6 +137,7 @@ private void selectProject(Project project) {`
`137`	`137`	`removeFileDownloaders(downloadButton);`
`138`	`138`	`}`
`139`	`139`	`if (project.projectStatus().totalCount() < 1) {`
	`140`	`+ samplesButton.setEnabled(false);`
`140`	`141`	`return;`
`141`	`142`	`}`
`142`	`143`	`samplesButton.setEnabled(true);`