Skip to content

Commit 96e7031

Browse files
trautmanetrautmane
committed
add maxNumberOfPairsToRemove parameter for cross resin match removal, return removal tile pair information instead of just a count for improved logging on the driver, rename RemoveMatchClient as RemoveMatchPairClient
1 parent 721d864 commit 96e7031

File tree

6 files changed

+207
-167
lines changed

6 files changed

+207
-167
lines changed

‎render-app/src/test/java/org/janelia/alignment/match/CanvasMatchesTest.java‎

Lines changed: 24 additions & 0 deletions
Large diffs are not rendered by default.

‎render-ws-java-client/src/main/java/org/janelia/render/client/match/RemoveMatchClient.java‎

Lines changed: 0 additions & 127 deletions
This file was deleted.

‎render-ws-java-client/src/main/java/org/janelia/render/client/match/RemoveMatchPairClient.java‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
package org.janelia.render.client.match;
2+
3+
import com.beust.jcommander.ParametersDelegate;
4+
5+
import java.io.IOException;
6+
import java.util.ArrayList;
7+
import java.util.List;
8+
9+
import org.janelia.alignment.match.CanvasMatches;
10+
import org.janelia.alignment.match.OrderedCanvasIdPair;
11+
import org.janelia.render.client.ClientRunner;
12+
import org.janelia.render.client.RenderDataClient;
13+
import org.janelia.render.client.parameter.CommandLineParameters;
14+
import org.janelia.render.client.parameter.MatchPairRemovalParameters;
15+
import org.janelia.render.client.parameter.MatchWebServiceParameters;
16+
import org.slf4j.Logger;
17+
import org.slf4j.LoggerFactory;
18+
19+
/**
20+
* Java client for removing pairs from a match collection.
21+
*/
22+
public class RemoveMatchPairClient {
23+
24+
public static class Parameters extends CommandLineParameters {
25+
26+
@ParametersDelegate
27+
MatchWebServiceParameters matchClient = new MatchWebServiceParameters();
28+
29+
@ParametersDelegate
30+
public MatchPairRemovalParameters matchRemoval = new MatchPairRemovalParameters();
31+
}
32+
33+
public static void main(final String[] args) {
34+
final ClientRunner clientRunner = new ClientRunner(args) {
35+
@Override
36+
public void runClient(final String[] args) throws Exception {
37+
38+
final Parameters parameters = new Parameters();
39+
parameters.parse(args);
40+
parameters.matchRemoval.validate();
41+
42+
LOG.info("runClient: entry, parameters={}", parameters);
43+
44+
final RemoveMatchPairClient client = new RemoveMatchPairClient();
45+
client.removeMatchPairs(parameters);
46+
}
47+
};
48+
clientRunner.run();
49+
}
50+
51+
public RemoveMatchPairClient() {
52+
}
53+
54+
private void removeMatchPairs(final Parameters parameters)
55+
throws IOException {
56+
57+
final RenderDataClient matchClient = parameters.matchClient.getDataClient();
58+
removeMatchPairsForCollection(matchClient, parameters.matchRemoval);
59+
}
60+
61+
public static List<OrderedCanvasIdPair> removeMatchPairsForCollection(final RenderDataClient matchClient,
62+
final MatchPairRemovalParameters matchRemoval)
63+
throws IOException {
64+
65+
final List<OrderedCanvasIdPair> pairsToRemove = new ArrayList<>();
66+
67+
final List<String> pGroupIds = matchClient.getMatchPGroupIds();
68+
for (final String pGroupId : pGroupIds) {
69+
pairsToRemove.addAll(
70+
findPairsToRemoveForPGroupId(matchClient, pGroupId, matchRemoval));
71+
}
72+
73+
if (pairsToRemove.isEmpty()) {
74+
75+
LOG.info("removeMatchPairsForCollection: no match pairs to remove from {}",
76+
matchClient.getProject());
77+
78+
} else if (pairsToRemove.size() <= matchRemoval.maxNumberOfPairsToRemove) {
79+
80+
for (final OrderedCanvasIdPair pair : pairsToRemove) {
81+
matchClient.deleteMatchPair(pair.getP(), pair.getQ());
82+
}
83+
84+
} else {
85+
86+
throw new IOException("found " + pairsToRemove + " pairs to remove from " + matchClient.getProject() +
87+
" which exceeds the maximum of " + matchRemoval.maxNumberOfPairsToRemove);
88+
89+
}
90+
91+
return pairsToRemove;
92+
}
93+
94+
private static List<OrderedCanvasIdPair> findPairsToRemoveForPGroupId(final RenderDataClient matchClient,
95+
final String pGroupId,
96+
final MatchPairRemovalParameters matchRemoval)
97+
throws IOException {
98+
99+
final List<OrderedCanvasIdPair> pairsToRemove = new ArrayList<>();
100+
101+
final List<CanvasMatches> crossLayerMatchPairs = matchClient.getMatchesOutsideGroup(pGroupId,
102+
false);
103+
if (! crossLayerMatchPairs.isEmpty()) {
104+
for (final CanvasMatches matchPair : crossLayerMatchPairs) {
105+
if (! matchPair.isPOffsetFromQ(matchRemoval.minCrossMatchPixelDistance)) {
106+
pairsToRemove.add(new OrderedCanvasIdPair(matchPair.getpCanvasId(),
107+
matchPair.getqCanvasId(),
108+
null));
109+
}
110+
}
111+
}
112+
113+
LOG.info("findPairsToRemoveForPGroupId: found {} pair(s) to remove for pGroupId {} in {}",
114+
pairsToRemove.size(), pGroupId, matchClient.getProject());
115+
116+
return pairsToRemove;
117+
}
118+
119+
private static final Logger LOG = LoggerFactory.getLogger(RemoveMatchPairClient.class);
120+
}

‎render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MatchPairRemovalParameters.java‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
package org.janelia.render.client.parameter;
2+
3+
import com.beust.jcommander.Parameter;
4+
import com.beust.jcommander.Parameters;
5+
6+
import java.io.Serializable;
7+
8+
/**
9+
* Parameters for removing match pairs.
10+
*/
11+
@Parameters
12+
public class MatchPairRemovalParameters
13+
implements Serializable {
14+
15+
@Parameter(
16+
names = "--minCrossMatchPixelDistance",
17+
description = "Remove any cross match pair that has match points with an average offset that is less than this distance.")
18+
public Double minCrossMatchPixelDistance;
19+
20+
@Parameter(
21+
names = "--maxNumberOfPairsToRemove",
22+
description = "Throw an exception if more than this number of pairs will be removed.")
23+
public Integer maxNumberOfPairsToRemove;
24+
25+
public void validate()
26+
throws IllegalArgumentException {
27+
28+
if (minCrossMatchPixelDistance != null) {
29+
30+
if (minCrossMatchPixelDistance < 0.1) {
31+
throw new IllegalArgumentException("when specified, --minCrossMatchPixelDistance must be >= 0.1");
32+
}
33+
34+
if ((maxNumberOfPairsToRemove == null) || (maxNumberOfPairsToRemove < 1))
35+
throw new IllegalArgumentException(
36+
"when --minCrossMatchPixelDistance is specified, --maxNumberOfPairsToRemove " +
37+
"must be specified as a value 1 or greater"); {
38+
}
39+
40+
}
41+
42+
}
43+
44+
}

‎render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MatchRemovalParameters.java‎

Lines changed: 0 additions & 29 deletions
This file was deleted.

‎render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVASTileClient.java‎

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import org.apache.spark.api.java.function.Function;
2121
import org.janelia.alignment.match.ConnectedTileClusterSummaryForStack;
2222
import org.janelia.alignment.match.MatchCollectionId;
23+
import org.janelia.alignment.match.OrderedCanvasIdPair;
2324
import org.janelia.alignment.match.parameters.MatchRunParameters;
2425
import org.janelia.alignment.multisem.LayerMFOV;
2526
import org.janelia.alignment.multisem.MultiSemUtilities;
@@ -33,15 +34,15 @@
3334
import org.janelia.render.client.ClientRunner;
3435
import org.janelia.render.client.ClusterCountClient;
3536
import org.janelia.render.client.RenderDataClient;
36-
import org.janelia.render.client.match.RemoveMatchClient;
37+
import org.janelia.render.client.match.RemoveMatchPairClient;
3738
import org.janelia.render.client.multisem.MFOVAsTileMontageMatchPatchClient;
3839
import org.janelia.render.client.multisem.MFOVAsTileStackClient;
3940
import org.janelia.render.client.newsolver.setup.AffineBlockSolverSetup;
4041
import org.janelia.render.client.parameter.CommandLineParameters;
4142
import org.janelia.render.client.parameter.MFOVAsTileParameters;
4243
import org.janelia.render.client.parameter.MFOVAsTileStackLists;
4344
import org.janelia.render.client.parameter.MFOVMontageMatchPatchParameters;
44-
import org.janelia.render.client.parameter.MatchRemovalParameters;
45+
import org.janelia.render.client.parameter.MatchPairRemovalParameters;
4546
import org.janelia.render.client.parameter.MultiProjectParameters;
4647
import org.janelia.render.client.parameter.TileClusterParameters;
4748
import org.janelia.render.client.parameter.TileRenderParameters;
@@ -443,7 +444,7 @@ private static void removeCrossResinMfovAsTileMatches(final JavaSparkContext spa
443444

444445
final JavaRDD<StackWithZValues> rddStacks = sparkContext.parallelize(renderedMfovStacksWithAllZ);
445446

446-
final Function<StackWithZValues, Integer> removalFunction = stackWithZValues -> {
447+
final Function<StackWithZValues, String> removalFunction = stackWithZValues -> {
447448

448449
LogUtilities.setupExecutorLog4j(stackWithZValues.getStackId().toDevString());
449450

@@ -453,19 +454,26 @@ private static void removeCrossResinMfovAsTileMatches(final JavaSparkContext spa
453454
matchCollectionId.getOwner(),
454455
matchCollectionId.getName());
455456

456-
final MatchRemovalParameters matchRemovalParameters = new MatchRemovalParameters();
457-
matchRemovalParameters.minCrossMatchPixelDistance = minCrossMatchPixelDistance;
457+
final MatchPairRemovalParameters matchPairRemovalParameters = new MatchPairRemovalParameters();
458+
matchPairRemovalParameters.minCrossMatchPixelDistance = minCrossMatchPixelDistance;
459+
matchPairRemovalParameters.maxNumberOfPairsToRemove = 1;
458460

459-
return RemoveMatchClient.removeMatchPairsForCollection(matchClient, matchRemovalParameters);
461+
final List<OrderedCanvasIdPair> removedPairs =
462+
RemoveMatchPairClient.removeMatchPairsForCollection(matchClient, matchPairRemovalParameters);
463+
464+
final String removedPairsString = removedPairs.stream()
465+
.map(OrderedCanvasIdPair::toString)
466+
.collect(Collectors.joining(", "));
467+
return stackId.toDevString() + " had " + removedPairs.size() + " pairs removed: " + removedPairsString;
460468
};
461469

462-
final JavaRDD<Integer> rddRemovedPairCounts = rddStacks.map(removalFunction);
470+
final List<String> removedPairsMessages = rddStacks.map(removalFunction).collect();
463471

464-
final long totalNumberOfRemovedPairs =
465-
rddRemovedPairCounts.collect().stream().mapToInt(Integer::intValue).sum();
472+
for (final String msg : removedPairsMessages) {
473+
LOG.info("removeCrossResinMfovAsTileMatches: {}", msg);
474+
}
466475

467-
LOG.info("removeCrossResinMfovAsTileMatches: removed {} match pairs across all stacks",
468-
totalNumberOfRemovedPairs);
476+
LOG.info("removeCrossResinMfovAsTileMatches: exit");
469477
}
470478

471479
private static void patchMissingMfovAsTileMatches(final JavaSparkContext sparkContext,

0 commit comments

Comments
 (0)