Skip to content

Commit 52e96a6

Browse files
HannesWellHannesWell
committed
[TP-Editor] Completely parallelize P2 metadata fetching
1 parent ad86de6 commit 52e96a6

File tree

6 files changed

+70
-106
lines changed

6 files changed

+70
-106
lines changed

‎ui/org.eclipse.pde.genericeditor.extension/src/org/eclipse/pde/internal/genericeditor/target/extension/command/UpdateUnitVersions.java

Lines changed: 9 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
import org.eclipse.core.commands.AbstractHandler;
2323
import org.eclipse.core.commands.ExecutionEvent;
2424
import org.eclipse.core.commands.ExecutionException;
25-
import org.eclipse.core.runtime.jobs.Job;
2625
import org.eclipse.equinox.p2.metadata.IVersionedId;
2726
import org.eclipse.jface.dialogs.IPageChangeProvider;
2827
import org.eclipse.jface.dialogs.MessageDialog;
@@ -33,7 +32,6 @@
3332
import org.eclipse.pde.internal.genericeditor.target.extension.model.RepositoryCache;
3433
import org.eclipse.pde.internal.genericeditor.target.extension.model.UnitNode;
3534
import org.eclipse.pde.internal.genericeditor.target.extension.model.xml.Parser;
36-
import org.eclipse.pde.internal.genericeditor.target.extension.p2.UpdateJob;
3735
import org.eclipse.swt.widgets.Display;
3836
import org.eclipse.ui.IEditorInput;
3937
import org.eclipse.ui.IEditorPart;
@@ -69,20 +67,19 @@ public Object execute(ExecutionEvent event) throws ExecutionException {
6967

7068
int offsetChange = 0;
7169
String documentText = document.get();
72-
for (Node n1 : locationsNode.get(0).getChildNodesByTag(ITargetConstants.LOCATION_TAG)) {
73-
LocationNode locationNode = (LocationNode) n1;
70+
71+
List<LocationNode> locationNodes = locationsNode.get(0).getChildNodesByTag(ITargetConstants.LOCATION_TAG)
72+
.stream().map(LocationNode.class::cast).toList();
73+
74+
// Fetch all repos at once to fetch pending metadata in parallel
75+
locationNodes.stream().map(LocationNode::getRepositoryLocations).flatMap(List::stream)
76+
.forEach(RepositoryCache::prefetchP2MetadataOfRepository);
77+
78+
for (LocationNode locationNode : locationNodes) {
7479
List<String> repositoryLocations = locationNode.getRepositoryLocations();
7580
if (repositoryLocations.isEmpty()) {
7681
continue;
7782
}
78-
if (!repositoryLocations.stream().allMatch(RepositoryCache::isUpToDate)) {
79-
try {
80-
updateCache(locationNode);
81-
} catch (InterruptedException e) {
82-
e.printStackTrace();
83-
continue;
84-
}
85-
}
8683
Map<String, List<IVersionedId>> repositoryUnits = RepositoryCache
8784
.fetchP2UnitsFromRepos(repositoryLocations);
8885
for (Node n2 : locationNode.getChildNodesByTag(ITargetConstants.UNIT_TAG)) {
@@ -129,15 +126,6 @@ public Object execute(ExecutionEvent event) throws ExecutionException {
129126
});
130127
}
131128

132-
private void updateCache(LocationNode locationNode) throws InterruptedException {
133-
Job job = new UpdateJob(locationNode);
134-
job.setUser(true);
135-
job.schedule();
136-
while (job.getResult() == null) {
137-
Thread.sleep(50);
138-
}
139-
}
140-
141129
private IDocument getDocument() {
142130
IEditorPart editor = PlatformUI.getWorkbench().getActiveWorkbenchWindow().getActivePage().getActiveEditor();
143131
IDocumentProvider provider = null;

‎ui/org.eclipse.pde.genericeditor.extension/src/org/eclipse/pde/internal/genericeditor/target/extension/model/RepositoryCache.java

Lines changed: 46 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,17 @@
2121
import java.util.LinkedHashMap;
2222
import java.util.List;
2323
import java.util.Map;
24+
import java.util.concurrent.CompletableFuture;
2425
import java.util.concurrent.ConcurrentHashMap;
26+
import java.util.concurrent.Future;
2527
import java.util.stream.Collectors;
2628
import java.util.stream.Stream;
2729

30+
import org.eclipse.core.runtime.ILog;
31+
import org.eclipse.core.runtime.jobs.Job;
2832
import org.eclipse.equinox.p2.metadata.IVersionedId;
33+
import org.eclipse.osgi.util.NLS;
34+
import org.eclipse.pde.internal.genericeditor.target.extension.p2.Messages;
2935
import org.eclipse.pde.internal.genericeditor.target.extension.p2.P2Fetcher;
3036

3137
/**
@@ -42,7 +48,7 @@ private RepositoryCache() {
4248
// avoid instantiation
4349
}
4450

45-
private static final Map<URI, Map<String, List<IVersionedId>>> CACHE = new ConcurrentHashMap<>();
51+
private static final Map<URI, CompletableFuture<Map<String, List<IVersionedId>>>> CACHE = new ConcurrentHashMap<>();
4652

4753
/**
4854
* Fetches information and caches it.
@@ -58,20 +64,46 @@ private RepositoryCache() {
5864
*/
5965
public static Map<String, List<IVersionedId>> fetchP2UnitsFromRepos(List<String> repositories) {
6066
if (repositories.size() == 1) {
61-
return fetchP2DataOfRepo(repositories.get(0));
67+
return getFutureValue(fetchP2DataOfRepo(repositories.get(0)));
6268
}
6369
var repos = repositories.stream().map(RepositoryCache::fetchP2DataOfRepo).toList();
64-
return toSortedMap(repos.stream().map(Map::values).flatMap(Collection::stream).flatMap(List::stream));
70+
// Fetch all repos at once to await pending metadata in parallel
71+
return toSortedMap(repos.stream().map(RepositoryCache::getFutureValue) //
72+
.map(Map::values).flatMap(Collection::stream).flatMap(List::stream));
6573
}
6674

67-
private static Map<String, List<IVersionedId>> fetchP2DataOfRepo(String repository) {
75+
public static void prefetchP2MetadataOfRepository(String repository) {
76+
fetchP2DataOfRepo(repository);
77+
}
78+
79+
private static Future<Map<String, List<IVersionedId>>> fetchP2DataOfRepo(String repository) {
6880
URI location;
6981
try {
7082
location = new URI(repository);
7183
} catch (URISyntaxException e) {
72-
return Map.of();
84+
return CompletableFuture.failedFuture(e);
7385
}
74-
return CACHE.computeIfAbsent(location, repo -> toSortedMap(P2Fetcher.fetchAvailableUnits(repo)));
86+
return CACHE.compute(location, (repo, f) -> {
87+
if (f != null && (!f.isDone() || !f.isCompletedExceptionally() && !f.isCancelled())) {
88+
return f; // computation is running or has succeeded
89+
}
90+
CompletableFuture<Map<String, List<IVersionedId>>> future = new CompletableFuture<>();
91+
// Fetching P2 repository information is a costly operation
92+
// time-wise. Thus it is done in a job.
93+
Job job = Job.create(NLS.bind(Messages.UpdateJob_P2DataFetch, repo), m -> {
94+
try {
95+
Map<String, List<IVersionedId>> units = toSortedMap(P2Fetcher.fetchAvailableUnits(repo, m));
96+
future.complete(units);
97+
} catch (Throwable e) {
98+
future.completeExceptionally(e);
99+
// Only log the failure, don't open an error-dialog.
100+
ILog.get().warn(e.getMessage(), e);
101+
}
102+
});
103+
job.setUser(true);
104+
job.schedule();
105+
return future;
106+
});
75107
}
76108

77109
private static final Comparator<IVersionedId> BY_ID_FIRST_THEN_DESCENDING_VERSION = Comparator
@@ -83,6 +115,14 @@ private static Map<String, List<IVersionedId>> toSortedMap(Stream<IVersionedId>
83115
Collectors.groupingBy(IVersionedId::getId, LinkedHashMap::new, Collectors.toUnmodifiableList()));
84116
}
85117

118+
private static Map<String, List<IVersionedId>> getFutureValue(Future<Map<String, List<IVersionedId>>> future) {
119+
try {
120+
return future.get();
121+
} catch (Exception e) { // interrupted, canceled or execution failure
122+
return Map.of();
123+
}
124+
}
125+
86126
/**
87127
*
88128
* Method used to narrow down proposals in case a prefix is provided.
@@ -129,15 +169,4 @@ public static List<IVersionedId> getUnitsBySearchTerm(String repo, String search
129169
return allUnits.values().stream().flatMap(List::stream) //
130170
.filter(unit -> unit.getId().contains(searchTerm)).toList();
131171
}
132-
133-
/**
134-
* Classic cache up-to-date check.
135-
*
136-
* @param repo
137-
* repository URL
138-
* @return whether the cache is up to date for this repo
139-
*/
140-
public static boolean isUpToDate(String repo) {
141-
return CACHE.get(URI.create(repo)) != null;
142-
}
143172
}

‎ui/org.eclipse.pde.genericeditor.extension/src/org/eclipse/pde/internal/genericeditor/target/extension/p2/Messages.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
public class Messages extends NLS {
66
private static final String BUNDLE_NAME = "org.eclipse.pde.internal.genericeditor.target.extension.p2.messages"; //$NON-NLS-1$
77
public static String UpdateJob_P2DataFetch;
8-
public static String UpdateJob_ErrorMessage;
98
static {
109
// initialize resource bundle
1110
NLS.initializeMessages(BUNDLE_NAME, Messages.class);

‎ui/org.eclipse.pde.genericeditor.extension/src/org/eclipse/pde/internal/genericeditor/target/extension/p2/P2Fetcher.java

Lines changed: 13 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,11 @@
1616
import java.net.URI;
1717
import java.util.stream.Stream;
1818

19-
import org.eclipse.core.runtime.ILog;
19+
import org.eclipse.core.runtime.CoreException;
20+
import org.eclipse.core.runtime.IProgressMonitor;
21+
import org.eclipse.core.runtime.SubMonitor;
2022
import org.eclipse.equinox.p2.core.IProvisioningAgent;
2123
import org.eclipse.equinox.p2.core.IProvisioningAgentProvider;
22-
import org.eclipse.equinox.p2.core.ProvisionException;
2324
import org.eclipse.equinox.p2.metadata.IInstallableUnit;
2425
import org.eclipse.equinox.p2.metadata.IVersionedId;
2526
import org.eclipse.equinox.p2.metadata.VersionedId;
@@ -45,28 +46,20 @@ public class P2Fetcher {
4546
* URL string of a p2 repository
4647
* @return List of available installable unit models. See {@link UnitNode}
4748
*/
48-
public static Stream<IVersionedId> fetchAvailableUnits(URI repositoryLocation) {
49+
public static Stream<IVersionedId> fetchAvailableUnits(URI repositoryLocation, IProgressMonitor monitor)
50+
throws CoreException {
51+
SubMonitor subMonitor = SubMonitor.convert(monitor, 31);
52+
BundleContext context = FrameworkUtil.getBundle(P2Fetcher.class).getBundleContext();
53+
ServiceReference<IProvisioningAgentProvider> sr = context.getServiceReference(IProvisioningAgentProvider.class);
4954
try {
50-
BundleContext context = FrameworkUtil.getBundle(P2Fetcher.class).getBundleContext();
51-
ServiceReference<IProvisioningAgentProvider> sr = context
52-
.getServiceReference(IProvisioningAgentProvider.class);
5355
IProvisioningAgentProvider agentProvider = context.getService(sr);
54-
IProvisioningAgent agent = null;
55-
try {
56-
agent = agentProvider.createAgent(null);
57-
} catch (ProvisionException e) {
58-
ILog.get().error("Failed to create provisioning-agent", e);
59-
} finally {
60-
context.ungetService(sr);
61-
}
56+
IProvisioningAgent agent = agentProvider.createAgent(null);
6257
IMetadataRepositoryManager manager = agent.getService(IMetadataRepositoryManager.class);
63-
IMetadataRepository repository = manager.loadRepository(repositoryLocation, null);
64-
IQueryResult<IInstallableUnit> allUnits = repository.query(QueryUtil.ALL_UNITS, null);
65-
58+
IMetadataRepository repository = manager.loadRepository(repositoryLocation, subMonitor.split(30));
59+
IQueryResult<IInstallableUnit> allUnits = repository.query(QueryUtil.ALL_UNITS, subMonitor.split(1));
6660
return allUnits.stream().map(iu -> new VersionedId(iu.getId(), iu.getVersion()));
67-
} catch (Exception e) {
68-
ILog.get().error("Failed to fetch metadata of repository: " + repositoryLocation, e);
69-
return Stream.empty();
61+
} finally {
62+
context.ungetService(sr);
7063
}
7164
}
7265

‎ui/org.eclipse.pde.genericeditor.extension/src/org/eclipse/pde/internal/genericeditor/target/extension/p2/UpdateJob.java

Lines changed: 0 additions & 44 deletions
This file was deleted.

‎ui/org.eclipse.pde.genericeditor.extension/src/org/eclipse/pde/internal/genericeditor/target/extension/p2/messages.properties

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
###############################################################################
2-
# Copyright (c) 2016 Red Hat Inc. and others
2+
# Copyright (c) 2016, 2024 Red Hat Inc. and others
33
#
44
# This program and the accompanying materials
55
# are made available under the terms of the Eclipse Public License 2.0
@@ -11,5 +11,4 @@
1111
# Contributors:
1212
# Sopot Cela (Red Hat Inc.) - initial implementation
1313
###############################################################################
14-
UpdateJob_P2DataFetch=Fetching p2 metadata from repository
15-
UpdateJob_ErrorMessage=Issue fetching data from repository. Please check URL or see log for even more details.
14+
UpdateJob_P2DataFetch=Fetching p2 metadata from repository: {0}

0 commit comments

Comments
 (0)