Improve local replays tab loading time (#3394)

Rogerstalker · web-flow · commit 2234cb5f77e9 · 2025-10-11T16:50:01.000-04:00
* Implemented replay caching for ReplayService
Parallelized local replay loading to improve performance
Fixed a bug, where two separate subscribed actions prompted the VaultEntityController to load the replays twice on changing the number of replays per page combobox.

* Adjust ReplayService and tests to java-common replay data type change

* Bump faf-java-commons to 317e19f

* Reduced local replay cache size and expiry
Added check for processor count for parallelization

* Change commons to mvn repo artifacts

* Revert to original single threaded local replays page loading

* Remove unused nThreads field

* Remove unused changePerPageCount method, adjust corresponding unit test
diff --git a/build.gradle b/build.gradle
@@ -256,13 +256,13 @@ dependencies {
     implementation("io.projectreactor.addons:reactor-extra")
     implementation("io.projectreactor:reactor-tools")
 
-    def commonsVersion = "07e30bcb925ff34273a9a4eb08c8c506b13d5d78"
+  def commonsVersion = "20250913-317e19f"
 
-    implementation("com.github.faforever.faf-java-commons:data:${commonsVersion}") {
+    implementation("com.faforever.commons:data:${commonsVersion}") {
         exclude module: 'guava'
     }
-    implementation("com.github.faforever.faf-java-commons:api:${commonsVersion}")
-    implementation("com.github.faforever.faf-java-commons:lobby:${commonsVersion}")
+    implementation("com.faforever.commons:api:${commonsVersion}")
+    implementation("com.faforever.commons:lobby:${commonsVersion}")
     implementation("com.google.guava:guava:33.5.0-jre")
     implementation("org.apache.commons:commons-compress:1.28.0")
     implementation("net.java.dev.jna:jna:5.18.1")
diff --git a/src/main/java/com/faforever/client/replay/ReplayService.java b/src/main/java/com/faforever/client/replay/ReplayService.java
@@ -34,9 +34,13 @@
 import com.faforever.commons.api.elide.ElideNavigatorOnId;
 import com.faforever.commons.replay.ReplayDataParser;
 import com.faforever.commons.replay.ReplayMetadata;
+import com.fasterxml.jackson.databind.util.ByteBufferBackedInputStream;
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.jetbrains.annotations.NotNull;
 import org.springframework.beans.factory.ObjectFactory;
 import org.springframework.cache.annotation.Cacheable;
 import org.springframework.context.annotation.Lazy;
@@ -45,11 +49,11 @@
 import reactor.core.publisher.Mono;
 import reactor.util.function.Tuple2;
 
-import java.io.ByteArrayInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
+import java.nio.ByteBuffer;
 import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -63,6 +67,7 @@
 import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -101,6 +106,10 @@ public class ReplayService {
   private final DataPrefs dataPrefs;
   private final ObjectFactory<ReplayDownloadTask> replayDownloadTaskFactory;
   private final ReplayHistoryPrefs replayHistory;
+  private final Cache<@NotNull Path, @NotNull Replay> replayCache = CacheBuilder.newBuilder()
+                                                                                .maximumSize(500)
+                                                                                .expireAfterWrite(20, TimeUnit.MINUTES)
+                                                                                .build();
 
   @VisibleForTesting
   static Integer parseSupComVersion(ReplayDataParser parser) {
@@ -179,16 +188,20 @@ public Mono<Tuple2<List<Replay>, Integer>> loadLocalReplayPage(int pageSize, int
 
       return Mono.fromFuture(CompletableFuture.allOf(replayFutures.toArray(new CompletableFuture[0]))
                                               .thenApply(_ -> replayFutures.stream()
-                                                                                     .map(CompletableFuture::join)
-                                                                                     .filter(Objects::nonNull)
-                                                                                     .collect(Collectors.toList())))
+                                                                           .map(CompletableFuture::join)
+                                                                           .filter(Objects::nonNull)
+                                                                           .collect(Collectors.toList())))
                  .zipWith(Mono.just(numPages));
     }
   }
 
-
   private CompletableFuture<Replay> tryLoadingLocalReplay(Path replayFile) {
     try {
+      final Replay cachedReplay = this.replayCache.getIfPresent(replayFile);
+      if (cachedReplay != null) {
+        return CompletableFuture.completedFuture(cachedReplay);
+      }
+
       ReplayDataParser replayData = replayFileReader.parseReplay(replayFile);
       ReplayMetadata replayMetadata = replayData.getMetadata();
 
@@ -203,7 +216,10 @@ private CompletableFuture<Replay> tryLoadingLocalReplay(Path replayFile) {
         if (mapVersion == null) {
           log.warn("Could not find map for replay file `{}`", replayFile);
         }
-        return replayMapper.map(replayData, replayFile, featuredMod, mapVersion);
+
+        final Replay replay = replayMapper.map(replayData, replayFile, featuredMod, mapVersion);
+        this.replayCache.put(replayFile, replay);
+        return replay;
       }).exceptionally(throwable -> {
         log.warn("Could not read replay file `{}`", replayFile, throwable);
         moveCorruptedReplayFile(replayFile);
@@ -287,7 +303,10 @@ public ReplayDetails loadReplayDetails(Path path) throws IOException {
     List<ChatMessage> chatMessages = replayDataParser.getChatMessages().stream().map(replayMapper::map).toList();
     List<GameOption> gameOptions = Stream.concat(
         Stream.of(new GameOption("FAF Version", String.valueOf(parseSupComVersion(replayDataParser)))),
-        replayDataParser.getGameOptions().stream().map(replayMapper::map).sorted(Comparator.comparing(GameOption::key, String.CASE_INSENSITIVE_ORDER))).toList();
+        replayDataParser.getGameOptions()
+                        .stream()
+                        .map(replayMapper::map)
+                        .sorted(Comparator.comparing(GameOption::key, String.CASE_INSENSITIVE_ORDER))).toList();
 
     String mapFolderName = parseMapFolderName(replayDataParser);
     Map map = new Map(null, mapFolderName, 0, null, false, null, null);
@@ -351,12 +370,13 @@ private void runOnlineReplay(int replayId) {
 
   private void runFafReplayFile(Path path) throws IOException {
     ReplayDataParser replayData = replayFileReader.parseReplay(path);
-    byte[] rawReplayBytes = replayData.getData();
+    ByteBuffer rawReplayByteBuffer = replayData.getData();
 
     Path tempSupComReplayFile = dataPrefs.getCacheDirectory().resolve(TEMP_SCFA_REPLAY_FILE_NAME);
 
     Files.createDirectories(tempSupComReplayFile.getParent());
-    Files.copy(new ByteArrayInputStream(rawReplayBytes), tempSupComReplayFile, StandardCopyOption.REPLACE_EXISTING);
+    Files.copy(new ByteBufferBackedInputStream(rawReplayByteBuffer), tempSupComReplayFile,
+               StandardCopyOption.REPLACE_EXISTING);
 
     ReplayMetadata replayMetadata = replayData.getMetadata();
     String gameType = replayMetadata.getFeaturedMod();
@@ -445,8 +465,7 @@ public Mono<Tuple2<List<Replay>, Integer>> findByQueryWithPageCount(SearchConfig
   @Cacheable(value = CacheNames.REPLAYS_SEARCH, sync = true)
   public Mono<Replay> findById(int id) {
     ElideNavigatorOnId<Game> navigator = ElideNavigator.of(Game.class).id(String.valueOf(id));
-    return fafApiAccessor.getOne(navigator).map(replayMapper::map)
-                         .cache();
+    return fafApiAccessor.getOne(navigator).map(replayMapper::map).cache();
   }
 
   @Cacheable(value = CacheNames.REPLAYS_MINE, sync = true)
@@ -467,10 +486,13 @@ private Mono<Tuple2<List<Replay>, Integer>> getReplayPage(ElideNavigatorOnCollec
   }
 
   public Flux<LeagueScoreJournal> getLeagueScoreJournalForReplay(Replay replay) {
-    ElideNavigatorOnCollection<com.faforever.commons.api.dto.LeagueScoreJournal> navigator = ElideNavigator.of(com.faforever.commons.api.dto.LeagueScoreJournal.class).collection()
-        .setFilter(qBuilder().intNum("gameId").eq(replay.id()));
-    return fafApiAccessor.getMany(navigator)
-        .map(replayMapper::map)
-        .cache();
+    ElideNavigatorOnCollection<com.faforever.commons.api.dto.LeagueScoreJournal> navigator = ElideNavigator.of(
+                                                                                                               com.faforever.commons.api.dto.LeagueScoreJournal.class)
+                                                                                                           .collection()
+                                                                                                           .setFilter(
+                                                                                                               qBuilder().intNum(
+                                                                                                                             "gameId")
+                                                                                                                         .eq(replay.id()));
+    return fafApiAccessor.getMany(navigator).map(replayMapper::map).cache();
   }
 }
diff --git a/src/main/java/com/faforever/client/vault/VaultEntityController.java b/src/main/java/com/faforever/client/vault/VaultEntityController.java
@@ -134,7 +134,6 @@ protected void onInitialize() {
                        ((observable, oldValue, newValue) -> onPerPageCountChanged(oldValue == null ? 0 : oldValue,
                                                                                   newValue)));
     perPageComboBox.setValue(20);
-    perPageComboBox.setOnAction((event -> changePerPageCount()));
     pageSize = perPageComboBox.getValue();
 
     initSearchController();
@@ -265,14 +264,6 @@ private VaultEntityShowRoomController loadShowRoom(ShowRoomCategory<T> showRoomC
     return vaultEntityShowRoomController;
   }
 
-  protected void changePerPageCount() {
-    pageSize = perPageComboBox.getValue();
-    if (state.get() == State.RESULT) {
-      SearchConfig searchConfig = searchController.getLastSearchConfig();
-      onPageChange(searchConfig, true);
-    }
-  }
-
   protected void onPageChange(SearchConfig searchConfig, boolean firstLoad) {
     enterSearchingState();
     setSupplier(searchConfig);
diff --git a/src/test/java/com/faforever/client/replay/ReplayFileReaderImplTest.java b/src/test/java/com/faforever/client/replay/ReplayFileReaderImplTest.java
@@ -28,6 +28,6 @@ public void readReplayData() throws Exception {
     try (InputStream inputStream = new BufferedInputStream(getClass().getResourceAsStream("/replay/test.fafreplay"))) {
       Files.copy(inputStream, tempFile);
     }
-    assertThat(instance.parseReplay(tempFile).getData().length, is(197007));
+    assertThat(instance.parseReplay(tempFile).getData().capacity(), is(197007));
   }
 }
diff --git a/src/test/java/com/faforever/client/replay/ReplayServiceTest.java b/src/test/java/com/faforever/client/replay/ReplayServiceTest.java
@@ -51,6 +51,7 @@
 import reactor.util.function.Tuple2;
 import reactor.util.function.Tuples;
 
+import java.nio.ByteBuffer;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.time.Instant;
@@ -185,7 +186,7 @@ public void setUp() throws Exception {
 
     lenient().when(replayFileReader.parseReplay(any())).thenReturn(replayDataParser);
     lenient().when(replayDataParser.getMetadata()).thenReturn(replayMetadata);
-    lenient().when(replayDataParser.getData()).thenReturn(REPLAY_FIRST_BYTES);
+    lenient().when(replayDataParser.getData()).thenReturn(ByteBuffer.wrap(REPLAY_FIRST_BYTES));
     lenient().when(replayDataParser.getChatMessages()).thenReturn(List.of());
     lenient().when(replayDataParser.getGameOptions()).thenReturn(List.of());
     lenient().when(replayDataParser.getMods()).thenReturn(Map.of());
diff --git a/src/test/java/com/faforever/client/vault/VaultEntityControllerTest.java b/src/test/java/com/faforever/client/vault/VaultEntityControllerTest.java
@@ -70,7 +70,7 @@ public class VaultEntityControllerTest extends PlatformTest {
 
   private List<Integer> getMockPageElements(List<Integer> elements, int pageSize, int page) {
     return elements.subList(Math.min((page) * pageSize, elements.size()),
-        Math.min((page + 1) * pageSize, elements.size()));
+                            Math.min((page + 1) * pageSize, elements.size()));
   }
 
   private Mono<Tuple2<List<Integer>, Integer>> mocksAsMono(List<Integer> elements, int pageSize, int page) {
@@ -94,7 +94,8 @@ public void setUp() throws Exception {
     when(vaultEntityShowRoomController.getPane()).thenReturn(showRoomPane);
 
     items = IntStream.range(0, 50).boxed().toList();
-    instance = new VaultEntityController<>(uiService, notificationService, i18n, reportingService, vaultPrefs, fxApplicationThreadExecutor) {
+    instance = new VaultEntityController<>(uiService, notificationService, i18n, reportingService, vaultPrefs,
+                                           fxApplicationThreadExecutor) {
       @Override
       protected void initSearchController() {
         //Do Nothing
@@ -259,8 +260,6 @@ public void testPageSizeChange() {
 
     instance.perPageComboBox.setValue(newPageSize);
     WaitForAsyncUtils.waitForFxEvents();
-    instance.changePerPageCount();
-    WaitForAsyncUtils.waitForFxEvents();
 
     moreButton.fire();
     WaitForAsyncUtils.waitForFxEvents();

Original file line number	Diff line number	Diff line change
`@@ -28,6 +28,6 @@ public void readReplayData() throws Exception {`
`28`	`28`	`try (InputStream inputStream = new BufferedInputStream(getClass().getResourceAsStream("/replay/test.fafreplay"))) {`
`29`	`29`	`Files.copy(inputStream, tempFile);`
`30`	`30`	`}`
`31`		`- assertThat(instance.parseReplay(tempFile).getData().length, is(197007));`
	`31`	`+ assertThat(instance.parseReplay(tempFile).getData().capacity(), is(197007));`
`32`	`32`	`}`
`33`	`33`	`}`