Use one FileDownloader in config server

FileDownloader keeps tracks of downloads in progress, this does not
work if we have more than one (an file received that is unknown because
the other downloader started downloading it will not be handled correctly).

Author: hmusum

configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java

@@ -12,6 +12,7 @@
 import com.yahoo.container.handler.VipStatus;
 import com.yahoo.container.jdisc.state.StateMonitor;
 import com.yahoo.vespa.config.server.filedistribution.FileDirectory;
+import com.yahoo.vespa.config.server.filedistribution.FileServer;
 import com.yahoo.vespa.config.server.maintenance.ConfigServerMaintenance;
 import com.yahoo.vespa.config.server.rpc.RpcServer;
 import com.yahoo.vespa.config.server.version.VersionState;
@@ -74,15 +75,15 @@ enum VipStatusMode { VIP_STATUS_FILE, VIP_STATUS_PROGRAMMATICALLY }
     @Inject
     public ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server,
                                  VersionState versionState, StateMonitor stateMonitor, VipStatus vipStatus,
-                                 FileDirectory fileDirectory) {
+                                 FileDirectory fileDirectory, FileServer fileServer) {
         this(applicationRepository, server, versionState, stateMonitor, vipStatus, EXIT_JVM,
-             vipStatusMode(applicationRepository), fileDirectory);
+             vipStatusMode(applicationRepository), fileDirectory, fileServer);
     }
 
     protected ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server,
                                     VersionState versionState, StateMonitor stateMonitor, VipStatus vipStatus,
                                     RedeployingApplicationsFails exitIfRedeployingApplicationsFails,
-                                    VipStatusMode vipStatusMode, FileDirectory fileDirectory) {
+                                    VipStatusMode vipStatusMode, FileDirectory fileDirectory, FileServer fileServer) {
         this.applicationRepository = applicationRepository;
         this.server = server;
         this.versionState = versionState;
@@ -94,7 +95,7 @@ protected ConfigServerBootstrap(ApplicationRepository applicationRepository, Rpc
         this.exitIfRedeployingApplicationsFails = exitIfRedeployingApplicationsFails;
         this.clock = applicationRepository.clock();
         rpcServerExecutor = Executors.newSingleThreadExecutor(new DaemonThreadFactory("config server RPC server"));
-        configServerMaintenance = new ConfigServerMaintenance(applicationRepository, fileDirectory);
+        configServerMaintenance = new ConfigServerMaintenance(applicationRepository, fileDirectory, fileServer);
         configServerMaintenance.startBeforeBootstrap();
         log.log(Level.FINE, () -> "VIP status mode: " + vipStatusMode);
         initializing(vipStatusMode);

configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java

@@ -3,18 +3,14 @@
 
 import com.yahoo.config.FileReference;
 import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.subscription.ConfigSourceSet;
-import com.yahoo.jrt.Supervisor;
-import com.yahoo.jrt.Transport;
-import com.yahoo.vespa.config.ConnectionPool;
 import com.yahoo.vespa.config.server.ApplicationRepository;
+import com.yahoo.vespa.config.server.filedistribution.FileServer;
 import com.yahoo.vespa.config.server.session.RemoteSession;
 import com.yahoo.vespa.config.server.session.Session;
 import com.yahoo.vespa.config.server.session.SessionRepository;
 import com.yahoo.vespa.config.server.tenant.Tenant;
 import com.yahoo.vespa.curator.Curator;
 import com.yahoo.vespa.defaults.Defaults;
-import com.yahoo.vespa.filedistribution.FileDistributionConnectionPool;
 import com.yahoo.vespa.filedistribution.FileDownloader;
 import com.yahoo.vespa.filedistribution.FileReferenceDownload;
 
@@ -28,7 +24,6 @@
 import java.util.logging.Logger;
 
 import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.fileReferenceExistsOnDisk;
-import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.getOtherConfigServersInCluster;
 import static com.yahoo.vespa.config.server.session.Session.Status.ACTIVATE;
 import static com.yahoo.vespa.config.server.session.Session.Status.PREPARE;
 
@@ -43,16 +38,14 @@
 public class ApplicationPackageMaintainer extends ConfigServerMaintainer {
 
     private static final Logger log = Logger.getLogger(ApplicationPackageMaintainer.class.getName());
-    private static final Duration fileDownloaderTimeout = Duration.ofSeconds(30);
 
     private final File downloadDirectory;
-    private final Supervisor supervisor = new Supervisor(new Transport("filedistribution-pool")).setDropEmptyBuffers(true);
     private final FileDownloader fileDownloader;
 
-    ApplicationPackageMaintainer(ApplicationRepository applicationRepository, Curator curator, Duration interval) {
+    ApplicationPackageMaintainer(ApplicationRepository applicationRepository, Curator curator, Duration interval, FileServer fileServer) {
         super(applicationRepository, curator, applicationRepository.flagSource(), applicationRepository.clock(), interval, false);
         this.downloadDirectory = new File(Defaults.getDefaults().underVespaHome(applicationRepository.configserverConfig().fileReferencesDir()));
-        this.fileDownloader = createFileDownloader(applicationRepository, downloadDirectory, supervisor);
+        this.fileDownloader = fileServer.downloader();
     }
 
     @Override
@@ -121,18 +114,8 @@ private Collection<RemoteSession> preparedAndActivatedSessions() {
                 .toList();
     }
 
-    private static FileDownloader createFileDownloader(ApplicationRepository applicationRepository,
-                                                       File downloadDirectory,
-                                                       Supervisor supervisor) {
-        List<String> otherConfigServersInCluster = getOtherConfigServersInCluster(applicationRepository.configserverConfig());
-        ConfigSourceSet configSourceSet = new ConfigSourceSet(otherConfigServersInCluster);
-        ConnectionPool connectionPool = new FileDistributionConnectionPool(configSourceSet, supervisor);
-        return new FileDownloader(connectionPool, supervisor, downloadDirectory, fileDownloaderTimeout);
-    }
-
     @Override
     public void awaitShutdown() {
-        supervisor.transport().shutdown().join();
         fileDownloader.close();
         super.awaitShutdown();
     }

configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java

@@ -5,6 +5,7 @@
 import com.yahoo.vespa.config.server.ApplicationRepository;
 import com.yahoo.vespa.config.server.application.ConfigConvergenceChecker;
 import com.yahoo.vespa.config.server.filedistribution.FileDirectory;
+import com.yahoo.vespa.config.server.filedistribution.FileServer;
 import com.yahoo.vespa.curator.Curator;
 
 import java.time.Clock;
@@ -28,19 +29,21 @@ public class ConfigServerMaintenance {
     private final Curator curator;
     private final ConfigConvergenceChecker convergenceChecker;
     private final FileDirectory fileDirectory;
+    private final FileServer fileServer;
     private final Duration interval;
 
-    public ConfigServerMaintenance(ApplicationRepository applicationRepository, FileDirectory fileDirectory) {
+    public ConfigServerMaintenance(ApplicationRepository applicationRepository, FileDirectory fileDirectory, FileServer fileServer) {
         this.applicationRepository = applicationRepository;
         this.curator = applicationRepository.tenantRepository().getCurator();
         this.convergenceChecker = applicationRepository.configConvergenceChecker();
         this.fileDirectory = fileDirectory;
+        this.fileServer = fileServer;
         this.interval = Duration.ofMinutes(applicationRepository.configserverConfig().maintainerIntervalMinutes());
     }
 
     public void startBeforeBootstrap() {
         if (moreThanOneConfigServer())
-            maintainers.add(new ApplicationPackageMaintainer(applicationRepository, curator, Duration.ofSeconds(15)));
+            maintainers.add(new ApplicationPackageMaintainer(applicationRepository, curator, Duration.ofSeconds(15), fileServer));
         maintainers.add(new TenantsMaintainer(applicationRepository, curator, interval, Clock.systemUTC()));
     }
 

configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java

@@ -21,6 +21,7 @@
 import com.yahoo.text.Utf8;
 import com.yahoo.vespa.config.server.deploy.DeployTester;
 import com.yahoo.vespa.config.server.filedistribution.FileDirectory;
+import com.yahoo.vespa.config.server.filedistribution.FileServer;
 import com.yahoo.vespa.config.server.rpc.RpcServer;
 import com.yahoo.vespa.config.server.version.VersionState;
 import com.yahoo.vespa.config.server.version.VespaVersion;
@@ -258,13 +259,17 @@ private Bootstrapper createBootstrapper(DeployTester tester,
                                             VersionState versionState) {
         StateMonitor stateMonitor = StateMonitor.createForTesting();
         VipStatus vipStatus = createVipStatus(stateMonitor);
+        ConfigserverConfig configserverConfig = tester.applicationRepository().configserverConfig();
+        FileDirectory fileDirectory = new FileDirectory(configserverConfig);
+        FileServer fileServer = new FileServer(configserverConfig, fileDirectory);
         return new Bootstrapper(tester.applicationRepository(),
                                 rpcServer,
                                 versionState,
                                 stateMonitor,
                                 vipStatus,
                                 vipStatusMode,
-                                new FileDirectory(tester.applicationRepository().configserverConfig()));
+                                fileDirectory,
+                                fileServer);
     }
 
     private void waitUntil(BooleanSupplier booleanSupplier, String messageIfWaitingFails) throws InterruptedException {
@@ -363,8 +368,10 @@ public Bootstrapper(ApplicationRepository applicationRepository,
                             StateMonitor stateMonitor,
                             VipStatus vipStatus,
                             VipStatusMode vipStatusMode,
-                            FileDirectory fileDirectory) {
-            super(applicationRepository, server, versionState, stateMonitor, vipStatus, CONTINUE, vipStatusMode, fileDirectory);
+                            FileDirectory fileDirectory,
+                            FileServer fileServer) {
+            super(applicationRepository, server, versionState, stateMonitor, vipStatus, CONTINUE, vipStatusMode,
+                  fileDirectory, fileServer);
         }
 
         @Override