Merge pull request #2676 from AlexDickerson/fasterResumableDownloads

Faster resumable downloads
wabbajack-tools · Jan 8, 2025 · 4b1e242 · 4b1e242
2 parents 8a1417a + ef55706
commit 4b1e242
Show file tree

Hide file tree

Showing 15 changed files with 114 additions and 390 deletions.
diff --git a/Wabbajack.App.Wpf/Settings.cs b/Wabbajack.App.Wpf/Settings.cs
@@ -17,38 +17,12 @@ public class Mo2ModlistInstallationSettings
     public class PerformanceSettings : ViewModel
     {
         private readonly Configuration.MainSettings _settings;
-        private readonly int _defaultMaximumMemoryPerDownloadThreadMb;
 
         public PerformanceSettings(Configuration.MainSettings settings, IResource<DownloadDispatcher> downloadResources, SystemParametersConstructor systemParams)
         {
             var p = systemParams.Create();
 
             _settings = settings;
-            // Split half of available memory among download threads
-            _defaultMaximumMemoryPerDownloadThreadMb = (int)(p.SystemMemorySize / downloadResources.MaxTasks / 1024 / 1024) / 2;
-            _maximumMemoryPerDownloadThreadMb = settings.PerformanceSettings.MaximumMemoryPerDownloadThreadMb;
-
-            if (MaximumMemoryPerDownloadThreadMb < 0)
-            {
-                ResetMaximumMemoryPerDownloadThreadMb();
-            }
-        }
-
-        private int _maximumMemoryPerDownloadThreadMb;
-
-        public int MaximumMemoryPerDownloadThreadMb
-        {
-            get => _maximumMemoryPerDownloadThreadMb;
-            set
-            {
-                RaiseAndSetIfChanged(ref _maximumMemoryPerDownloadThreadMb, value);
-                _settings.PerformanceSettings.MaximumMemoryPerDownloadThreadMb = value;
-            }
-        }
-
-        public void ResetMaximumMemoryPerDownloadThreadMb()
-        {
-            MaximumMemoryPerDownloadThreadMb = _defaultMaximumMemoryPerDownloadThreadMb;
         }
     }
 }
diff --git a/Wabbajack.App.Wpf/Views/Settings/PerformanceSettingsView.xaml b/Wabbajack.App.Wpf/Views/Settings/PerformanceSettingsView.xaml
@@ -47,30 +47,6 @@
             <Button x:Name="EditResourceSettings" Grid.Row="2">
                 <TextBlock FontSize="14" FontWeight="Bold">Edit Resource Usage Settings and Close Wabbajack</TextBlock>
             </Button>
-            <TextBlock
-                Grid.Row="4"
-                FontSize="14"
-                Foreground="{StaticResource ForegroundBrush}"
-                Text="Maximum RAM per download thread (MB)"
-                ToolTip="Defines the maximum amount of RAM each download thread can use for buffering. Set to 0 to disable this limit completely." />
-            <xwpf:IntegerUpDown
-                x:Name="MaximumMemoryPerDownloadThreadIntegerUpDown"
-                Grid.Row="4"
-                Grid.Column="2"
-                Width="80"
-                HorizontalAlignment="Left"
-                Foreground="{StaticResource ForegroundBrush}"
-                Maximum="10000"
-                Minimum="0" />
-            <Button
-                x:Name="ResetMaximumMemoryPerDownloadThread"
-                Grid.Row="4"
-                Grid.Column="3"
-                Margin="20,0,0,0"
-                Padding="10,0"
-                HorizontalAlignment="Left">
-                <TextBlock FontSize="14">Reset</TextBlock>
-            </Button>
         </Grid>
     </Border>
 </rxui:ReactiveUserControl>
diff --git a/Wabbajack.App.Wpf/Views/Settings/PerformanceSettingsView.xaml.cs b/Wabbajack.App.Wpf/Views/Settings/PerformanceSettingsView.xaml.cs
@@ -15,22 +15,13 @@ public PerformanceSettingsView()
             InitializeComponent();
 
             this.WhenActivated(disposable =>
-            {
-                this.BindStrict(
-                        ViewModel,
-                        x => x.MaximumMemoryPerDownloadThreadMb,
-                        x => x.MaximumMemoryPerDownloadThreadIntegerUpDown.Value)
-                    .DisposeWith(disposable);
+            {       
                 this.EditResourceSettings.Command = ReactiveCommand.Create(() =>
                 {
                     UIUtils.OpenFile(
                         KnownFolders.WabbajackAppLocal.Combine("saved_settings", "resource_settings.json"));
                     Environment.Exit(0);
                 });
-                ResetMaximumMemoryPerDownloadThread.Command = ReactiveCommand.Create(() =>
-                {
-                    ViewModel.ResetMaximumMemoryPerDownloadThreadMb();
-                });
             });
         }
     }

diff --git a/Wabbajack.CLI/Program.cs b/Wabbajack.CLI/Program.cs
@@ -32,7 +32,7 @@ private static async Task<int> Main(string[] args)
             {
                 services.AddSingleton(new JsonSerializerOptions());
                 services.AddSingleton<HttpClient, HttpClient>();
-                services.AddSingleton<IHttpDownloader, SingleThreadedDownloader>();
+                services.AddResumableHttpDownloader();
                 services.AddSingleton<IConsole, SystemConsole>();
                 services.AddSingleton<CommandLineBuilder, CommandLineBuilder>();
                 services.AddSingleton<TemporaryFileManager>();

diff --git a/Wabbajack.Configuration/MainSettings.cs b/Wabbajack.Configuration/MainSettings.cs
@@ -7,20 +7,13 @@ public class MainSettings
 
     public int CurrentSettingsVersion { get; set; }
 
-    public PerformanceSettings PerformanceSettings { get; set; } = new();
-
     public bool Upgrade()
     {
         if (CurrentSettingsVersion == SettingsVersion)
         {
             return false;
         }
 
-        if (CurrentSettingsVersion < 1)
-        {
-            PerformanceSettings.MaximumMemoryPerDownloadThreadMb = -1;
-        }
-
         CurrentSettingsVersion = SettingsVersion;
         return true;
     }

diff --git a/Wabbajack.Configuration/PerformanceSettings.cs b/Wabbajack.Configuration/PerformanceSettings.cs
diff --git a/Wabbajack.Downloaders.GoogleDrive/GoogleDriveDownloader.cs b/Wabbajack.Downloaders.GoogleDrive/GoogleDriveDownloader.cs
@@ -47,15 +47,15 @@ public override Task<bool> Prepare()
 
     public override bool IsAllowed(ServerAllowList allowList, IDownloadState state)
     {
-        return allowList.GoogleIDs.Contains(((DTOs.DownloadStates.GoogleDrive) state).Id);
+        return allowList.GoogleIDs.Contains(((DTOs.DownloadStates.GoogleDrive)state).Id);
     }
 
     public IDownloadState? Parse(Uri uri)
     {
         if (uri.Host != "drive.google.com") return null;
         var match = GDriveRegex.Match(uri.ToString());
         if (match.Success)
-            return new DTOs.DownloadStates.GoogleDrive {Id = match.ToString()};
+            return new DTOs.DownloadStates.GoogleDrive { Id = match.ToString() };
         _logger.LogWarning($"Tried to parse drive.google.com Url but couldn't get an id from: {uri}");
         return null;
     }
@@ -102,7 +102,7 @@ public override async Task<bool> Verify(Archive archive, DTOs.DownloadStates.Goo
 
     public override IEnumerable<string> MetaIni(Archive a, DTOs.DownloadStates.GoogleDrive state)
     {
-        return new[] {$"directURL=https://drive.google.com/uc?id={state.Id}&export=download"};
+        return new[] { $"directURL=https://drive.google.com/uc?id={state.Id}&export=download" };
     }
 
     private async Task<HttpRequestMessage?> ToMessage(DTOs.DownloadStates.GoogleDrive state, bool download,

diff --git a/Wabbajack.Installer/AInstaller.cs b/Wabbajack.Installer/AInstaller.cs
@@ -370,26 +370,14 @@ public async Task DownloadMissingArchives(List<Archive> missing, CancellationTok
                 UpdateProgress(1);
             }
         }
-        
+
         await missing
             .Shuffle()
             .Where(a => a.State is not Manual)
             .PDoAll(async archive =>
             {
                 _logger.LogInformation("Downloading {Archive}", archive.Name);
                 var outputPath = _configuration.Downloads.Combine(archive.Name);
-                var downloadPackagePath = outputPath.WithExtension(Ext.DownloadPackage);
-
-                if (download)
-                    if (outputPath.FileExists() && !downloadPackagePath.FileExists())
-                    {
-                        var origName = Path.GetFileNameWithoutExtension(archive.Name);
-                        var ext = Path.GetExtension(archive.Name);
-                        var uniqueKey = archive.State.PrimaryKeyString.StringSha256Hex();
-                        outputPath = _configuration.Downloads.Combine(origName + "_" + uniqueKey + "_" + ext);
-                        outputPath.Delete();
-                    }
-
                 var hash = await DownloadArchive(archive, download, token, outputPath);
                 UpdateProgress(1);
             });

diff --git a/Wabbajack.Launcher/Program.cs b/Wabbajack.Launcher/Program.cs
@@ -59,8 +59,8 @@ public static void Main(string[] args)
                 services.AddSingleton<ITokenProvider<NexusOAuthState>, NexusApiTokenProvider>();
                 services.AddSingleton<HttpDownloader>();
                 services.AddAllSingleton<IResource, IResource<HttpClient>>(s => new Resource<HttpClient>("Web Requests", 4));
-                services.AddAllSingleton<IHttpDownloader, SingleThreadedDownloader>();
-                
+                services.AddResumableHttpDownloader();
+
                 var version =
                     $"{ThisAssembly.Git.SemVer.Major}.{ThisAssembly.Git.SemVer.Major}.{ThisAssembly.Git.SemVer.Patch}{ThisAssembly.Git.SemVer.DashLabel}";
                 services.AddSingleton(s => new ApplicationInfo