DownloadTask.cs

using System.Collections.Concurrent;
using System.Diagnostics;
using System.Net.Http.Headers;
using System.Security;
using System.Text;
using Blake3;
using Dalamud.Interface.ImGuiNotification;
using DequeNet;
using gfoidl.Base64;
using Heliosphere.Exceptions;
using Heliosphere.Model;
using Heliosphere.Model.Api;
using Heliosphere.Model.Generated;
using Heliosphere.Model.Penumbra;
using Heliosphere.Ui;
using Heliosphere.Ui.Dialogs;
using Heliosphere.Util;
using Microsoft.VisualBasic.FileIO;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Penumbra.Api.Enums;
using StrawberryShake;
using ZstdSharp;

namespace Heliosphere;

internal class DownloadTask : IDisposable {
    #if LOCAL
    internal const string ApiBase = "http://127.0.0.1:42011";
    #else
    internal const string ApiBase = "https://heliosphere.app/api";
    #endif

    internal Guid TaskId { get; } = Guid.NewGuid();
    internal required Plugin Plugin { get; init; }
    internal required string PenumbraRoot { get; init; }
    internal required Guid PackageId { get; init; }
    internal required Guid VariantId { get; init; }
    internal required Guid VersionId { get; init; }
    internal required bool IncludeTags { get; init; }
    internal required bool OpenInPenumbra { get; init; }
    internal required Guid? PenumbraCollection { get; init; }
    internal required string? PenumbraFolderOverride { get; init; }
    internal required IActiveNotification? Notification { get; set; }
    internal required LoginUpdateMode? LoginUpdateMode { get; set; }
    internal required PackageSettings.UpdateSetting ManualUpdateMode { get; set; }

    private string? PenumbraModPath { get; set; }
    private string? FilesPath { get; set; }
    /// <summary>
    /// Path to where existing files from before this task started are located.
    /// This is not guaranteed to exist!
    /// </summary>
    private string? OldFilesPath { get; set; }
    internal string? PackageName { get; private set; }
    internal string? VariantName { get; private set; }

    internal CancellationTokenSource CancellationToken { get; } = new();
    internal State State { get; private set; } = State.NotStarted;
    private uint _stateData;

    internal uint StateData => Interlocked.CompareExchange(ref this._stateData, 0, 0);

    internal uint StateDataMax { get; private set; }
    internal Exception? Error { get; private set; }
    private ConcurrentDeque<Measurement> Entries { get; } = new();
    private Util.SentryTransaction? Transaction { get; set; }
    private bool SupportsHardLinks { get; set; }
    private SemaphoreSlim DuplicateMutex { get; } = new(1, 1);
    private bool RequiresDuplicateMutex { get; set; }

    /// <summary>
    /// Mapping of hash to fully-qualified path.
    /// </summary>
    private Dictionary<string, string> ExistingHashes { get; } = [];

    /// <summary>
    /// A list of files expected by the group jsons made by this task. These
    /// paths should be relative to the files directory.
    /// </summary>
    private HashSet<string> ExpectedFiles { get; } = [];

    private Dictionary<Guid, (string, string)> ContainerNames { get; } = [];
    private HashSet<string> CrashingPapPaths { get; } = [];

    private const double Window = 5;
    private const string DefaultFolder = "_default";

    internal double BytesPerSecond {
        get {
            if (this.Entries.Count == 0) {
                return 0;
            }

            var total = 0u;
            var removeTo = 0;
            foreach (var entry in this.Entries) {
                if (Stopwatch.GetElapsedTime(entry.Ticks) > TimeSpan.FromSeconds(Window)) {
                    removeTo += 1;
                    continue;
                }

                total += entry.Data;
            }

            for (var i = 0; i < removeTo; i++) {
                this.Entries.TryPopLeft(out _);
            }

            return total / Window;
        }
    }

    private bool _disposed;

    /// This is non-null when a directory exists in the Penumbra directory that
    /// starts with hs- and ends with the variant/package IDs, and it does not
    /// equal the expected mod installation path. Essentially only true for
    /// version updates (not reinstalls).
    private string? _oldModName;

    private bool _reinstall;

    ~DownloadTask() {
        this.Dispose();
    }

    public void Dispose() {
        if (this._disposed) {
            return;
        }

        this._disposed = true;
        GC.SuppressFinalize(this);
        this.CancellationToken.Dispose();
        this.DuplicateMutex.Dispose();
    }

    internal Task Start() {
        return Task.Run(this.Run, this.CancellationToken.Token);
    }

    private async Task Run() {
        using var setNull = new OnDispose(() => this.Transaction = null);
        using var transaction = SentryHelper.StartTransaction(
            this.GetType().Name,
            nameof(this.Run)
        );
        this.Transaction = transaction;

        this.Transaction?.Inner.SetExtras(new Dictionary<string, object?> {
            [nameof(this.VersionId)] = this.VersionId.ToCrockford(),
            [nameof(this.IncludeTags)] = this.IncludeTags,
        });

        SentrySdk.AddBreadcrumb("Started download", "user", data: new Dictionary<string, string> {
            [nameof(this.VersionId)] = this.VersionId.ToCrockford(),
            [nameof(this.PenumbraModPath)] = this.PenumbraModPath ?? "<null>",
            [nameof(this.PenumbraCollection)] = this.PenumbraCollection?.ToString("N") ?? "<null>",
        });

        try {
            var info = await this.GetPackageInfo();

            this.Transaction?.Inner.SetExtra("Package", info.Variant.Package.Id.ToCrockford());
            this.Transaction?.Inner.SetExtra("Variant", info.Variant.Id.ToCrockford());

            this.PackageName = info.Variant.Package.Name;
            this.VariantName = info.Variant.Name;
            this.GenerateModDirectoryPath(info);
            this.DetermineIfUpdate(info);
            this.CreateDirectories();
            await this.TestHardLinks();
            this.CheckOutputPaths(info);
            await this.HashExistingFiles();
            await this.DownloadFiles(info);
            await this.ConstructModPack(info);
            this.RemoveWorkingDirectories();
            this.RemoveOldFiles();
            await this.AddMod(info);
            this.UpdateConfig(info);

            // before setting state to finished, set the directory name

            this.State = State.Finished;
            Interlocked.Exchange(ref this._stateData, 1);
            this.StateDataMax = 1;

            if (!this.Plugin.Config.UseNotificationProgress) {
                this.Notification = this.Notification.AddOrUpdate(
                    this.Plugin.NotificationManager,
                    type: NotificationType.Success,
                    title: "Install successful",
                    content: $"{this.PackageName} was installed in Penumbra.",
                    autoDuration: true
                );
                this.Notification.Click += async _ => await this.OpenModInPenumbra();
            }

            SentrySdk.AddBreadcrumb("Finished download", data: new Dictionary<string, string> {
                [nameof(this.VersionId)] = this.VersionId.ToCrockford(),
            });

            if (this.OpenInPenumbra) {
                await this.OpenModInPenumbra();
            }

            // refresh the manager package list after install finishes
            using (this.Transaction?.StartChild(nameof(this.Plugin.State.UpdatePackages))) {
                await this.Plugin.State.UpdatePackages(false, this.CancellationToken.Token);
            }
        } catch (Exception ex) when (ex is TaskCanceledException or OperationCanceledException) {
            this.State = State.Cancelled;
            Interlocked.Exchange(ref this._stateData, 0);
            this.StateDataMax = 0;

            if (this.Transaction?.Inner is { } inner) {
                inner.Status = SpanStatus.Cancelled;
            }
        } catch (Exception ex) {
            this.State = State.Errored;
            Interlocked.Exchange(ref this._stateData, 0);
            this.StateDataMax = 0;
            this.Error = ex;
            this.Notification = this.Notification.AddOrUpdate(
                this.Plugin.NotificationManager,
                type: NotificationType.Error,
                title: "Install failed",
                content: $"Failed to install {this.PackageName ?? "mod"}.",
                autoDuration: true
            );

            if (this.Transaction?.Inner is { } inner) {
                inner.Status = SpanStatus.InternalError;
            }

            this.Transaction?.Inner?.SetExtra("WasAntivirus", false);

            // probably antivirus (ioexception is being used by other process or
            // access denied)
            if (ex.IsAntiVirus()) {
                this.Plugin.PluginUi.OpenAntiVirusWarning();
                Plugin.Log.Warning(ex, $"[AV] Error downloading version {this.VersionId.ToCrockford()}");

                this.Transaction?.Inner?.SetExtra("WasAntivirus", true);
            } else if (ex is MultipleModDirectoriesException multiple) {
                await this.Plugin.PluginUi.AddToDrawAsync(new MultipleModDirectoriesDialog(
                    this.Plugin,
                    multiple
                ));
            } else {
                ErrorHelper.Handle(ex, $"Error downloading version {this.VersionId.ToCrockford()}", this.Transaction?.LatestChild()?.Inner ?? this.Transaction?.Inner);
            }
        }
    }

    private void SetStateData(uint current, uint max) {
        Interlocked.Exchange(ref this._stateData, current);
        this.StateDataMax = max;
    }

    internal static async Task<HttpResponseMessage> GetImage(Guid id, int imageId, CancellationToken token = default) {
        var resp = await Plugin.Client.GetAsync2($"{ApiBase}/web/package/{id:N}/image/{imageId}", HttpCompletionOption.ResponseHeadersRead, token);
        resp.EnsureSuccessStatusCode();
        return resp;
    }

    private async Task<IDownloadTask_GetVersion> GetPackageInfo() {
        using var span = this.Transaction?.StartChild(nameof(this.GetPackageInfo));

        this.State = State.DownloadingPackageInfo;
        this.SetStateData(0, 1);

        var downloadKind = DownloadKind.Install;
        var installed = await this.Plugin.State.GetInstalled(this.CancellationToken.Token);
        if (installed.TryGetValue(this.PackageId, out var pkg)) {
            if (pkg.Variants.Any(meta => meta.VariantId == this.VariantId)) {
                downloadKind = DownloadKind.Update;
            }
        }

        var resp = await Plugin.GraphQl.DownloadTask.ExecuteAsync(this.VersionId, downloadKind, this.CancellationToken.Token);
        resp.EnsureNoErrors();

        var version = resp.Data?.GetVersion ?? throw new MissingVersionException(this.VersionId);
        // sort needed files for dedupe consistency
        foreach (var files in version.NeededFiles.Files.Files.Values) {
            foreach (var list in files.Values) {
                list.Sort((a, b) => string.Compare($"{a.GamePath}:{a.ArchivePath}", $"{b.GamePath}:{b.ArchivePath}", StringComparison.Ordinal));
            }
        }

        // get the group/option names for containers
        var groups = GroupsUtil.Convert(version.Groups);
        foreach (var group in groups) {
            if (group is StandardGroup standard) {
                foreach (var option in standard.Inner.Options) {
                    this.ContainerNames[option.HsId] = (group.Name, option.Name);
                }
            } else if (group is CombiningGroup combining) {
                for (int i = 0; i < combining.Inner.Containers.Count; i++) {
                    var container = combining.Inner.Containers[i];
                    var name = string.IsNullOrWhiteSpace(container.Name)
                        ? $"container-{i + 1}"
                        : container.Name;
                    this.ContainerNames[container.HsId] = (group.Name, name);
                }
            }
        }

        Interlocked.Increment(ref this._stateData);
        return version;
    }

    private void GenerateModDirectoryPath(IDownloadTask_GetVersion info) {
        var dirName = HeliosphereMeta.ModDirectoryName(info.Variant.Package.Name, info.Version, (uint) info.Variant.ShortId);
        this.PenumbraModPath = Path.Join(this.PenumbraRoot, dirName);
    }

    private void CreateDirectories() {
        this.FilesPath = Path.GetFullPath(Path.Join(this.PenumbraModPath, "files"));
        this.OldFilesPath = Path.GetFullPath(Path.Join(this.PenumbraModPath, ".hs-old"));

        // this gets "Attempt to update previously set global instance." sometimes
        // https://github.com/microsoft/CsWinRT/issues/623
        // https://github.com/microsoft/winget-cli/pull/3253
        // https://github.com/microsoft/CsWinRT/issues/1371
        // https://github.com/dotnet/runtime/issues/45632
        // perhaps an issue due to plugin loading assembly context shit
        // await Plugin.Resilience.ExecuteAsync(async (token) => {
        //     try {
        //         var folder = await Windows.Storage.StorageFolder.GetFolderFromPathAsync(this.PenumbraRoot).AsTask(token);
        //         if (folder.Provider.Id is "Network" or "OneDrive") {
        //             throw new ModPathNetworkedException(this.PenumbraModPath!, folder.Provider);
        //         }
        //     } catch (COMException) {
        //         // no-op
        //     }
        // });

        // for now, just check if "OneDrive" is mentioned, even though this will
        // miss networked documents folders
        if (!this.Plugin.Config.Unsupported.AllowNetworkedInstalls && this.PenumbraRoot.Contains("\\OneDrive\\")) {
            throw new ModPathNetworkedException(this.PenumbraRoot, "OneDrive");
        }

        if (Path.Exists(this.FilesPath) && Path.Exists(this.OldFilesPath)) {
            // an update was interrupted
            Plugin.Log.Warning($"Va:{this.VariantId.ToCrockford()} appears to have an interrupted update. Keeping the directory with more files.");
            var filesCount = DirectoryHelper.GetFilesRecursive(this.FilesPath).Count();
            var oldFilesCount = DirectoryHelper.GetFilesRecursive(this.OldFilesPath).Count();

            // no extra action is necessary if the opposite is true
            if (oldFilesCount > filesCount) {
                // move old files path to files path, deleting files path
                Plugin.Resilience.Execute(() => Directory.Delete(this.FilesPath, true));
                Plugin.Resilience.Execute(() => Directory.Move(this.OldFilesPath, this.FilesPath));
            }
        }

        Plugin.Resilience.Execute(() => {
            try {
                Directory.Delete(this.OldFilesPath, true);
            } catch (DirectoryNotFoundException) {
                // ignore
            }
        });

        // if the files path already exists, this is an update/reinstall, so
        // move it to the old files path instead
        if (Path.Exists(this.FilesPath)) {
            Plugin.Resilience.Execute(() => Directory.Move(this.FilesPath, this.OldFilesPath));
            Plugin.Resilience.Execute(() => {
                var di = new DirectoryInfo(this.OldFilesPath);
                di.Attributes |= FileAttributes.Hidden;
            });
        }

        Plugin.Resilience.Execute(() => Directory.CreateDirectory(this.FilesPath));
    }

    private async Task TestHardLinks() {
        string? a = null;
        string? b = null;

        try {
            a = Path.Join(this.PenumbraModPath, Path.GetRandomFileName());
            await FileHelper.Create(a, true).DisposeAsync();

            b = Path.Join(this.PenumbraModPath, Path.GetRandomFileName());
            FileHelper.CreateHardLink(a, b);

            this.SupportsHardLinks = true;
        } catch (InvalidOperationException) {
            this.SupportsHardLinks = false;
        } finally {
            if (a != null) {
                try {
                    File.Delete(a);
                } catch (Exception ex) {
                    Plugin.Log.Warning(ex, "Could not delete temp files");
                }
            }

            if (b != null) {
                try {
                    File.Delete(b);
                } catch (Exception ex) {
                    Plugin.Log.Warning(ex, "Could not delete temp files");
                }
            }
        }
    }

    private void DetermineIfUpdate(IDownloadTask_GetVersion info) {
        var directories = Directory.EnumerateDirectories(this.PenumbraRoot)
            .Select(Path.GetFileName)
            .Where(path => !string.IsNullOrEmpty(path))
            .Cast<string>()
            .Where(path =>
                HeliosphereMeta.ParseDirectory(path) is { ShortVariantId: var shortVariantId }
                && shortVariantId == (uint) info.Variant.ShortId
            )
            .ToArray();

        if (directories.Length == 1) {
            var oldName = Path.Join(this.PenumbraRoot, directories[0]!);
            if (oldName == this.PenumbraModPath) {
                // the path found is what we expect it to be, so this is not a
                // version change but a reinstall
                this._reinstall = true;
            } else {
                // the path found is not what we expect it to be, so the version
                // has changed. rename the directory to the new version
                this._oldModName = directories[0];
                Directory.Move(oldName, this.PenumbraModPath!);
            }
        } else if (directories.Length > 1) {
            var rejoined = directories
                .Select(name => Path.Join(this.PenumbraRoot, name))
                .ToArray();

            throw new MultipleModDirectoriesException(
                info.Variant.Package.Name,
                info.Variant.Name,
                info.Version,
                rejoined
            );
        }
    }

    private void CheckOutputPaths(IDownloadTask_GetVersion info) {
        var neededFiles = info.NeededFiles.Files.Files;

        var outputToHash = new Dictionary<string, string>();
        foreach (var (hash, file) in neededFiles) {
            foreach (var outputPath in this.GetOutputPaths(file)) {
                if (outputToHash.TryGetValue(outputPath, out var stored) && stored != hash) {
                    Plugin.Log.Warning($"V:{this.VersionId.ToCrockford()} has the same output path linked to multiple hashes, will use slow duplication");
                    this.RequiresDuplicateMutex = true;
                    return;
                }

                outputToHash[outputPath] = hash;
            }

            // also check for pap crashes while we're here
            foreach (var (containerId, files) in file) {
                foreach (var needed in files) {
                    var ext = Path.GetExtension(needed.ArchivePath);
                    if (!".pap".Equals(ext, StringComparison.OrdinalIgnoreCase)) {
                        continue;
                    }

                    var outputPath = this.GetOutputPath(containerId, needed);
                    var fullPath = Path.Join(this.PenumbraModPath, "files", outputPath);
                    if (fullPath.Length <= 260) {
                        continue;
                    }

                    this.CrashingPapPaths.Add(fullPath);
                }
            }
        }
    }

    private async Task HashExistingFiles() {
        this.State = State.CheckingExistingFiles;
        this.SetStateData(0, 0);

        if (this.OldFilesPath == null) {
            throw new Exception("old files path was null");
        }

        // hash => path
        var hashes = new ConcurrentDictionary<string, string>();
        List<string> allFiles;
        try {
            allFiles = DirectoryHelper.GetFilesRecursive(this.OldFilesPath).ToList();
        } catch (DirectoryNotFoundException) {
            return;
        }

        this.StateDataMax = (uint) allFiles.Count;

        await Parallel.ForEachAsync(
            allFiles,
            new ParallelOptions {
                CancellationToken = this.CancellationToken.Token,
            },
            async (path, token) => {
                using var blake3 = new Blake3HashAlgorithm();
                blake3.Initialize();
                await using var file = FileHelper.OpenSharedReadIfExists(path);
                if (file == null) {
                    return;
                }

                await blake3.ComputeHashAsync(file, token);
                var hash = Base64.Url.Encode(blake3.Hash);

                hashes.TryAdd(hash, path);
                Interlocked.Increment(ref this._stateData);
            }
        );

        foreach (var (hash, path) in hashes) {
            this.ExistingHashes.Add(hash, path);
        }
    }

    private async Task DownloadFiles(IDownloadTask_GetVersion info) {
        using var span = this.Transaction?.StartChild(nameof(this.DownloadFiles));

        var task = info.Batched
            ? this.DownloadBatchedFiles(info.NeededFiles, info.Batches, this.FilesPath!)
            : this.DownloadNormalFiles(info.NeededFiles, this.FilesPath!);
        await task;
    }

    private Task DownloadNormalFiles(IDownloadTask_GetVersion_NeededFiles neededFiles, string filesPath) {
        using var span = this.Transaction?.StartChild(nameof(this.DownloadNormalFiles));

        this.State = State.DownloadingFiles;
        this.SetStateData(0, (uint) neededFiles.Files.Files.Count);

        return Parallel.ForEachAsync(
            neededFiles.Files.Files,
            new ParallelOptions {
                CancellationToken = this.CancellationToken.Token,
            },
            async (pair, token) => {
                var (hash, files) = pair;
                var outputPaths = this.GetOutputPaths(files);

                using (await SemaphoreGuard.WaitAsync(Plugin.DownloadSemaphore, token)) {
                    await this.DownloadFile(new Uri(neededFiles.BaseUri), filesPath, outputPaths, hash);
                }
            }
        );
    }

    private Task DownloadBatchedFiles(IDownloadTask_GetVersion_NeededFiles neededFiles, BatchList batches, string filesPath) {
        using var span = this.Transaction?.StartChild(nameof(this.DownloadBatchedFiles));

        this.State = State.DownloadingFiles;
        this.SetStateData(0, 0);

        var neededHashes = neededFiles.Files.Files.Keys.ToList();
        var clonedBatches = batches.Files.ToDictionary(pair => pair.Key, pair => pair.Value.ToDictionary(pair => pair.Key, pair => pair.Value));
        var seenHashes = new List<string>();
        foreach (var (batch, files) in batches.Files) {
            // remove any hashes that aren't needed
            foreach (var hash in files.Keys) {
                if (neededHashes.Contains(hash) && !seenHashes.Contains(hash)) {
                    seenHashes.Add(hash);
                } else {
                    clonedBatches[batch].Remove(hash);
                }
            }

            // remove any empty batches
            if (clonedBatches[batch].Count == 0) {
                clonedBatches.Remove(batch);
            }
        }

        this.StateDataMax = (uint) neededFiles.Files.Files.Count;

        return Parallel.ForEachAsync(
            clonedBatches,
            new ParallelOptions {
                CancellationToken = this.CancellationToken.Token,
            },
            async (pair, token) => {
                var (batch, batchedFiles) = pair;

                // find which files from this batch we already have a hash for
                var toDuplicate = new Dictionary<string, string>();
                foreach (var hash in batchedFiles.Keys) {
                    if (this.ExistingHashes.TryGetValue(hash, out var path)) {
                        toDuplicate.Add(hash, path);
                    }
                }

                // sort files in batch by offset, removing already-downloaded files
                var listOfFiles = batchedFiles
                    .Select(pair => (Hash: pair.Key, Info: pair.Value))
                    .Where(pair => !this.ExistingHashes.ContainsKey(pair.Hash))
                    .OrderBy(pair => pair.Info.Offset).ToList();

                if (listOfFiles.Count > 0) {
                    // calculate ranges
                    var ranges = new List<(ulong, ulong)>();
                    var begin = 0ul;
                    var end = 0ul;
                    var chunk = new List<string>();
                    var chunks = new List<List<string>>();
                    foreach (var (hash, info) in listOfFiles) {
                        if (begin == 0 && end == 0) {
                            // first item, so set begin and end
                            begin = info.Offset;
                            end = info.Offset + info.SizeCompressed;
                            // add the hash to this chunk
                            chunk.Add(hash);

                            continue;
                        }

                        if (info.Offset == end) {
                            // there's no gap, so extend the end of this range
                            end += info.SizeCompressed;
                            // add the hash to this chunk
                            chunk.Add(hash);
                            continue;
                        }

                        // there is a gap
                        // add this chunk to the list of chunks
                        chunks.Add(chunk);
                        // make a new chunk
                        chunk = [];

                        // add the range to the list of ranges
                        ranges.Add((begin, end));

                        // start a new range after the gap
                        begin = info.Offset;
                        end = info.Offset + info.SizeCompressed;

                        // add the hash to the new chunk
                        chunk.Add(hash);
                    }

                    if (end != 0) {
                        // add the last range if necessary
                        ranges.Add((begin, end));

                        if (chunk.Count > 0) {
                            chunks.Add(chunk);
                        }
                    }

                    // check if we're just downloading the whole file - cf cache
                    // won't kick in for range requests
                    var totalBatchSize = batchedFiles.Values
                        .Select(file => file.SizeCompressed)
                        // no Sum function for ulong
                        .Aggregate((total, size) => total + size);

                    RangeHeaderValue? rangeHeader;
                    if (ranges is [{ Item1: 0, Item2: var rangeEnd }] && rangeEnd == totalBatchSize) {
                        rangeHeader = null;
                    } else {
                        // construct the header
                        var minOffset = ulong.MaxValue;
                        var maxOffset = 0ul;

                        foreach (var (from, to) in ranges) {
                            minOffset = Math.Min(minOffset, from);
                            maxOffset = Math.Max(maxOffset, to);
                        }

                        if (minOffset > maxOffset) {
                            throw new Exception("invalid offsets");
                        }

                        rangeHeader = new RangeHeaderValue((long) minOffset, (long) maxOffset);
                    }

                    // construct the uri
                    var baseUri = new Uri(new Uri(neededFiles.BaseUri), "../batches/");
                    var uri = new Uri(baseUri, batch);

                    using (await SemaphoreGuard.WaitAsync(Plugin.DownloadSemaphore, token)) {
                        var counter = new StateCounter();
                        await Plugin.Resilience.ExecuteAsync(
                            async _ => {
                                // if we're retrying, remove the files that this task added
                                Interlocked.Add(ref this._stateData, UintHelper.OverflowSubtractValue(counter.Added));
                                counter.Added = 0;

                                await this.DownloadBatchedFile(neededFiles, filesPath, uri, rangeHeader, chunks, ranges, batchedFiles, counter);
                            },
                            token
                        );
                    }
                }

                foreach (var (hash, path) in toDuplicate) {
                    var gamePaths = neededFiles.Files.Files[hash];
                    var outputPaths = this.GetOutputPaths(gamePaths);

                    await this.DuplicateFile(filesPath, outputPaths, path);

                    Interlocked.Increment(ref this._stateData);
                }
            }
        );
    }

    private class StateCounter {
        internal uint Added { get; set; }
    }

    private async Task DownloadBatchedFile(
        IDownloadTask_GetVersion_NeededFiles neededFiles,
        string filesPath,
        Uri uri,
        RangeHeaderValue? rangeHeader,
        IReadOnlyList<List<string>> chunks,
        IReadOnlyList<(ulong, ulong)> ranges,
        IReadOnlyDictionary<string, BatchedFile> batchedFiles,
        StateCounter counter
    ) {
        using var span = this.Transaction?.StartChild(nameof(this.DownloadBatchedFile), true);
        span?.Inner.SetExtras(new Dictionary<string, object?> {
            [nameof(uri)] = uri,
            [nameof(rangeHeader)] = rangeHeader,
            [nameof(chunks)] = chunks,
            [nameof(batchedFiles)] = batchedFiles,
        });

        // construct the request
        using var req = new HttpRequestMessage(HttpMethod.Get, uri);
        req.Headers.Range = rangeHeader;

        // send the request
        using var resp = await Plugin.Client.SendAsync2(req, HttpCompletionOption.ResponseHeadersRead, this.CancellationToken.Token);
        resp.EnsureSuccessStatusCode();

        // cloudflare is completely fucking up multiple range cached responses,
        // so now we're just requesting the min-max offset as a single range.
        // check if there's more than one chunk or not
        IMultipartProvider multipart;
        if (chunks.Count > 1) {
            // this is a min-max offset range request to work around cf
            multipart = new SingleMultipleMultipartProvider(resp.Content, ranges);
        } else {
            // this is a single chunk - one file
            multipart = new SingleMultipartProvider(resp.Content);
        }

        // // if only one chunk is requested, it's not multipart, so check
        // // for that
        // IMultipartProvider multipart;
        // if (resp.Content.IsMimeMultipartContent()) {
        //     var boundary = resp.Content.Headers.ContentType
        //         ?.Parameters
        //         .Find(p => p.Name == "boundary")
        //         ?.Value;
        //     if (boundary == null) {
        //         throw new Exception("missing boundary in multipart response");
        //     }

        //     multipart = new StandardMultipartProvider(boundary, resp.Content);
        // } else if (chunks.Count == 1) {
        //     multipart = new SingleMultipartProvider(resp.Content);
        // } else if (rangeHeader != null && resp.StatusCode == HttpStatusCode.PartialContent) {
        //     // we requested multiple chunks, but we didn't get the expected
        //     // multipart response. this is a cloudflare bug. work around this by
        //     // requesting a range containing the lowest and highest offsets.
        //     resp.Dispose();

        //     var minOffset = rangeHeader.Ranges.Select(range => range.From).Min();
        //     var maxOffset = rangeHeader.Ranges.Select(range => range.To).Max();
        //     var adjustedRangeHeader = new RangeHeaderValue(minOffset, maxOffset);

        //     // construct the request with the adjusted header
        //     using var request = new HttpRequestMessage(HttpMethod.Get, uri);
        //     request.Headers.Range = adjustedRangeHeader;

        //     // NOTE: this is not disposed as a convenience. making this
        //     // workaround able to cleanly dispose this response would be a pain
        //     // in the ass. fortunately, as of the time of writing (2025-01-20),
        //     // all a Response.Dispose call does is call Dispose on the Content
        //     // if necessary. since we have the disposeMultipart handler below,
        //     // this doesn't actually matter.
        //     var response = await Plugin.Client.SendAsync2(request, HttpCompletionOption.ResponseHeadersRead, this.CancellationToken.Token);

        //     // this is a special wrapper that will return a wrapped stream that
        //     // emulates the server returning the proper response. this will
        //     // waste bandwidth.
        //     multipart = new SingleMultipleMultipartProvider(response.Content, rangeHeader.Ranges);
        // } else {
        //     throw new Exception("unexpected download response state");
        // }

        using var disposeMultipart = new OnDispose(multipart.Dispose);

        foreach (var chunk in chunks) {
            await using var rawStream = await multipart.GetNextStreamAsync(this.CancellationToken.Token);
            if (rawStream == null) {
                throw new Exception("did not download correct number of chunks");
            }

            await using var stream = new GloballyThrottledStream(
                rawStream,
                this.Entries
            );

            // now we're going to read each file in the chunk out,
            // decompress it, and write it to the disk
            var buffer = new byte[81_920];
            foreach (var hash in chunk) {
                // firstly, we now need to figure out which extensions and
                // discriminators to use for this specific file
                var gamePaths = neededFiles.Files.Files[hash];
                var outputPaths = this.GetOutputPaths(gamePaths);
                if (outputPaths.Length == 0) {
                    Plugin.Log.Warning($"file with hash {hash} has no output paths");
                    continue;
                }

                var batchedFileInfo = batchedFiles[hash];
                var path = Path.Join(filesPath, outputPaths[0]);
                await using var file = FileHelper.Create(path, true);
                // make a stream that's only capable of reading the
                // amount of compressed bytes
                await using var limited = new LimitedStream(stream, (int) batchedFileInfo.SizeCompressed);
                await using var decompressor = new DecompressionStream(limited);

                // make sure we only read *this* file - one file is only
                // part of the multipart chunk
                var total = 0ul;
                while (total < batchedFileInfo.SizeUncompressed) {
                    var leftToRead = Math.Min(
                        (ulong) buffer.Length,
                        batchedFileInfo.SizeUncompressed - total
                    );
                    var read = await decompressor.ReadAsync(buffer.AsMemory(0, (int) leftToRead), this.CancellationToken.Token);
                    total += (ulong) read;

                    if (read == 0) {
                        break;
                    }

                    await file.WriteAsync(buffer.AsMemory()[..read], this.CancellationToken.Token);
                }

                // make sure we read all the bytes before moving on to
                // the next file
                await limited.ReadToEndAsync(buffer, this.CancellationToken.Token);

                // flush the file and close it
                await file.FlushAsync(this.CancellationToken.Token);
                // ReSharper disable once DisposeOnUsingVariable
                await file.DisposeAsync();

                // the file is now fully written to, so duplicate it if
                // necessary
                await this.DuplicateFile(filesPath, outputPaths, path);

                Interlocked.Increment(ref this._stateData);
                counter.Added += 1;
            }
        }
    }

    private static string MakePathPartsSafe(string input) {
        var cleaned = input
            .Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar, '\\', '/')
            .Select(MakeFileNameSafe);
        return string.Join(Path.DirectorySeparatorChar, cleaned);
    }

    private static string MakeFileNameSafe(string input) {
        var invalid = Path.GetInvalidFileNameChars();

        var sb = new StringBuilder(input.Length);
        foreach (var ch in input) {
            sb.Append(
                Array.IndexOf(invalid, ch) == -1
                    ? ch
                    : '-'
            );
        }

        var path = sb.ToString();
        return path.TrimEnd('.', ' ');
    }

    private string GetOutputPath(Guid containerId, NeededFile file) {
        var outputPath = file.ArchivePath;
        if (outputPath != null) {
            if (Path.GetExtension(outputPath) == string.Empty) {
                // we need to add an extension or this can cause a crash
                outputPath = Path.ChangeExtension(outputPath, Path.GetExtension(file.GamePath));
            }

            return MakePathPartsSafe(outputPath);
        }

        var groupName = DefaultFolder;
        var optionName = DefaultFolder;
        if (this.ContainerNames.TryGetValue(containerId, out var names)) {
            groupName = names.Item1;
            optionName = names.Item2;
        }

        var group = MakeFileNameSafe(groupName ?? DefaultFolder);
        var option = MakeFileNameSafe(optionName ?? DefaultFolder);
        var gamePath = MakePathPartsSafe(file.GamePath);

        return Path.Join(group, option, gamePath);
    }

    private string[] GetOutputPaths(Dictionary<Guid, List<NeededFile>> files) {
        return [.. files
            .SelectMany(entry => {
                var (containerId, files) = entry;

                return files.Select(file => this.GetOutputPath(containerId, file));
            })
            .Where(file => !string.IsNullOrEmpty(file))
            .Order()];
    }

    private async Task DuplicateFile(string filesDir, IEnumerable<string> outputPaths, string path) {
        using var guard = this.RequiresDuplicateMutex
            ? await SemaphoreGuard.WaitAsync(this.DuplicateMutex, this.CancellationToken.Token)
            : null;

        if (!this.SupportsHardLinks) {
            // If hard links aren't supported, move the path to the first output
            // path.
            // This is done because things reference the first output path
            // assuming it will exist.
            var firstPath = outputPaths.FirstOrDefault();
            if (firstPath == null) {
                return;
            }

            var dest = Path.Join(filesDir, firstPath);
            if (dest.Equals(path, StringComparison.InvariantCultureIgnoreCase)) {
                return;
            }

            var parent = PathHelper.GetParent(dest);
            Plugin.Resilience.Execute(() => Directory.CreateDirectory(parent));

            if (!await PathHelper.WaitForDelete(dest)) {
                throw new DeleteFileException(dest);
            }

            // ReSharper disable once AccessToModifiedClosure
            Plugin.Resilience.Execute(() => File.Move(path, dest));
            return;
        }

        foreach (var outputPath in outputPaths) {
            await DuplicateInner(outputPath);
        }

        return;

        async Task DuplicateInner(string dest) {
            dest = Path.Join(filesDir, dest);
            if (path.Equals(dest, StringComparison.InvariantCultureIgnoreCase)) {
                return;
            }

            if (!Path.IsPathFullyQualified(path) || !Path.IsPathFullyQualified(dest)) {
                throw new Exception($"{path} or {dest} was not fully qualified");
            }

            if (!await PathHelper.WaitForDelete(dest)) {
                throw new DeleteFileException(dest);
            }

            var parent = PathHelper.GetParent(dest);
            Plugin.Resilience.Execute(() => Directory.CreateDirectory(parent));

            Plugin.Resilience.Execute(() => FileHelper.CreateHardLink(path, dest));
        }
    }

    private void RemoveOldFiles() {
        using var span = this.Transaction?.StartChild(nameof(this.RemoveOldFiles));

        this.State = State.RemovingOldFiles;
        this.SetStateData(0, 0);

        // find old, normal files no longer being used to remove
        var presentFiles = DirectoryHelper.GetFilesRecursive(this.FilesPath!)
            .Select(path => PathHelper.MakeRelativeSub(this.FilesPath!, path))
            .Where(path => !string.IsNullOrEmpty(path))
            .Cast<string>()
            .Select(path => path.ToLowerInvariant())
            .ToHashSet();

        // remove the files that we expect from the list of already-existing
        // files - these are the files to remove now
        presentFiles.ExceptWith(this.ExpectedFiles);

        var total = (uint) presentFiles.Count;
        this.SetStateData(0, total);

        var done = 0u;
        Parallel.ForEach(
            presentFiles,
            extra => {
                var extraPath = Path.Join(this.FilesPath, extra);
                Plugin.Log.Info($"removing extra file {extraPath}");
                Plugin.Resilience.Execute(() => {
                    if (this.Plugin.Config.UseRecycleBin) {
                        try {
                            FileSystem.DeleteFile(extraPath, UIOption.OnlyErrorDialogs, RecycleOption.SendToRecycleBin);
                        } catch (Exception ex) when (ex is IOException { HResult: Consts.UsedByAnotherProcess } io) {
                            var procs = RestartManager.GetLockingProcesses(extraPath);
                            throw new AlreadyInUseException(io, extraPath, procs);
                        }
                    } else {
                        FileHelper.Delete(extraPath);
                    }
                });

                done += 1;
                this.SetStateData(done, total);
            }
        );

        // remove any empty directories
        DirectoryHelper.RemoveEmptyDirectories(this.FilesPath!);
    }

    private async Task DownloadFile(Uri baseUri, string filesPath, string[] outputPaths, string hash) {
        using var span = this.Transaction?.StartChild(nameof(this.DownloadFile), true);
        span?.Inner.SetExtras(new Dictionary<string, object?> {
            [nameof(hash)] = hash,
            [nameof(outputPaths)] = outputPaths,
        });

        if (outputPaths.Length == 0) {
            return;
        }

        // check each path for containment breaks when joining
        foreach (var outputPath in outputPaths) {
            var joined = Path.GetFullPath(Path.Join(filesPath, outputPath));
            // check that this path is under the files path still
            if (PathHelper.MakeRelativeSub(filesPath, joined) == null) {
                throw new SecurityException($"path from mod was attempting to leave the files directory: '{joined}' is not within '{filesPath}'");
            }
        }

        // find an existing path that has this hash
        if (this.ExistingHashes.TryGetValue(hash, out var validPath)) {
            goto Duplicate;
        }

        // no valid, existing file, so download instead
        var path = Path.Join(filesPath, outputPaths[0]);
        validPath = path;

        await Plugin.Resilience.ExecuteAsync(
            async _ => {
                var uri = new Uri(baseUri, hash).ToString();
                using var resp = await Plugin.Client.GetAsync2(uri, HttpCompletionOption.ResponseHeadersRead, this.CancellationToken.Token);
                resp.EnsureSuccessStatusCode();

                await using var file = FileHelper.Create(path, true);
                await using var stream = new GloballyThrottledStream(
                    await resp.Content.ReadAsStreamAsync(this.CancellationToken.Token),
                    this.Entries
                );
                await using var decompress = new DecompressionStream(stream);
                await decompress.CopyToAsync(file, this.CancellationToken.Token);

                span?.Inner.SetMeasurement("Decompressed", file.Position, MeasurementUnit.Information.Byte);
            },
            this.CancellationToken.Token
        );

        Duplicate:
        await this.DuplicateFile(filesPath, outputPaths, validPath);

        Interlocked.Increment(ref this._stateData);
    }

    private async Task ConstructModPack(IDownloadTask_GetVersion info) {
        using var span = this.Transaction?.StartChild(nameof(this.ConstructModPack));

        this.State = State.ConstructingModPack;
        this.SetStateData(0, 5);
        var hsMeta = await this.ConstructHeliosphereMeta(info);
        await this.ConstructMeta(info, hsMeta);
        var defaultMod = await this.ConstructDefaultMod(info);
        var groups = await this.ConstructGroups(info);
        await this.DuplicateUiFiles(defaultMod, groups);
    }

    private string GenerateModName(IDownloadTask_GetVersion info) {
        var pkgName = info.Variant.Package.Name.Replace('/', '-');
        var name = $"{this.Plugin.Config.TitlePrefix}{pkgName}";

        if (!this.Plugin.Config.HideDefaultVariant || info.Variant.Name != Consts.DefaultVariant) {
            var varName = info.Variant.Name.Replace('/', '-');
            name += $" ({varName})";
        }

        return name;
    }

    private async Task ConstructMeta(IDownloadTask_GetVersion info, HeliosphereMeta hsMeta) {
        using var span = this.Transaction?.StartChild(nameof(this.ConstructMeta));

        var tags = this.IncludeTags
            ? info.Variant.Package.Tags.Select(tag => tag.Slug).ToList()
            : [];

        var metaPath = Path.Join(this.PenumbraModPath, "meta.json");

        // keep old name if configured to
        var modName = this.GenerateModName(info);
        if (!this.Plugin.Config.ReplaceModName) {
            try {
                var oldJson = await File.ReadAllTextAsync(metaPath);
                var oldMeta = JsonConvert.DeserializeObject<ModMeta>(oldJson);
                if (oldMeta != null) {
                    modName = oldMeta.Name;
                }
            } catch (Exception) {
                // ignore
            }
        }

        var meta = new ModMeta {
            Name = modName,
            Author = info.Variant.Package.User.Username,
            Description = info.MetaInfo.Description ?? "",
            Version = info.Version,
            Website = info.MetaInfo.Url ?? "",
            ModTags = [.. tags],
            DefaultPreferredItems = info.DefaultPreferredItemIds == null
                ? null
                : [.. info.DefaultPreferredItemIds.Select(id => unchecked((ulong) id))],
        };
        var json = JsonConvert.SerializeObject(meta, Formatting.Indented);

        var path = Path.Join(this.PenumbraModPath, "meta.json");
        await using var file = FileHelper.Create(path);
        await file.WriteAsync(Encoding.UTF8.GetBytes(json), this.CancellationToken.Token);
        Interlocked.Increment(ref this._stateData);
    }

    private async Task<HeliosphereMeta> ConstructHeliosphereMeta(IDownloadTask_GetVersion info) {
        using var span = this.Transaction?.StartChild(nameof(this.ConstructHeliosphereMeta));

        var meta = new HeliosphereMeta {
            Id = info.Variant.Package.Id,
            Name = info.Variant.Package.Name,
            Tagline = info.Variant.Package.Tagline,
            Description = info.Variant.Package.Description,
            Author = info.Variant.Package.User.Username,
            AuthorId = info.Variant.Package.User.Id,
            Variant = info.Variant.Name,
            VariantId = info.Variant.Id,
            ShortVariantId = (uint) info.Variant.ShortId,
            Version = info.Version,
            VersionId = this.VersionId,
            IncludeTags = this.IncludeTags,
        };

        var metaJson = JsonConvert.SerializeObject(meta, Formatting.Indented);
        var path = Path.Join(this.PenumbraModPath, "heliosphere.json");
        await using var file = FileHelper.Create(path);
        await file.WriteAsync(Encoding.UTF8.GetBytes(metaJson), this.CancellationToken.Token);

        // save cover image
        if (info.Variant.Package.Images.Count > 0) {
            var coverImage = info.Variant.Package.Images[0];
            var coverPath = Path.Join(this.PenumbraModPath, "cover.webp");

            try {
                using var image = await GetImage(info.Variant.Package.Id, coverImage.Id, this.CancellationToken.Token);
                await using var cover = FileHelper.Create(coverPath);
                await image.Content.CopyToAsync(cover, this.CancellationToken.Token);
            } catch (Exception ex) {
                ErrorHelper.Handle(ex, "Could not download cover image");
            }
        }

        if (this.CrashingPapPaths.Count > 0) {
            var paths = this.CrashingPapPaths.ToArray();
            Array.Sort(paths);
            this.Plugin.PluginUi.AddIfNotPresent(new PapCrashWarning(this.Plugin, meta, this.PenumbraRoot, paths));
        }

        Interlocked.Increment(ref this._stateData);

        return meta;
    }

    private static string GetReplacedPath(string? groupName, string? optionName, string gamePath, string? archivePath) {
        var replacedPath = archivePath == null
            ? Path.Join(
                MakeFileNameSafe(groupName ?? DefaultFolder),
                MakeFileNameSafe(optionName ?? DefaultFolder),
                MakePathPartsSafe(gamePath)
            )
            : MakePathPartsSafe(archivePath);

        if (Path.GetExtension(replacedPath) == string.Empty) {
            replacedPath = Path.ChangeExtension(replacedPath, Path.GetExtension(gamePath));
        }

        return replacedPath;
    }

    private async Task<DefaultMod> ConstructDefaultMod(IDownloadTask_GetVersion info) {
        using var span = this.Transaction?.StartChild(nameof(this.ConstructDefaultMod));

        var defaultMod = new DefaultMod {
            Manipulations = ManipTokensForOption(info.NeededFiles.DefaultManipulations),
            FileSwaps = info.DefaultOption?.FileSwaps.Swaps ?? [],
        };

        foreach (var neededContainers in info.NeededFiles.Files.Files.Values) {
            if (!neededContainers.TryGetValue(Guid.Empty, out var defaultFiles)) {
                continue;
            }

            foreach (var file in defaultFiles) {
                var replacedPath = this.SupportsHardLinks
                    ? GetReplacedPath(null, null, file.GamePath, file.ArchivePath)
                    : GetReplacedPath(null, null, defaultFiles[0].GamePath, defaultFiles[0].ArchivePath);

                defaultMod.Files[file.GamePath] = Path.Join("files", replacedPath);
                this.ExpectedFiles.Add(replacedPath.ToLowerInvariant());
            }
        }

        await this.SaveDefaultMod(defaultMod);
        Interlocked.Increment(ref this._stateData);

        return defaultMod;
    }

    private async Task SaveDefaultMod(DefaultMod defaultMod) {
        var json = JsonConvert.SerializeObject(defaultMod, Formatting.Indented);

        var path = Path.Join(this.PenumbraModPath, "default_mod.json");
        await using var output = FileHelper.Create(path);
        await output.WriteAsync(Encoding.UTF8.GetBytes(json), this.CancellationToken.Token);
    }

    private async Task<List<ModGroup>> ConstructGroups(IDownloadTask_GetVersion info) {
        using var span = this.Transaction?.StartChild(nameof(this.ConstructGroups));

        // remove any groups that already exist
        var existingGroups = Directory.EnumerateFiles(this.PenumbraModPath!)
            .Where(file => {
                var name = Path.GetFileName(file);
                return name.StartsWith("group_") && name.EndsWith(".json");
            });

        var oldGroups = new List<ModGroup>();
        foreach (var existing in existingGroups) {
            try {
                var text = await FileHelper.ReadAllTextAsync(existing, this.CancellationToken.Token);
                ModGroup? group;
                try {
                    group = JsonConvert.DeserializeObject<StandardModGroup>(text);
                } catch {
                    try {
                        group = JsonConvert.DeserializeObject<ImcModGroup>(text);
                    } catch {
                        group = JsonConvert.DeserializeObject<CombiningModGroup>(text);
                    }
                }

                if (group == null) {
                    Plugin.Log.Warning("Could not deserialise old group (was null)");
                    continue;
                }

                oldGroups.Add(group);
            } catch (Exception ex) {
                Plugin.Log.Warning(ex, "Could not deserialise old group");
            }

            FileHelper.Delete(existing);
        }

        var rawGroups = GroupsUtil.Convert(info.Groups).ToList();
        var containers = new Dictionary<Guid, (string, string, IContainer)>();
        var modGroups = new Dictionary<string, ModGroup>(rawGroups.Count);
        foreach (var group in rawGroups) {
            ModGroup modGroup;
            switch (group) {
                case StandardGroup { Inner: var inner }: {
                    var standard = new StandardModGroup(group.Name, group.Description, group.GroupType.ToString()) {
                        Priority = group.Priority,
                        DefaultSettings = group.DefaultSettings,
                        OriginalIndex = (group.OriginalIndex, 0),
                    };

                    foreach (var option in inner.Options) {
                        var manipulations = ManipTokensForOption(option.Manipulations);
                        var item = new OptionItem {
                            Name = option.Name,
                            Description = option.Description,
                            Priority = option.Priority,
                            Manipulations = manipulations,
                            FileSwaps = option.FileSwaps.Swaps,
                            IsDefault = option.IsDefault,
                        };
                        standard.Options.Add(item);
                        containers[option.HsId] = (group.Name, option.Name, item);
                    }

                    modGroup = standard;

                    break;
                }
                case ImcGroup { Inner: var inner }: {
                    var identifier = JToken.Parse(inner.Identifier.GetRawText());
                    var defaultEntry = JToken.Parse(inner.DefaultEntry.GetRawText());
                    var imc = new ImcModGroup(group.Name, group.Description, identifier, inner.AllVariants, inner.OnlyAttributes, defaultEntry) {
                        Priority = group.Priority,
                        DefaultSettings = group.DefaultSettings,
                        OriginalIndex = (group.OriginalIndex, 0),
                    };

                    foreach (var option in inner.Options) {
                        imc.Options.Add(new ImcOption {
                            Name = option.Name,
                            Description = option.Description,
                            IsDisableSubMod = option.IsDisableSubMod,
                            AttributeMask = option.AttributeMask,
                        });
                    }

                    modGroup = imc;
                    break;
                }
                case CombiningGroup { Inner: var inner }: {
                    var combining = new CombiningModGroup(group.Name, group.Description, group.GroupType.ToString()) {
                        Priority = group.Priority,
                        DefaultSettings = group.DefaultSettings,
                        OriginalIndex = (group.OriginalIndex, 0),
                    };

                    foreach (var option in inner.Options) {
                        combining.Options.Add(new CombiningOption {
                            Name = option.Name,
                            Description = option.Description,
                            IsDefault = option.IsDefault,
                        });
                    }

                    for (int i = 0; i < inner.Containers.Count; i++) {
                        var container = inner.Containers[i];
                        var manipulations = ManipTokensForOption(container.Manipulations);
                        var item = new CombiningContainer {
                            Name = container.Name,
                            Manipulations = manipulations,
                            FileSwaps = container.FileSwaps.Swaps,
                        };
                        combining.Containers.Add(item);

                        var optionName = string.IsNullOrWhiteSpace(container.Name)
                            ? $"container-{i + 1}"
                            : container.Name;

                        containers[container.HsId] = (group.Name, optionName, item);
                    }

                    modGroup = combining;

                    break;
                }
                default:
                    throw new Exception("unknown mod group type");
            }

            modGroups[group.Name] = modGroup;
        }

        // add files to their respective containers
        foreach (var (_, neededContainers) in info.NeededFiles.Files.Files) {
            foreach (var (containerId, files) in neededContainers) {
                if (!containers.TryGetValue(containerId, out var entry)) {
                    continue;
                }

                var (groupName, optionName, container) = entry;

                foreach (var file in files) {
                    var replacedPath = this.SupportsHardLinks
                        ? GetReplacedPath(groupName, optionName, file.GamePath, file.ArchivePath)
                        : GetReplacedPath(groupName, optionName, files[0].GamePath, files[0].ArchivePath);
                    container.AddFile(file.GamePath, Path.Join("files", replacedPath));
                    this.ExpectedFiles.Add(replacedPath.ToLowerInvariant());
                }
            }
        }

        // split groups that have more than 32 options
        var splitGroups = SplitGroups(modGroups.Values);

        var list = splitGroups
            .OrderBy(group => group.OriginalIndex)
            .ToList();

        if (this.Plugin.Config.WarnAboutBreakingChanges && this._oldModName != null) {
            var oldVersion = "???";
            var installedPkgs = await this.Plugin.State.GetInstalled(this.CancellationToken.Token);
            if (installedPkgs.TryGetValue(info.Variant.Package.Id, out var meta)) {
                var variant = meta.Variants.Find(v => v.VariantId == info.Variant.Id);
                if (variant != null) {
                    oldVersion = variant.Version;
                }
            }

            var change = new BreakingChange {
                ModName = info.Variant.Package.Name,
                VariantName = info.Variant.Name,
                OldVersion = oldVersion,
                NewVersion = info.Version,
                ModPath = HeliosphereMeta.ModDirectoryName(info.Variant.Package.Name, info.Version, (uint) info.Variant.ShortId),
            };

            var settings = await this.Plugin.Framework.RunOnFrameworkThread(() => {
                var collections = this.Plugin.Penumbra.GetCollections();
                if (collections == null) {
                    return [];
                }

                var allSettings = new Dictionary<string, HashSet<string>>();
                foreach (var (collectionId, collectionName) in collections) {
                    var gcms = this.Plugin.Penumbra.GetCurrentModSettings(collectionId, this._oldModName, false);
                    if (gcms == null) {
                        continue;
                    }

                    var (result, settings) = gcms.Value;
                    if (result != PenumbraApiEc.Success || settings == null) {
                        continue;
                    }

                    foreach (var (group, options) in settings.Value.EnabledOptions) {
                        if (!change.OldSelectedOptions.TryGetValue(collectionName, out var collection)) {
                            collection = [];
                            change.OldSelectedOptions.Add(collectionName, collection);
                        }

                        collection.Add((group, options));

                        if (!allSettings.TryGetValue(group, out var value)) {
                            value = [];
                            allSettings.Add(group, value);
                        }

                        foreach (var option in options) {
                            value.Add(option);
                        }
                    }
                }

                return allSettings;
            });

            foreach (var oldGroup in oldGroups) {
                if (!settings.TryGetValue(oldGroup.Name, out var currentOptions)) {
                    // no settings for this group, so breaking changes don't matter
                    continue;
                }

                if (currentOptions.Count == 0) {
                    // not using this group, so breaking changes don't matter
                    continue;
                }

                var newGroup = list.Find(g => g.Name == oldGroup.Name);
                if (newGroup == null) {
                    // an old group is missing, so penumbra won't have any settings
                    // saved anymore
                    change.RemovedGroups.Add(oldGroup.Name);
                    continue;
                }

                if (newGroup.Type != oldGroup.Type) {
                    change.ChangedType.Add(oldGroup.Name);
                    // the rest of these changes don't matter, since this is a
                    // large breaking change
                    continue;
                }

                var newOptions = (newGroup switch {
                    StandardModGroup { Options: var options } => options.Select(o => o.Name),
                    ImcModGroup { Options: var options } => options.Select(o => o.Name),
                    CombiningModGroup { Options: var options } => options.Select(o => o.Name),
                    _ => throw new Exception("unexpected mod group type"),
                }).ToArray();
                var oldOptions = (oldGroup switch {
                    StandardModGroup { Options: var options } => options.Select(o => o.Name),
                    ImcModGroup { Options: var options } => options.Select(o => o.Name),
                    CombiningModGroup { Options: var options } => options.Select(o => o.Name),
                    _ => throw new Exception("unexpected mod group type"),
                }).ToArray();
                if (newOptions.Length < oldOptions.Length) {
                    var missingOptions = oldOptions.Skip(newOptions.Length).ToArray();
                    if (missingOptions.Any(opt => currentOptions.Contains(opt))) {
                        change.TruncatedOptions.Add((oldGroup.Name, missingOptions));
                    }
                }

                var smallest = Math.Min(newOptions.Length, oldOptions.Length);

                var sameNames = oldOptions.Take(smallest).Order()
                    .SequenceEqual(newOptions.Take(smallest).Order());
                var sameOrder = oldOptions.Take(smallest)
                    .SequenceEqual(newOptions.Take(smallest));

                var addList = sameNames switch {
                    true when !sameOrder => change.ChangedOptionOrder,
                    false => change.DifferentOptionNames,
                    _ => null,
                };

                if (addList != null) {
                    for (var i = 0; i < smallest; i++) {
                        // skip any same options or non-enabled options
                        if (oldOptions[i] == newOptions[i] || !currentOptions.Contains(oldOptions[i])) {
                            continue;
                        }

                        addList.Add((oldGroup.Name, oldOptions, newOptions));
                        break;
                    }
                }
            }

            if (change.HasChanges) {
                using var handle = await this.Plugin.PluginUi.BreakingChangeWindow.BreakingChanges.WaitAsync(this.CancellationToken.Token);
                handle.Data.Add(change);

                // write text file
                var breakingPath = Path.Join(this.PenumbraModPath, "breaking");
                Plugin.Resilience.Execute(() => Directory.CreateDirectory(breakingPath));
                var date = DateTime.Now;
                var safeOldVersion = oldVersion == "???" ? "Unknown" : oldVersion;
                var txtPath = Path.Join(breakingPath, $"{date:yyyy-MM-dd HH-mm-ss} - {safeOldVersion} to {info.Version}.txt");
                var txt = new StringBuilder($@"Breaking change report
======================
Mod name........: {info.Variant.Package.Name} ({info.Variant.Name})
Previous version: {oldVersion}
New version.....: {info.Version}

Breaking changes were introduced in version {info.Version}. A summary of the changes
and a list of your settings before the changes appear below.

Summary of changes
------------------");
                if (change.RemovedGroups.Count > 0) {
                    txt.Append("\n\n- Removed option groups -\nThese option groups are no longer available (but may be available under a\ndifferent name), which means your settings for this group are no longer applied.\n");
                    foreach (var group in change.RemovedGroups) {
                        txt.Append($"\n    - {group}");
                    }
                }

                if (change.ChangedType.Count > 0) {
                    txt.Append("\n\n- Changed group type -\nThese option groups have gone from single-select to multi-select or vice versa,\nwhich can change your selected options in unexpected ways.\n");
                    foreach (var group in change.ChangedType) {
                        txt.Append($"\n    - {group}");
                    }
                }

                if (change.TruncatedOptions.Count > 0) {
                    txt.Append("\n\n- Removed options -\nThese option groups have had options removed from the end, which has unselected\noptions you had enabled.");
                    foreach (var (group, options) in change.TruncatedOptions) {
                        txt.Append($"\n    - {group}");
                        foreach (var option in options) {
                            txt.Append($"\n        - {option}");
                        }
                    }
                }

                if (change.DifferentOptionNames.Count > 0) {
                    txt.Append("\n\n- Changed option names -\nThese option groups have had their option names changed, which may have\nunexpectedly changed what options you have selected.");
                    foreach (var (group, _, _) in change.DifferentOptionNames) {
                        txt.Append($"\n    - {group}");
                    }
                }

                if (change.ChangedOptionOrder.Count > 0) {
                    txt.Append("\n\n- Changed option order -\nThese option groups have had their options reordered, which may have\nunexpectedly changed what options you have selected.");
                    foreach (var (group, _, _) in change.ChangedOptionOrder) {
                        txt.Append($"\n    - {group}");
                    }
                }

                if (change.OldSelectedOptions.Count > 0) {
                    txt.Append("\n\nSettings before breaking changes\n--------------------------------");

                    foreach (var (collection, groups) in change.OldSelectedOptions) {
                        txt.Append($"\n\n- {collection} -");
                        foreach (var (group, options) in groups) {
                            txt.Append($"\n    - {group}");
                            foreach (var option in options) {
                                txt.Append($"\n        - {option}");
                            }
                        }
                    }
                }

                txt.AppendLine();
                await Plugin.Resilience.ExecuteAsync(
                    async token => await File.WriteAllTextAsync(txtPath, txt.ToString(), token),
                    this.CancellationToken.Token
                );
            }
        }

        for (var i = 0; i < list.Count; i++) {
            using var innerSpan = this.Transaction?.StartChild("SaveGroup");
            innerSpan?.Inner.SetExtras(new Dictionary<string, object?> {
                ["name"] = list[i].Name,
            });

            await this.SaveGroup(i, list[i]);
            Interlocked.Increment(ref this._stateData);
        }

        return list;
    }

    private async Task SaveGroup(int index, ModGroup group) {
        var invalidChars = Path.GetInvalidFileNameChars();
        var slug = group.Name.ToLowerInvariant()
            .Select(c => invalidChars.Contains(c) ? '-' : c)
            .Aggregate(new StringBuilder(), (sb, c) => sb.Append(c))
            .ToString();
        var json = JsonConvert.SerializeObject(group, Formatting.Indented);
        var path = Path.Join(this.PenumbraModPath, $"group_{index + 1:000}_{slug}.json");
        await using var file = FileHelper.Create(path);
        await file.WriteAsync(Encoding.UTF8.GetBytes(json), this.CancellationToken.Token);
    }

    private static IEnumerable<ModGroup> SplitGroups(IEnumerable<ModGroup> groups) {
        return groups.SelectMany(SplitGroup);
    }

    private static IEnumerable<ModGroup> SplitGroup(ModGroup rawGroup) {
        const int perGroup = 32;

        if (rawGroup is not StandardModGroup group) {
            return [rawGroup];
        }

        if (group.Type != "Multi" || group.Options.Count <= perGroup) {
            return [group];
        }

        var newGroups = new List<StandardModGroup>();
        for (var i = 0; i < group.Options.Count; i++) {
            var option = group.Options[i];
            var groupIdx = i / perGroup;
            var optionIdx = i % perGroup;

            if (optionIdx == 0) {
                newGroups.Add(new StandardModGroup($"{group.Name}, Part {groupIdx + 1}", group.Description, group.Type) {
                    Priority = group.Priority,
                    OriginalIndex = (group.OriginalIndex.Item1, (uint) groupIdx + 1),
                });
            }

            var newGroup = newGroups[groupIdx];
            newGroup.Options.Add(option);
            if (option.IsDefault) {
                newGroup.DefaultSettings |= unchecked(1ul << optionIdx);
            }
        }

        return newGroups;
    }

    private static List<JToken> ManipTokensForOption(IReadOnlyList<System.Text.Json.JsonElement> rawManipulations) {
        var manipulations = rawManipulations
            .Select(manip => {
                var token = JToken.Parse(manip.GetRawText());
                if (token is JObject jObject) {
                    var type = jObject["Type"];
                    jObject.Remove("Type");
                    jObject.AddFirst(new JProperty("Type", type));
                }

                return token;
            })
            .ToList();

        return manipulations ?? [];
    }

    private async Task DuplicateUiFiles(DefaultMod defaultMod, List<ModGroup> modGroups) {
        // check for any game path that starts with the ui prefix that is
        // referenced in more than one option. make its file name unique for
        // each reference.
        const string uiPrefix = "ui/";

        // first record unique references
        var references = new Dictionary<string, (uint, List<Action<string>>)>();
        UpdateReferences(defaultMod.Files);
        foreach (var group in modGroups) {
            if (group is StandardModGroup standard) {
                foreach (var option in standard.Options) {
                    UpdateReferences(option.Files);
                }
            } else if (group is CombiningModGroup combining) {
                foreach (var container in combining.Containers) {
                    UpdateReferences(container.Files);
                }
            }
        }

        // then find any uniquely referenced more than once
        foreach (var (joinedOutputPath, (refs, updatePathActions)) in references) {
            var outputPath = joinedOutputPath[6..];
            if (refs < 2) {
                continue;
            }

            // At this point, we have identified a path on disk that is
            // referenced more than once by differing options. This path needs
            // to be duplicated with a different file name to avoid crashes.
            // This process can be done using hard links if they're supported;
            // otherwise copy the file.

            Action<string, string> duplicateMethod = this.SupportsHardLinks
                ? FileHelper.CreateHardLink
                : File.Copy;

            var src = Path.Join(this.FilesPath, outputPath);
            for (var i = 0; i < refs; i++) {
                var ext = $".{i + 1}" + Path.GetExtension(outputPath);
                var newRelative = Path.ChangeExtension(outputPath, ext);
                var dst = Path.Join(this.FilesPath, newRelative);

                Plugin.Resilience.Execute(() => {
                    FileHelper.DeleteIfExists(dst);
                    duplicateMethod(src, dst);
                });

                // update the path
                updatePathActions[i](Path.Join("files", newRelative));
                this.ExpectedFiles.Add(newRelative.ToLowerInvariant());
            }

            // remove the original file
            Plugin.Resilience.Execute(() => File.Delete(src));
            this.ExpectedFiles.Remove(outputPath);
        }

        await this.SaveDefaultMod(defaultMod);

        for (var i = 0; i < modGroups.Count; i++) {
            await this.SaveGroup(i, modGroups[i]);
        }

        Interlocked.Increment(ref this._stateData);
        return;

        void UpdateReferences(Dictionary<string, string> files) {
            foreach (var (gamePath, outputPath) in files) {
                if (!gamePath.StartsWith(uiPrefix)) {
                    continue;
                }

                // normalise case of output path
                var normalised = outputPath.ToLowerInvariant();

                if (!references.TryGetValue(normalised, out var refs)) {
                    refs = (0, []);
                }

                refs.Item2.Add(path => {
                    files[gamePath] = path;
                });
                refs.Item1 += 1;

                references[normalised] = refs;
            }
        }
    }

    private void RemoveWorkingDirectories() {
        if (this.OldFilesPath == null) {
            return;
        }

        Plugin.Resilience.Execute(() => {
            try {
                Directory.Delete(this.OldFilesPath, true);
            } catch (DirectoryNotFoundException) {
                // ignore
            }
        });
    }

    private async Task AddMod(IDownloadTask_GetVersion info) {
        using var span = this.Transaction?.StartChild(nameof(this.AddMod));

        this.State = State.AddingMod;
        this.SetStateData(0, 1);

        await this.Plugin.Framework.RunOnFrameworkThread(() => {
            using var span = this.Transaction?.StartChild($"{nameof(this.AddMod)} - {nameof(this.Plugin.Framework.RunOnFrameworkThread)}");

            SentrySdk.AddBreadcrumb("Adding mod", data: new Dictionary<string, string> {
                ["_oldModName"] = this._oldModName ?? "<null>",
            });

            string? oldPath = null;
            if (this._oldModName != null) {
                oldPath = this.Plugin.Penumbra.GetModPath(this._oldModName);
                if (oldPath != null && this.Plugin.Config.ReplaceSortName) {
                    var parts = oldPath.Split('/');
                    parts[^1] = this.GenerateModName(info);
                    oldPath = string.Join('/', parts);
                }

                this.Plugin.Penumbra.DeleteMod(this._oldModName);
            }

            var modPath = Path.GetFileName(this.PenumbraModPath!);
            if (this.Plugin.Penumbra.AddMod(modPath) is { } result && result != PenumbraApiEc.Success) {
                throw new Exception($"could not add mod to Penumbra ({Enum.GetName(result)}): \"{modPath}\"");
            }

            if (this._reinstall) {
                this.Plugin.Penumbra.ReloadMod(modPath);
            }

            // put mod in folder
            var folder = this.PenumbraFolderOverride ?? this.Plugin.Config.PenumbraFolder;
            if (oldPath == null && !string.IsNullOrWhiteSpace(folder)) {
                var modName = this.GenerateModName(info);
                this.Plugin.Penumbra.SetModPath(modPath, $"{folder}/{modName}");
            } else if (oldPath != null) {
                this.Plugin.Penumbra.SetModPath(modPath, oldPath);
            }

            if (this._oldModName != null) {
                this.Plugin.Penumbra.CopyModSettings(this._oldModName, modPath);
            }

            if (this.PenumbraCollection != null) {
                this.Plugin.Penumbra.TrySetMod(this.PenumbraCollection.Value, modPath, true);
            }

            Interlocked.Increment(ref this._stateData);
        });
    }

    private void UpdateConfig(IDownloadTask_GetVersion info) {
        if (!this.Plugin.Config.TryGetPackageSettings(info.Variant.Package.Id, out var settings)) {
            settings = PackageSettings.NewDefault;
        }

        settings.LoginUpdateMode = this.LoginUpdateMode;
        settings.Update = this.ManualUpdateMode;
    }

    /// <summary>
    /// Open the mod in Penumbra if it was successfully installed. Runs on
    /// framework thread, so no need to call this from within
    /// <see cref="Dalamud.Plugin.Services.IFramework.RunOnFrameworkThread(Action)"/>.
    /// </summary>
    internal Task OpenModInPenumbra() {
        if (this.State != State.Finished || this.PenumbraModPath is not { } path) {
            return Task.CompletedTask;
        }

        return this.Plugin.Framework.RunOnFrameworkThread(() => {
            this.Plugin.Penumbra.OpenMod(Path.GetFileName(path));
        });
    }

    internal string? GetErrorInformation() {
        if (this.Error is not { } error) {
            return null;
        }

        return error.ToBbCode();
    }

    internal struct Measurement {
        internal long Ticks;
        internal uint Data;
    }
}

internal enum State {
    NotStarted,
    DownloadingPackageInfo,
    CheckingExistingFiles,
    SettingUpExistingFiles,
    DownloadingFiles,
    ConstructingModPack,
    RemovingOldFiles,
    AddingMod,
    Finished,
    Errored,
    Cancelled,
}

internal static class StateExt {
    internal static string Name(this State state) {
        return state switch {
            State.NotStarted => "Not started",
            State.DownloadingPackageInfo => "Downloading package info",
            State.CheckingExistingFiles => "Checking existing files",
            State.SettingUpExistingFiles => "Setting up existing files",
            State.DownloadingFiles => "Downloading files",
            State.ConstructingModPack => "Constructing mod pack",
            State.AddingMod => "Adding mod",
            State.RemovingOldFiles => "Removing old files",
            State.Finished => "Finished",
            State.Errored => "Errored",
            State.Cancelled => "Cancelled",
            _ => throw new ArgumentOutOfRangeException(nameof(state), state, null),
        };
    }

    internal static bool IsDone(this State state) {
        return state switch {
            State.Finished => true,
            State.Errored => true,
            State.Cancelled => true,
            _ => false,
        };
    }

    internal static Stream GetIconStream(this State state) {
        return state switch {
            State.NotStarted => Resourcer.Resource.AsStream("Heliosphere.Resources.clock.png"),
            State.DownloadingPackageInfo => Resourcer.Resource.AsStream("Heliosphere.Resources.magnifying-glass.png"),
            State.CheckingExistingFiles
                or State.SettingUpExistingFiles => Resourcer.Resource.AsStream("Heliosphere.Resources.hard-drives.png"),
            State.DownloadingFiles => Resourcer.Resource.AsStream("Heliosphere.Resources.cloud-arrow-down.png"),
            State.ConstructingModPack => Resourcer.Resource.AsStream("Heliosphere.Resources.package.png"),
            State.AddingMod => Resourcer.Resource.AsStream("Heliosphere.Resources.file-plus.png"),
            State.RemovingOldFiles => Resourcer.Resource.AsStream("Heliosphere.Resources.trash-simple.png"),
            State.Finished => Resourcer.Resource.AsStream("Heliosphere.Resources.check.png"),
            State.Errored => Resourcer.Resource.AsStream("Heliosphere.Resources.warning.png"),
            State.Cancelled => Resourcer.Resource.AsStream("Heliosphere.Resources.prohibit-inset.png"),
            _ => throw new ArgumentOutOfRangeException(nameof(state), state, null),
        };
    }
}

internal class LimitedStream : Stream {
    private readonly Stream _inner;
    private readonly int _maxRead;

    private int _read;

    internal int ReadAmount => this._read;

    internal LimitedStream(Stream inner, int maxRead) {
        this._inner = inner;
        this._maxRead = maxRead;
    }

    public override void Flush() {
        this._inner.Flush();
    }

    /// <summary>
    /// Reads until hitting the read limit. Note that this does not allow valid
    /// reads from the buffer, as it is overwritten with multiple read calls.
    /// </summary>
    /// <param name="buffer">a buffer of bytes</param>
    public void ReadToEnd(byte[] buffer) {
        while (this._read < this._maxRead) {
            var leftToRead = Math.Min(
                buffer.Length,
                this._maxRead - this._read
            );

            _ = this.Read(buffer, 0, leftToRead);
        }
    }

    /// <summary>
    /// Reads until hitting the read limit. Note that this does not allow valid
    /// reads from the buffer, as it is overwritten with multiple read calls.
    /// </summary>
    /// <param name="buffer">a buffer of bytes</param>
    public async Task ReadToEndAsync(byte[] buffer, CancellationToken token = default) {
        while (this._read < this._maxRead) {
            var leftToRead = Math.Min(
                buffer.Length,
                this._maxRead - this._read
            );

            _ = await this.ReadAsync(buffer.AsMemory(0, leftToRead), token);
        }
    }

    public override int Read(byte[] buffer, int offset, int count) {
        if (this._read >= this._maxRead) {
            return 0;
        }

        if (count + this._read > this._maxRead) {
            count = this._maxRead - this._read;
        }

        var read = this._inner.Read(buffer, offset, count);
        this._read += read;
        return read;
    }

    public override long Seek(long offset, SeekOrigin origin) {
        return this._inner.Seek(offset, origin);
    }

    public override void SetLength(long value) {
        this._inner.SetLength(value);
    }

    public override void Write(byte[] buffer, int offset, int count) {
        this._inner.Write(buffer, offset, count);
    }

    public override bool CanRead => this._inner.CanRead;
    public override bool CanSeek => this._inner.CanSeek;
    public override bool CanWrite => this._inner.CanWrite;
    public override long Length => this._inner.Length;

    public override long Position {
        get => this._inner.Position;
        set => this._inner.Position = value;
    }
}