Task/rdmp 265 version data loads

CHANGELOG.md

@@ -4,11 +4,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased]
+## [8.4.3] - Unreleased
 
 - Build on and target .Net 9 rather than 8
 - Simplify DB Patching Interface
 - Fix issue with Simple File Extractor pipeline component checking
+- Add Load Metadata versioning
 
 ## [8.4.2] - 2024-12-18
 

Rdmp.Core.Tests/CommandExecution/ExecuteCommandCloneLoadMetadataTests.cs

@@ -0,0 +1,45 @@
+using NUnit.Framework;
+using Rdmp.Core.Curation.Data.DataLoad;
+using Rdmp.Core.Curation.Data;
+using Rdmp.Core.Repositories;
+using Tests.Common;
+using Rdmp.Core.DataLoad.Modules.Attachers;
+using Rdmp.Core.MapsDirectlyToDatabaseTable;
+using System.Linq;
+
+namespace Rdmp.Core.Tests.CommandExecution
+{
+    public class ExecuteCommandCloneLoadMetadataTests: DatabaseTests
+    {
+
+        [Test]
+        public void TestCloneLoadMetadata()
+        {
+            var lmd1 = new LoadMetadata(CatalogueRepository, "MyLmd");
+            lmd1.Description = "Desc!";
+            var cata = new Catalogue(CatalogueRepository, "myCata")
+            {
+                LoggingDataTask = "B"
+            };
+            cata.SaveToDatabase();
+            lmd1.LinkToCatalogue(cata);
+            var pt1 = new ProcessTask(CatalogueRepository, lmd1, LoadStage.Mounting)
+            {
+                ProcessTaskType = ProcessTaskType.Attacher,
+                LoadStage = LoadStage.Mounting,
+                Path = typeof(AnySeparatorFileAttacher).FullName
+            };
+            pt1.SaveToDatabase();
+
+            pt1.CreateArgumentsForClassIfNotExists(typeof(AnySeparatorFileAttacher));
+            var pta = pt1.ProcessTaskArguments.Single(pt => pt.Name == "Separator");
+            pta.SetValue(",");
+            pta.SaveToDatabase();
+            LoadMetadata clonedLmd;
+            clonedLmd = lmd1.Clone();
+            Assert.That(clonedLmd.ProcessTasks.Count(), Is.EqualTo(1));
+            Assert.That(clonedLmd.Description, Is.EqualTo(lmd1.Description));
+            Assert.That(clonedLmd.ProcessTasks.First().ProcessTaskArguments.First().Value, Is.EqualTo(lmd1.ProcessTasks.First().ProcessTaskArguments.First().Value));
+        }
+    }
+}

Rdmp.Core.Tests/CommandExecution/ExecuteCommandCreateLoadMetadataVersionTests.cs

@@ -0,0 +1,46 @@
+using NUnit.Framework;
+using Rdmp.Core.Curation.Data.DataLoad;
+using Rdmp.Core.Curation.Data;
+using Rdmp.Core.Repositories;
+using Tests.Common;
+using Rdmp.Core.DataLoad.Modules.Attachers;
+using Rdmp.Core.MapsDirectlyToDatabaseTable;
+using System.Linq;
+
+namespace Rdmp.Core.Tests.CommandExecution
+{
+    public class ExecuteCommandCreateLoadMetadataVersionTests : DatabaseTests
+    {
+
+        [Test]
+        public void TestCreateLoadMetadataVersion()
+        {
+            var lmd1 = new LoadMetadata(CatalogueRepository, "MyLmd");
+            lmd1.Description = "Desc!";
+            var cata = new Catalogue(CatalogueRepository, "myCata")
+            {
+                LoggingDataTask = "B"
+            };
+            cata.SaveToDatabase();
+            lmd1.LinkToCatalogue(cata);
+            var pt1 = new ProcessTask(CatalogueRepository, lmd1, LoadStage.Mounting)
+            {
+                ProcessTaskType = ProcessTaskType.Attacher,
+                LoadStage = LoadStage.Mounting,
+                Path = typeof(AnySeparatorFileAttacher).FullName
+            };
+            pt1.SaveToDatabase();
+
+            pt1.CreateArgumentsForClassIfNotExists(typeof(AnySeparatorFileAttacher));
+            var pta = pt1.ProcessTaskArguments.Single(pt => pt.Name == "Separator");
+            pta.SetValue(",");
+            pta.SaveToDatabase();
+            LoadMetadata clonedLmd;
+            clonedLmd = (LoadMetadata)lmd1.SaveNewVersion();
+            Assert.That(clonedLmd.ProcessTasks.Count(), Is.EqualTo(1));
+            Assert.That(clonedLmd.RootLoadMetadata_ID, Is.EqualTo(lmd1.ID));
+            Assert.That(clonedLmd.Description, Is.EqualTo(lmd1.Description));
+            Assert.That(clonedLmd.ProcessTasks.First().ProcessTaskArguments.First().Value, Is.EqualTo(lmd1.ProcessTasks.First().ProcessTaskArguments.First().Value));
+        }
+    }
+}

Rdmp.Core.Tests/CommandExecution/ExecuteCommandRestoreLoadMetadataTests.cs

@@ -0,0 +1,58 @@
+using NUnit.Framework;
+using Rdmp.Core.Curation.Data.DataLoad;
+using Rdmp.Core.Curation.Data;
+using Rdmp.Core.Repositories;
+using Tests.Common;
+using Rdmp.Core.DataLoad.Modules.Attachers;
+using Rdmp.Core.MapsDirectlyToDatabaseTable;
+using System.Linq;
+using Rdmp.Core.CommandExecution.AtomicCommands;
+using Rdmp.Core.CommandLine.Interactive;
+using Rdmp.Core.ReusableLibraryCode.Checks;
+using Rdmp.Core.CommandExecution;
+
+namespace Rdmp.Core.Tests.CommandExecution
+{
+    public class ExecuteCommandRestoreLoadMetadataTests : DatabaseTests
+    {
+
+        [Test]
+        public void TestRestoreLoadMetadataVersion()
+        {
+            var lmd1 = new LoadMetadata(CatalogueRepository, "MyLmd");
+            lmd1.Description = "Desc!";
+            var cata = new Catalogue(CatalogueRepository, "myCata")
+            {
+                LoggingDataTask = "B"
+            };
+            cata.SaveToDatabase();
+            lmd1.LinkToCatalogue(cata);
+            var pt1 = new ProcessTask(CatalogueRepository, lmd1, LoadStage.Mounting)
+            {
+                ProcessTaskType = ProcessTaskType.Attacher,
+                LoadStage = LoadStage.Mounting,
+                Path = typeof(AnySeparatorFileAttacher).FullName
+            };
+            pt1.SaveToDatabase();
+
+            pt1.CreateArgumentsForClassIfNotExists(typeof(AnySeparatorFileAttacher));
+            var pta = pt1.ProcessTaskArguments.Single(pt => pt.Name == "Separator");
+            pta.SetValue(",");
+            pta.SaveToDatabase();
+            LoadMetadata clonedLmd;
+            clonedLmd = (LoadMetadata)lmd1.SaveNewVersion();
+            Assert.That(clonedLmd.ProcessTasks.Count(), Is.EqualTo(1));
+            Assert.That(clonedLmd.RootLoadMetadata_ID, Is.EqualTo(lmd1.ID));
+            Assert.That(clonedLmd.Description, Is.EqualTo(lmd1.Description));
+            Assert.That(clonedLmd.ProcessTasks.First().ProcessTaskArguments.First().Value, Is.EqualTo(lmd1.ProcessTasks.First().ProcessTaskArguments.First().Value));
+            pt1.DeleteInDatabase();
+            var fetchedlmd = CatalogueRepository.GetObjectByID<LoadMetadata>(lmd1.ID);
+            Assert.That(fetchedlmd.ProcessTasks.Count(), Is.EqualTo(0));
+            var activator = new ThrowImmediatelyActivator(RepositoryLocator);
+            var cmd = new ExecuteCommandRestoreLoadMetadataVersion(activator, clonedLmd);
+            Assert.DoesNotThrow(()=>cmd.Execute());
+            fetchedlmd = CatalogueRepository.GetObjectByID<LoadMetadata>(lmd1.ID);
+            Assert.That(fetchedlmd.ProcessTasks.Count(), Is.EqualTo(1));
+        }
+    }
+}

Rdmp.Core/CommandExecution/AtomicCommandFactory.cs

@@ -502,39 +502,60 @@ public IEnumerable<IAtomicCommand> CreateCommands(object o)
 
         if (Is(o, out LoadMetadata lmd))
         {
-            yield return new ExecuteCommandExportObjectsToFile(_activator, new IMapsDirectlyToDatabaseTable[] { lmd });
-
-            yield return new ExecuteCommandOverrideRawServer(_activator, lmd);
-            yield return new ExecuteCommandCreateNewLoadMetadata(_activator);
-            var reservedTest = lmd.AllowReservedPrefix ? "Drop" : "Allow";
-            yield return new ExecuteCommandToggleAllowReservedPrefixForLoadMetadata(lmd)
+            if (lmd.RootLoadMetadata_ID is null)
             {
-                OverrideCommandName=$"{reservedTest} Reserved Prefix Columns"
-            };
+                yield return new ExecuteCommandExportObjectsToFile(_activator, new IMapsDirectlyToDatabaseTable[] { lmd });
 
-            yield return new ExecuteCommandSetGlobalDleIgnorePattern(_activator) { SuggestedCategory = Advanced };
-            yield return new ExecuteCommandSetIgnoredColumns(_activator, lmd) { SuggestedCategory = Advanced };
-            yield return new ExecuteCommandSetIgnoredColumns(_activator, lmd, null)
-            { OverrideCommandName = "Clear Ignored Columns", SuggestedCategory = Advanced };
+                yield return new ExecuteCommandOverrideRawServer(_activator, lmd);
+                yield return new ExecuteCommandCreateNewLoadMetadata(_activator);
+                var reservedTest = lmd.AllowReservedPrefix ? "Drop" : "Allow";
+                yield return new ExecuteCommandToggleAllowReservedPrefixForLoadMetadata(lmd)
+                {
+                    OverrideCommandName = $"{reservedTest} Reserved Prefix Columns"
+                };
+                yield return new ExecuteCommandCreateLoadMetadataVersion(_activator, lmd)
+                {
+                    OverrideCommandName = "Save Version"
+                };
+                yield return new ExecuteCommandCloneLoadMetadata(_activator, lmd)
+                {
+                    OverrideCommandName = "Clone Load Metadata"
+                };
 
-            yield return new ExecuteCommandSetExtendedProperty(_activator, new[] { lmd },
-                ExtendedProperty.PersistentRaw, null)
-            {
-                OverrideCommandName = "Persistent RAW",
-                PromptForValue = true,
-                PromptForValueTaskDescription = ExtendedProperty.PersistentRawDescription,
-                SuggestedCategory = Advanced
-            };
+                yield return new ExecuteCommandSetGlobalDleIgnorePattern(_activator) { SuggestedCategory = Advanced };
+                yield return new ExecuteCommandSetIgnoredColumns(_activator, lmd) { SuggestedCategory = Advanced };
+                yield return new ExecuteCommandSetIgnoredColumns(_activator, lmd, null)
+                { OverrideCommandName = "Clear Ignored Columns", SuggestedCategory = Advanced };
 
-            yield return new ExecuteCommandSet(_activator, lmd,
-                typeof(LoadMetadata).GetProperty(nameof(LoadMetadata.IgnoreTrigger)))
+                yield return new ExecuteCommandSetExtendedProperty(_activator, new[] { lmd },
+                    ExtendedProperty.PersistentRaw, null)
+                {
+                    OverrideCommandName = "Persistent RAW",
+                    PromptForValue = true,
+                    PromptForValueTaskDescription = ExtendedProperty.PersistentRawDescription,
+                    SuggestedCategory = Advanced
+                };
+
+                yield return new ExecuteCommandSet(_activator, lmd,
+                    typeof(LoadMetadata).GetProperty(nameof(LoadMetadata.IgnoreTrigger)))
+                {
+                    OverrideCommandName = $"Ignore Trigger (Current value:{lmd.IgnoreTrigger})",
+                    SuggestedCategory = Advanced
+                };
+            }
+            else
             {
-                OverrideCommandName = $"Ignore Trigger (Current value:{lmd.IgnoreTrigger})",
-                SuggestedCategory = Advanced
-            };
+                yield return new ExecuteCommandRestoreLoadMetadataVersion(_activator, lmd)
+                {
+                    OverrideCommandName = "Restore Version"
+                };
+                yield return new ExecuteCommandCloneLoadMetadata(_activator, lmd)
+                {
+                    OverrideCommandName = "Clone Load Metadata"
+                };
+            }
         }
 
-
         if (Is(o, out LoadMetadataScheduleNode scheduleNode))
             yield return new ExecuteCommandCreateNewLoadProgress(_activator, scheduleNode.LoadMetadata);
 

Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCloneLoadMetadata.cs

@@ -0,0 +1,26 @@
+using Rdmp.Core.Curation.Data.DataLoad;
+using Rdmp.Core.Curation.Data;
+
+namespace Rdmp.Core.CommandExecution.AtomicCommands;
+
+public class ExecuteCommandCloneLoadMetadata : BasicCommandExecution
+{
+    private readonly LoadMetadata _loadMetadata;
+    private readonly IBasicActivateItems _activator;
+    public ExecuteCommandCloneLoadMetadata(IBasicActivateItems activator,[DemandsInitialization("The LoadMetadata to clone")] LoadMetadata loadMetadata)
+    {
+
+        _loadMetadata = loadMetadata;
+        _activator = activator;
+    }
+
+    public override void Execute()
+    {
+        base.Execute();
+        var lmd = _loadMetadata.Clone();
+        lmd.Name = lmd.Name + " (Clone)";
+        lmd.SaveToDatabase();
+        _activator.Publish(lmd);
+
+    }
+}

Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateLoadMetadataVersion.cs

@@ -0,0 +1,29 @@
+using Rdmp.Core.Curation.Data.DataLoad;
+using Rdmp.Core.Curation.Data;
+using System;
+
+namespace Rdmp.Core.CommandExecution.AtomicCommands;
+
+public class ExecuteCommandCreateLoadMetadataVersion: BasicCommandExecution
+{
+    private readonly LoadMetadata _loadMetadata;
+    private readonly IBasicActivateItems _activator;
+    public ExecuteCommandCreateLoadMetadataVersion(IBasicActivateItems activator,[DemandsInitialization("The LoadMetadata to version")] LoadMetadata loadMetadata)
+    {
+
+        _loadMetadata = loadMetadata;
+        _activator = activator;
+    }
+
+    public override void Execute()
+    {
+        base.Execute();
+        if(_loadMetadata.RootLoadMetadata_ID != null)
+        {
+            throw new Exception("Must Use Root LoadMetadata to create Version");
+        }
+        var lmd = _loadMetadata.SaveNewVersion();
+        lmd.SaveToDatabase();
+        _activator.Publish(lmd);
+    }
+}

Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewClassBasedProcessTask.cs

@@ -72,7 +72,6 @@ public override void Execute()
             else
                 return;
         }
-
         var newTask = new ProcessTask(BasicActivator.RepositoryLocator.CatalogueRepository, _loadMetadata, _loadStage)
         {
             Path = _type.FullName,

Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewFileBasedProcessTask.cs

@@ -96,7 +96,6 @@ public override void Execute()
                 throw new ArgumentOutOfRangeException($"Unexpected _taskType:{_taskType}");
             }
         }
-
         var task = new ProcessTask((ICatalogueRepository)_loadMetadata.Repository, _loadMetadata, _loadStage)
         {
             ProcessTaskType = _taskType,

Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDelete.cs

@@ -48,7 +48,6 @@ public ExecuteCommandDelete(IBasicActivateItems activator,
         _allowDeleteMany = deleteMany;
         if (_deletables.Any(d => d is CohortAggregateContainer c && c.IsRootContainer()))
             SetImpossible("Cannot delete root containers");
-
         var reason = "";
 
         if (_deletables.Any(d => d is IMightBeReadOnly ro && ro.ShouldBeReadOnly(out reason)))

Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreLoadMetadataVersion.cs

@@ -0,0 +1,39 @@
+using Rdmp.Core.Curation.Data.DataLoad;
+using Rdmp.Core.Curation.Data;
+using System;
+using System.Linq;
+
+namespace Rdmp.Core.CommandExecution.AtomicCommands;
+
+public class ExecuteCommandRestoreLoadMetadataVersion : BasicCommandExecution
+{
+    private readonly LoadMetadata _loadMetadata;
+    private readonly IBasicActivateItems _activator;
+    public ExecuteCommandRestoreLoadMetadataVersion(IBasicActivateItems activator, [DemandsInitialization("The LoadMetadata to version")] LoadMetadata loadMetadata)
+    {
+
+        _loadMetadata = loadMetadata;
+        _activator = activator;
+    }
+
+    public override void Execute()
+    {
+        if (_activator.IsInteractive && !_activator.YesNo("Replace root Load Metadata with this configuration?", "Resotre Load Metadata Version")) return;
+        base.Execute();
+        if (_loadMetadata.RootLoadMetadata_ID is null)
+        {
+            throw new Exception("Must Use a versioned LoadMetadata to create Version");
+        }
+        LoadMetadata lmd = (LoadMetadata)_activator.RepositoryLocator.CatalogueRepository.GetObjectByID(typeof(LoadMetadata), (int)_loadMetadata.RootLoadMetadata_ID) ?? throw new Exception("Could not find root load metadata");
+        foreach (ProcessTask task in lmd.ProcessTasks.Cast<ProcessTask>())
+        {
+            task.DeleteInDatabase();
+        }
+        foreach (ProcessTask task in _loadMetadata.ProcessTasks.Cast<ProcessTask>())
+        {
+            task.Clone(lmd);
+        }
+        lmd.SaveToDatabase();
+        _activator.Publish(lmd);
+    }
+}

Rdmp.Core/Curation/Data/DataLoad/ILoadMetadata.cs

@@ -45,6 +45,11 @@ public interface ILoadMetadata : INamed, ILoggedActivityRootObject
     /// </summary>
     string LocationOfCacheDirectory { get; set; }
 
+    /// <summary>
+    /// The ID of the base Load Metadata that is version is based on
+    /// </summary>
+    int? RootLoadMetadata_ID { get; }
+
 
     /// <summary>
     /// Set to true to ignore the requirement for live tables to need the backup archive trigger