From eb4288151bf5997a5c9c084e9ce3ac7b49b5533f Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 13 Jan 2025 11:52:12 +0000 Subject: [PATCH 01/15] basic versions --- ...ExecuteCommandCreateLoadMetadataVersion.cs | 30 ++++++++++++++++ .../Curation/Data/DataLoad/ILoadMetadata.cs | 5 +++ .../Curation/Data/DataLoad/IProcessTask.cs | 3 ++ .../Curation/Data/DataLoad/LoadMetadata.cs | 35 +++++++++++++++++-- .../Curation/Data/DataLoad/ProcessTask.cs | 15 +++++--- .../CreateCatalogue.sql | 2 ++ .../up/089_AddDataLoadVersioning.sql | 12 +++++++ Rdmp.Core/Rdmp.Core.csproj | 1 + SharedAssemblyInfo.cs | 6 ++-- 9 files changed, 99 insertions(+), 10 deletions(-) create mode 100644 Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateLoadMetadataVersion.cs create mode 100644 Rdmp.Core/Databases/CatalogueDatabase/up/089_AddDataLoadVersioning.sql diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateLoadMetadataVersion.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateLoadMetadataVersion.cs new file mode 100644 index 0000000000..f5c6cc87d8 --- /dev/null +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateLoadMetadataVersion.cs @@ -0,0 +1,30 @@ +using Rdmp.Core.Curation.Data.DataLoad; +using Rdmp.Core.Curation.Data; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.CommandExecution.AtomicCommands; + +public class ExecuteCommandCreateLoadMetadataVersion: BasicCommandExecution +{ + private LoadMetadata _loadMetadata; + public ExecuteCommandCreateLoadMetadataVersion([DemandsInitialization("The LoadMetadata to update")] LoadMetadata loadMetadata) + { + + _loadMetadata = loadMetadata; + } + + public override void Execute() + { + base.Execute(); + if(_loadMetadata.RootLoadMetadata_ID != null) + { + throw new Exception("Must Use Root LoadMetadata to create Version"); + } + var lmd = _loadMetadata.Clone(); + lmd.SaveToDatabase(); + } +} diff --git a/Rdmp.Core/Curation/Data/DataLoad/ILoadMetadata.cs b/Rdmp.Core/Curation/Data/DataLoad/ILoadMetadata.cs index 89bc7eace3..31061e2aad 100644 --- a/Rdmp.Core/Curation/Data/DataLoad/ILoadMetadata.cs +++ b/Rdmp.Core/Curation/Data/DataLoad/ILoadMetadata.cs @@ -45,6 +45,11 @@ public interface ILoadMetadata : INamed, ILoggedActivityRootObject /// string LocationOfCacheDirectory { get; set; } + /// + /// The ID of the base Load Metadata that is version is based on + /// + int? RootLoadMetadata_ID { get; } + /// /// Set to true to ignore the requirement for live tables to need the backup archive trigger diff --git a/Rdmp.Core/Curation/Data/DataLoad/IProcessTask.cs b/Rdmp.Core/Curation/Data/DataLoad/IProcessTask.cs index b974fe66f2..baec117ecb 100644 --- a/Rdmp.Core/Curation/Data/DataLoad/IProcessTask.cs +++ b/Rdmp.Core/Curation/Data/DataLoad/IProcessTask.cs @@ -48,4 +48,7 @@ public interface IProcessTask : IRevertable, IArgumentHost, IOrderable, IDisable /// #nullable enable string? SerialisableConfiguration { get; } + + + int LoadMetadataVersion { get; } } \ No newline at end of file diff --git a/Rdmp.Core/Curation/Data/DataLoad/LoadMetadata.cs b/Rdmp.Core/Curation/Data/DataLoad/LoadMetadata.cs index 2b02f72d40..287a7d8d2c 100644 --- a/Rdmp.Core/Curation/Data/DataLoad/LoadMetadata.cs +++ b/Rdmp.Core/Curation/Data/DataLoad/LoadMetadata.cs @@ -11,6 +11,7 @@ using System.Linq; using FAnsi.Discovery; using FAnsi.Discovery.QuerySyntax; +using MongoDB.Driver; using Rdmp.Core.Curation.Data.Cache; using Rdmp.Core.Curation.Data.Defaults; using Rdmp.Core.Curation.Data.ImportExport; @@ -61,6 +62,7 @@ public class LoadMetadata : DatabaseEntity, ILoadMetadata, IHasDependencies, IHa private string _folder; private DateTime? _lastLoadTime; private bool _allowReservedPrefix; + private int? _rootLoadMetadata_ID; public string DefaultForLoadingPath = Path.Combine("Data", "ForLoading"); public string DefaultForArchivingPath = Path.Combine("Data", "ForArchiving"); @@ -192,6 +194,9 @@ public DateTime? LastLoadTime set => SetField(ref _lastLoadTime, value); } + /// + public int? RootLoadMetadata_ID { get => _rootLoadMetadata_ID; private set => SetField(ref _rootLoadMetadata_ID, value); } + #endregion @@ -231,16 +236,17 @@ public LoadMetadata() /// /// /// - public LoadMetadata(ICatalogueRepository repository, string name = null) + /// + public LoadMetadata(ICatalogueRepository repository, string name = null, LoadMetadata rootLoadMetadata = null) { name ??= $"NewLoadMetadata{Guid.NewGuid()}"; - repository.InsertAndHydrate(this, new Dictionary { { "Name", name }, { "IgnoreTrigger", false /*todo could be system global default here*/ }, { "Folder", FolderHelper.Root }, - {"LastLoadTime", null } + {"LastLoadTime", null }, + {"RootLoadMetadata_ID", rootLoadMetadata is null? null: rootLoadMetadata.ID } }); } @@ -261,6 +267,29 @@ internal LoadMetadata(ICatalogueRepository repository, DbDataReader r) Folder = r["Folder"] as string ?? FolderHelper.Root; LastLoadTime = string.IsNullOrWhiteSpace(r["LastLoadTime"].ToString()) ? null : DateTime.Parse(r["LastLoadTime"].ToString()); AllowReservedPrefix = ObjectToNullableBool(r["AllowReservedPrefix"]) ?? false; + RootLoadMetadata_ID = ObjectToNullableInt(r["RootLoadMetadata_ID"]); + } + + + public LoadMetadata Clone() + { + return new LoadMetadata() + { + LocationOfForLoadingDirectory = LocationOfForLoadingDirectory, + LocationOfForArchivingDirectory = LocationOfForArchivingDirectory, + LocationOfCacheDirectory = LocationOfCacheDirectory, + LocationOfExecutablesDirectory = LocationOfExecutablesDirectory, + AnonymisationEngineClass = AnonymisationEngineClass, + Name = Name, + Description = Description, + CacheArchiveType = CacheArchiveType, + AllowReservedPrefix = AllowReservedPrefix, + LastLoadTime = LastLoadTime, + OverrideRAWServer_ID = OverrideRAWServer_ID, + IgnoreTrigger = IgnoreTrigger, + Folder = Folder, + RootLoadMetadata_ID = RootLoadMetadata_ID != null?RootLoadMetadata_ID: this.ID + }; } internal LoadMetadata(ShareManager shareManager, ShareDefinition shareDefinition) : base() diff --git a/Rdmp.Core/Curation/Data/DataLoad/ProcessTask.cs b/Rdmp.Core/Curation/Data/DataLoad/ProcessTask.cs index eee3cca355..42ea719a8c 100644 --- a/Rdmp.Core/Curation/Data/DataLoad/ProcessTask.cs +++ b/Rdmp.Core/Curation/Data/DataLoad/ProcessTask.cs @@ -47,6 +47,8 @@ public class ProcessTask : DatabaseEntity, IProcessTask, IOrderable, INamed, ICh #nullable enable private string? _SerialisableConfiguration; #nullable disable + private int _loadMetadataVersion; + /// /// The load the process task exists as part of /// @@ -118,6 +120,9 @@ public string? SerialisableConfiguration get => _SerialisableConfiguration; set => SetField(ref _SerialisableConfiguration, value); } + + + public int LoadMetadataVersion { get => _loadMetadataVersion; set => SetField(ref _loadMetadataVersion, value); } #nullable disable #endregion @@ -158,7 +163,8 @@ public ProcessTask(ICatalogueRepository repository, ILoadMetadata parent, LoadSt repository.InsertAndHydrate(this, new Dictionary { - { "LoadMetadata_ID", parent.ID }, + { "LoadMetadata_ID", parent.RootLoadMetadata_ID??parent.ID }, + { "LoadMetadataVersion", parent.RootLoadMetadata_ID != null?parent.ID:0 }, { "ProcessTaskType", ProcessTaskType.Executable.ToString() }, { "LoadStage", stage }, { "Name", $"New Process{Guid.NewGuid()}" }, @@ -180,7 +186,8 @@ public ProcessTask(ICatalogueRepository repository, ILoadMetadata parent, LoadSt repository.InsertAndHydrate(this, new Dictionary { - { "LoadMetadata_ID", parent.ID }, + { "LoadMetadata_ID", parent.RootLoadMetadata_ID??parent.ID }, + { "LoadMetadataVersion", parent.RootLoadMetadata_ID != null?parent.ID:0 }, { "ProcessTaskType", ProcessTaskType.Executable.ToString() }, { "LoadStage", stage }, { "Name", $"New Process{Guid.NewGuid()}" }, @@ -200,7 +207,7 @@ internal ProcessTask(ICatalogueRepository repository, DbDataReader r) Path = r["Path"] as string; Name = r["Name"] as string; Order = int.Parse(r["Order"].ToString()); - + LoadMetadataVersion = int.Parse(r["LoadMetadataVersion"].ToString()); if (Enum.TryParse(r["ProcessTaskType"] as string, out ProcessTaskType processTaskType)) ProcessTaskType = processTaskType; else @@ -212,7 +219,7 @@ internal ProcessTask(ICatalogueRepository repository, DbDataReader r) throw new Exception($"Could not parse LoadStage:{r["LoadStage"]}"); IsDisabled = Convert.ToBoolean(r["IsDisabled"]); - if(r["SerialisableConfiguration"] is not null) + if (r["SerialisableConfiguration"] is not null) SerialisableConfiguration = r["SerialisableConfiguration"].ToString(); } diff --git a/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql b/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql index 3d3c8ce877..c5df74b2ad 100644 --- a/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql +++ b/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql @@ -601,6 +601,8 @@ CREATE TABLE [dbo].[LoadMetadata]( [CacheArchiveType] [int] NOT NULL, [SoftwareVersion] [nvarchar](50) NOT NULL, [AllowReservedPrefix] [bit] NOT NULL default 0, + [RootLoadMetadata_ID] [int] NULL, + CONSTRAINT [fk_loadMetadataRootReference] FOREIGN KEY([RootLoadMetadata_ID]) REFERENCES [dbo].[LoadMetadata](id), CONSTRAINT [PK_LoadMetadata] PRIMARY KEY CLUSTERED ( [ID] ASC diff --git a/Rdmp.Core/Databases/CatalogueDatabase/up/089_AddDataLoadVersioning.sql b/Rdmp.Core/Databases/CatalogueDatabase/up/089_AddDataLoadVersioning.sql new file mode 100644 index 0000000000..2d51823cf4 --- /dev/null +++ b/Rdmp.Core/Databases/CatalogueDatabase/up/089_AddDataLoadVersioning.sql @@ -0,0 +1,12 @@ +--Version: 8.4.3 +--Description: Add data load versioning + + if not exists (select 1 from sys.columns where name = 'RootLoadMetadata_ID' and OBJECT_NAME(object_id) = 'LoadMetadata') + BEGIN + ALTER TABLE [dbo].[LoadMetadata] + ADD RootLoadMetadata_ID [int] NULL, + CONSTRAINT [fk_loadMetadataRootReference] FOREIGN KEY(RootLoadMetadata_ID) REFERENCES [dbo].[LoadMetadata](id) + END + + + diff --git a/Rdmp.Core/Rdmp.Core.csproj b/Rdmp.Core/Rdmp.Core.csproj index 4abe04fb29..77e04c2aef 100644 --- a/Rdmp.Core/Rdmp.Core.csproj +++ b/Rdmp.Core/Rdmp.Core.csproj @@ -256,6 +256,7 @@ + diff --git a/SharedAssemblyInfo.cs b/SharedAssemblyInfo.cs index 55f72e959f..851d4a2363 100644 --- a/SharedAssemblyInfo.cs +++ b/SharedAssemblyInfo.cs @@ -10,6 +10,6 @@ [assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] -[assembly: AssemblyVersion("8.4.2")] -[assembly: AssemblyFileVersion("8.4.2")] -[assembly: AssemblyInformationalVersion("8.4.2")] +[assembly: AssemblyVersion("8.4.3")] +[assembly: AssemblyFileVersion("8.4.3")] +[assembly: AssemblyInformationalVersion("8.4.3")] From 89036d02f622d9d3663d8caf3d493ebb9425c6df Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 16 Jan 2025 08:26:59 +0000 Subject: [PATCH 02/15] add basic versioning --- ...teCommandCreateNewClassBasedProcessTask.cs | 2 +- ...uteCommandCreateNewFileBasedProcessTask.cs | 2 +- .../ExecuteCommandDisableOrEnable.cs | 16 +++++++++-- ...oggleAllowReservedPrefixForLoadMetadata.cs | 1 + .../Curation/Data/DataLoad/IProcessTask.cs | 3 --- .../Curation/Data/DataLoad/LoadMetadata.cs | 27 ++++++++++++++++--- .../Curation/Data/DataLoad/ProcessTask.cs | 24 ++++++++++++++--- .../CreateCatalogue.sql | 1 + Rdmp.Core/Providers/CatalogueChildProvider.cs | 4 +-- 9 files changed, 64 insertions(+), 16 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewClassBasedProcessTask.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewClassBasedProcessTask.cs index 258322a77a..9c2639e310 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewClassBasedProcessTask.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewClassBasedProcessTask.cs @@ -72,7 +72,7 @@ public override void Execute() else return; } - + _loadMetadata.Clone(); var newTask = new ProcessTask(BasicActivator.RepositoryLocator.CatalogueRepository, _loadMetadata, _loadStage) { Path = _type.FullName, diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewFileBasedProcessTask.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewFileBasedProcessTask.cs index 880f8e8fad..60406d1a73 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewFileBasedProcessTask.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewFileBasedProcessTask.cs @@ -96,7 +96,7 @@ public override void Execute() throw new ArgumentOutOfRangeException($"Unexpected _taskType:{_taskType}"); } } - + _loadMetadata.Clone(); var task = new ProcessTask((ICatalogueRepository)_loadMetadata.Repository, _loadMetadata, _loadStage) { ProcessTaskType = _taskType, diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDisableOrEnable.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDisableOrEnable.cs index d2008fff56..e33b9b0545 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDisableOrEnable.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDisableOrEnable.cs @@ -8,7 +8,9 @@ using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.Data.Aggregation; using Rdmp.Core.Curation.Data.Cohort; +using Rdmp.Core.Curation.Data.DataLoad; using Rdmp.Core.MapsDirectlyToDatabaseTable; +using Rdmp.Core.ReusableLibraryCode; namespace Rdmp.Core.CommandExecution.AtomicCommands; @@ -72,8 +74,18 @@ public override void Execute() foreach (var d in _targets) { - d.IsDisabled = !d.IsDisabled; - d.SaveToDatabase(); + var x = d.GetType(); + if (d.GetType() == typeof(ProcessTask)) + { + var newVersion = (IDisableable)((IVersionable)d).SaveNewVersion(); + newVersion.IsDisabled = true; + newVersion.SaveToDatabase(); + } + else + { + d.IsDisabled = !d.IsDisabled; + d.SaveToDatabase(); + } } var toRefresh = _targets.FirstOrDefault(); diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandToggleAllowReservedPrefixForLoadMetadata.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandToggleAllowReservedPrefixForLoadMetadata.cs index 16aba34c68..53e68eb461 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandToggleAllowReservedPrefixForLoadMetadata.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandToggleAllowReservedPrefixForLoadMetadata.cs @@ -24,6 +24,7 @@ public ExecuteCommandToggleAllowReservedPrefixForLoadMetadata([DemandsInitializa public override void Execute() { base.Execute(); + _loadMetadata.Clone(); _loadMetadata.AllowReservedPrefix = !_loadMetadata.AllowReservedPrefix; _loadMetadata.SaveToDatabase(); } diff --git a/Rdmp.Core/Curation/Data/DataLoad/IProcessTask.cs b/Rdmp.Core/Curation/Data/DataLoad/IProcessTask.cs index baec117ecb..b974fe66f2 100644 --- a/Rdmp.Core/Curation/Data/DataLoad/IProcessTask.cs +++ b/Rdmp.Core/Curation/Data/DataLoad/IProcessTask.cs @@ -48,7 +48,4 @@ public interface IProcessTask : IRevertable, IArgumentHost, IOrderable, IDisable /// #nullable enable string? SerialisableConfiguration { get; } - - - int LoadMetadataVersion { get; } } \ No newline at end of file diff --git a/Rdmp.Core/Curation/Data/DataLoad/LoadMetadata.cs b/Rdmp.Core/Curation/Data/DataLoad/LoadMetadata.cs index 287a7d8d2c..d3bec51560 100644 --- a/Rdmp.Core/Curation/Data/DataLoad/LoadMetadata.cs +++ b/Rdmp.Core/Curation/Data/DataLoad/LoadMetadata.cs @@ -45,7 +45,7 @@ public enum CacheArchiveType /// public class LoadMetadata : DatabaseEntity, ILoadMetadata, IHasDependencies, IHasQuerySyntaxHelper, - ILoggedActivityRootObject, IHasFolder + ILoggedActivityRootObject, IHasFolder, IVersionable { #region Database Properties @@ -273,7 +273,7 @@ internal LoadMetadata(ICatalogueRepository repository, DbDataReader r) public LoadMetadata Clone() { - return new LoadMetadata() + var lmd = new LoadMetadata(CatalogueRepository, this.Name) { LocationOfForLoadingDirectory = LocationOfForLoadingDirectory, LocationOfForArchivingDirectory = LocationOfForArchivingDirectory, @@ -283,13 +283,22 @@ public LoadMetadata Clone() Name = Name, Description = Description, CacheArchiveType = CacheArchiveType, - AllowReservedPrefix = AllowReservedPrefix, + AllowReservedPrefix = false, LastLoadTime = LastLoadTime, OverrideRAWServer_ID = OverrideRAWServer_ID, IgnoreTrigger = IgnoreTrigger, Folder = Folder, - RootLoadMetadata_ID = RootLoadMetadata_ID != null?RootLoadMetadata_ID: this.ID + RootLoadMetadata_ID = this.ID }; + lmd.SaveToDatabase(); + //link to catalogue + //process task + var pts = CatalogueRepository.GetAllObjectsWhere("LoadMetadata_ID", this.ID); + foreach(ProcessTask pt in pts) + { + pt.Clone(lmd); + } + return lmd; } internal LoadMetadata(ShareManager shareManager, ShareDefinition shareDefinition) : base() @@ -494,4 +503,14 @@ public static bool UsesPersistentRaw(ILoadMetadata loadMetadata) return loadMetadata.CatalogueRepository.GetExtendedProperties(ExtendedProperty.PersistentRaw, loadMetadata).Any(p => p.Value == "true"); } + + + /// + public DatabaseEntity SaveNewVersion() + { + var lmd = this.Clone(); + lmd.RootLoadMetadata_ID = this.RootLoadMetadata_ID != null ? this.RootLoadMetadata_ID : this.ID; + lmd.SaveToDatabase(); + return lmd; + } } \ No newline at end of file diff --git a/Rdmp.Core/Curation/Data/DataLoad/ProcessTask.cs b/Rdmp.Core/Curation/Data/DataLoad/ProcessTask.cs index 42ea719a8c..300d0d61ef 100644 --- a/Rdmp.Core/Curation/Data/DataLoad/ProcessTask.cs +++ b/Rdmp.Core/Curation/Data/DataLoad/ProcessTask.cs @@ -16,6 +16,7 @@ using Rdmp.Core.MapsDirectlyToDatabaseTable; using Rdmp.Core.MapsDirectlyToDatabaseTable.Attributes; using Rdmp.Core.Repositories; +using Rdmp.Core.ReusableLibraryCode; using Rdmp.Core.ReusableLibraryCode.Annotations; using Rdmp.Core.ReusableLibraryCode.Checks; @@ -47,7 +48,6 @@ public class ProcessTask : DatabaseEntity, IProcessTask, IOrderable, INamed, ICh #nullable enable private string? _SerialisableConfiguration; #nullable disable - private int _loadMetadataVersion; /// /// The load the process task exists as part of @@ -122,7 +122,6 @@ public string? SerialisableConfiguration } - public int LoadMetadataVersion { get => _loadMetadataVersion; set => SetField(ref _loadMetadataVersion, value); } #nullable disable #endregion @@ -207,7 +206,6 @@ internal ProcessTask(ICatalogueRepository repository, DbDataReader r) Path = r["Path"] as string; Name = r["Name"] as string; Order = int.Parse(r["Order"].ToString()); - LoadMetadataVersion = int.Parse(r["LoadMetadataVersion"].ToString()); if (Enum.TryParse(r["ProcessTaskType"] as string, out ProcessTaskType processTaskType)) ProcessTaskType = processTaskType; else @@ -450,4 +448,24 @@ public void SetArgumentValue(string parameterName, object o) matchingArgument.SetValue(o); matchingArgument.SaveToDatabase(); } + + + public ProcessTask Clone(LoadMetadata lmd) + { + var pt = new ProcessTask(this.CatalogueRepository, lmd, this.LoadStage) { + ProcessTaskType = this.ProcessTaskType, + Order = this.Order, + IsDisabled = this.IsDisabled, + SerialisableConfiguration = this.SerialisableConfiguration, + Path = this.Path, + Name= this.Name, + }; + pt.LoadMetadata_ID = lmd.ID; + pt.SaveToDatabase(); + foreach(var pta in ProcessTaskArguments) + { + pta.ShallowClone(pt); + } + return pt; + } } \ No newline at end of file diff --git a/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql b/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql index c5df74b2ad..b4423e7c03 100644 --- a/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql +++ b/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql @@ -774,6 +774,7 @@ CREATE TABLE [dbo].[ProcessTask]( [SoftwareVersion] [nvarchar](50) NOT NULL, [IsDisabled] [bit] NOT NULL, [SerialisableConfiguration] [varchar](max), + [LoadMetadataVersion] [int], CONSTRAINT [PK_ProcessTask] PRIMARY KEY CLUSTERED ( [ID] ASC diff --git a/Rdmp.Core/Providers/CatalogueChildProvider.cs b/Rdmp.Core/Providers/CatalogueChildProvider.cs index fd83e6369e..091f1616ae 100644 --- a/Rdmp.Core/Providers/CatalogueChildProvider.cs +++ b/Rdmp.Core/Providers/CatalogueChildProvider.cs @@ -998,8 +998,8 @@ private void AddChildren(AllProcessTasksUsedByLoadMetadataNode allProcessTasksUs private void AddChildren(LoadStageNode loadStageNode, DescendancyList descendancy) { var tasks = AllProcessTasks.Where( - p => p.LoadMetadata_ID == loadStageNode.LoadMetadata.ID && p.LoadStage == loadStageNode.LoadStage) - .OrderBy(o => o.Order).ToArray(); + p => p.LoadMetadata_ID == loadStageNode.LoadMetadata.ID && p.LoadStage == loadStageNode.LoadStage) + .OrderBy(o => o.Order).ToArray(); foreach (var processTask in tasks) AddChildren(processTask, descendancy.Add(processTask)); From 24b337709dc31d4f7d10321d3b34faf9cfe6ead0 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 27 Jan 2025 09:29:27 +0000 Subject: [PATCH 03/15] add versioning --- .../CommandExecution/AtomicCommandFactory.cs | 68 ++++++++++++------- .../ExecuteCommandCloneLoadMetadata.cs | 31 +++++++++ ...ExecuteCommandCreateLoadMetadataVersion.cs | 7 +- ...teCommandCreateNewClassBasedProcessTask.cs | 1 - ...uteCommandCreateNewFileBasedProcessTask.cs | 1 - .../ExecuteCommandDisableOrEnable.cs | 14 +--- ...xecuteCommandRestoreLoadMetadataVersion.cs | 48 +++++++++++++ ...oggleAllowReservedPrefixForLoadMetadata.cs | 1 - .../Curation/Data/DataLoad/LoadMetadata.cs | 5 ++ .../Curation/Data/DataLoad/ProcessTask.cs | 2 - .../Icons/IconProvision/CatalogueIcons.resx | 3 + Rdmp.Core/Icons/IconProvision/RDMPConcept.cs | 1 + Rdmp.Core/Providers/CatalogueChildProvider.cs | 14 +++- .../LoadMetadataVersionNodes.cs | 49 +++++++++++++ Rdmp.Core/ReusableLibraryCode/IVersionable.cs | 18 +++++ 15 files changed, 216 insertions(+), 47 deletions(-) create mode 100644 Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCloneLoadMetadata.cs create mode 100644 Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreLoadMetadataVersion.cs create mode 100644 Rdmp.Core/Providers/Nodes/LoadMetadataNodes/LoadMetadataVersionNodes.cs create mode 100644 Rdmp.Core/ReusableLibraryCode/IVersionable.cs diff --git a/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs b/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs index e2e35c627b..382df303d1 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs @@ -502,39 +502,55 @@ public IEnumerable CreateCommands(object o) if (Is(o, out LoadMetadata lmd)) { - yield return new ExecuteCommandExportObjectsToFile(_activator, new IMapsDirectlyToDatabaseTable[] { lmd }); - - yield return new ExecuteCommandOverrideRawServer(_activator, lmd); - yield return new ExecuteCommandCreateNewLoadMetadata(_activator); - var reservedTest = lmd.AllowReservedPrefix ? "Drop" : "Allow"; - yield return new ExecuteCommandToggleAllowReservedPrefixForLoadMetadata(lmd) + if (lmd.RootLoadMetadata_ID is null) { - OverrideCommandName=$"{reservedTest} Reserved Prefix Columns" - }; + yield return new ExecuteCommandExportObjectsToFile(_activator, new IMapsDirectlyToDatabaseTable[] { lmd }); - yield return new ExecuteCommandSetGlobalDleIgnorePattern(_activator) { SuggestedCategory = Advanced }; - yield return new ExecuteCommandSetIgnoredColumns(_activator, lmd) { SuggestedCategory = Advanced }; - yield return new ExecuteCommandSetIgnoredColumns(_activator, lmd, null) - { OverrideCommandName = "Clear Ignored Columns", SuggestedCategory = Advanced }; + yield return new ExecuteCommandOverrideRawServer(_activator, lmd); + yield return new ExecuteCommandCreateNewLoadMetadata(_activator); + var reservedTest = lmd.AllowReservedPrefix ? "Drop" : "Allow"; + yield return new ExecuteCommandToggleAllowReservedPrefixForLoadMetadata(lmd) + { + OverrideCommandName = $"{reservedTest} Reserved Prefix Columns" + }; + yield return new ExecuteCommandCreateLoadMetadataVersion(_activator, lmd) + { + OverrideCommandName = "Save Version" + }; + yield return new ExecuteCommandCloneLoadMetadata(_activator, lmd) + { + OverrideCommandName = "Clone Load Metadata" + }; - yield return new ExecuteCommandSetExtendedProperty(_activator, new[] { lmd }, - ExtendedProperty.PersistentRaw, null) - { - OverrideCommandName = "Persistent RAW", - PromptForValue = true, - PromptForValueTaskDescription = ExtendedProperty.PersistentRawDescription, - SuggestedCategory = Advanced - }; + yield return new ExecuteCommandSetGlobalDleIgnorePattern(_activator) { SuggestedCategory = Advanced }; + yield return new ExecuteCommandSetIgnoredColumns(_activator, lmd) { SuggestedCategory = Advanced }; + yield return new ExecuteCommandSetIgnoredColumns(_activator, lmd, null) + { OverrideCommandName = "Clear Ignored Columns", SuggestedCategory = Advanced }; - yield return new ExecuteCommandSet(_activator, lmd, - typeof(LoadMetadata).GetProperty(nameof(LoadMetadata.IgnoreTrigger))) + yield return new ExecuteCommandSetExtendedProperty(_activator, new[] { lmd }, + ExtendedProperty.PersistentRaw, null) + { + OverrideCommandName = "Persistent RAW", + PromptForValue = true, + PromptForValueTaskDescription = ExtendedProperty.PersistentRawDescription, + SuggestedCategory = Advanced + }; + + yield return new ExecuteCommandSet(_activator, lmd, + typeof(LoadMetadata).GetProperty(nameof(LoadMetadata.IgnoreTrigger))) + { + OverrideCommandName = $"Ignore Trigger (Current value:{lmd.IgnoreTrigger})", + SuggestedCategory = Advanced + }; + } + else { - OverrideCommandName = $"Ignore Trigger (Current value:{lmd.IgnoreTrigger})", - SuggestedCategory = Advanced - }; + yield return new ExecuteCommandRestoreLoadMetadataVersion(_activator, lmd) + { + OverrideCommandName = "Restore Version"}; + } } - if (Is(o, out LoadMetadataScheduleNode scheduleNode)) yield return new ExecuteCommandCreateNewLoadProgress(_activator, scheduleNode.LoadMetadata); diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCloneLoadMetadata.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCloneLoadMetadata.cs new file mode 100644 index 0000000000..aebb36fee2 --- /dev/null +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCloneLoadMetadata.cs @@ -0,0 +1,31 @@ +using Rdmp.Core.Curation.Data.DataLoad; +using Rdmp.Core.Curation.Data; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.CommandExecution.AtomicCommands; + +public class ExecuteCommandCloneLoadMetadata : BasicCommandExecution +{ + private LoadMetadata _loadMetadata; + private IBasicActivateItems _activator; + public ExecuteCommandCloneLoadMetadata(IBasicActivateItems activator,[DemandsInitialization("The LoadMetadata to clone")] LoadMetadata loadMetadata) + { + + _loadMetadata = loadMetadata; + _activator = activator; + } + + public override void Execute() + { + base.Execute(); + var lmd = _loadMetadata.Clone(); + lmd.Name = lmd.Name + " (Clone)"; + lmd.SaveToDatabase(); + _activator.Publish(lmd); + + } +} diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateLoadMetadataVersion.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateLoadMetadataVersion.cs index f5c6cc87d8..85b410bb9e 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateLoadMetadataVersion.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateLoadMetadataVersion.cs @@ -11,10 +11,12 @@ namespace Rdmp.Core.CommandExecution.AtomicCommands; public class ExecuteCommandCreateLoadMetadataVersion: BasicCommandExecution { private LoadMetadata _loadMetadata; - public ExecuteCommandCreateLoadMetadataVersion([DemandsInitialization("The LoadMetadata to update")] LoadMetadata loadMetadata) + private IBasicActivateItems _activator; + public ExecuteCommandCreateLoadMetadataVersion(IBasicActivateItems activator,[DemandsInitialization("The LoadMetadata to version")] LoadMetadata loadMetadata) { _loadMetadata = loadMetadata; + _activator = activator; } public override void Execute() @@ -24,7 +26,8 @@ public override void Execute() { throw new Exception("Must Use Root LoadMetadata to create Version"); } - var lmd = _loadMetadata.Clone(); + var lmd = _loadMetadata.SaveNewVersion(); lmd.SaveToDatabase(); + _activator.Publish(lmd); } } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewClassBasedProcessTask.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewClassBasedProcessTask.cs index 9c2639e310..b4afaa4220 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewClassBasedProcessTask.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewClassBasedProcessTask.cs @@ -72,7 +72,6 @@ public override void Execute() else return; } - _loadMetadata.Clone(); var newTask = new ProcessTask(BasicActivator.RepositoryLocator.CatalogueRepository, _loadMetadata, _loadStage) { Path = _type.FullName, diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewFileBasedProcessTask.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewFileBasedProcessTask.cs index 60406d1a73..cc42d6cfcf 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewFileBasedProcessTask.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewFileBasedProcessTask.cs @@ -96,7 +96,6 @@ public override void Execute() throw new ArgumentOutOfRangeException($"Unexpected _taskType:{_taskType}"); } } - _loadMetadata.Clone(); var task = new ProcessTask((ICatalogueRepository)_loadMetadata.Repository, _loadMetadata, _loadStage) { ProcessTaskType = _taskType, diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDisableOrEnable.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDisableOrEnable.cs index e33b9b0545..6965c81c10 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDisableOrEnable.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDisableOrEnable.cs @@ -74,18 +74,8 @@ public override void Execute() foreach (var d in _targets) { - var x = d.GetType(); - if (d.GetType() == typeof(ProcessTask)) - { - var newVersion = (IDisableable)((IVersionable)d).SaveNewVersion(); - newVersion.IsDisabled = true; - newVersion.SaveToDatabase(); - } - else - { - d.IsDisabled = !d.IsDisabled; - d.SaveToDatabase(); - } + d.IsDisabled = !d.IsDisabled; + d.SaveToDatabase(); } var toRefresh = _targets.FirstOrDefault(); diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreLoadMetadataVersion.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreLoadMetadataVersion.cs new file mode 100644 index 0000000000..e9354c6175 --- /dev/null +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreLoadMetadataVersion.cs @@ -0,0 +1,48 @@ +using Rdmp.Core.Curation.Data.DataLoad; +using Rdmp.Core.Curation.Data; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Org.BouncyCastle.Pqc.Crypto.Lms; + +namespace Rdmp.Core.CommandExecution.AtomicCommands; + +public class ExecuteCommandRestoreLoadMetadataVersion : BasicCommandExecution +{ + private LoadMetadata _loadMetadata; + private IBasicActivateItems _activator; + public ExecuteCommandRestoreLoadMetadataVersion(IBasicActivateItems activator, [DemandsInitialization("The LoadMetadata to version")] LoadMetadata loadMetadata) + { + + _loadMetadata = loadMetadata; + _activator = activator; + } + + public override void Execute() + { + if (!_activator.YesNo("Replace root Load Metadata with this configuration?","Resotre Load Metadata Version")) return; + base.Execute(); + if (_loadMetadata.RootLoadMetadata_ID is null) + { + throw new Exception("Must Use a versioned LoadMetadata to create Version"); + } + LoadMetadata lmd = (LoadMetadata)_activator.RepositoryLocator.CatalogueRepository.GetObjectByID(typeof(LoadMetadata), (int)_loadMetadata.RootLoadMetadata_ID); + if (lmd is null) + { + throw new Exception("Could not find root load metadata"); + + } + foreach (ProcessTask task in lmd.ProcessTasks) + { + task.DeleteInDatabase(); + } + foreach(ProcessTask task in _loadMetadata.ProcessTasks) + { + task.Clone(lmd); + } + lmd.SaveToDatabase(); + _activator.Publish(lmd); + } +} diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandToggleAllowReservedPrefixForLoadMetadata.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandToggleAllowReservedPrefixForLoadMetadata.cs index 53e68eb461..16aba34c68 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandToggleAllowReservedPrefixForLoadMetadata.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandToggleAllowReservedPrefixForLoadMetadata.cs @@ -24,7 +24,6 @@ public ExecuteCommandToggleAllowReservedPrefixForLoadMetadata([DemandsInitializa public override void Execute() { base.Execute(); - _loadMetadata.Clone(); _loadMetadata.AllowReservedPrefix = !_loadMetadata.AllowReservedPrefix; _loadMetadata.SaveToDatabase(); } diff --git a/Rdmp.Core/Curation/Data/DataLoad/LoadMetadata.cs b/Rdmp.Core/Curation/Data/DataLoad/LoadMetadata.cs index d3bec51560..d78da70a8a 100644 --- a/Rdmp.Core/Curation/Data/DataLoad/LoadMetadata.cs +++ b/Rdmp.Core/Curation/Data/DataLoad/LoadMetadata.cs @@ -292,6 +292,10 @@ public LoadMetadata Clone() }; lmd.SaveToDatabase(); //link to catalogue + foreach(var catalogue in this.GetAllCatalogues()) + { + lmd.LinkToCatalogue(catalogue); + } //process task var pts = CatalogueRepository.GetAllObjectsWhere("LoadMetadata_ID", this.ID); foreach(ProcessTask pt in pts) @@ -510,6 +514,7 @@ public DatabaseEntity SaveNewVersion() { var lmd = this.Clone(); lmd.RootLoadMetadata_ID = this.RootLoadMetadata_ID != null ? this.RootLoadMetadata_ID : this.ID; + lmd.Name = $"{this.Name} - {DateTime.Now}"; lmd.SaveToDatabase(); return lmd; } diff --git a/Rdmp.Core/Curation/Data/DataLoad/ProcessTask.cs b/Rdmp.Core/Curation/Data/DataLoad/ProcessTask.cs index 300d0d61ef..1f1bf8825a 100644 --- a/Rdmp.Core/Curation/Data/DataLoad/ProcessTask.cs +++ b/Rdmp.Core/Curation/Data/DataLoad/ProcessTask.cs @@ -163,7 +163,6 @@ public ProcessTask(ICatalogueRepository repository, ILoadMetadata parent, LoadSt repository.InsertAndHydrate(this, new Dictionary { { "LoadMetadata_ID", parent.RootLoadMetadata_ID??parent.ID }, - { "LoadMetadataVersion", parent.RootLoadMetadata_ID != null?parent.ID:0 }, { "ProcessTaskType", ProcessTaskType.Executable.ToString() }, { "LoadStage", stage }, { "Name", $"New Process{Guid.NewGuid()}" }, @@ -186,7 +185,6 @@ public ProcessTask(ICatalogueRepository repository, ILoadMetadata parent, LoadSt repository.InsertAndHydrate(this, new Dictionary { { "LoadMetadata_ID", parent.RootLoadMetadata_ID??parent.ID }, - { "LoadMetadataVersion", parent.RootLoadMetadata_ID != null?parent.ID:0 }, { "ProcessTaskType", ProcessTaskType.Executable.ToString() }, { "LoadStage", stage }, { "Name", $"New Process{Guid.NewGuid()}" }, diff --git a/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx b/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx index 6abf0c1c75..db264af676 100644 --- a/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx +++ b/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx @@ -262,6 +262,9 @@ ..\CatalogueFolder.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + ..\CatalogueFolder.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + ..\CatalogueItem.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 diff --git a/Rdmp.Core/Icons/IconProvision/RDMPConcept.cs b/Rdmp.Core/Icons/IconProvision/RDMPConcept.cs index 3fc3fbb0d4..b3988b6401 100644 --- a/Rdmp.Core/Icons/IconProvision/RDMPConcept.cs +++ b/Rdmp.Core/Icons/IconProvision/RDMPConcept.cs @@ -122,6 +122,7 @@ public enum RDMPConcept AllProcessTasksUsedByLoadMetadataNode, AllCataloguesUsedByLoadMetadataNode, LoadMetadataScheduleNode, + LoadMetadataVersionNode, Logging, GetFilesStage, diff --git a/Rdmp.Core/Providers/CatalogueChildProvider.cs b/Rdmp.Core/Providers/CatalogueChildProvider.cs index 091f1616ae..c8988d0374 100644 --- a/Rdmp.Core/Providers/CatalogueChildProvider.cs +++ b/Rdmp.Core/Providers/CatalogueChildProvider.cs @@ -397,7 +397,7 @@ public CatalogueChildProvider(ICatalogueRepository repository, IChildProvider[] ReportProgress("Build Catalogue Folder Root"); - LoadMetadataRootFolder = FolderHelper.BuildFolderTree(AllLoadMetadatas); + LoadMetadataRootFolder = FolderHelper.BuildFolderTree(AllLoadMetadatas.Where(lmd => lmd.RootLoadMetadata_ID is null).ToArray()); AddChildren(LoadMetadataRootFolder, new DescendancyList(LoadMetadataRootFolder)); CohortIdentificationConfigurationRootFolder = @@ -856,7 +856,7 @@ private void AddChildren(FolderNode folder, DescendancyList descen AddChildren(child, descendancy.Add(child)); //add loads in folder - foreach (var lmd in folder.ChildObjects) AddChildren(lmd, descendancy.Add(lmd)); + foreach (var lmd in folder.ChildObjects.Where(lmd => lmd.RootLoadMetadata_ID == null).ToArray()) AddChildren(lmd, descendancy.Add(lmd)); // Children are the folders + objects AddToDictionaries(new HashSet( folder.ChildFolders.Cast() @@ -929,6 +929,10 @@ private void AddChildren(LoadMetadata lmd, DescendancyList descendancy) AddChildren(processTasksNode, descendancy.Add(processTasksNode)); childObjects.Add(processTasksNode); + var versionsNode = new LoadMetadataVersionNode(lmd); + AddChildren(versionsNode, descendancy.Add(versionsNode)); + childObjects.Add(versionsNode); + childObjects.Add(new LoadDirectoryNode(lmd)); AddToDictionaries(new HashSet(childObjects), descendancy); @@ -1017,6 +1021,12 @@ private void AddChildren(ProcessTask procesTask, DescendancyList descendancy) AddToDictionaries(new HashSet(args), descendancy); } + private void AddChildren(LoadMetadataVersionNode LoadMetadataVersionNode, DescendancyList descendancy) + { + LoadMetadataVersionNode.LoadMetadataVersions = AllLoadMetadatas.Where(lmd => lmd.RootLoadMetadata_ID == LoadMetadataVersionNode.LoadMetadata.ID).ToList(); + AddToDictionaries(LoadMetadataVersionNode.LoadMetadataVersions.Cast().ToHashSet(), descendancy); + } + private void AddChildren(AllCataloguesUsedByLoadMetadataNode allCataloguesUsedByLoadMetadataNode, DescendancyList descendancy) { diff --git a/Rdmp.Core/Providers/Nodes/LoadMetadataNodes/LoadMetadataVersionNodes.cs b/Rdmp.Core/Providers/Nodes/LoadMetadataNodes/LoadMetadataVersionNodes.cs new file mode 100644 index 0000000000..a6c1e8e402 --- /dev/null +++ b/Rdmp.Core/Providers/Nodes/LoadMetadataNodes/LoadMetadataVersionNodes.cs @@ -0,0 +1,49 @@ +// Copyright (c) The University of Dundee 2018-2019 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using System; +using System.Collections.Generic; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.Cohort; +using Rdmp.Core.Curation.Data.DataLoad; + +namespace Rdmp.Core.Providers.Nodes.LoadMetadataNodes; + +/// +/// Collection of all the s which are currently associated with a given . This governs +/// which tables are created in RAW=>STAGING=>LIVE. +/// +public class LoadMetadataVersionNode : Node, IOrderable +{ + public LoadMetadata LoadMetadata { get; } + + public int Order + { + get => 1; + set { } + } + + public List LoadMetadataVersions { get; set; } + + public LoadMetadataVersionNode(LoadMetadata lmd) + { + LoadMetadata = lmd; + } + + public override string ToString() => "Versions"; + + protected bool Equals(LoadMetadataVersionNode other) => Equals(LoadMetadata, other.LoadMetadata); + + public override bool Equals(object obj) + { + if (obj is null) return false; + if (ReferenceEquals(this, obj)) return true; + if (obj.GetType() != GetType()) return false; + return Equals((LoadMetadataVersionNode)obj); + } + + public override int GetHashCode() => HashCode.Combine(LoadMetadata); +} \ No newline at end of file diff --git a/Rdmp.Core/ReusableLibraryCode/IVersionable.cs b/Rdmp.Core/ReusableLibraryCode/IVersionable.cs new file mode 100644 index 0000000000..53011c2972 --- /dev/null +++ b/Rdmp.Core/ReusableLibraryCode/IVersionable.cs @@ -0,0 +1,18 @@ +using Rdmp.Core.Curation.Data; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.ReusableLibraryCode; + +internal interface IVersionable +{ + + /// + /// + /// + /// + DatabaseEntity SaveNewVersion(); +} From f7541b668d77fc6575e015b57f689475c3e07a17 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 27 Jan 2025 09:52:10 +0000 Subject: [PATCH 04/15] rename file --- .../{LoadMetadataVersionNodes.cs => LoadMetadataVersionNode.cs} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Rdmp.Core/Providers/Nodes/LoadMetadataNodes/{LoadMetadataVersionNodes.cs => LoadMetadataVersionNode.cs} (100%) diff --git a/Rdmp.Core/Providers/Nodes/LoadMetadataNodes/LoadMetadataVersionNodes.cs b/Rdmp.Core/Providers/Nodes/LoadMetadataNodes/LoadMetadataVersionNode.cs similarity index 100% rename from Rdmp.Core/Providers/Nodes/LoadMetadataNodes/LoadMetadataVersionNodes.cs rename to Rdmp.Core/Providers/Nodes/LoadMetadataNodes/LoadMetadataVersionNode.cs From 61e2c69bfc7f4ae75533cea80b8d74b6a806d583 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 27 Jan 2025 10:55:57 +0000 Subject: [PATCH 05/15] update changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3dec610b5..08fff9c9d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,11 +4,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [8.4.3] - Unreleased - Build on and target .Net 9 rather than 8 - Simplify DB Patching Interface - Fix issue with Simple File Extractor pipeline component checking +- Add Load Metadata versioning ## [8.4.2] - 2024-12-18 From cdc93b6cbee7a717c352012dec546485652c7e09 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 27 Jan 2025 11:34:56 +0000 Subject: [PATCH 06/15] fix catalogue --- Rdmp.Core/Curation/KeywordHelp.txt | 3 ++- .../runAfterCreateDatabase/CreateCatalogue.sql | 2 ++ .../CatalogueDatabase/up/089_AddDataLoadVersioning.sql | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Rdmp.Core/Curation/KeywordHelp.txt b/Rdmp.Core/Curation/KeywordHelp.txt index b1859e7d59..7082134c87 100644 --- a/Rdmp.Core/Curation/KeywordHelp.txt +++ b/Rdmp.Core/Curation/KeywordHelp.txt @@ -98,4 +98,5 @@ UNIQUE_SettingKey: Ensures the uniqueness of key values in the setting table FK_Redaction_RedactionConfiguration_ID: Prevents you from deleting a redaction configuration if it's in use FK_Redaction_ColumnInfo_ID: Prevents redactions from becoming orphaned from their associated catalogue columns FK_RedactionKey_Redaction_ID: Prevents redaction primary keys from being orphaned from the associated redaction -FK_RedactionKey_ColumnInfo_ID: Prevents redaction keys from becoming orphaned from their associated catalogue columns \ No newline at end of file +FK_RedactionKey_ColumnInfo_ID: Prevents redaction keys from becoming orphaned from their associated catalogue columns +FK_LoadMetadataRootReference: Links versions of a load metadata back to the source load metadata \ No newline at end of file diff --git a/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql b/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql index b4423e7c03..6b21d1645d 100644 --- a/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql +++ b/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql @@ -943,6 +943,8 @@ ALTER TABLE [dbo].[ColumnInfo] ADD CONSTRAINT [FK_Column_Info_Dataset] FOREIGN K GO ALTER TABLE [dbo].[LoadMetadata] ADD LastLoadTime [datetime] NULL; GO +ALTER TABLE [dbo].[LoadMetadata] ADD CONSTRAINT [FK_LoadMetadataRootReference] FOREIGN KEY(RootLoadMetadata_ID) REFERENCES [dbo].[LoadMetadata](id) ON DELETE CASCADE +GO SET ANSI_PADDING OFF GO ALTER TABLE [dbo].[ANOTable] ADD CONSTRAINT [DF_ANOTable_SoftwareVersion] DEFAULT ([dbo].[GetSoftwareVersion]()) FOR [SoftwareVersion] diff --git a/Rdmp.Core/Databases/CatalogueDatabase/up/089_AddDataLoadVersioning.sql b/Rdmp.Core/Databases/CatalogueDatabase/up/089_AddDataLoadVersioning.sql index 2d51823cf4..c3f852ac4b 100644 --- a/Rdmp.Core/Databases/CatalogueDatabase/up/089_AddDataLoadVersioning.sql +++ b/Rdmp.Core/Databases/CatalogueDatabase/up/089_AddDataLoadVersioning.sql @@ -5,7 +5,7 @@ BEGIN ALTER TABLE [dbo].[LoadMetadata] ADD RootLoadMetadata_ID [int] NULL, - CONSTRAINT [fk_loadMetadataRootReference] FOREIGN KEY(RootLoadMetadata_ID) REFERENCES [dbo].[LoadMetadata](id) + CONSTRAINT [FK_LoadMetadataRootReference] FOREIGN KEY(RootLoadMetadata_ID) REFERENCES [dbo].[LoadMetadata](id) ON DELETE CASCADE END From ebfd124ba0c594cd8ee854eb91264974564124cb Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 27 Jan 2025 11:49:18 +0000 Subject: [PATCH 07/15] fix sql --- .../runAfterCreateDatabase/CreateCatalogue.sql | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql b/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql index 6b21d1645d..fdcb9a242b 100644 --- a/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql +++ b/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql @@ -602,7 +602,7 @@ CREATE TABLE [dbo].[LoadMetadata]( [SoftwareVersion] [nvarchar](50) NOT NULL, [AllowReservedPrefix] [bit] NOT NULL default 0, [RootLoadMetadata_ID] [int] NULL, - CONSTRAINT [fk_loadMetadataRootReference] FOREIGN KEY([RootLoadMetadata_ID]) REFERENCES [dbo].[LoadMetadata](id), + CONSTRAINT [fk_loadMetadataRootReference] FOREIGN KEY([RootLoadMetadata_ID]) REFERENCES [dbo].[LoadMetadata](id) ON DELETE CASCADE, CONSTRAINT [PK_LoadMetadata] PRIMARY KEY CLUSTERED ( [ID] ASC @@ -943,8 +943,6 @@ ALTER TABLE [dbo].[ColumnInfo] ADD CONSTRAINT [FK_Column_Info_Dataset] FOREIGN K GO ALTER TABLE [dbo].[LoadMetadata] ADD LastLoadTime [datetime] NULL; GO -ALTER TABLE [dbo].[LoadMetadata] ADD CONSTRAINT [FK_LoadMetadataRootReference] FOREIGN KEY(RootLoadMetadata_ID) REFERENCES [dbo].[LoadMetadata](id) ON DELETE CASCADE -GO SET ANSI_PADDING OFF GO ALTER TABLE [dbo].[ANOTable] ADD CONSTRAINT [DF_ANOTable_SoftwareVersion] DEFAULT ([dbo].[GetSoftwareVersion]()) FOR [SoftwareVersion] From 8407f8357fc7dea533eb2a693e7ba3524b42055a Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 27 Jan 2025 12:01:24 +0000 Subject: [PATCH 08/15] fix up sql --- .../runAfterCreateDatabase/CreateCatalogue.sql | 2 +- .../CatalogueDatabase/up/089_AddDataLoadVersioning.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql b/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql index fdcb9a242b..b4423e7c03 100644 --- a/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql +++ b/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql @@ -602,7 +602,7 @@ CREATE TABLE [dbo].[LoadMetadata]( [SoftwareVersion] [nvarchar](50) NOT NULL, [AllowReservedPrefix] [bit] NOT NULL default 0, [RootLoadMetadata_ID] [int] NULL, - CONSTRAINT [fk_loadMetadataRootReference] FOREIGN KEY([RootLoadMetadata_ID]) REFERENCES [dbo].[LoadMetadata](id) ON DELETE CASCADE, + CONSTRAINT [fk_loadMetadataRootReference] FOREIGN KEY([RootLoadMetadata_ID]) REFERENCES [dbo].[LoadMetadata](id), CONSTRAINT [PK_LoadMetadata] PRIMARY KEY CLUSTERED ( [ID] ASC diff --git a/Rdmp.Core/Databases/CatalogueDatabase/up/089_AddDataLoadVersioning.sql b/Rdmp.Core/Databases/CatalogueDatabase/up/089_AddDataLoadVersioning.sql index c3f852ac4b..febdc440c1 100644 --- a/Rdmp.Core/Databases/CatalogueDatabase/up/089_AddDataLoadVersioning.sql +++ b/Rdmp.Core/Databases/CatalogueDatabase/up/089_AddDataLoadVersioning.sql @@ -5,7 +5,7 @@ BEGIN ALTER TABLE [dbo].[LoadMetadata] ADD RootLoadMetadata_ID [int] NULL, - CONSTRAINT [FK_LoadMetadataRootReference] FOREIGN KEY(RootLoadMetadata_ID) REFERENCES [dbo].[LoadMetadata](id) ON DELETE CASCADE + CONSTRAINT [FK_LoadMetadataRootReference] FOREIGN KEY(RootLoadMetadata_ID) REFERENCES [dbo].[LoadMetadata](id) END From 2fcd762f1a7d690a2104744840b2223df18014be Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 27 Jan 2025 15:13:38 +0000 Subject: [PATCH 09/15] update delete --- .../AtomicCommands/ExecuteCommandDelete.cs | 3 ++- .../Curation/Data/DataLoad/LoadMetadata.cs | 21 ++++++++++++++++--- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDelete.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDelete.cs index 91886cc5fa..207b194924 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDelete.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDelete.cs @@ -9,7 +9,9 @@ using System.Linq; using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.Data.Cohort; +using Rdmp.Core.Curation.Data.DataLoad; using Rdmp.Core.MapsDirectlyToDatabaseTable; +using Rdmp.Core.Repositories; using Rdmp.Core.Repositories.Construction; using Rdmp.Core.ReusableLibraryCode; @@ -48,7 +50,6 @@ public ExecuteCommandDelete(IBasicActivateItems activator, _allowDeleteMany = deleteMany; if (_deletables.Any(d => d is CohortAggregateContainer c && c.IsRootContainer())) SetImpossible("Cannot delete root containers"); - var reason = ""; if (_deletables.Any(d => d is IMightBeReadOnly ro && ro.ShouldBeReadOnly(out reason))) diff --git a/Rdmp.Core/Curation/Data/DataLoad/LoadMetadata.cs b/Rdmp.Core/Curation/Data/DataLoad/LoadMetadata.cs index d78da70a8a..9e36dca63c 100644 --- a/Rdmp.Core/Curation/Data/DataLoad/LoadMetadata.cs +++ b/Rdmp.Core/Curation/Data/DataLoad/LoadMetadata.cs @@ -292,13 +292,13 @@ public LoadMetadata Clone() }; lmd.SaveToDatabase(); //link to catalogue - foreach(var catalogue in this.GetAllCatalogues()) + foreach (var catalogue in this.GetAllCatalogues()) { lmd.LinkToCatalogue(catalogue); } //process task var pts = CatalogueRepository.GetAllObjectsWhere("LoadMetadata_ID", this.ID); - foreach(ProcessTask pt in pts) + foreach (ProcessTask pt in pts) { pt.Clone(lmd); } @@ -329,10 +329,25 @@ public override void DeleteInDatabase() { var firstOrDefault = GetAllCatalogues().FirstOrDefault(); - if (firstOrDefault != null) + if (firstOrDefault != null && this.RootLoadMetadata_ID == null) throw new Exception( $"This load is used by {firstOrDefault.Name} so cannot be deleted (Disassociate it first)"); + var versions = Repository.GetAllObjectsWhere("RootLoadMetadata_ID", ID); + foreach (var version in versions) + { + version.DeleteInDatabase(); + } + if (this.RootLoadMetadata_ID != null) + { + var catalogueLinkIDs = Repository.GetAllObjectsWhere("LoadMetadataID", ID); + foreach (var link in catalogueLinkIDs) + { + link.DeleteInDatabase(); + } + } + + base.DeleteInDatabase(); } From 93c45fa85860495eb160064fba3c537b51f71e6e Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 28 Jan 2025 08:51:57 +0000 Subject: [PATCH 10/15] add tests --- .../ExecuteCommandCloneLoadMetadataTests.cs | 45 ++++++++++++++ ...teCommandCreateLoadMetadataVersionTests.cs | 46 +++++++++++++++ .../ExecuteCommandRestoreLoadMetadataTests.cs | 58 +++++++++++++++++++ .../ExecuteCommandDisableOrEnable.cs | 2 - ...xecuteCommandRestoreLoadMetadataVersion.cs | 13 ++--- 5 files changed, 153 insertions(+), 11 deletions(-) create mode 100644 Rdmp.Core.Tests/CommandExecution/ExecuteCommandCloneLoadMetadataTests.cs create mode 100644 Rdmp.Core.Tests/CommandExecution/ExecuteCommandCreateLoadMetadataVersionTests.cs create mode 100644 Rdmp.Core.Tests/CommandExecution/ExecuteCommandRestoreLoadMetadataTests.cs diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandCloneLoadMetadataTests.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandCloneLoadMetadataTests.cs new file mode 100644 index 0000000000..f3ae07b43d --- /dev/null +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandCloneLoadMetadataTests.cs @@ -0,0 +1,45 @@ +using NUnit.Framework; +using Rdmp.Core.Curation.Data.DataLoad; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Repositories; +using Tests.Common; +using Rdmp.Core.DataLoad.Modules.Attachers; +using Rdmp.Core.MapsDirectlyToDatabaseTable; +using System.Linq; + +namespace Rdmp.Core.Tests.CommandExecution +{ + public class ExecuteCommandCloneLoadMetadataTests: DatabaseTests + { + + [Test] + public void TestCloneLoadMetadata() + { + var lmd1 = new LoadMetadata(CatalogueRepository, "MyLmd"); + lmd1.Description = "Desc!"; + var cata = new Catalogue(CatalogueRepository, "myCata") + { + LoggingDataTask = "B" + }; + cata.SaveToDatabase(); + lmd1.LinkToCatalogue(cata); + var pt1 = new ProcessTask(CatalogueRepository, lmd1, LoadStage.Mounting) + { + ProcessTaskType = ProcessTaskType.Attacher, + LoadStage = LoadStage.Mounting, + Path = typeof(AnySeparatorFileAttacher).FullName + }; + pt1.SaveToDatabase(); + + pt1.CreateArgumentsForClassIfNotExists(typeof(AnySeparatorFileAttacher)); + var pta = pt1.ProcessTaskArguments.Single(pt => pt.Name == "Separator"); + pta.SetValue(","); + pta.SaveToDatabase(); + LoadMetadata clonedLmd; + clonedLmd = lmd1.Clone(); + Assert.That(clonedLmd.ProcessTasks.Count(), Is.EqualTo(1)); + Assert.That(clonedLmd.Description, Is.EqualTo(lmd1.Description)); + Assert.That(clonedLmd.ProcessTasks.First().ProcessTaskArguments.First().Value, Is.EqualTo(lmd1.ProcessTasks.First().ProcessTaskArguments.First().Value)); + } + } +} diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandCreateLoadMetadataVersionTests.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandCreateLoadMetadataVersionTests.cs new file mode 100644 index 0000000000..2354444a6a --- /dev/null +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandCreateLoadMetadataVersionTests.cs @@ -0,0 +1,46 @@ +using NUnit.Framework; +using Rdmp.Core.Curation.Data.DataLoad; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Repositories; +using Tests.Common; +using Rdmp.Core.DataLoad.Modules.Attachers; +using Rdmp.Core.MapsDirectlyToDatabaseTable; +using System.Linq; + +namespace Rdmp.Core.Tests.CommandExecution +{ + public class ExecuteCommandCreateLoadMetadataVersionTests : DatabaseTests + { + + [Test] + public void TestCreateLoadMetadataVersion() + { + var lmd1 = new LoadMetadata(CatalogueRepository, "MyLmd"); + lmd1.Description = "Desc!"; + var cata = new Catalogue(CatalogueRepository, "myCata") + { + LoggingDataTask = "B" + }; + cata.SaveToDatabase(); + lmd1.LinkToCatalogue(cata); + var pt1 = new ProcessTask(CatalogueRepository, lmd1, LoadStage.Mounting) + { + ProcessTaskType = ProcessTaskType.Attacher, + LoadStage = LoadStage.Mounting, + Path = typeof(AnySeparatorFileAttacher).FullName + }; + pt1.SaveToDatabase(); + + pt1.CreateArgumentsForClassIfNotExists(typeof(AnySeparatorFileAttacher)); + var pta = pt1.ProcessTaskArguments.Single(pt => pt.Name == "Separator"); + pta.SetValue(","); + pta.SaveToDatabase(); + LoadMetadata clonedLmd; + clonedLmd = (LoadMetadata)lmd1.SaveNewVersion(); + Assert.That(clonedLmd.ProcessTasks.Count(), Is.EqualTo(1)); + Assert.That(clonedLmd.RootLoadMetadata_ID, Is.EqualTo(lmd1.ID)); + Assert.That(clonedLmd.Description, Is.EqualTo(lmd1.Description)); + Assert.That(clonedLmd.ProcessTasks.First().ProcessTaskArguments.First().Value, Is.EqualTo(lmd1.ProcessTasks.First().ProcessTaskArguments.First().Value)); + } + } +} diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandRestoreLoadMetadataTests.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandRestoreLoadMetadataTests.cs new file mode 100644 index 0000000000..888b67bf31 --- /dev/null +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandRestoreLoadMetadataTests.cs @@ -0,0 +1,58 @@ +using NUnit.Framework; +using Rdmp.Core.Curation.Data.DataLoad; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Repositories; +using Tests.Common; +using Rdmp.Core.DataLoad.Modules.Attachers; +using Rdmp.Core.MapsDirectlyToDatabaseTable; +using System.Linq; +using Rdmp.Core.CommandExecution.AtomicCommands; +using Rdmp.Core.CommandLine.Interactive; +using Rdmp.Core.ReusableLibraryCode.Checks; +using Rdmp.Core.CommandExecution; + +namespace Rdmp.Core.Tests.CommandExecution +{ + public class ExecuteCommandRestoreLoadMetadataTests : DatabaseTests + { + + [Test] + public void TestRestoreLoadMetadataVersion() + { + var lmd1 = new LoadMetadata(CatalogueRepository, "MyLmd"); + lmd1.Description = "Desc!"; + var cata = new Catalogue(CatalogueRepository, "myCata") + { + LoggingDataTask = "B" + }; + cata.SaveToDatabase(); + lmd1.LinkToCatalogue(cata); + var pt1 = new ProcessTask(CatalogueRepository, lmd1, LoadStage.Mounting) + { + ProcessTaskType = ProcessTaskType.Attacher, + LoadStage = LoadStage.Mounting, + Path = typeof(AnySeparatorFileAttacher).FullName + }; + pt1.SaveToDatabase(); + + pt1.CreateArgumentsForClassIfNotExists(typeof(AnySeparatorFileAttacher)); + var pta = pt1.ProcessTaskArguments.Single(pt => pt.Name == "Separator"); + pta.SetValue(","); + pta.SaveToDatabase(); + LoadMetadata clonedLmd; + clonedLmd = (LoadMetadata)lmd1.SaveNewVersion(); + Assert.That(clonedLmd.ProcessTasks.Count(), Is.EqualTo(1)); + Assert.That(clonedLmd.RootLoadMetadata_ID, Is.EqualTo(lmd1.ID)); + Assert.That(clonedLmd.Description, Is.EqualTo(lmd1.Description)); + Assert.That(clonedLmd.ProcessTasks.First().ProcessTaskArguments.First().Value, Is.EqualTo(lmd1.ProcessTasks.First().ProcessTaskArguments.First().Value)); + pt1.DeleteInDatabase(); + var fetchedlmd = CatalogueRepository.GetObjectByID(lmd1.ID); + Assert.That(fetchedlmd.ProcessTasks.Count(), Is.EqualTo(0)); + var activator = new ThrowImmediatelyActivator(RepositoryLocator); + var cmd = new ExecuteCommandRestoreLoadMetadataVersion(activator, clonedLmd); + Assert.DoesNotThrow(()=>cmd.Execute()); + fetchedlmd = CatalogueRepository.GetObjectByID(lmd1.ID); + Assert.That(fetchedlmd.ProcessTasks.Count(), Is.EqualTo(1)); + } + } +} diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDisableOrEnable.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDisableOrEnable.cs index 6965c81c10..d2008fff56 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDisableOrEnable.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDisableOrEnable.cs @@ -8,9 +8,7 @@ using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.Data.Aggregation; using Rdmp.Core.Curation.Data.Cohort; -using Rdmp.Core.Curation.Data.DataLoad; using Rdmp.Core.MapsDirectlyToDatabaseTable; -using Rdmp.Core.ReusableLibraryCode; namespace Rdmp.Core.CommandExecution.AtomicCommands; diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreLoadMetadataVersion.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreLoadMetadataVersion.cs index e9354c6175..e69c5dbcc1 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreLoadMetadataVersion.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreLoadMetadataVersion.cs @@ -22,23 +22,18 @@ public ExecuteCommandRestoreLoadMetadataVersion(IBasicActivateItems activator, [ public override void Execute() { - if (!_activator.YesNo("Replace root Load Metadata with this configuration?","Resotre Load Metadata Version")) return; + if (_activator.IsInteractive && !_activator.YesNo("Replace root Load Metadata with this configuration?", "Resotre Load Metadata Version")) return; base.Execute(); if (_loadMetadata.RootLoadMetadata_ID is null) { throw new Exception("Must Use a versioned LoadMetadata to create Version"); } - LoadMetadata lmd = (LoadMetadata)_activator.RepositoryLocator.CatalogueRepository.GetObjectByID(typeof(LoadMetadata), (int)_loadMetadata.RootLoadMetadata_ID); - if (lmd is null) - { - throw new Exception("Could not find root load metadata"); - - } - foreach (ProcessTask task in lmd.ProcessTasks) + LoadMetadata lmd = (LoadMetadata)_activator.RepositoryLocator.CatalogueRepository.GetObjectByID(typeof(LoadMetadata), (int)_loadMetadata.RootLoadMetadata_ID) ?? throw new Exception("Could not find root load metadata"); + foreach (ProcessTask task in lmd.ProcessTasks.Cast()) { task.DeleteInDatabase(); } - foreach(ProcessTask task in _loadMetadata.ProcessTasks) + foreach (ProcessTask task in _loadMetadata.ProcessTasks.Cast()) { task.Clone(lmd); } From d565cee5292f65486124f5ce68cf21702695f89c Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 28 Jan 2025 08:55:33 +0000 Subject: [PATCH 11/15] tidy up --- .../AtomicCommands/ExecuteCommandCloneLoadMetadata.cs | 9 ++------- .../ExecuteCommandCreateLoadMetadataVersion.cs | 8 ++------ .../AtomicCommands/ExecuteCommandDelete.cs | 2 -- .../ExecuteCommandRestoreLoadMetadataVersion.cs | 8 ++------ Rdmp.Core/ReusableLibraryCode/IVersionable.cs | 4 ++-- 5 files changed, 8 insertions(+), 23 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCloneLoadMetadata.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCloneLoadMetadata.cs index aebb36fee2..968a51ad3a 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCloneLoadMetadata.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCloneLoadMetadata.cs @@ -1,17 +1,12 @@ using Rdmp.Core.Curation.Data.DataLoad; using Rdmp.Core.Curation.Data; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace Rdmp.Core.CommandExecution.AtomicCommands; public class ExecuteCommandCloneLoadMetadata : BasicCommandExecution { - private LoadMetadata _loadMetadata; - private IBasicActivateItems _activator; + private readonly LoadMetadata _loadMetadata; + private readonly IBasicActivateItems _activator; public ExecuteCommandCloneLoadMetadata(IBasicActivateItems activator,[DemandsInitialization("The LoadMetadata to clone")] LoadMetadata loadMetadata) { diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateLoadMetadataVersion.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateLoadMetadataVersion.cs index 85b410bb9e..57bc43a652 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateLoadMetadataVersion.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateLoadMetadataVersion.cs @@ -1,17 +1,13 @@ using Rdmp.Core.Curation.Data.DataLoad; using Rdmp.Core.Curation.Data; using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace Rdmp.Core.CommandExecution.AtomicCommands; public class ExecuteCommandCreateLoadMetadataVersion: BasicCommandExecution { - private LoadMetadata _loadMetadata; - private IBasicActivateItems _activator; + private readonly LoadMetadata _loadMetadata; + private readonly IBasicActivateItems _activator; public ExecuteCommandCreateLoadMetadataVersion(IBasicActivateItems activator,[DemandsInitialization("The LoadMetadata to version")] LoadMetadata loadMetadata) { diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDelete.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDelete.cs index 207b194924..05ea7479fe 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDelete.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDelete.cs @@ -9,9 +9,7 @@ using System.Linq; using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.Data.Cohort; -using Rdmp.Core.Curation.Data.DataLoad; using Rdmp.Core.MapsDirectlyToDatabaseTable; -using Rdmp.Core.Repositories; using Rdmp.Core.Repositories.Construction; using Rdmp.Core.ReusableLibraryCode; diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreLoadMetadataVersion.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreLoadMetadataVersion.cs index e69c5dbcc1..4ff1e1ba33 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreLoadMetadataVersion.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreLoadMetadataVersion.cs @@ -1,18 +1,14 @@ using Rdmp.Core.Curation.Data.DataLoad; using Rdmp.Core.Curation.Data; using System; -using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Org.BouncyCastle.Pqc.Crypto.Lms; namespace Rdmp.Core.CommandExecution.AtomicCommands; public class ExecuteCommandRestoreLoadMetadataVersion : BasicCommandExecution { - private LoadMetadata _loadMetadata; - private IBasicActivateItems _activator; + private readonly LoadMetadata _loadMetadata; + private readonly IBasicActivateItems _activator; public ExecuteCommandRestoreLoadMetadataVersion(IBasicActivateItems activator, [DemandsInitialization("The LoadMetadata to version")] LoadMetadata loadMetadata) { diff --git a/Rdmp.Core/ReusableLibraryCode/IVersionable.cs b/Rdmp.Core/ReusableLibraryCode/IVersionable.cs index 53011c2972..b56910eed1 100644 --- a/Rdmp.Core/ReusableLibraryCode/IVersionable.cs +++ b/Rdmp.Core/ReusableLibraryCode/IVersionable.cs @@ -11,8 +11,8 @@ internal interface IVersionable { /// - /// + /// Interface to provide the SaveNewVersion function /// - /// + /// DatabaseEntity DatabaseEntity SaveNewVersion(); } From e52f91f9c53c6e0f3d4c7005d403abc1d4113b1b Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 4 Feb 2025 15:51:15 +0000 Subject: [PATCH 12/15] fiz clone --- Rdmp.Core/Curation/Data/DataLoad/LoadMetadata.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rdmp.Core/Curation/Data/DataLoad/LoadMetadata.cs b/Rdmp.Core/Curation/Data/DataLoad/LoadMetadata.cs index 9e36dca63c..09dbe89945 100644 --- a/Rdmp.Core/Curation/Data/DataLoad/LoadMetadata.cs +++ b/Rdmp.Core/Curation/Data/DataLoad/LoadMetadata.cs @@ -288,7 +288,7 @@ public LoadMetadata Clone() OverrideRAWServer_ID = OverrideRAWServer_ID, IgnoreTrigger = IgnoreTrigger, Folder = Folder, - RootLoadMetadata_ID = this.ID + //RootLoadMetadata_ID = this.ID }; lmd.SaveToDatabase(); //link to catalogue From c91e7815f19877aab78365d8aa915e139e368efa Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 5 Feb 2025 09:49:05 +0000 Subject: [PATCH 13/15] fix up tree structure --- Rdmp.Core/Providers/CatalogueChildProvider.cs | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/Rdmp.Core/Providers/CatalogueChildProvider.cs b/Rdmp.Core/Providers/CatalogueChildProvider.cs index c8988d0374..2e83383547 100644 --- a/Rdmp.Core/Providers/CatalogueChildProvider.cs +++ b/Rdmp.Core/Providers/CatalogueChildProvider.cs @@ -906,7 +906,7 @@ private void AddChildren(Curation.Data.Dataset lmd, DescendancyList descendancy) #region Load Metadata - private void AddChildren(LoadMetadata lmd, DescendancyList descendancy) + private void AddChildren(LoadMetadata lmd, DescendancyList descendancy, bool includeSchedule = true, bool includeVersions = true) { var childObjects = new List(); @@ -916,10 +916,12 @@ private void AddChildren(LoadMetadata lmd, DescendancyList descendancy) var usage = new OverrideRawServerNode(lmd, server); childObjects.Add(usage); } - - var allSchedulesNode = new LoadMetadataScheduleNode(lmd); - AddChildren(allSchedulesNode, descendancy.Add(allSchedulesNode)); - childObjects.Add(allSchedulesNode); + if (includeSchedule) + { + var allSchedulesNode = new LoadMetadataScheduleNode(lmd); + AddChildren(allSchedulesNode, descendancy.Add(allSchedulesNode)); + childObjects.Add(allSchedulesNode); + } var allCataloguesNode = new AllCataloguesUsedByLoadMetadataNode(lmd); AddChildren(allCataloguesNode, descendancy.Add(allCataloguesNode)); @@ -929,9 +931,12 @@ private void AddChildren(LoadMetadata lmd, DescendancyList descendancy) AddChildren(processTasksNode, descendancy.Add(processTasksNode)); childObjects.Add(processTasksNode); - var versionsNode = new LoadMetadataVersionNode(lmd); - AddChildren(versionsNode, descendancy.Add(versionsNode)); - childObjects.Add(versionsNode); + if (includeVersions) + { + var versionsNode = new LoadMetadataVersionNode(lmd); + AddChildren(versionsNode, descendancy.Add(versionsNode)); + childObjects.Add(versionsNode); + } childObjects.Add(new LoadDirectoryNode(lmd)); @@ -1024,7 +1029,16 @@ private void AddChildren(ProcessTask procesTask, DescendancyList descendancy) private void AddChildren(LoadMetadataVersionNode LoadMetadataVersionNode, DescendancyList descendancy) { LoadMetadataVersionNode.LoadMetadataVersions = AllLoadMetadatas.Where(lmd => lmd.RootLoadMetadata_ID == LoadMetadataVersionNode.LoadMetadata.ID).ToList(); - AddToDictionaries(LoadMetadataVersionNode.LoadMetadataVersions.Cast().ToHashSet(), descendancy); + //AddToDictionaries(LoadMetadataVersionNode.LoadMetadataVersions.Cast().ToHashSet(), descendancy); + var childObjects = new List(); + + foreach (var lmd in LoadMetadataVersionNode.LoadMetadataVersions) + { + AddChildren(lmd, descendancy.Add(lmd), false, false); + childObjects.Add(lmd); + } + AddToDictionaries(new HashSet(childObjects), descendancy); + } private void AddChildren(AllCataloguesUsedByLoadMetadataNode allCataloguesUsedByLoadMetadataNode, From 9ce267045aef287d1b784d0439a3cb4676266aab Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 5 Feb 2025 10:13:45 +0000 Subject: [PATCH 14/15] clone version --- Rdmp.Core/CommandExecution/AtomicCommandFactory.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs b/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs index 382df303d1..38d67ded93 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs @@ -547,7 +547,12 @@ public IEnumerable CreateCommands(object o) { yield return new ExecuteCommandRestoreLoadMetadataVersion(_activator, lmd) { - OverrideCommandName = "Restore Version"}; + OverrideCommandName = "Restore Version" + }; + yield return new ExecuteCommandCloneLoadMetadata(_activator, lmd) + { + OverrideCommandName = "Clone Load Metadata" + }; } } From cebe13140210807690dc4c0bccfd26d0ef60aa16 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 5 Feb 2025 14:20:38 +0000 Subject: [PATCH 15/15] tidy up --- .../Curation/Data/DataLoad/LoadMetadata.cs | 1 - .../CreateCatalogue.sql | 1 - Rdmp.Core/Providers/CatalogueChildProvider.cs | 3448 +++++++++-------- 3 files changed, 1725 insertions(+), 1725 deletions(-) diff --git a/Rdmp.Core/Curation/Data/DataLoad/LoadMetadata.cs b/Rdmp.Core/Curation/Data/DataLoad/LoadMetadata.cs index 09dbe89945..835426d04e 100644 --- a/Rdmp.Core/Curation/Data/DataLoad/LoadMetadata.cs +++ b/Rdmp.Core/Curation/Data/DataLoad/LoadMetadata.cs @@ -288,7 +288,6 @@ public LoadMetadata Clone() OverrideRAWServer_ID = OverrideRAWServer_ID, IgnoreTrigger = IgnoreTrigger, Folder = Folder, - //RootLoadMetadata_ID = this.ID }; lmd.SaveToDatabase(); //link to catalogue diff --git a/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql b/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql index b4423e7c03..c5df74b2ad 100644 --- a/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql +++ b/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql @@ -774,7 +774,6 @@ CREATE TABLE [dbo].[ProcessTask]( [SoftwareVersion] [nvarchar](50) NOT NULL, [IsDisabled] [bit] NOT NULL, [SerialisableConfiguration] [varchar](max), - [LoadMetadataVersion] [int], CONSTRAINT [PK_ProcessTask] PRIMARY KEY CLUSTERED ( [ID] ASC diff --git a/Rdmp.Core/Providers/CatalogueChildProvider.cs b/Rdmp.Core/Providers/CatalogueChildProvider.cs index 2e83383547..fab8b0811b 100644 --- a/Rdmp.Core/Providers/CatalogueChildProvider.cs +++ b/Rdmp.Core/Providers/CatalogueChildProvider.cs @@ -53,1914 +53,1916 @@ namespace Rdmp.Core.Providers; /// public class CatalogueChildProvider : ICoreChildProvider { - //Load System - public LoadMetadata[] AllLoadMetadatas { get; set; } + //Load System + public LoadMetadata[] AllLoadMetadatas { get; set; } - private LoadMetadataCatalogueLinkage[] AllLoadMetadataLinkage { get; set; } - public ProcessTask[] AllProcessTasks { get; set; } - public ProcessTaskArgument[] AllProcessTasksArguments { get; set; } + private LoadMetadataCatalogueLinkage[] AllLoadMetadataLinkage { get; set; } + public ProcessTask[] AllProcessTasks { get; set; } + public ProcessTaskArgument[] AllProcessTasksArguments { get; set; } - public LoadProgress[] AllLoadProgresses { get; set; } - public CacheProgress[] AllCacheProgresses { get; set; } - public PermissionWindow[] AllPermissionWindows { get; set; } + public LoadProgress[] AllLoadProgresses { get; set; } + public CacheProgress[] AllCacheProgresses { get; set; } + public PermissionWindow[] AllPermissionWindows { get; set; } - //Catalogue side of things - public Catalogue[] AllCatalogues { get; set; } - public Curation.Data.Dataset[] AllDatasets { get; set; } - public Dictionary AllCataloguesDictionary { get; private set; } + //Catalogue side of things + public Catalogue[] AllCatalogues { get; set; } + public Curation.Data.Dataset[] AllDatasets { get; set; } + public Dictionary AllCataloguesDictionary { get; private set; } - public SupportingDocument[] AllSupportingDocuments { get; set; } - public SupportingSQLTable[] AllSupportingSQL { get; set; } + public SupportingDocument[] AllSupportingDocuments { get; set; } + public SupportingSQLTable[] AllSupportingSQL { get; set; } - //tells you the imediate children of a given node. Do not add to this directly instead add using AddToDictionaries unless you want the Key to be an 'on the sly' no known descendency child - private ConcurrentDictionary> _childDictionary = new(); + //tells you the imediate children of a given node. Do not add to this directly instead add using AddToDictionaries unless you want the Key to be an 'on the sly' no known descendency child + private ConcurrentDictionary> _childDictionary = new(); - //This is the reverse of _childDictionary in some ways. _childDictionary tells you the immediate children while - //this tells you for a given child object what the navigation tree down to get to it is e.g. ascendancy[child] would return [root,grandParent,parent] - private ConcurrentDictionary _descendancyDictionary = new(); + //This is the reverse of _childDictionary in some ways. _childDictionary tells you the immediate children while + //this tells you for a given child object what the navigation tree down to get to it is e.g. ascendancy[child] would return [root,grandParent,parent] + private ConcurrentDictionary _descendancyDictionary = new(); - public IEnumerable AllCatalogueItems => AllCatalogueItemsDictionary.Values; + public IEnumerable AllCatalogueItems => AllCatalogueItemsDictionary.Values; - private Dictionary> _catalogueToCatalogueItems; - public Dictionary AllCatalogueItemsDictionary { get; private set; } + private Dictionary> _catalogueToCatalogueItems; + public Dictionary AllCatalogueItemsDictionary { get; private set; } - private Dictionary _allColumnInfos; + private Dictionary _allColumnInfos; - public AggregateConfiguration[] AllAggregateConfigurations { get; private set; } - public AggregateDimension[] AllAggregateDimensions { get; private set; } + public AggregateConfiguration[] AllAggregateConfigurations { get; private set; } + public AggregateDimension[] AllAggregateDimensions { get; private set; } - public AggregateContinuousDateAxis[] AllAggregateContinuousDateAxis { get; private set; } + public AggregateContinuousDateAxis[] AllAggregateContinuousDateAxis { get; private set; } - public AllRDMPRemotesNode AllRDMPRemotesNode { get; private set; } - public RemoteRDMP[] AllRemoteRDMPs { get; set; } + public AllRDMPRemotesNode AllRDMPRemotesNode { get; private set; } + public RemoteRDMP[] AllRemoteRDMPs { get; set; } - public AllDashboardsNode AllDashboardsNode { get; set; } - public DashboardLayout[] AllDashboards { get; set; } + public AllDashboardsNode AllDashboardsNode { get; set; } + public DashboardLayout[] AllDashboards { get; set; } - public AllObjectSharingNode AllObjectSharingNode { get; private set; } - public ObjectImport[] AllImports { get; set; } - public ObjectExport[] AllExports { get; set; } + public AllObjectSharingNode AllObjectSharingNode { get; private set; } + public ObjectImport[] AllImports { get; set; } + public ObjectExport[] AllExports { get; set; } - public AllStandardRegexesNode AllStandardRegexesNode { get; private set; } - public AllPipelinesNode AllPipelinesNode { get; private set; } - public OtherPipelinesNode OtherPipelinesNode { get; private set; } - public Pipeline[] AllPipelines { get; set; } - public PipelineComponent[] AllPipelineComponents { get; set; } + public AllStandardRegexesNode AllStandardRegexesNode { get; private set; } + public AllPipelinesNode AllPipelinesNode { get; private set; } + public OtherPipelinesNode OtherPipelinesNode { get; private set; } + public Pipeline[] AllPipelines { get; set; } + public PipelineComponent[] AllPipelineComponents { get; set; } - public PipelineComponentArgument[] AllPipelineComponentsArguments { get; set; } + public PipelineComponentArgument[] AllPipelineComponentsArguments { get; set; } - public StandardRegex[] AllStandardRegexes { get; set; } + public StandardRegex[] AllStandardRegexes { get; set; } - //TableInfo side of things - public AllANOTablesNode AllANOTablesNode { get; private set; } - public ANOTable[] AllANOTables { get; set; } + //TableInfo side of things + public AllANOTablesNode AllANOTablesNode { get; private set; } + public ANOTable[] AllANOTables { get; set; } - public ExternalDatabaseServer[] AllExternalServers { get; private set; } - public TableInfoServerNode[] AllServers { get; private set; } - public TableInfo[] AllTableInfos { get; private set; } + public ExternalDatabaseServer[] AllExternalServers { get; private set; } + public TableInfoServerNode[] AllServers { get; private set; } + public TableInfo[] AllTableInfos { get; private set; } - public AllDataAccessCredentialsNode AllDataAccessCredentialsNode { get; set; } + public AllDataAccessCredentialsNode AllDataAccessCredentialsNode { get; set; } - public AllExternalServersNode AllExternalServersNode { get; private set; } - public AllServersNode AllServersNode { get; private set; } + public AllExternalServersNode AllExternalServersNode { get; private set; } + public AllServersNode AllServersNode { get; private set; } - public DataAccessCredentials[] AllDataAccessCredentials { get; set; } - public Dictionary> AllDataAccessCredentialUsages { get; set; } + public DataAccessCredentials[] AllDataAccessCredentials { get; set; } + public Dictionary> AllDataAccessCredentialUsages { get; set; } - public Dictionary> TableInfosToColumnInfos { get; private set; } - public ColumnInfo[] AllColumnInfos { get; private set; } - public PreLoadDiscardedColumn[] AllPreLoadDiscardedColumns { get; private set; } + public Dictionary> TableInfosToColumnInfos { get; private set; } + public ColumnInfo[] AllColumnInfos { get; private set; } + public PreLoadDiscardedColumn[] AllPreLoadDiscardedColumns { get; private set; } - public Lookup[] AllLookups { get; set; } + public Lookup[] AllLookups { get; set; } - public JoinInfo[] AllJoinInfos { get; set; } + public JoinInfo[] AllJoinInfos { get; set; } - public AnyTableSqlParameter[] AllAnyTableParameters; + public AnyTableSqlParameter[] AllAnyTableParameters; - //Filter / extraction side of things - public IEnumerable AllExtractionInformations => AllExtractionInformationsDictionary.Values; + //Filter / extraction side of things + public IEnumerable AllExtractionInformations => AllExtractionInformationsDictionary.Values; - public AllPermissionWindowsNode AllPermissionWindowsNode { get; set; } - public FolderNode LoadMetadataRootFolder { get; set; } + public AllPermissionWindowsNode AllPermissionWindowsNode { get; set; } + public FolderNode LoadMetadataRootFolder { get; set; } - public FolderNode DatasetRootFolder { get; set; } - public FolderNode CohortIdentificationConfigurationRootFolder { get; set; } - public FolderNode CohortIdentificationConfigurationRootFolderWithoutVersionedConfigurations { get; set; } + public FolderNode DatasetRootFolder { get; set; } + public FolderNode CohortIdentificationConfigurationRootFolder { get; set; } + public FolderNode CohortIdentificationConfigurationRootFolderWithoutVersionedConfigurations { get; set; } - public AllConnectionStringKeywordsNode AllConnectionStringKeywordsNode { get; set; } - public ConnectionStringKeyword[] AllConnectionStringKeywords { get; set; } + public AllConnectionStringKeywordsNode AllConnectionStringKeywordsNode { get; set; } + public ConnectionStringKeyword[] AllConnectionStringKeywords { get; set; } - public Dictionary AllExtractionInformationsDictionary { get; private set; } - protected Dictionary _extractionInformationsByCatalogueItem; + public Dictionary AllExtractionInformationsDictionary { get; private set; } + protected Dictionary _extractionInformationsByCatalogueItem; - private IFilterManager _aggregateFilterManager; + private IFilterManager _aggregateFilterManager; - //Filters for Aggregates (includes filter containers (AND/OR) - public Dictionary AllAggregateContainersDictionary { get; private set; } - public AggregateFilterContainer[] AllAggregateContainers => AllAggregateContainersDictionary.Values.ToArray(); + //Filters for Aggregates (includes filter containers (AND/OR) + public Dictionary AllAggregateContainersDictionary { get; private set; } + public AggregateFilterContainer[] AllAggregateContainers => AllAggregateContainersDictionary.Values.ToArray(); - public AggregateFilter[] AllAggregateFilters { get; private set; } - public AggregateFilterParameter[] AllAggregateFilterParameters { get; private set; } + public AggregateFilter[] AllAggregateFilters { get; private set; } + public AggregateFilterParameter[] AllAggregateFilterParameters { get; private set; } - //Catalogue master filters (does not include any support for filter containers (AND/OR) - private ExtractionFilter[] AllCatalogueFilters; - public ExtractionFilterParameter[] AllCatalogueParameters; - public ExtractionFilterParameterSet[] AllCatalogueValueSets; - public ExtractionFilterParameterSetValue[] AllCatalogueValueSetValues; + //Catalogue master filters (does not include any support for filter containers (AND/OR) + private ExtractionFilter[] AllCatalogueFilters; + public ExtractionFilterParameter[] AllCatalogueParameters; + public ExtractionFilterParameterSet[] AllCatalogueValueSets; + public ExtractionFilterParameterSetValue[] AllCatalogueValueSetValues; - private ICohortContainerManager _cohortContainerManager; + private ICohortContainerManager _cohortContainerManager; - public CohortIdentificationConfiguration[] AllCohortIdentificationConfigurations { get; private set; } - public CohortAggregateContainer[] AllCohortAggregateContainers { get; set; } - public JoinableCohortAggregateConfiguration[] AllJoinables { get; set; } - public JoinableCohortAggregateConfigurationUse[] AllJoinUses { get; set; } + public CohortIdentificationConfiguration[] AllCohortIdentificationConfigurations { get; private set; } + public CohortAggregateContainer[] AllCohortAggregateContainers { get; set; } + public JoinableCohortAggregateConfiguration[] AllJoinables { get; set; } + public JoinableCohortAggregateConfigurationUse[] AllJoinUses { get; set; } - /// - /// Collection of all objects for which there are masqueraders - /// - public ConcurrentDictionary> AllMasqueraders { get; private set; } + /// + /// Collection of all objects for which there are masqueraders + /// + public ConcurrentDictionary> AllMasqueraders { get; private set; } - private IChildProvider[] _pluginChildProviders; - private readonly ICatalogueRepository _catalogueRepository; - private readonly ICheckNotifier _errorsCheckNotifier; - private readonly List _blockedPlugins = new(); + private IChildProvider[] _pluginChildProviders; + private readonly ICatalogueRepository _catalogueRepository; + private readonly ICheckNotifier _errorsCheckNotifier; + private readonly List _blockedPlugins = new(); - public AllGovernanceNode AllGovernanceNode { get; private set; } - public GovernancePeriod[] AllGovernancePeriods { get; private set; } - public GovernanceDocument[] AllGovernanceDocuments { get; private set; } - public Dictionary> GovernanceCoverage { get; private set; } + public AllGovernanceNode AllGovernanceNode { get; private set; } + public GovernancePeriod[] AllGovernancePeriods { get; private set; } + public GovernanceDocument[] AllGovernanceDocuments { get; private set; } + public Dictionary> GovernanceCoverage { get; private set; } - private CommentStore _commentStore; + private CommentStore _commentStore; - public JoinableCohortAggregateConfigurationUse[] AllJoinableCohortAggregateConfigurationUse { get; private set; } - public AllPluginsNode AllPluginsNode { get; private set; } - public HashSet PipelineUseCases { get; set; } = new(); + public JoinableCohortAggregateConfigurationUse[] AllJoinableCohortAggregateConfigurationUse { get; private set; } + public AllPluginsNode AllPluginsNode { get; private set; } + public HashSet PipelineUseCases { get; set; } = new(); - /// - /// Lock for changes to Child provider - /// - protected object WriteLock = new(); + /// + /// Lock for changes to Child provider + /// + protected object WriteLock = new(); - public AllOrphanAggregateConfigurationsNode OrphanAggregateConfigurationsNode { get; set; } = new(); - public AllTemplateAggregateConfigurationsNode TemplateAggregateConfigurationsNode { get; set; } = new(); - public FolderNode CatalogueRootFolder { get; private set; } + public AllOrphanAggregateConfigurationsNode OrphanAggregateConfigurationsNode { get; set; } = new(); + public AllTemplateAggregateConfigurationsNode TemplateAggregateConfigurationsNode { get; set; } = new(); + public FolderNode CatalogueRootFolder { get; private set; } - public AllDatasetsNode AllDatasetsNode { get; set; } + public AllDatasetsNode AllDatasetsNode { get; set; } - public RegexRedactionConfiguration[] AllRegexRedactionConfigurations { get; set; } - public AllRegexRedactionConfigurationsNode AllRegexRedactionConfigurationsNode { get; set; } + public RegexRedactionConfiguration[] AllRegexRedactionConfigurations { get; set; } + public AllRegexRedactionConfigurationsNode AllRegexRedactionConfigurationsNode { get; set; } - public HashSet OrphanAggregateConfigurations; - public AggregateConfiguration[] TemplateAggregateConfigurations; + public HashSet OrphanAggregateConfigurations; + public AggregateConfiguration[] TemplateAggregateConfigurations; - protected Stopwatch ProgressStopwatch = Stopwatch.StartNew(); - private int _progress; + protected Stopwatch ProgressStopwatch = Stopwatch.StartNew(); + private int _progress; - /// - /// - /// - /// - /// - /// Where to report errors building the hierarchy e.g. when crash. Set to null for - /// Previous child provider state if you know it otherwise null - public CatalogueChildProvider(ICatalogueRepository repository, IChildProvider[] pluginChildProviders, - ICheckNotifier errorsCheckNotifier, CatalogueChildProvider previousStateIfKnown) - { - _commentStore = repository.CommentStore; - _catalogueRepository = repository; - _catalogueRepository?.EncryptionManager?.ClearAllInjections(); + /// + /// + /// + /// + /// + /// Where to report errors building the hierarchy e.g. when crash. Set to null for + /// Previous child provider state if you know it otherwise null + public CatalogueChildProvider(ICatalogueRepository repository, IChildProvider[] pluginChildProviders, + ICheckNotifier errorsCheckNotifier, CatalogueChildProvider previousStateIfKnown) + { + _commentStore = repository.CommentStore; + _catalogueRepository = repository; + _catalogueRepository?.EncryptionManager?.ClearAllInjections(); - _errorsCheckNotifier = errorsCheckNotifier ?? IgnoreAllErrorsCheckNotifier.Instance; + _errorsCheckNotifier = errorsCheckNotifier ?? IgnoreAllErrorsCheckNotifier.Instance; - if (UserSettings.DebugPerformance) - _errorsCheckNotifier.OnCheckPerformed(new CheckEventArgs( - $"Refresh generated by:{Environment.NewLine}{Environment.StackTrace}", CheckResult.Success)); + if (UserSettings.DebugPerformance) + _errorsCheckNotifier.OnCheckPerformed(new CheckEventArgs( + $"Refresh generated by:{Environment.NewLine}{Environment.StackTrace}", CheckResult.Success)); - // all the objects which are - AllMasqueraders = new ConcurrentDictionary>(); + // all the objects which are + AllMasqueraders = new ConcurrentDictionary>(); - _pluginChildProviders = pluginChildProviders ?? Array.Empty(); + _pluginChildProviders = pluginChildProviders ?? Array.Empty(); - ReportProgress("Before object fetches"); + ReportProgress("Before object fetches"); - AllAnyTableParameters = GetAllObjects(repository); + AllAnyTableParameters = GetAllObjects(repository); - AllANOTables = GetAllObjects(repository); - AllANOTablesNode = new AllANOTablesNode(); - AddChildren(AllANOTablesNode); + AllANOTables = GetAllObjects(repository); + AllANOTablesNode = new AllANOTablesNode(); + AddChildren(AllANOTablesNode); - AllCatalogues = GetAllObjects(repository); - AllCataloguesDictionary = AllCatalogues.ToDictionaryEx(i => i.ID, o => o); + AllCatalogues = GetAllObjects(repository); + AllCataloguesDictionary = AllCatalogues.ToDictionaryEx(i => i.ID, o => o); - AllDatasets = GetAllObjects(repository); + AllDatasets = GetAllObjects(repository); - AllLoadMetadatas = GetAllObjects(repository); - AllLoadMetadataLinkage = GetAllObjects(repository); - AllProcessTasks = GetAllObjects(repository); - AllProcessTasksArguments = GetAllObjects(repository); - AllLoadProgresses = GetAllObjects(repository); - AllCacheProgresses = GetAllObjects(repository); + AllLoadMetadatas = GetAllObjects(repository); + AllLoadMetadataLinkage = GetAllObjects(repository); + AllProcessTasks = GetAllObjects(repository); + AllProcessTasksArguments = GetAllObjects(repository); + AllLoadProgresses = GetAllObjects(repository); + AllCacheProgresses = GetAllObjects(repository); - AllPermissionWindows = GetAllObjects(repository); - AllPermissionWindowsNode = new AllPermissionWindowsNode(); - AddChildren(AllPermissionWindowsNode); + AllPermissionWindows = GetAllObjects(repository); + AllPermissionWindowsNode = new AllPermissionWindowsNode(); + AddChildren(AllPermissionWindowsNode); - AllRemoteRDMPs = GetAllObjects(repository); + AllRemoteRDMPs = GetAllObjects(repository); - AllExternalServers = GetAllObjects(repository); + AllExternalServers = GetAllObjects(repository); - AllTableInfos = GetAllObjects(repository); - AllDataAccessCredentials = GetAllObjects(repository); - AllDataAccessCredentialsNode = new AllDataAccessCredentialsNode(); - AddChildren(AllDataAccessCredentialsNode); + AllTableInfos = GetAllObjects(repository); + AllDataAccessCredentials = GetAllObjects(repository); + AllDataAccessCredentialsNode = new AllDataAccessCredentialsNode(); + AddChildren(AllDataAccessCredentialsNode); - AllConnectionStringKeywordsNode = new AllConnectionStringKeywordsNode(); - AllConnectionStringKeywords = GetAllObjects(repository).ToArray(); - AddToDictionaries(new HashSet(AllConnectionStringKeywords), - new DescendancyList(AllConnectionStringKeywordsNode)); + AllConnectionStringKeywordsNode = new AllConnectionStringKeywordsNode(); + AllConnectionStringKeywords = GetAllObjects(repository).ToArray(); + AddToDictionaries(new HashSet(AllConnectionStringKeywords), + new DescendancyList(AllConnectionStringKeywordsNode)); - ReportProgress("after basic object fetches"); + ReportProgress("after basic object fetches"); - Task.WaitAll( - //which TableInfos use which Credentials under which DataAccessContexts - Task.Factory.StartNew(() => - { - AllDataAccessCredentialUsages = - repository.TableInfoCredentialsManager.GetAllCredentialUsagesBy(AllDataAccessCredentials, - AllTableInfos); - }), - Task.Factory.StartNew(() => { AllColumnInfos = GetAllObjects(repository); }) - ); + Task.WaitAll( + //which TableInfos use which Credentials under which DataAccessContexts + Task.Factory.StartNew(() => + { + AllDataAccessCredentialUsages = + repository.TableInfoCredentialsManager.GetAllCredentialUsagesBy(AllDataAccessCredentials, + AllTableInfos); + }), + Task.Factory.StartNew(() => { AllColumnInfos = GetAllObjects(repository); }) + ); - ReportProgress("After credentials"); + ReportProgress("After credentials"); - TableInfosToColumnInfos = AllColumnInfos.GroupBy(c => c.TableInfo_ID) - .ToDictionaryEx(gdc => gdc.Key, gdc => gdc.ToList()); + TableInfosToColumnInfos = AllColumnInfos.GroupBy(c => c.TableInfo_ID) + .ToDictionaryEx(gdc => gdc.Key, gdc => gdc.ToList()); - ReportProgress("After TableInfo to ColumnInfo mapping"); + ReportProgress("After TableInfo to ColumnInfo mapping"); - AllPreLoadDiscardedColumns = GetAllObjects(repository); + AllPreLoadDiscardedColumns = GetAllObjects(repository); - AllSupportingDocuments = GetAllObjects(repository); - AllSupportingSQL = GetAllObjects(repository); + AllSupportingDocuments = GetAllObjects(repository); + AllSupportingSQL = GetAllObjects(repository); - AllCohortIdentificationConfigurations = GetAllObjects(repository); + AllCohortIdentificationConfigurations = GetAllObjects(repository); - FetchCatalogueItems(); + FetchCatalogueItems(); - ReportProgress("After CatalogueItem injection"); + ReportProgress("After CatalogueItem injection"); - FetchExtractionInformations(); + FetchExtractionInformations(); - ReportProgress("After ExtractionInformation injection"); + ReportProgress("After ExtractionInformation injection"); - BuildAggregateConfigurations(); + BuildAggregateConfigurations(); - BuildCohortCohortAggregateContainers(); + BuildCohortCohortAggregateContainers(); - AllJoinables = GetAllObjects(repository); - AllJoinUses = GetAllObjects(repository); + AllJoinables = GetAllObjects(repository); + AllJoinUses = GetAllObjects(repository); - AllCatalogueFilters = GetAllObjects(repository); - AllCatalogueParameters = GetAllObjects(repository); - AllCatalogueValueSets = GetAllObjects(repository); - AllCatalogueValueSetValues = GetAllObjects(repository); + AllCatalogueFilters = GetAllObjects(repository); + AllCatalogueParameters = GetAllObjects(repository); + AllCatalogueValueSets = GetAllObjects(repository); + AllCatalogueValueSetValues = GetAllObjects(repository); - ReportProgress("After Filter and Joinable fetching"); + ReportProgress("After Filter and Joinable fetching"); - AllLookups = GetAllObjects(repository); + AllLookups = GetAllObjects(repository); - foreach (var l in AllLookups) - l.SetKnownColumns(_allColumnInfos[l.PrimaryKey_ID], _allColumnInfos[l.ForeignKey_ID], - _allColumnInfos[l.Description_ID]); + foreach (var l in AllLookups) + l.SetKnownColumns(_allColumnInfos[l.PrimaryKey_ID], _allColumnInfos[l.ForeignKey_ID], + _allColumnInfos[l.Description_ID]); - AllJoinInfos = repository.GetAllObjects(); + AllJoinInfos = repository.GetAllObjects(); - foreach (var j in AllJoinInfos) - j.SetKnownColumns(_allColumnInfos[j.PrimaryKey_ID], _allColumnInfos[j.ForeignKey_ID]); + foreach (var j in AllJoinInfos) + j.SetKnownColumns(_allColumnInfos[j.PrimaryKey_ID], _allColumnInfos[j.ForeignKey_ID]); - ReportProgress("After SetKnownColumns"); + ReportProgress("After SetKnownColumns"); - AllExternalServersNode = new AllExternalServersNode(); - AddChildren(AllExternalServersNode); + AllExternalServersNode = new AllExternalServersNode(); + AddChildren(AllExternalServersNode); - AllRDMPRemotesNode = new AllRDMPRemotesNode(); - AddChildren(AllRDMPRemotesNode); + AllRDMPRemotesNode = new AllRDMPRemotesNode(); + AddChildren(AllRDMPRemotesNode); - AllDashboardsNode = new AllDashboardsNode(); - AllDashboards = GetAllObjects(repository); - AddChildren(AllDashboardsNode); + AllDashboardsNode = new AllDashboardsNode(); + AllDashboards = GetAllObjects(repository); + AddChildren(AllDashboardsNode); - AllObjectSharingNode = new AllObjectSharingNode(); - AllExports = GetAllObjects(repository); - AllImports = GetAllObjects(repository); + AllObjectSharingNode = new AllObjectSharingNode(); + AllExports = GetAllObjects(repository); + AllImports = GetAllObjects(repository); - AddChildren(AllObjectSharingNode); + AddChildren(AllObjectSharingNode); - ReportProgress("After Object Sharing discovery"); + ReportProgress("After Object Sharing discovery"); - //Pipelines setup (see also DataExportChildProvider for calls to AddPipelineUseCases) - //Root node for all pipelines - AllPipelinesNode = new AllPipelinesNode(); + //Pipelines setup (see also DataExportChildProvider for calls to AddPipelineUseCases) + //Root node for all pipelines + AllPipelinesNode = new AllPipelinesNode(); - //Pipelines not found to be part of any use case after AddPipelineUseCases - OtherPipelinesNode = new OtherPipelinesNode(); - AllPipelines = GetAllObjects(repository); - AllPipelineComponents = GetAllObjects(repository); - AllPipelineComponentsArguments = GetAllObjects(repository); + //Pipelines not found to be part of any use case after AddPipelineUseCases + OtherPipelinesNode = new OtherPipelinesNode(); + AllPipelines = GetAllObjects(repository); + AllPipelineComponents = GetAllObjects(repository); + AllPipelineComponentsArguments = GetAllObjects(repository); - foreach (var p in AllPipelines) - p.InjectKnown(AllPipelineComponents.Where(pc => pc.Pipeline_ID == p.ID).ToArray()); + foreach (var p in AllPipelines) + p.InjectKnown(AllPipelineComponents.Where(pc => pc.Pipeline_ID == p.ID).ToArray()); - AllStandardRegexesNode = new AllStandardRegexesNode(); - AllStandardRegexes = GetAllObjects(repository); - AddToDictionaries(new HashSet(AllStandardRegexes), new DescendancyList(AllStandardRegexesNode)); + AllStandardRegexesNode = new AllStandardRegexesNode(); + AllStandardRegexes = GetAllObjects(repository); + AddToDictionaries(new HashSet(AllStandardRegexes), new DescendancyList(AllStandardRegexesNode)); - ReportProgress("After Pipelines setup"); + ReportProgress("After Pipelines setup"); - //All the things for TableInfoCollectionUI - BuildServerNodes(); + //All the things for TableInfoCollectionUI + BuildServerNodes(); - ReportProgress("BuildServerNodes"); + ReportProgress("BuildServerNodes"); - //add a new CatalogueItemNodes - InjectCatalogueItems(); + //add a new CatalogueItemNodes + InjectCatalogueItems(); - CatalogueRootFolder = FolderHelper.BuildFolderTree(AllCatalogues); - AddChildren(CatalogueRootFolder, new DescendancyList(CatalogueRootFolder)); + CatalogueRootFolder = FolderHelper.BuildFolderTree(AllCatalogues); + AddChildren(CatalogueRootFolder, new DescendancyList(CatalogueRootFolder)); - DatasetRootFolder = FolderHelper.BuildFolderTree(AllDatasets); - AddChildren(DatasetRootFolder, new DescendancyList(DatasetRootFolder)); + DatasetRootFolder = FolderHelper.BuildFolderTree(AllDatasets); + AddChildren(DatasetRootFolder, new DescendancyList(DatasetRootFolder)); - ReportProgress("Build Catalogue Folder Root"); + ReportProgress("Build Catalogue Folder Root"); - LoadMetadataRootFolder = FolderHelper.BuildFolderTree(AllLoadMetadatas.Where(lmd => lmd.RootLoadMetadata_ID is null).ToArray()); - AddChildren(LoadMetadataRootFolder, new DescendancyList(LoadMetadataRootFolder)); + LoadMetadataRootFolder = FolderHelper.BuildFolderTree(AllLoadMetadatas.Where(lmd => lmd.RootLoadMetadata_ID is null).ToArray()); + AddChildren(LoadMetadataRootFolder, new DescendancyList(LoadMetadataRootFolder)); - CohortIdentificationConfigurationRootFolder = - FolderHelper.BuildFolderTree(AllCohortIdentificationConfigurations); - AddChildren(CohortIdentificationConfigurationRootFolder, - new DescendancyList(CohortIdentificationConfigurationRootFolder)); + CohortIdentificationConfigurationRootFolder = + FolderHelper.BuildFolderTree(AllCohortIdentificationConfigurations); + AddChildren(CohortIdentificationConfigurationRootFolder, + new DescendancyList(CohortIdentificationConfigurationRootFolder)); - CohortIdentificationConfigurationRootFolderWithoutVersionedConfigurations = FolderHelper.BuildFolderTree(AllCohortIdentificationConfigurations.Where(cic => cic.Version is null).ToArray()); - AddChildren(CohortIdentificationConfigurationRootFolderWithoutVersionedConfigurations, - new DescendancyList(CohortIdentificationConfigurationRootFolderWithoutVersionedConfigurations)); - var templateAggregateConfigurationIds = - new HashSet( - repository.GetExtendedProperties(ExtendedProperty.IsTemplate) - .Where(p => p.ReferencedObjectType.Equals(nameof(AggregateConfiguration))) - .Select(r => r.ReferencedObjectID)); + CohortIdentificationConfigurationRootFolderWithoutVersionedConfigurations = FolderHelper.BuildFolderTree(AllCohortIdentificationConfigurations.Where(cic => cic.Version is null).ToArray()); + AddChildren(CohortIdentificationConfigurationRootFolderWithoutVersionedConfigurations, + new DescendancyList(CohortIdentificationConfigurationRootFolderWithoutVersionedConfigurations)); + var templateAggregateConfigurationIds = + new HashSet( + repository.GetExtendedProperties(ExtendedProperty.IsTemplate) + .Where(p => p.ReferencedObjectType.Equals(nameof(AggregateConfiguration))) + .Select(r => r.ReferencedObjectID)); - TemplateAggregateConfigurations = AllAggregateConfigurations - .Where(ac => templateAggregateConfigurationIds.Contains(ac.ID)).ToArray(); + TemplateAggregateConfigurations = AllAggregateConfigurations + .Where(ac => templateAggregateConfigurationIds.Contains(ac.ID)).ToArray(); - //add the orphans under the orphan folder - AddToDictionaries(new HashSet(OrphanAggregateConfigurations), - new DescendancyList(OrphanAggregateConfigurationsNode)); + //add the orphans under the orphan folder + AddToDictionaries(new HashSet(OrphanAggregateConfigurations), + new DescendancyList(OrphanAggregateConfigurationsNode)); - var dec = new DescendancyList(TemplateAggregateConfigurationsNode); - dec.SetBetterRouteExists(); - AddToDictionaries(new HashSet(TemplateAggregateConfigurations), dec); + var dec = new DescendancyList(TemplateAggregateConfigurationsNode); + dec.SetBetterRouteExists(); + AddToDictionaries(new HashSet(TemplateAggregateConfigurations), dec); - //Some AggregateConfigurations are 'Patient Index Tables', this happens when there is an existing JoinableCohortAggregateConfiguration declared where - //the AggregateConfiguration_ID is the AggregateConfiguration.ID. We can inject this knowledge now so to avoid database lookups later (e.g. at icon provision time) - var joinableDictionaryByAggregateConfigurationId = - AllJoinables.ToDictionaryEx(j => j.AggregateConfiguration_ID, v => v); + //Some AggregateConfigurations are 'Patient Index Tables', this happens when there is an existing JoinableCohortAggregateConfiguration declared where + //the AggregateConfiguration_ID is the AggregateConfiguration.ID. We can inject this knowledge now so to avoid database lookups later (e.g. at icon provision time) + var joinableDictionaryByAggregateConfigurationId = + AllJoinables.ToDictionaryEx(j => j.AggregateConfiguration_ID, v => v); - foreach (var ac in AllAggregateConfigurations) //if there's a joinable - ac.InjectKnown( //inject that we know the joinable (and what it is) - joinableDictionaryByAggregateConfigurationId.GetValueOrDefault(ac.ID)); //otherwise inject that it is not a joinable (suppresses database checking later) + foreach (var ac in AllAggregateConfigurations) //if there's a joinable + ac.InjectKnown( //inject that we know the joinable (and what it is) + joinableDictionaryByAggregateConfigurationId.GetValueOrDefault(ac.ID)); //otherwise inject that it is not a joinable (suppresses database checking later) - ReportProgress("After AggregateConfiguration injection"); + ReportProgress("After AggregateConfiguration injection"); - AllGovernanceNode = new AllGovernanceNode(); - AllGovernancePeriods = GetAllObjects(repository); - AllGovernanceDocuments = GetAllObjects(repository); - GovernanceCoverage = repository.GovernanceManager.GetAllGovernedCataloguesForAllGovernancePeriods(); + AllGovernanceNode = new AllGovernanceNode(); + AllGovernancePeriods = GetAllObjects(repository); + AllGovernanceDocuments = GetAllObjects(repository); + GovernanceCoverage = repository.GovernanceManager.GetAllGovernedCataloguesForAllGovernancePeriods(); - AddChildren(AllGovernanceNode); + AddChildren(AllGovernanceNode); - ReportProgress("After Governance"); + ReportProgress("After Governance"); - AllPluginsNode = new AllPluginsNode(); - AddChildren(AllPluginsNode); + AllPluginsNode = new AllPluginsNode(); + AddChildren(AllPluginsNode); - ReportProgress("After Plugins"); + ReportProgress("After Plugins"); - AllRegexRedactionConfigurations = GetAllObjects(repository); - AllRegexRedactionConfigurationsNode = new AllRegexRedactionConfigurationsNode(); - AddChildren(AllRegexRedactionConfigurationsNode); + AllRegexRedactionConfigurations = GetAllObjects(repository); + AllRegexRedactionConfigurationsNode = new AllRegexRedactionConfigurationsNode(); + AddChildren(AllRegexRedactionConfigurationsNode); - AllDatasets = GetAllObjects(repository); - AllDatasetsNode = new AllDatasetsNode(); - AddChildren(AllDatasetsNode); + AllDatasets = GetAllObjects(repository); + AllDatasetsNode = new AllDatasetsNode(); + AddChildren(AllDatasetsNode); - ReportProgress("After Configurations"); + ReportProgress("After Configurations"); - var searchables = new Dictionary>(); + var searchables = new Dictionary>(); - foreach (var o in _descendancyDictionary.Keys.OfType()) - { - if (!searchables.ContainsKey(o.ID)) - searchables.Add(o.ID, new HashSet()); + foreach (var o in _descendancyDictionary.Keys.OfType()) + { + if (!searchables.ContainsKey(o.ID)) + searchables.Add(o.ID, new HashSet()); - searchables[o.ID].Add(o); - } + searchables[o.ID].Add(o); + } - ReportProgress("After building Searchables"); + ReportProgress("After building Searchables"); - foreach (var e in AllExports) - { - if (!searchables.TryGetValue(e.ReferencedObjectID, out var searchable)) - continue; + foreach (var e in AllExports) + { + if (!searchables.TryGetValue(e.ReferencedObjectID, out var searchable)) + continue; - var known = searchable - .FirstOrDefault(s => e.ReferencedObjectType == s.GetType().FullName); + var known = searchable + .FirstOrDefault(s => e.ReferencedObjectType == s.GetType().FullName); - if (known != null) - e.InjectKnown(known); - } + if (known != null) + e.InjectKnown(known); + } - ReportProgress("After building exports"); - } + ReportProgress("After building exports"); + } - private void FetchCatalogueItems() - { - AllCatalogueItemsDictionary = - GetAllObjects(_catalogueRepository).ToDictionaryEx(i => i.ID, o => o); + private void FetchCatalogueItems() + { + AllCatalogueItemsDictionary = + GetAllObjects(_catalogueRepository).ToDictionaryEx(i => i.ID, o => o); - ReportProgress("After CatalogueItem getting"); + ReportProgress("After CatalogueItem getting"); - _catalogueToCatalogueItems = AllCatalogueItems.GroupBy(c => c.Catalogue_ID) - .ToDictionaryEx(gdc => gdc.Key, gdc => gdc.ToList()); - _allColumnInfos = AllColumnInfos.ToDictionaryEx(i => i.ID, o => o); + _catalogueToCatalogueItems = AllCatalogueItems.GroupBy(c => c.Catalogue_ID) + .ToDictionaryEx(gdc => gdc.Key, gdc => gdc.ToList()); + _allColumnInfos = AllColumnInfos.ToDictionaryEx(i => i.ID, o => o); - ReportProgress("After CatalogueItem Dictionary building"); - - //Inject known ColumnInfos into CatalogueItems - Parallel.ForEach(AllCatalogueItems, ci => - { - if (ci.ColumnInfo_ID != null && _allColumnInfos.TryGetValue(ci.ColumnInfo_ID.Value, out var col)) - ci.InjectKnown(col); - else - ci.InjectKnown((ColumnInfo)null); - }); - } - - private void FetchExtractionInformations() - { - AllExtractionInformationsDictionary = GetAllObjects(_catalogueRepository) - .ToDictionaryEx(i => i.ID, o => o); - _extractionInformationsByCatalogueItem = - AllExtractionInformationsDictionary.Values.ToDictionaryEx(k => k.CatalogueItem_ID, v => v); - - //Inject known CatalogueItems into ExtractionInformations - foreach (var ei in AllExtractionInformationsDictionary.Values) - if (AllCatalogueItemsDictionary.TryGetValue(ei.CatalogueItem_ID, out var ci)) - { - ei.InjectKnown(ci.ColumnInfo); - ei.InjectKnown(ci); - } - } - - private void BuildCohortCohortAggregateContainers() - { - AllCohortAggregateContainers = GetAllObjects(_catalogueRepository); - - - //if we have a database repository then we should get answers from the caching version CohortContainerManagerFromChildProvider otherwise - //just use the one that is configured on the repository. - - _cohortContainerManager = _catalogueRepository is CatalogueRepository cataRepo - ? new CohortContainerManagerFromChildProvider(cataRepo, this) - : _catalogueRepository.CohortContainerManager; - } - - private void BuildAggregateConfigurations() - { - AllJoinableCohortAggregateConfigurationUse = - GetAllObjects(_catalogueRepository); - AllAggregateConfigurations = GetAllObjects(_catalogueRepository); - - BuildAggregateDimensions(); - - //to start with all aggregates are orphans (we prune this as we determine descendency in AddChildren methods - OrphanAggregateConfigurations = - new HashSet( - AllAggregateConfigurations.Where(ac => ac.IsCohortIdentificationAggregate)); - - foreach (var configuration in AllAggregateConfigurations) - { - configuration.InjectKnown(AllCataloguesDictionary[configuration.Catalogue_ID]); - configuration.InjectKnown(AllAggregateDimensions.Where(d => d.AggregateConfiguration_ID == configuration.ID) - .ToArray()); - } - - foreach (var d in AllAggregateDimensions) - d.InjectKnown(AllExtractionInformationsDictionary[d.ExtractionInformation_ID]); - - ReportProgress("AggregateDimension injections"); - - BuildAggregateFilterContainers(); - } - - private void BuildAggregateDimensions() - { - AllAggregateDimensions = GetAllObjects(_catalogueRepository); - AllAggregateContinuousDateAxis = GetAllObjects(_catalogueRepository); - } - - private void BuildAggregateFilterContainers() - { - AllAggregateContainersDictionary = GetAllObjects(_catalogueRepository) - .ToDictionaryEx(o => o.ID, o2 => o2); - AllAggregateFilters = GetAllObjects(_catalogueRepository); - AllAggregateFilterParameters = GetAllObjects(_catalogueRepository); - - _aggregateFilterManager = _catalogueRepository is CatalogueRepository cataRepo - ? new FilterManagerFromChildProvider(cataRepo, this) - : _catalogueRepository.FilterManager; - } - - - protected void ReportProgress(string desc) - { - if (UserSettings.DebugPerformance) - { - _errorsCheckNotifier.OnCheckPerformed(new CheckEventArgs( - $"ChildProvider Stage {_progress++} ({desc}):{ProgressStopwatch.ElapsedMilliseconds}ms", - CheckResult.Success)); - ProgressStopwatch.Restart(); - } - } - - private void AddChildren(AllPluginsNode allPluginsNode) - { - var children = new HashSet(LoadModuleAssembly.Assemblies); - var descendancy = new DescendancyList(allPluginsNode); - AddToDictionaries(children, descendancy); - } - - private void AddChildren(AllRegexRedactionConfigurationsNode allRegexRedactionConfigurationsNode) - { - var children = new HashSet(AllRegexRedactionConfigurations); - var descendancy = new DescendancyList(allRegexRedactionConfigurationsNode); - AddToDictionaries(children, descendancy); - } - - private void AddChildren(AllDatasetsNode allDatasetsNode) - { - var children = new HashSet(AllDatasets); - var descendancy = new DescendancyList(allDatasetsNode); - AddToDictionaries(children, descendancy); - } - - private void AddChildren(AllGovernanceNode allGovernanceNode) - { - var children = new HashSet(); - var descendancy = new DescendancyList(allGovernanceNode); - - foreach (var gp in AllGovernancePeriods) - { - children.Add(gp); - AddChildren(gp, descendancy.Add(gp)); - } - - AddToDictionaries(children, descendancy); - } - - private void AddChildren(GovernancePeriod governancePeriod, DescendancyList descendancy) - { - var children = new HashSet(); - - foreach (var doc in AllGovernanceDocuments.Where(d => d.GovernancePeriod_ID == governancePeriod.ID)) - children.Add(doc); - - AddToDictionaries(children, descendancy); - } - - private void AddChildren(AllPermissionWindowsNode allPermissionWindowsNode) - { - var descendancy = new DescendancyList(allPermissionWindowsNode); - - foreach (var permissionWindow in AllPermissionWindows) - AddChildren(permissionWindow, descendancy.Add(permissionWindow)); - - - AddToDictionaries(new HashSet(AllPermissionWindows), descendancy); - } - - private void AddChildren(PermissionWindow permissionWindow, DescendancyList descendancy) - { - var children = new HashSet(); - - foreach (var cacheProgress in AllCacheProgresses) - if (cacheProgress.PermissionWindow_ID == permissionWindow.ID) - children.Add(new PermissionWindowUsedByCacheProgressNode(cacheProgress, permissionWindow, false)); + ReportProgress("After CatalogueItem Dictionary building"); + + //Inject known ColumnInfos into CatalogueItems + Parallel.ForEach(AllCatalogueItems, ci => + { + if (ci.ColumnInfo_ID != null && _allColumnInfos.TryGetValue(ci.ColumnInfo_ID.Value, out var col)) + ci.InjectKnown(col); + else + ci.InjectKnown((ColumnInfo)null); + }); + } + + private void FetchExtractionInformations() + { + AllExtractionInformationsDictionary = GetAllObjects(_catalogueRepository) + .ToDictionaryEx(i => i.ID, o => o); + _extractionInformationsByCatalogueItem = + AllExtractionInformationsDictionary.Values.ToDictionaryEx(k => k.CatalogueItem_ID, v => v); + + //Inject known CatalogueItems into ExtractionInformations + foreach (var ei in AllExtractionInformationsDictionary.Values) + if (AllCatalogueItemsDictionary.TryGetValue(ei.CatalogueItem_ID, out var ci)) + { + ei.InjectKnown(ci.ColumnInfo); + ei.InjectKnown(ci); + } + } + + private void BuildCohortCohortAggregateContainers() + { + AllCohortAggregateContainers = GetAllObjects(_catalogueRepository); + + + //if we have a database repository then we should get answers from the caching version CohortContainerManagerFromChildProvider otherwise + //just use the one that is configured on the repository. + + _cohortContainerManager = _catalogueRepository is CatalogueRepository cataRepo + ? new CohortContainerManagerFromChildProvider(cataRepo, this) + : _catalogueRepository.CohortContainerManager; + } + + private void BuildAggregateConfigurations() + { + AllJoinableCohortAggregateConfigurationUse = + GetAllObjects(_catalogueRepository); + AllAggregateConfigurations = GetAllObjects(_catalogueRepository); + + BuildAggregateDimensions(); + + //to start with all aggregates are orphans (we prune this as we determine descendency in AddChildren methods + OrphanAggregateConfigurations = + new HashSet( + AllAggregateConfigurations.Where(ac => ac.IsCohortIdentificationAggregate)); + + foreach (var configuration in AllAggregateConfigurations) + { + configuration.InjectKnown(AllCataloguesDictionary[configuration.Catalogue_ID]); + configuration.InjectKnown(AllAggregateDimensions.Where(d => d.AggregateConfiguration_ID == configuration.ID) + .ToArray()); + } + + foreach (var d in AllAggregateDimensions) + d.InjectKnown(AllExtractionInformationsDictionary[d.ExtractionInformation_ID]); + + ReportProgress("AggregateDimension injections"); + + BuildAggregateFilterContainers(); + } + + private void BuildAggregateDimensions() + { + AllAggregateDimensions = GetAllObjects(_catalogueRepository); + AllAggregateContinuousDateAxis = GetAllObjects(_catalogueRepository); + } + + private void BuildAggregateFilterContainers() + { + AllAggregateContainersDictionary = GetAllObjects(_catalogueRepository) + .ToDictionaryEx(o => o.ID, o2 => o2); + AllAggregateFilters = GetAllObjects(_catalogueRepository); + AllAggregateFilterParameters = GetAllObjects(_catalogueRepository); + + _aggregateFilterManager = _catalogueRepository is CatalogueRepository cataRepo + ? new FilterManagerFromChildProvider(cataRepo, this) + : _catalogueRepository.FilterManager; + } + + + protected void ReportProgress(string desc) + { + if (UserSettings.DebugPerformance) + { + _errorsCheckNotifier.OnCheckPerformed(new CheckEventArgs( + $"ChildProvider Stage {_progress++} ({desc}):{ProgressStopwatch.ElapsedMilliseconds}ms", + CheckResult.Success)); + ProgressStopwatch.Restart(); + } + } + + private void AddChildren(AllPluginsNode allPluginsNode) + { + var children = new HashSet(LoadModuleAssembly.Assemblies); + var descendancy = new DescendancyList(allPluginsNode); + AddToDictionaries(children, descendancy); + } + + private void AddChildren(AllRegexRedactionConfigurationsNode allRegexRedactionConfigurationsNode) + { + var children = new HashSet(AllRegexRedactionConfigurations); + var descendancy = new DescendancyList(allRegexRedactionConfigurationsNode); + AddToDictionaries(children, descendancy); + } + + private void AddChildren(AllDatasetsNode allDatasetsNode) + { + var children = new HashSet(AllDatasets); + var descendancy = new DescendancyList(allDatasetsNode); + AddToDictionaries(children, descendancy); + } + + private void AddChildren(AllGovernanceNode allGovernanceNode) + { + var children = new HashSet(); + var descendancy = new DescendancyList(allGovernanceNode); + + foreach (var gp in AllGovernancePeriods) + { + children.Add(gp); + AddChildren(gp, descendancy.Add(gp)); + } + + AddToDictionaries(children, descendancy); + } + + private void AddChildren(GovernancePeriod governancePeriod, DescendancyList descendancy) + { + var children = new HashSet(); + + foreach (var doc in AllGovernanceDocuments.Where(d => d.GovernancePeriod_ID == governancePeriod.ID)) + children.Add(doc); + + AddToDictionaries(children, descendancy); + } + + private void AddChildren(AllPermissionWindowsNode allPermissionWindowsNode) + { + var descendancy = new DescendancyList(allPermissionWindowsNode); + + foreach (var permissionWindow in AllPermissionWindows) + AddChildren(permissionWindow, descendancy.Add(permissionWindow)); + + + AddToDictionaries(new HashSet(AllPermissionWindows), descendancy); + } + + private void AddChildren(PermissionWindow permissionWindow, DescendancyList descendancy) + { + var children = new HashSet(); + + foreach (var cacheProgress in AllCacheProgresses) + if (cacheProgress.PermissionWindow_ID == permissionWindow.ID) + children.Add(new PermissionWindowUsedByCacheProgressNode(cacheProgress, permissionWindow, false)); - AddToDictionaries(children, descendancy); - } + AddToDictionaries(children, descendancy); + } - private void AddChildren(AllExternalServersNode allExternalServersNode) - { - AddToDictionaries(new HashSet(AllExternalServers), new DescendancyList(allExternalServersNode)); - } - - private void AddChildren(AllRDMPRemotesNode allRDMPRemotesNode) - { - AddToDictionaries(new HashSet(AllRemoteRDMPs), new DescendancyList(allRDMPRemotesNode)); - } + private void AddChildren(AllExternalServersNode allExternalServersNode) + { + AddToDictionaries(new HashSet(AllExternalServers), new DescendancyList(allExternalServersNode)); + } + + private void AddChildren(AllRDMPRemotesNode allRDMPRemotesNode) + { + AddToDictionaries(new HashSet(AllRemoteRDMPs), new DescendancyList(allRDMPRemotesNode)); + } - private void AddChildren(AllDashboardsNode allDashboardsNode) - { - AddToDictionaries(new HashSet(AllDashboards), new DescendancyList(allDashboardsNode)); - } + private void AddChildren(AllDashboardsNode allDashboardsNode) + { + AddToDictionaries(new HashSet(AllDashboards), new DescendancyList(allDashboardsNode)); + } - private void AddChildren(AllObjectSharingNode allObjectSharingNode) - { - var descendancy = new DescendancyList(allObjectSharingNode); + private void AddChildren(AllObjectSharingNode allObjectSharingNode) + { + var descendancy = new DescendancyList(allObjectSharingNode); - var allExportsNode = new AllObjectExportsNode(); - var allImportsNode = new AllObjectImportsNode(); + var allExportsNode = new AllObjectExportsNode(); + var allImportsNode = new AllObjectImportsNode(); - AddToDictionaries(new HashSet(AllExports), descendancy.Add(allExportsNode)); - AddToDictionaries(new HashSet(AllImports), descendancy.Add(allImportsNode)); + AddToDictionaries(new HashSet(AllExports), descendancy.Add(allExportsNode)); + AddToDictionaries(new HashSet(AllImports), descendancy.Add(allImportsNode)); - AddToDictionaries(new HashSet(new object[] { allExportsNode, allImportsNode }), descendancy); - } + AddToDictionaries(new HashSet(new object[] { allExportsNode, allImportsNode }), descendancy); + } - /// - /// Creates new s and fills it with all compatible Pipelines - do not call this method more than once - /// - protected void AddPipelineUseCases(Dictionary useCases) - { - var descendancy = new DescendancyList(AllPipelinesNode); - var children = new HashSet(); + /// + /// Creates new s and fills it with all compatible Pipelines - do not call this method more than once + /// + protected void AddPipelineUseCases(Dictionary useCases) + { + var descendancy = new DescendancyList(AllPipelinesNode); + var children = new HashSet(); - //pipelines not found to be part of any StandardPipelineUseCase - var unknownPipelines = new HashSet(AllPipelines); + //pipelines not found to be part of any StandardPipelineUseCase + var unknownPipelines = new HashSet(AllPipelines); - foreach (var useCase in useCases) - { - var node = new StandardPipelineUseCaseNode(useCase.Key, useCase.Value, _commentStore); + foreach (var useCase in useCases) + { + var node = new StandardPipelineUseCaseNode(useCase.Key, useCase.Value, _commentStore); - //keep track of all the use cases - PipelineUseCases.Add(node); + //keep track of all the use cases + PipelineUseCases.Add(node); - foreach (var pipeline in AddChildren(node, descendancy.Add(node))) - unknownPipelines.Remove(pipeline); + foreach (var pipeline in AddChildren(node, descendancy.Add(node))) + unknownPipelines.Remove(pipeline); - children.Add(node); - } + children.Add(node); + } - children.Add(OtherPipelinesNode); - OtherPipelinesNode.Pipelines.AddRange(unknownPipelines.Cast()); - AddToDictionaries(unknownPipelines, descendancy.Add(OtherPipelinesNode)); + children.Add(OtherPipelinesNode); + OtherPipelinesNode.Pipelines.AddRange(unknownPipelines.Cast()); + AddToDictionaries(unknownPipelines, descendancy.Add(OtherPipelinesNode)); - //it is the first standard use case - AddToDictionaries(children, descendancy); - } + //it is the first standard use case + AddToDictionaries(children, descendancy); + } - private IEnumerable AddChildren(StandardPipelineUseCaseNode node, DescendancyList descendancy) - { - var children = new HashSet(); + private IEnumerable AddChildren(StandardPipelineUseCaseNode node, DescendancyList descendancy) + { + var children = new HashSet(); - var repo = new MemoryRepository(); + var repo = new MemoryRepository(); - //Could be an issue here if a pipeline becomes compatible with multiple use cases. - //Should be impossible currently but one day it could be an issue especially if we were to - //support plugin use cases in this hierarchy + //Could be an issue here if a pipeline becomes compatible with multiple use cases. + //Should be impossible currently but one day it could be an issue especially if we were to + //support plugin use cases in this hierarchy - //find compatible pipelines useCase.Value - foreach (var compatiblePipeline in AllPipelines.Where(node.UseCase.GetContext().IsAllowable)) - { - var useCaseNode = new PipelineCompatibleWithUseCaseNode(repo, compatiblePipeline, node.UseCase); + //find compatible pipelines useCase.Value + foreach (var compatiblePipeline in AllPipelines.Where(node.UseCase.GetContext().IsAllowable)) + { + var useCaseNode = new PipelineCompatibleWithUseCaseNode(repo, compatiblePipeline, node.UseCase); - AddChildren(useCaseNode, descendancy.Add(useCaseNode)); + AddChildren(useCaseNode, descendancy.Add(useCaseNode)); - node.Pipelines.Add(compatiblePipeline); - children.Add(useCaseNode); - } + node.Pipelines.Add(compatiblePipeline); + children.Add(useCaseNode); + } - //it is the first standard use case - AddToDictionaries(children, descendancy); + //it is the first standard use case + AddToDictionaries(children, descendancy); - return children.Cast().Select(u => u.Pipeline); - } + return children.Cast().Select(u => u.Pipeline); + } - private void AddChildren(PipelineCompatibleWithUseCaseNode pipelineNode, DescendancyList descendancy) - { - var components = AllPipelineComponents.Where(c => c.Pipeline_ID == pipelineNode.Pipeline.ID) - .OrderBy(o => o.Order) - .ToArray(); + private void AddChildren(PipelineCompatibleWithUseCaseNode pipelineNode, DescendancyList descendancy) + { + var components = AllPipelineComponents.Where(c => c.Pipeline_ID == pipelineNode.Pipeline.ID) + .OrderBy(o => o.Order) + .ToArray(); - foreach (var component in components) - AddChildren(component, descendancy.Add(component)); + foreach (var component in components) + AddChildren(component, descendancy.Add(component)); - var children = new HashSet(components); + var children = new HashSet(components); - AddToDictionaries(children, descendancy); - } + AddToDictionaries(children, descendancy); + } - private void AddChildren(PipelineComponent pipelineComponent, DescendancyList descendancy) - { - var components = AllPipelineComponentsArguments.Where(c => c.PipelineComponent_ID == pipelineComponent.ID) - .ToArray(); + private void AddChildren(PipelineComponent pipelineComponent, DescendancyList descendancy) + { + var components = AllPipelineComponentsArguments.Where(c => c.PipelineComponent_ID == pipelineComponent.ID) + .ToArray(); - var children = new HashSet(components); + var children = new HashSet(components); - AddToDictionaries(children, descendancy); - } + AddToDictionaries(children, descendancy); + } - private void BuildServerNodes() - { - //add a root node for all the servers to be children of - AllServersNode = new AllServersNode(); + private void BuildServerNodes() + { + //add a root node for all the servers to be children of + AllServersNode = new AllServersNode(); - var descendancy = new DescendancyList(AllServersNode); - var allServers = new List(); + var descendancy = new DescendancyList(AllServersNode); + var allServers = new List(); - foreach (var typeGroup in AllTableInfos.GroupBy(t => t.DatabaseType)) - { - var dbType = typeGroup.Key; - IEnumerable tables = typeGroup; + foreach (var typeGroup in AllTableInfos.GroupBy(t => t.DatabaseType)) + { + var dbType = typeGroup.Key; + IEnumerable tables = typeGroup; - var serversByName = tables - .GroupBy(c => c.Server ?? TableInfoServerNode.NullServerNode, StringComparer.CurrentCultureIgnoreCase) - .Select(s => new TableInfoServerNode(s.Key, dbType, s)); + var serversByName = tables + .GroupBy(c => c.Server ?? TableInfoServerNode.NullServerNode, StringComparer.CurrentCultureIgnoreCase) + .Select(s => new TableInfoServerNode(s.Key, dbType, s)); - foreach (var server in serversByName) - { - allServers.Add(server); - AddChildren(server, descendancy.Add(server)); - } - } + foreach (var server in serversByName) + { + allServers.Add(server); + AddChildren(server, descendancy.Add(server)); + } + } - //create the server nodes - AllServers = allServers.ToArray(); + //create the server nodes + AllServers = allServers.ToArray(); - //record the fact that all the servers are children of the all servers node - AddToDictionaries(new HashSet(AllServers), descendancy); - } + //record the fact that all the servers are children of the all servers node + AddToDictionaries(new HashSet(AllServers), descendancy); + } - private void AddChildren(AllDataAccessCredentialsNode allDataAccessCredentialsNode) - { - var children = new HashSet(); + private void AddChildren(AllDataAccessCredentialsNode allDataAccessCredentialsNode) + { + var children = new HashSet(); - var isKeyMissing = false; - if (_catalogueRepository.EncryptionManager is PasswordEncryptionKeyLocation keyLocation) - isKeyMissing = string.IsNullOrWhiteSpace(keyLocation.GetKeyFileLocation()); + var isKeyMissing = false; + if (_catalogueRepository.EncryptionManager is PasswordEncryptionKeyLocation keyLocation) + isKeyMissing = string.IsNullOrWhiteSpace(keyLocation.GetKeyFileLocation()); - children.Add(new DecryptionPrivateKeyNode(isKeyMissing)); - - foreach (var creds in AllDataAccessCredentials) - children.Add(creds); - - - AddToDictionaries(children, new DescendancyList(allDataAccessCredentialsNode)); - } - - private void AddChildren(AllANOTablesNode anoTablesNode) - { - AddToDictionaries(new HashSet(AllANOTables), new DescendancyList(anoTablesNode)); - } - - private void AddChildren(FolderNode folder, DescendancyList descendancy) - { - foreach (var child in folder.ChildFolders) - //add subfolder children - AddChildren(child, descendancy.Add(child)); - - //add catalogues in folder - foreach (var c in folder.ChildObjects) AddChildren(c, descendancy.Add(c)); - - // Children are the folders + objects - AddToDictionaries(new HashSet( - folder.ChildFolders.Cast() - .Union(folder.ChildObjects)), descendancy - ); - } - - private void AddChildren(FolderNode folder, DescendancyList descendancy) - { - foreach (var child in folder.ChildFolders) - //add subfolder children - AddChildren(child, descendancy.Add(child)); - - //add loads in folder - foreach (var lmd in folder.ChildObjects.Where(lmd => lmd.RootLoadMetadata_ID == null).ToArray()) AddChildren(lmd, descendancy.Add(lmd)); - // Children are the folders + objects - AddToDictionaries(new HashSet( - folder.ChildFolders.Cast() - .Union(folder.ChildObjects)), descendancy - ); - } - - private void AddChildren(FolderNode folder, DescendancyList descendancy) - { - foreach (var child in folder.ChildFolders) - //add subfolder children - AddChildren(child, descendancy.Add(child)); - - //add loads in folder - foreach (var ds in folder.ChildObjects) AddChildren(ds, descendancy.Add(ds)); - - // Children are the folders + objects - AddToDictionaries(new HashSet( - folder.ChildFolders.Cast() - .Union(folder.ChildObjects)), descendancy - ); - } - - private void AddChildren(FolderNode folder, DescendancyList descendancy) - { - foreach (var child in folder.ChildFolders) - //add subfolder children - AddChildren(child, descendancy.Add(child)); - - - //add cics in folder - foreach (var cic in folder.ChildObjects) AddChildren(cic, descendancy.Add(cic)); - - // Children are the folders + objects - AddToDictionaries(new HashSet( - folder.ChildFolders.Cast() - .Union(folder.ChildObjects)), descendancy - ); - } - - private void AddChildren(Curation.Data.Dataset lmd, DescendancyList descendancy) - { - var childObjects = new List(); - AddToDictionaries(new HashSet(childObjects), descendancy); - } - - - #region Load Metadata - - private void AddChildren(LoadMetadata lmd, DescendancyList descendancy, bool includeSchedule = true, bool includeVersions = true) - { - var childObjects = new List(); - - if (lmd.OverrideRAWServer_ID.HasValue) - { - var server = AllExternalServers.Single(s => s.ID == lmd.OverrideRAWServer_ID.Value); - var usage = new OverrideRawServerNode(lmd, server); - childObjects.Add(usage); - } - if (includeSchedule) - { - var allSchedulesNode = new LoadMetadataScheduleNode(lmd); - AddChildren(allSchedulesNode, descendancy.Add(allSchedulesNode)); - childObjects.Add(allSchedulesNode); - } - - var allCataloguesNode = new AllCataloguesUsedByLoadMetadataNode(lmd); - AddChildren(allCataloguesNode, descendancy.Add(allCataloguesNode)); - childObjects.Add(allCataloguesNode); - - var processTasksNode = new AllProcessTasksUsedByLoadMetadataNode(lmd); - AddChildren(processTasksNode, descendancy.Add(processTasksNode)); - childObjects.Add(processTasksNode); - - if (includeVersions) - { - var versionsNode = new LoadMetadataVersionNode(lmd); - AddChildren(versionsNode, descendancy.Add(versionsNode)); - childObjects.Add(versionsNode); - } - - childObjects.Add(new LoadDirectoryNode(lmd)); - - AddToDictionaries(new HashSet(childObjects), descendancy); - } - - private void AddChildren(LoadMetadataScheduleNode allSchedulesNode, DescendancyList descendancy) - { - var childObjects = new HashSet(); - - var lmd = allSchedulesNode.LoadMetadata; - - foreach (var lp in AllLoadProgresses.Where(p => p.LoadMetadata_ID == lmd.ID)) - { - AddChildren(lp, descendancy.Add(lp)); - childObjects.Add(lp); - } - - if (childObjects.Any()) - AddToDictionaries(childObjects, descendancy); - } - - private void AddChildren(LoadProgress loadProgress, DescendancyList descendancy) - { - var cacheProgresses = AllCacheProgresses.Where(cp => cp.LoadProgress_ID == loadProgress.ID).ToArray(); - - foreach (var cacheProgress in cacheProgresses) - AddChildren(cacheProgress, descendancy.Add(cacheProgress)); - - if (cacheProgresses.Any()) - AddToDictionaries(new HashSet(cacheProgresses), descendancy); - } - - private void AddChildren(CacheProgress cacheProgress, DescendancyList descendancy) - { - var children = new HashSet(); - - if (cacheProgress.PermissionWindow_ID != null) - { - var window = AllPermissionWindows.Single(w => w.ID == cacheProgress.PermissionWindow_ID); - var windowNode = new PermissionWindowUsedByCacheProgressNode(cacheProgress, window, true); - - children.Add(windowNode); - } - - if (children.Any()) - AddToDictionaries(children, descendancy); - } - - private void AddChildren(AllProcessTasksUsedByLoadMetadataNode allProcessTasksUsedByLoadMetadataNode, - DescendancyList descendancy) - { - var childObjects = new HashSet(); - - var lmd = allProcessTasksUsedByLoadMetadataNode.LoadMetadata; - childObjects.Add(new LoadStageNode(lmd, LoadStage.GetFiles)); - childObjects.Add(new LoadStageNode(lmd, LoadStage.Mounting)); - childObjects.Add(new LoadStageNode(lmd, LoadStage.AdjustRaw)); - childObjects.Add(new LoadStageNode(lmd, LoadStage.AdjustStaging)); - childObjects.Add(new LoadStageNode(lmd, LoadStage.PostLoad)); - - foreach (LoadStageNode node in childObjects) - AddChildren(node, descendancy.Add(node)); - - AddToDictionaries(childObjects, descendancy); - } - - private void AddChildren(LoadStageNode loadStageNode, DescendancyList descendancy) - { - var tasks = AllProcessTasks.Where( - p => p.LoadMetadata_ID == loadStageNode.LoadMetadata.ID && p.LoadStage == loadStageNode.LoadStage) - .OrderBy(o => o.Order).ToArray(); - - foreach (var processTask in tasks) - AddChildren(processTask, descendancy.Add(processTask)); - - if (tasks.Any()) - AddToDictionaries(new HashSet(tasks), descendancy); - } - - private void AddChildren(ProcessTask procesTask, DescendancyList descendancy) - { - var args = AllProcessTasksArguments.Where( - a => a.ProcessTask_ID == procesTask.ID).ToArray(); - - if (args.Any()) - AddToDictionaries(new HashSet(args), descendancy); - } - - private void AddChildren(LoadMetadataVersionNode LoadMetadataVersionNode, DescendancyList descendancy) - { - LoadMetadataVersionNode.LoadMetadataVersions = AllLoadMetadatas.Where(lmd => lmd.RootLoadMetadata_ID == LoadMetadataVersionNode.LoadMetadata.ID).ToList(); - //AddToDictionaries(LoadMetadataVersionNode.LoadMetadataVersions.Cast().ToHashSet(), descendancy); - var childObjects = new List(); - - foreach (var lmd in LoadMetadataVersionNode.LoadMetadataVersions) - { - AddChildren(lmd, descendancy.Add(lmd), false, false); - childObjects.Add(lmd); - } - AddToDictionaries(new HashSet(childObjects), descendancy); - - } - - private void AddChildren(AllCataloguesUsedByLoadMetadataNode allCataloguesUsedByLoadMetadataNode, - DescendancyList descendancy) - { - var loadMetadataId = allCataloguesUsedByLoadMetadataNode.LoadMetadata.ID; - var linkedCatalogueIDs = AllLoadMetadataLinkage.Where(link => link.LoadMetadataID == loadMetadataId).Select(static link => link.CatalogueID); - var usedCatalogues = linkedCatalogueIDs.Select(catalogueId => AllCatalogues.FirstOrDefault(c => c.ID == catalogueId)).Where(static foundCatalogue => foundCatalogue is not null).ToList(); - allCataloguesUsedByLoadMetadataNode.UsedCatalogues = usedCatalogues; - var childObjects = usedCatalogues.Select(foundCatalogue => new CatalogueUsedByLoadMetadataNode(allCataloguesUsedByLoadMetadataNode.LoadMetadata, foundCatalogue)).Cast().ToHashSet(); - - AddToDictionaries(childObjects, descendancy); - } - - #endregion - - protected void AddChildren(Catalogue c, DescendancyList descendancy) - { - var childObjects = new List(); - - var catalogueAggregates = AllAggregateConfigurations.Where(a => a.Catalogue_ID == c.ID).ToArray(); - var cohortAggregates = catalogueAggregates.Where(a => a.IsCohortIdentificationAggregate).ToArray(); - var regularAggregates = catalogueAggregates.Except(cohortAggregates).ToArray(); - - //get all the CatalogueItems for this Catalogue (TryGet because Catalogue may not have any items - var cis = _catalogueToCatalogueItems.TryGetValue(c.ID, out var result) - ? result.ToArray() - : Array.Empty(); - - //tell the CatalogueItems that we are are their parent - foreach (var ci in cis) - ci.InjectKnown(c); - - // core includes project specific which basically means the same thing - var core = new CatalogueItemsNode(c, - cis.Where(ci => ci.ExtractionInformation?.ExtractionCategory == ExtractionCategory.Core || - ci.ExtractionInformation?.ExtractionCategory == ExtractionCategory.ProjectSpecific) - , ExtractionCategory.Core); - - c.InjectKnown(cis); - - var deprecated = new CatalogueItemsNode(c, - cis.Where(ci => ci.ExtractionInformation?.ExtractionCategory == ExtractionCategory.Deprecated), - ExtractionCategory.Deprecated); - var special = new CatalogueItemsNode(c, - cis.Where(ci => ci.ExtractionInformation?.ExtractionCategory == ExtractionCategory.SpecialApprovalRequired), - ExtractionCategory.SpecialApprovalRequired); - var intern = new CatalogueItemsNode(c, - cis.Where(ci => ci.ExtractionInformation?.ExtractionCategory == ExtractionCategory.Internal), - ExtractionCategory.Internal); - var supplemental = new CatalogueItemsNode(c, - cis.Where(ci => ci.ExtractionInformation?.ExtractionCategory == ExtractionCategory.Supplemental), - ExtractionCategory.Supplemental); - var notExtractable = new CatalogueItemsNode(c, cis.Where(ci => ci.ExtractionInformation == null), null); - - AddChildren(core, descendancy.Add(core)); - childObjects.Add(core); - - foreach (var optional in new[] { deprecated, special, intern, supplemental, notExtractable }) - if (optional.CatalogueItems.Any()) - { - AddChildren(optional, descendancy.Add(optional)); - childObjects.Add(optional); - } - - //do we have any foreign key fields into this lookup table - var lookups = AllLookups.Where(l => c.CatalogueItems.Any(ci => ci.ColumnInfo_ID == l.ForeignKey_ID)).ToArray(); - - var docs = AllSupportingDocuments.Where(d => d.Catalogue_ID == c.ID).ToArray(); - var sql = AllSupportingSQL.Where(d => d.Catalogue_ID == c.ID).ToArray(); - - //if there are supporting documents or supporting sql files then add documentation node - if (docs.Any() || sql.Any()) - { - var documentationNode = new DocumentationNode(c, docs, sql); - - //add the documentations node - childObjects.Add(documentationNode); - - //record the children - AddToDictionaries(new HashSet(docs.Cast().Union(sql)), descendancy.Add(documentationNode)); - } - - if (lookups.Any()) - { - var lookupsNode = new CatalogueLookupsNode(c, lookups); - //add the documentations node - childObjects.Add(lookupsNode); - - - //record the children - AddToDictionaries(new HashSet(lookups.Select(l => new CatalogueLookupUsageNode(c, l))), - descendancy.Add(lookupsNode)); - } - - if (regularAggregates.Any()) - { - var aggregatesNode = new AggregatesNode(c, regularAggregates); - childObjects.Add(aggregatesNode); - - var nodeDescendancy = descendancy.Add(aggregatesNode); - AddToDictionaries(new HashSet(regularAggregates), nodeDescendancy); - - foreach (var regularAggregate in regularAggregates) - AddChildren(regularAggregate, nodeDescendancy.Add(regularAggregate)); - } - - //finalise - AddToDictionaries(new HashSet(childObjects), descendancy); - } - - private void InjectCatalogueItems() - { - foreach (var ci in AllCatalogueItems) - if (_extractionInformationsByCatalogueItem.TryGetValue(ci.ID, out var ei)) - ci.InjectKnown(ei); - else - ci.InjectKnown((ExtractionInformation)null); - } - - private void AddChildren(CatalogueItemsNode node, DescendancyList descendancyList) - { - AddToDictionaries(new HashSet(node.CatalogueItems), descendancyList); - - foreach (var ci in node.CatalogueItems) - AddChildren(ci, descendancyList.Add(ci)); - } - - private void AddChildren(AggregateConfiguration aggregateConfiguration, DescendancyList descendancy) - { - var childrenObjects = new HashSet(); - - var parameters = AllAnyTableParameters.Where(p => p.IsReferenceTo(aggregateConfiguration)).Cast() - .ToArray(); - - foreach (var p in parameters) - childrenObjects.Add(p); - - // show the dimensions in the tree - foreach (var dim in aggregateConfiguration.AggregateDimensions) childrenObjects.Add(dim); - - // show the axis (if any) in the tree. If there are multiple axis in this tree then that is bad but maybe the user can delete one of them to fix the situation - foreach (var axis in AllAggregateContinuousDateAxis.Where(a => - aggregateConfiguration.AggregateDimensions.Any(d => d.ID == a.AggregateDimension_ID))) - childrenObjects.Add(axis); - - //we can step into this twice, once via Catalogue children and once via CohortIdentificationConfiguration children - //if we get in via Catalogue children then descendancy will be Ignore=true we don't end up emphasising into CatalogueCollectionUI when - //really user wants to see it in CohortIdentificationCollectionUI - if (aggregateConfiguration.RootFilterContainer_ID != null) - { - var container = AllAggregateContainersDictionary[(int)aggregateConfiguration.RootFilterContainer_ID]; - - AddChildren(container, descendancy.Add(container)); - childrenObjects.Add(container); - } - - AddToDictionaries(childrenObjects, descendancy); - } - - private void AddChildren(AggregateFilterContainer container, DescendancyList descendancy) - { - var childrenObjects = new List(); - - var subcontainers = _aggregateFilterManager.GetSubContainers(container); - var filters = _aggregateFilterManager.GetFilters(container); - - foreach (AggregateFilterContainer subcontainer in subcontainers) - { - //one of our children is this subcontainer - childrenObjects.Add(subcontainer); - - //but also document its children - AddChildren(subcontainer, descendancy.Add(subcontainer)); - } - - //also add the filters for the container - foreach (var f in filters) - { - // for filters add the parameters under them - AddChildren((AggregateFilter)f, descendancy.Add(f)); - childrenObjects.Add(f); - } - - //add our children to the dictionary - AddToDictionaries(new HashSet(childrenObjects), descendancy); - } - - private void AddChildren(AggregateFilter f, DescendancyList descendancy) - { - AddToDictionaries(new HashSet(AllAggregateFilterParameters.Where(p => p.AggregateFilter_ID == f.ID)), - descendancy); - } - - private void AddChildren(CatalogueItem ci, DescendancyList descendancy) - { - var childObjects = new List(); - - var ei = ci.ExtractionInformation; - if (ei != null) - { - childObjects.Add(ei); - AddChildren(ei, descendancy.Add(ei)); - } - else - { - ci.InjectKnown( - (ExtractionInformation)null); // we know the CatalogueItem has no ExtractionInformation child because it's not in the dictionary - } - - if (ci.ColumnInfo_ID.HasValue && _allColumnInfos.TryGetValue(ci.ColumnInfo_ID.Value, out var col)) - childObjects.Add(new LinkedColumnInfoNode(ci, col)); - - AddToDictionaries(new HashSet(childObjects), descendancy); - } - - private void AddChildren(ExtractionInformation extractionInformation, DescendancyList descendancy) - { - var children = new HashSet(); - - foreach (var filter in AllCatalogueFilters.Where(f => f.ExtractionInformation_ID == extractionInformation.ID)) - { - //add the filter as a child of the - children.Add(filter); - AddChildren(filter, descendancy.Add(filter)); - } - - AddToDictionaries(children, descendancy); - } - - private void AddChildren(ExtractionFilter filter, DescendancyList descendancy) - { - var children = new HashSet(); - var parameters = AllCatalogueParameters.Where(p => p.ExtractionFilter_ID == filter.ID).ToArray(); - var parameterSets = AllCatalogueValueSets.Where(vs => vs.ExtractionFilter_ID == filter.ID).ToArray(); - - filter.InjectKnown(parameterSets); - - foreach (var p in parameters) - children.Add(p); - - foreach (var set in parameterSets) - { - children.Add(set); - AddChildren(set, descendancy.Add(set), parameters); - } - - if (children.Any()) - AddToDictionaries(children, descendancy); - } - - private void AddChildren(ExtractionFilterParameterSet set, DescendancyList descendancy, - ExtractionFilterParameter[] filterParameters) - { - var children = new HashSet(); - - foreach (var setValue in AllCatalogueValueSetValues.Where(v => v.ExtractionFilterParameterSet_ID == set.ID)) - { - setValue.InjectKnown(filterParameters.SingleOrDefault(p => p.ID == setValue.ExtractionFilterParameter_ID)); - children.Add(setValue); - } - - AddToDictionaries(children, descendancy); - } - - private void AddChildren(CohortIdentificationConfiguration cic, DescendancyList descendancy) - { - var children = new HashSet(); - - //it has an associated query cache - if (cic.QueryCachingServer_ID != null) - children.Add(new QueryCacheUsedByCohortIdentificationNode(cic, - AllExternalServers.Single(s => s.ID == cic.QueryCachingServer_ID))); - - var parameters = AllAnyTableParameters.Where(p => p.IsReferenceTo(cic)).Cast().ToArray(); - foreach (var p in parameters) children.Add(p); - - //if it has a root container - if (cic.RootCohortAggregateContainer_ID != null) - { - var container = AllCohortAggregateContainers.Single(c => c.ID == cic.RootCohortAggregateContainer_ID); - AddChildren(container, descendancy.Add(container).SetBetterRouteExists()); - children.Add(container); - } - - //get the patient index tables - var joinableNode = new JoinableCollectionNode(cic, - AllJoinables.Where(j => j.CohortIdentificationConfiguration_ID == cic.ID).ToArray()); - AddChildren(joinableNode, descendancy.Add(joinableNode).SetBetterRouteExists()); - children.Add(joinableNode); - - AddToDictionaries(children, descendancy.SetBetterRouteExists()); - } - - private void AddChildren(JoinableCollectionNode joinablesNode, DescendancyList descendancy) - { - var children = new HashSet(); - - foreach (var joinable in joinablesNode.Joinables) - try - { - var agg = AllAggregateConfigurations.Single(ac => ac.ID == joinable.AggregateConfiguration_ID); - ForceAggregateNaming(agg, descendancy); - children.Add(agg); - - //it's no longer an orphan because it's in a known cic (as a patient index table) - OrphanAggregateConfigurations.Remove(agg); - - AddChildren(agg, descendancy.Add(agg)); - } - catch (Exception e) - { - throw new Exception( - $"JoinableCohortAggregateConfiguration (patient index table) object (ID={joinable.ID}) references AggregateConfiguration_ID {joinable.AggregateConfiguration_ID} but that AggregateConfiguration was not found", - e); - } - - AddToDictionaries(children, descendancy); - } - - private void AddChildren(CohortAggregateContainer container, DescendancyList descendancy) - { - //get subcontainers - var subcontainers = _cohortContainerManager.GetChildren(container).OfType().ToList(); - - //if there are subcontainers - foreach (var subcontainer in subcontainers) - AddChildren(subcontainer, descendancy.Add(subcontainer)); - - //get our configurations - var configurations = _cohortContainerManager.GetChildren(container).OfType().ToList(); - - //record the configurations children including full descendancy - foreach (var configuration in configurations) - { - ForceAggregateNaming(configuration, descendancy); - AddChildren(configuration, descendancy.Add(configuration)); - - //it's no longer an orphan because it's in a known cic - OrphanAggregateConfigurations.Remove(configuration); - } - - //all our children (containers and aggregates) - //children are all aggregates and containers at the current hierarchy level in order - var children = subcontainers.Union(configurations.Cast()).OrderBy(o => o.Order).ToList(); - - AddToDictionaries(new HashSet(children), descendancy); - } - - private void ForceAggregateNaming(AggregateConfiguration configuration, DescendancyList descendancy) - { - //configuration has the wrong name - if (!configuration.IsCohortIdentificationAggregate) - { - _errorsCheckNotifier.OnCheckPerformed(new CheckEventArgs( - $"Had to fix naming of configuration '{configuration}' because it didn't start with correct cic prefix", - CheckResult.Warning)); - descendancy.Parents.OfType().Single() - .EnsureNamingConvention(configuration); - configuration.SaveToDatabase(); - } - } - - private void AddChildren(TableInfoServerNode serverNode, DescendancyList descendancy) - { - //add empty hashset - var children = new HashSet(); - - var databases = - serverNode.Tables.GroupBy( - k => k.Database ?? TableInfoDatabaseNode.NullDatabaseNode, StringComparer.CurrentCultureIgnoreCase) - .Select(g => new TableInfoDatabaseNode(g.Key, serverNode, g)); - - foreach (var db in databases) - { - children.Add(db); - AddChildren(db, descendancy.Add(db)); - } - - //now we have recorded all the children add them with descendancy - AddToDictionaries(children, descendancy); - } - - private void AddChildren(TableInfoDatabaseNode dbNode, DescendancyList descendancy) - { - //add empty hashset - var children = new HashSet(); - - foreach (var t in dbNode.Tables) - { - //record the children of the table infos (mostly column infos) - children.Add(t); - - //the all servers node=>the TableInfoServerNode => the t - AddChildren(t, descendancy.Add(t)); - } - - //now we have recorded all the children add them with descendancy - AddToDictionaries(children, descendancy); - } - - private void AddChildren(TableInfo tableInfo, DescendancyList descendancy) - { - //add empty hashset - var children = new HashSet(); - - //if the table has an identifier dump listed - if (tableInfo.IdentifierDumpServer_ID != null) - { - //if there is a dump (e.g. for dilution and dumping - not appearing in the live table) - var server = AllExternalServers.Single(s => s.ID == tableInfo.IdentifierDumpServer_ID.Value); - - children.Add(new IdentifierDumpServerUsageNode(tableInfo, server)); - } - - //get the discarded columns in this table - var discardedCols = new HashSet(AllPreLoadDiscardedColumns.Where(c => c.TableInfo_ID == tableInfo.ID)); - - //tell the column who thier parent is so they don't need to look up the database - foreach (PreLoadDiscardedColumn discardedCol in discardedCols) - discardedCol.InjectKnown(tableInfo); - - //if there are discarded columns - if (discardedCols.Any()) - { - var identifierDumpNode = new PreLoadDiscardedColumnsNode(tableInfo); - - //record that the usage is a child of TableInfo - children.Add(identifierDumpNode); - - //record that the discarded columns are children of identifier dump usage node - AddToDictionaries(discardedCols, descendancy.Add(identifierDumpNode)); - } - - //if it is a table valued function - if (tableInfo.IsTableValuedFunction) - { - //that has parameters - var parameters = tableInfo.GetAllParameters(); - - foreach (var p in parameters) children.Add(p); - } - - //next add the column infos - if (TableInfosToColumnInfos.TryGetValue(tableInfo.ID, out var result)) - foreach (var c in result) - { - children.Add(c); - c.InjectKnown(tableInfo); - AddChildren(c, descendancy.Add(c).SetBetterRouteExists()); - } - - //finally add any credentials objects - if (AllDataAccessCredentialUsages.TryGetValue(tableInfo, out var nodes)) - foreach (var node in nodes) - children.Add(node); - - //now we have recorded all the children add them with descendancy via the TableInfo descendancy - AddToDictionaries(children, descendancy); - } - - private void AddChildren(ColumnInfo columnInfo, DescendancyList descendancy) - { - var lookups = AllLookups.Where(l => l.Description_ID == columnInfo.ID).ToArray(); - var joinInfos = AllJoinInfos.Where(j => j.PrimaryKey_ID == columnInfo.ID); - - var children = new HashSet(); - - foreach (var l in lookups) - children.Add(l); - - foreach (var j in joinInfos) - children.Add(j); - - if (children.Any()) - AddToDictionaries(children, descendancy); - } - - protected void AddToDictionaries(HashSet children, DescendancyList list) - { - if (list.IsEmpty) - throw new ArgumentException("DescendancyList cannot be empty", nameof(list)); - - //document that the last parent has these as children - var parent = list.Last(); - - _childDictionary.AddOrUpdate(parent, - children, (p, s) => children); - - //now document the entire parent order to reach each child object i.e. 'Root=>Grandparent=>Parent' is how you get to 'Child' - foreach (var o in children) - _descendancyDictionary.AddOrUpdate(o, list, (k, v) => HandleDescendancyCollision(k, v, list)); - - - foreach (var masquerader in children.OfType()) - { - var key = masquerader.MasqueradingAs(); - - if (!AllMasqueraders.ContainsKey(key)) - AllMasqueraders.AddOrUpdate(key, new HashSet(), (o, set) => set); - - lock (AllMasqueraders) - { - AllMasqueraders[key].Add(masquerader); - } - } - } - - private static DescendancyList HandleDescendancyCollision(object key, DescendancyList oldRoute, - DescendancyList newRoute) - { - //if the new route is the best best - if (newRoute.NewBestRoute && !oldRoute.NewBestRoute) - return newRoute; - - // If the new one is marked BetterRouteExists just throw away the new one - return newRoute.BetterRouteExists ? oldRoute : newRoute; - // If in doubt use the newest one - } - - private HashSet GetAllObjects() - { - //anything which has children or is a child of someone else (distinct because HashSet) - return new HashSet(_childDictionary.SelectMany(kvp => kvp.Value).Union(_childDictionary.Keys)); - } - - public virtual object[] GetChildren(object model) - { - lock (WriteLock) - { - //if we have a record of any children in the child dictionary for the parent model object - if (_childDictionary.TryGetValue(model, out var cached)) - return cached.OrderBy(static o => o.ToString()).ToArray(); - - return model switch - { - //if they want the children of a Pipeline (which we don't track) just serve the components - Pipeline p => p.PipelineComponents.ToArray(), - //if they want the children of a PipelineComponent (which we don't track) just serve the arguments - PipelineComponent pc => pc.PipelineComponentArguments.ToArray(), - _ => Array.Empty() - }; - } - } - - public IEnumerable GetAllObjects(Type type, bool unwrapMasqueraders) - { - lock (WriteLock) - { - //things that are a match on Type but not IMasqueradeAs - var exactMatches = GetAllSearchables().Keys.Where(t => t is not IMasqueradeAs).Where(type.IsInstanceOfType); - - //Union the unwrapped masqueraders - return unwrapMasqueraders - ? exactMatches.Union( - AllMasqueraders - .Select(kvp => kvp.Key) - .OfType() - .Where(type.IsInstanceOfType)) - .Distinct() - : exactMatches; - } - } - - public DescendancyList GetDescendancyListIfAnyFor(object model) - { - lock (WriteLock) - { - return _descendancyDictionary.GetValueOrDefault(model); - } - } - - - public object GetRootObjectOrSelf(object objectToEmphasise) - { - lock (WriteLock) - { - var descendancy = GetDescendancyListIfAnyFor(objectToEmphasise); - - return descendancy != null && descendancy.Parents.Any() ? descendancy.Parents[0] : objectToEmphasise; - } - } - - - public virtual Dictionary GetAllSearchables() - { - lock (WriteLock) - { - var toReturn = new Dictionary(); - - foreach (var kvp in _descendancyDictionary.Where(kvp => kvp.Key is IMapsDirectlyToDatabaseTable)) - toReturn.Add((IMapsDirectlyToDatabaseTable)kvp.Key, kvp.Value); - - return toReturn; - } - } - - public IEnumerable GetAllChildrenRecursively(object o) - { - lock (WriteLock) - { - var toReturn = new List(); - - foreach (var child in GetChildren(o)) - { - toReturn.Add(child); - toReturn.AddRange(GetAllChildrenRecursively(child)); - } - - return toReturn; - } - } - - /// - /// Asks all plugins to provide the child objects for every object we have found so far. This method is recursive, call it with null the first time to use all objects. It will then - /// call itself with all the new objects that were sent back by the plugin (so that new objects found can still have children). - /// - /// - protected void GetPluginChildren(HashSet objectsToAskAbout = null) - { - lock (WriteLock) - { - var newObjectsFound = new HashSet(); - - var sw = new Stopwatch(); - - var providers = _pluginChildProviders.Except(_blockedPlugins).ToArray(); - - //for every object found so far - if (providers.Any()) - foreach (var o in objectsToAskAbout ?? GetAllObjects()) - //for every plugin loaded (that is not forbidlisted) - foreach (var plugin in providers) - //ask about the children - try - { - sw.Restart(); - //otherwise ask plugin what its children are - var pluginChildren = plugin.GetChildren(o); - - //if the plugin takes too long to respond we need to stop - if (sw.ElapsedMilliseconds > 1000) - { - _blockedPlugins.Add(plugin); - throw new Exception( - $"Plugin '{plugin}' was forbidlisted for taking too long to respond to GetChildren(o) where o was a '{o.GetType().Name}' ('{o}')"); - } - - //it has children - if (pluginChildren != null && pluginChildren.Any()) - { - //get the descendancy of the parent - var parentDescendancy = GetDescendancyListIfAnyFor(o); - var newDescendancy = parentDescendancy == null - ? new DescendancyList(new[] { o }) - : //if the parent is a root level object start a new descendancy list from it - parentDescendancy - .Add(o); //otherwise keep going down, returns a new DescendancyList so doesn't corrupt the dictionary one - newDescendancy = - parentDescendancy - .Add(o); //otherwise keep going down, returns a new DescendancyList so doesn't corrupt the dictionary one - - //record that - foreach (var pluginChild in pluginChildren) - { - //if the parent didn't have any children before - if (!_childDictionary.ContainsKey(o)) - _childDictionary.AddOrUpdate(o, new HashSet(), - (o1, set) => set); //it does now - - - //add us to the parent objects child collection - _childDictionary[o].Add(pluginChild); - - //add to the child collection of the parent object kvp.Key - _descendancyDictionary.AddOrUpdate(pluginChild, newDescendancy, - (s, e) => newDescendancy); - - //we have found a new object so we must ask other plugins about it (chances are a plugin will have a whole tree of sub objects) - newObjectsFound.Add(pluginChild); - } - } - } - catch (Exception e) - { - _errorsCheckNotifier.OnCheckPerformed(new CheckEventArgs(e.Message, CheckResult.Fail, e)); - } - - if (newObjectsFound.Any()) - GetPluginChildren(newObjectsFound); - } - } - - public IEnumerable GetMasqueradersOf(object o) - { - lock (WriteLock) - { - return AllMasqueraders.TryGetValue(o, out var result) ? result : Array.Empty(); - } - } - - protected T[] GetAllObjects(IRepository repository) where T : IMapsDirectlyToDatabaseTable - { - lock (WriteLock) - { - return repository.GetAllObjects(); - } - } - - - protected void AddToReturnSearchablesWithNoDecendancy( - Dictionary toReturn, - IEnumerable toAdd) - { - lock (WriteLock) - { - foreach (var m in toAdd) - toReturn.Add(m, null); - } - } - - public virtual void UpdateTo(ICoreChildProvider other) - { - ArgumentNullException.ThrowIfNull(other); - - if (other is not CatalogueChildProvider otherCat) - throw new NotSupportedException( - $"Did not know how to UpdateTo ICoreChildProvider of type {other.GetType().Name}"); - - AllLoadMetadatas = otherCat.AllLoadMetadatas; - AllProcessTasks = otherCat.AllProcessTasks; - AllProcessTasksArguments = otherCat.AllProcessTasksArguments; - AllLoadProgresses = otherCat.AllLoadProgresses; - AllCacheProgresses = otherCat.AllCacheProgresses; - AllPermissionWindows = otherCat.AllPermissionWindows; - AllCatalogues = otherCat.AllCatalogues; - AllCataloguesDictionary = otherCat.AllCataloguesDictionary; - AllSupportingDocuments = otherCat.AllSupportingDocuments; - AllSupportingSQL = otherCat.AllSupportingSQL; - _childDictionary = otherCat._childDictionary; - _descendancyDictionary = otherCat._descendancyDictionary; - _catalogueToCatalogueItems = otherCat._catalogueToCatalogueItems; - AllCatalogueItemsDictionary = otherCat.AllCatalogueItemsDictionary; - _allColumnInfos = otherCat._allColumnInfos; - AllAggregateConfigurations = otherCat.AllAggregateConfigurations; - AllAggregateDimensions = otherCat.AllAggregateDimensions; - AllAggregateContinuousDateAxis = otherCat.AllAggregateContinuousDateAxis; - AllRDMPRemotesNode = otherCat.AllRDMPRemotesNode; - AllRemoteRDMPs = otherCat.AllRemoteRDMPs; - AllDashboardsNode = otherCat.AllDashboardsNode; - AllDashboards = otherCat.AllDashboards; - AllObjectSharingNode = otherCat.AllObjectSharingNode; - AllImports = otherCat.AllImports; - AllExports = otherCat.AllExports; - AllStandardRegexesNode = otherCat.AllStandardRegexesNode; - AllPipelinesNode = otherCat.AllPipelinesNode; - OtherPipelinesNode = otherCat.OtherPipelinesNode; - AllPipelines = otherCat.AllPipelines; - AllPipelineComponents = otherCat.AllPipelineComponents; - AllPipelineComponentsArguments = otherCat.AllPipelineComponentsArguments; - AllStandardRegexes = otherCat.AllStandardRegexes; - AllANOTablesNode = otherCat.AllANOTablesNode; - AllANOTables = otherCat.AllANOTables; - AllExternalServers = otherCat.AllExternalServers; - AllServers = otherCat.AllServers; - AllTableInfos = otherCat.AllTableInfos; - AllDataAccessCredentialsNode = otherCat.AllDataAccessCredentialsNode; - AllExternalServersNode = otherCat.AllExternalServersNode; - AllServersNode = otherCat.AllServersNode; - AllDataAccessCredentials = otherCat.AllDataAccessCredentials; - AllDataAccessCredentialUsages = otherCat.AllDataAccessCredentialUsages; - TableInfosToColumnInfos = otherCat.TableInfosToColumnInfos; - AllColumnInfos = otherCat.AllColumnInfos; - AllPreLoadDiscardedColumns = otherCat.AllPreLoadDiscardedColumns; - AllLookups = otherCat.AllLookups; - AllJoinInfos = otherCat.AllJoinInfos; - AllAnyTableParameters = otherCat.AllAnyTableParameters; - AllMasqueraders = otherCat.AllMasqueraders; - AllExtractionInformationsDictionary = otherCat.AllExtractionInformationsDictionary; - _pluginChildProviders = otherCat._pluginChildProviders; - AllPermissionWindowsNode = otherCat.AllPermissionWindowsNode; - LoadMetadataRootFolder = otherCat.LoadMetadataRootFolder; - CatalogueRootFolder = otherCat.CatalogueRootFolder; - CohortIdentificationConfigurationRootFolder = otherCat.CohortIdentificationConfigurationRootFolder; - AllConnectionStringKeywordsNode = otherCat.AllConnectionStringKeywordsNode; - AllConnectionStringKeywords = otherCat.AllConnectionStringKeywords; - AllAggregateContainersDictionary = otherCat.AllAggregateContainersDictionary; - AllAggregateFilters = otherCat.AllAggregateFilters; - AllAggregateFilterParameters = otherCat.AllAggregateFilterParameters; - AllCohortIdentificationConfigurations = otherCat.AllCohortIdentificationConfigurations; - AllCohortAggregateContainers = otherCat.AllCohortAggregateContainers; - AllJoinables = otherCat.AllJoinables; - AllJoinUses = otherCat.AllJoinUses; - AllGovernanceNode = otherCat.AllGovernanceNode; - AllGovernancePeriods = otherCat.AllGovernancePeriods; - AllGovernanceDocuments = otherCat.AllGovernanceDocuments; - GovernanceCoverage = otherCat.GovernanceCoverage; - AllJoinableCohortAggregateConfigurationUse = otherCat.AllJoinableCohortAggregateConfigurationUse; - AllPluginsNode = otherCat.AllPluginsNode; - PipelineUseCases = otherCat.PipelineUseCases; - OrphanAggregateConfigurationsNode = otherCat.OrphanAggregateConfigurationsNode; - TemplateAggregateConfigurationsNode = otherCat.TemplateAggregateConfigurationsNode; - AllCatalogueParameters = otherCat.AllCatalogueParameters; - AllCatalogueValueSets = otherCat.AllCatalogueValueSets; - AllCatalogueValueSetValues = otherCat.AllCatalogueValueSetValues; - OrphanAggregateConfigurations = otherCat.OrphanAggregateConfigurations; - } - - public virtual bool SelectiveRefresh(IMapsDirectlyToDatabaseTable databaseEntity) - { - ProgressStopwatch.Restart(); - - return databaseEntity switch - { - AggregateFilterParameter afp => SelectiveRefresh(afp.AggregateFilter), - AggregateFilter af => SelectiveRefresh(af), - AggregateFilterContainer afc => SelectiveRefresh(afc), - CohortAggregateContainer cac => SelectiveRefresh(cac), - ExtractionInformation ei => SelectiveRefresh(ei), - CatalogueItem ci => SelectiveRefresh(ci), - _ => false - }; - } - - - public bool SelectiveRefresh(CatalogueItem ci) - { - var descendancy = GetDescendancyListIfAnyFor(ci.Catalogue); - if (descendancy == null) return false; - - FetchCatalogueItems(); - FetchExtractionInformations(); - AddChildren(ci.Catalogue, descendancy.Add(ci.Catalogue)); - return true; - } - - public bool SelectiveRefresh(ExtractionInformation ei) - { - var descendancy = GetDescendancyListIfAnyFor(ei); - var cata = descendancy?.Parents.OfType().LastOrDefault() ?? ei.CatalogueItem.Catalogue; - - if (cata == null) - return false; - - var cataDescendancy = GetDescendancyListIfAnyFor(cata); - - if (cataDescendancy == null) - return false; - - FetchCatalogueItems(); - - foreach (var ci in AllCatalogueItems.Where(ci => ci.ID == ei.CatalogueItem_ID)) ci.ClearAllInjections(); - - // property changes or deleting the ExtractionInformation - FetchExtractionInformations(); - - // refresh the Catalogue - AddChildren(cata, cataDescendancy.Add(cata)); - return true; - } - - public bool SelectiveRefresh(CohortAggregateContainer container) - { - var parentContainer = container.GetParentContainerIfAny(); - if (parentContainer != null) - { - var descendancy = GetDescendancyListIfAnyFor(parentContainer); - - if (descendancy != null) - { - BuildAggregateConfigurations(); - - BuildCohortCohortAggregateContainers(); - AddChildren(parentContainer, descendancy.Add(parentContainer)); - return true; - } - } - - var cic = container.GetCohortIdentificationConfiguration(); - - if (cic != null) - { - var descendancy = GetDescendancyListIfAnyFor(cic); - - if (descendancy != null) - { - BuildAggregateConfigurations(); - BuildCohortCohortAggregateContainers(); - AddChildren(cic, descendancy.Add(cic)); - return true; - } - } - - return false; - } - - public bool SelectiveRefresh(AggregateFilter f) - { - var knownContainer = GetDescendancyListIfAnyFor(f.FilterContainer); - if (knownContainer == null) return false; - - BuildAggregateFilterContainers(); - AddChildren((AggregateFilterContainer)f.FilterContainer, knownContainer.Add(f.FilterContainer)); - return true; - } + children.Add(new DecryptionPrivateKeyNode(isKeyMissing)); + + foreach (var creds in AllDataAccessCredentials) + children.Add(creds); + + + AddToDictionaries(children, new DescendancyList(allDataAccessCredentialsNode)); + } + + private void AddChildren(AllANOTablesNode anoTablesNode) + { + AddToDictionaries(new HashSet(AllANOTables), new DescendancyList(anoTablesNode)); + } + + private void AddChildren(FolderNode folder, DescendancyList descendancy) + { + foreach (var child in folder.ChildFolders) + //add subfolder children + AddChildren(child, descendancy.Add(child)); + + //add catalogues in folder + foreach (var c in folder.ChildObjects) AddChildren(c, descendancy.Add(c)); + + // Children are the folders + objects + AddToDictionaries(new HashSet( + folder.ChildFolders.Cast() + .Union(folder.ChildObjects)), descendancy + ); + } + + private void AddChildren(FolderNode folder, DescendancyList descendancy) + { + foreach (var child in folder.ChildFolders) + //add subfolder children + AddChildren(child, descendancy.Add(child)); + + //add loads in folder + foreach (var lmd in folder.ChildObjects.Where(lmd => lmd.RootLoadMetadata_ID == null).ToArray()) AddChildren(lmd, descendancy.Add(lmd)); + // Children are the folders + objects + AddToDictionaries(new HashSet( + folder.ChildFolders.Cast() + .Union(folder.ChildObjects)), descendancy + ); + } + + private void AddChildren(FolderNode folder, DescendancyList descendancy) + { + foreach (var child in folder.ChildFolders) + //add subfolder children + AddChildren(child, descendancy.Add(child)); + + //add loads in folder + foreach (var ds in folder.ChildObjects) AddChildren(ds, descendancy.Add(ds)); + + // Children are the folders + objects + AddToDictionaries(new HashSet( + folder.ChildFolders.Cast() + .Union(folder.ChildObjects)), descendancy + ); + } + + private void AddChildren(FolderNode folder, DescendancyList descendancy) + { + foreach (var child in folder.ChildFolders) + //add subfolder children + AddChildren(child, descendancy.Add(child)); + + + //add cics in folder + foreach (var cic in folder.ChildObjects) AddChildren(cic, descendancy.Add(cic)); + + // Children are the folders + objects + AddToDictionaries(new HashSet( + folder.ChildFolders.Cast() + .Union(folder.ChildObjects)), descendancy + ); + } + + private void AddChildren(Curation.Data.Dataset lmd, DescendancyList descendancy) + { + var childObjects = new List(); + AddToDictionaries(new HashSet(childObjects), descendancy); + } + + + #region Load Metadata + + private void AddChildren(LoadMetadata lmd, DescendancyList descendancy, bool includeSchedule = true, bool includeCatalogues = true, bool includeVersions = true) + { + var childObjects = new List(); + + if (lmd.OverrideRAWServer_ID.HasValue) + { + var server = AllExternalServers.Single(s => s.ID == lmd.OverrideRAWServer_ID.Value); + var usage = new OverrideRawServerNode(lmd, server); + childObjects.Add(usage); + } + if (includeSchedule) + { + var allSchedulesNode = new LoadMetadataScheduleNode(lmd); + AddChildren(allSchedulesNode, descendancy.Add(allSchedulesNode)); + childObjects.Add(allSchedulesNode); + } + + if (includeCatalogues) + { + var allCataloguesNode = new AllCataloguesUsedByLoadMetadataNode(lmd); + AddChildren(allCataloguesNode, descendancy.Add(allCataloguesNode)); + childObjects.Add(allCataloguesNode); + } + + var processTasksNode = new AllProcessTasksUsedByLoadMetadataNode(lmd); + AddChildren(processTasksNode, descendancy.Add(processTasksNode)); + childObjects.Add(processTasksNode); + + if (includeVersions) + { + var versionsNode = new LoadMetadataVersionNode(lmd); + AddChildren(versionsNode, descendancy.Add(versionsNode)); + childObjects.Add(versionsNode); + } + + childObjects.Add(new LoadDirectoryNode(lmd)); + + AddToDictionaries(new HashSet(childObjects), descendancy); + } + + private void AddChildren(LoadMetadataScheduleNode allSchedulesNode, DescendancyList descendancy) + { + var childObjects = new HashSet(); + + var lmd = allSchedulesNode.LoadMetadata; + + foreach (var lp in AllLoadProgresses.Where(p => p.LoadMetadata_ID == lmd.ID)) + { + AddChildren(lp, descendancy.Add(lp)); + childObjects.Add(lp); + } + + if (childObjects.Any()) + AddToDictionaries(childObjects, descendancy); + } + + private void AddChildren(LoadProgress loadProgress, DescendancyList descendancy) + { + var cacheProgresses = AllCacheProgresses.Where(cp => cp.LoadProgress_ID == loadProgress.ID).ToArray(); + + foreach (var cacheProgress in cacheProgresses) + AddChildren(cacheProgress, descendancy.Add(cacheProgress)); + + if (cacheProgresses.Any()) + AddToDictionaries(new HashSet(cacheProgresses), descendancy); + } + + private void AddChildren(CacheProgress cacheProgress, DescendancyList descendancy) + { + var children = new HashSet(); + + if (cacheProgress.PermissionWindow_ID != null) + { + var window = AllPermissionWindows.Single(w => w.ID == cacheProgress.PermissionWindow_ID); + var windowNode = new PermissionWindowUsedByCacheProgressNode(cacheProgress, window, true); + + children.Add(windowNode); + } + + if (children.Any()) + AddToDictionaries(children, descendancy); + } + + private void AddChildren(AllProcessTasksUsedByLoadMetadataNode allProcessTasksUsedByLoadMetadataNode, + DescendancyList descendancy) + { + var childObjects = new HashSet(); + + var lmd = allProcessTasksUsedByLoadMetadataNode.LoadMetadata; + childObjects.Add(new LoadStageNode(lmd, LoadStage.GetFiles)); + childObjects.Add(new LoadStageNode(lmd, LoadStage.Mounting)); + childObjects.Add(new LoadStageNode(lmd, LoadStage.AdjustRaw)); + childObjects.Add(new LoadStageNode(lmd, LoadStage.AdjustStaging)); + childObjects.Add(new LoadStageNode(lmd, LoadStage.PostLoad)); + + foreach (LoadStageNode node in childObjects) + AddChildren(node, descendancy.Add(node)); + + AddToDictionaries(childObjects, descendancy); + } + + private void AddChildren(LoadStageNode loadStageNode, DescendancyList descendancy) + { + var tasks = AllProcessTasks.Where( + p => p.LoadMetadata_ID == loadStageNode.LoadMetadata.ID && p.LoadStage == loadStageNode.LoadStage) + .OrderBy(o => o.Order).ToArray(); + + foreach (var processTask in tasks) + AddChildren(processTask, descendancy.Add(processTask)); + + if (tasks.Any()) + AddToDictionaries(new HashSet(tasks), descendancy); + } + + private void AddChildren(ProcessTask procesTask, DescendancyList descendancy) + { + var args = AllProcessTasksArguments.Where( + a => a.ProcessTask_ID == procesTask.ID).ToArray(); + + if (args.Any()) + AddToDictionaries(new HashSet(args), descendancy); + } + + private void AddChildren(LoadMetadataVersionNode LoadMetadataVersionNode, DescendancyList descendancy) + { + LoadMetadataVersionNode.LoadMetadataVersions = AllLoadMetadatas.Where(lmd => lmd.RootLoadMetadata_ID == LoadMetadataVersionNode.LoadMetadata.ID).ToList(); + var childObjects = new List(); + + foreach (var lmd in LoadMetadataVersionNode.LoadMetadataVersions) + { + AddChildren(lmd, descendancy.Add(lmd), false, false, false); + childObjects.Add(lmd); + } + AddToDictionaries(new HashSet(childObjects), descendancy); + + } + + private void AddChildren(AllCataloguesUsedByLoadMetadataNode allCataloguesUsedByLoadMetadataNode, + DescendancyList descendancy) + { + var loadMetadataId = allCataloguesUsedByLoadMetadataNode.LoadMetadata.ID; + var linkedCatalogueIDs = AllLoadMetadataLinkage.Where(link => link.LoadMetadataID == loadMetadataId).Select(static link => link.CatalogueID); + var usedCatalogues = linkedCatalogueIDs.Select(catalogueId => AllCatalogues.FirstOrDefault(c => c.ID == catalogueId)).Where(static foundCatalogue => foundCatalogue is not null).ToList(); + allCataloguesUsedByLoadMetadataNode.UsedCatalogues = usedCatalogues; + var childObjects = usedCatalogues.Select(foundCatalogue => new CatalogueUsedByLoadMetadataNode(allCataloguesUsedByLoadMetadataNode.LoadMetadata, foundCatalogue)).Cast().ToHashSet(); + + AddToDictionaries(childObjects, descendancy); + } + + #endregion + + protected void AddChildren(Catalogue c, DescendancyList descendancy) + { + var childObjects = new List(); + + var catalogueAggregates = AllAggregateConfigurations.Where(a => a.Catalogue_ID == c.ID).ToArray(); + var cohortAggregates = catalogueAggregates.Where(a => a.IsCohortIdentificationAggregate).ToArray(); + var regularAggregates = catalogueAggregates.Except(cohortAggregates).ToArray(); + + //get all the CatalogueItems for this Catalogue (TryGet because Catalogue may not have any items + var cis = _catalogueToCatalogueItems.TryGetValue(c.ID, out var result) + ? result.ToArray() + : Array.Empty(); + + //tell the CatalogueItems that we are are their parent + foreach (var ci in cis) + ci.InjectKnown(c); + + // core includes project specific which basically means the same thing + var core = new CatalogueItemsNode(c, + cis.Where(ci => ci.ExtractionInformation?.ExtractionCategory == ExtractionCategory.Core || + ci.ExtractionInformation?.ExtractionCategory == ExtractionCategory.ProjectSpecific) + , ExtractionCategory.Core); + + c.InjectKnown(cis); + + var deprecated = new CatalogueItemsNode(c, + cis.Where(ci => ci.ExtractionInformation?.ExtractionCategory == ExtractionCategory.Deprecated), + ExtractionCategory.Deprecated); + var special = new CatalogueItemsNode(c, + cis.Where(ci => ci.ExtractionInformation?.ExtractionCategory == ExtractionCategory.SpecialApprovalRequired), + ExtractionCategory.SpecialApprovalRequired); + var intern = new CatalogueItemsNode(c, + cis.Where(ci => ci.ExtractionInformation?.ExtractionCategory == ExtractionCategory.Internal), + ExtractionCategory.Internal); + var supplemental = new CatalogueItemsNode(c, + cis.Where(ci => ci.ExtractionInformation?.ExtractionCategory == ExtractionCategory.Supplemental), + ExtractionCategory.Supplemental); + var notExtractable = new CatalogueItemsNode(c, cis.Where(ci => ci.ExtractionInformation == null), null); + + AddChildren(core, descendancy.Add(core)); + childObjects.Add(core); + + foreach (var optional in new[] { deprecated, special, intern, supplemental, notExtractable }) + if (optional.CatalogueItems.Any()) + { + AddChildren(optional, descendancy.Add(optional)); + childObjects.Add(optional); + } + + //do we have any foreign key fields into this lookup table + var lookups = AllLookups.Where(l => c.CatalogueItems.Any(ci => ci.ColumnInfo_ID == l.ForeignKey_ID)).ToArray(); + + var docs = AllSupportingDocuments.Where(d => d.Catalogue_ID == c.ID).ToArray(); + var sql = AllSupportingSQL.Where(d => d.Catalogue_ID == c.ID).ToArray(); + + //if there are supporting documents or supporting sql files then add documentation node + if (docs.Any() || sql.Any()) + { + var documentationNode = new DocumentationNode(c, docs, sql); + + //add the documentations node + childObjects.Add(documentationNode); + + //record the children + AddToDictionaries(new HashSet(docs.Cast().Union(sql)), descendancy.Add(documentationNode)); + } + + if (lookups.Any()) + { + var lookupsNode = new CatalogueLookupsNode(c, lookups); + //add the documentations node + childObjects.Add(lookupsNode); + + + //record the children + AddToDictionaries(new HashSet(lookups.Select(l => new CatalogueLookupUsageNode(c, l))), + descendancy.Add(lookupsNode)); + } + + if (regularAggregates.Any()) + { + var aggregatesNode = new AggregatesNode(c, regularAggregates); + childObjects.Add(aggregatesNode); + + var nodeDescendancy = descendancy.Add(aggregatesNode); + AddToDictionaries(new HashSet(regularAggregates), nodeDescendancy); + + foreach (var regularAggregate in regularAggregates) + AddChildren(regularAggregate, nodeDescendancy.Add(regularAggregate)); + } + + //finalise + AddToDictionaries(new HashSet(childObjects), descendancy); + } + + private void InjectCatalogueItems() + { + foreach (var ci in AllCatalogueItems) + if (_extractionInformationsByCatalogueItem.TryGetValue(ci.ID, out var ei)) + ci.InjectKnown(ei); + else + ci.InjectKnown((ExtractionInformation)null); + } + + private void AddChildren(CatalogueItemsNode node, DescendancyList descendancyList) + { + AddToDictionaries(new HashSet(node.CatalogueItems), descendancyList); + + foreach (var ci in node.CatalogueItems) + AddChildren(ci, descendancyList.Add(ci)); + } + + private void AddChildren(AggregateConfiguration aggregateConfiguration, DescendancyList descendancy) + { + var childrenObjects = new HashSet(); + + var parameters = AllAnyTableParameters.Where(p => p.IsReferenceTo(aggregateConfiguration)).Cast() + .ToArray(); + + foreach (var p in parameters) + childrenObjects.Add(p); + + // show the dimensions in the tree + foreach (var dim in aggregateConfiguration.AggregateDimensions) childrenObjects.Add(dim); + + // show the axis (if any) in the tree. If there are multiple axis in this tree then that is bad but maybe the user can delete one of them to fix the situation + foreach (var axis in AllAggregateContinuousDateAxis.Where(a => + aggregateConfiguration.AggregateDimensions.Any(d => d.ID == a.AggregateDimension_ID))) + childrenObjects.Add(axis); + + //we can step into this twice, once via Catalogue children and once via CohortIdentificationConfiguration children + //if we get in via Catalogue children then descendancy will be Ignore=true we don't end up emphasising into CatalogueCollectionUI when + //really user wants to see it in CohortIdentificationCollectionUI + if (aggregateConfiguration.RootFilterContainer_ID != null) + { + var container = AllAggregateContainersDictionary[(int)aggregateConfiguration.RootFilterContainer_ID]; + + AddChildren(container, descendancy.Add(container)); + childrenObjects.Add(container); + } + + AddToDictionaries(childrenObjects, descendancy); + } + + private void AddChildren(AggregateFilterContainer container, DescendancyList descendancy) + { + var childrenObjects = new List(); + + var subcontainers = _aggregateFilterManager.GetSubContainers(container); + var filters = _aggregateFilterManager.GetFilters(container); + + foreach (AggregateFilterContainer subcontainer in subcontainers) + { + //one of our children is this subcontainer + childrenObjects.Add(subcontainer); + + //but also document its children + AddChildren(subcontainer, descendancy.Add(subcontainer)); + } + + //also add the filters for the container + foreach (var f in filters) + { + // for filters add the parameters under them + AddChildren((AggregateFilter)f, descendancy.Add(f)); + childrenObjects.Add(f); + } + + //add our children to the dictionary + AddToDictionaries(new HashSet(childrenObjects), descendancy); + } + + private void AddChildren(AggregateFilter f, DescendancyList descendancy) + { + AddToDictionaries(new HashSet(AllAggregateFilterParameters.Where(p => p.AggregateFilter_ID == f.ID)), + descendancy); + } + + private void AddChildren(CatalogueItem ci, DescendancyList descendancy) + { + var childObjects = new List(); + + var ei = ci.ExtractionInformation; + if (ei != null) + { + childObjects.Add(ei); + AddChildren(ei, descendancy.Add(ei)); + } + else + { + ci.InjectKnown( + (ExtractionInformation)null); // we know the CatalogueItem has no ExtractionInformation child because it's not in the dictionary + } + + if (ci.ColumnInfo_ID.HasValue && _allColumnInfos.TryGetValue(ci.ColumnInfo_ID.Value, out var col)) + childObjects.Add(new LinkedColumnInfoNode(ci, col)); + + AddToDictionaries(new HashSet(childObjects), descendancy); + } + + private void AddChildren(ExtractionInformation extractionInformation, DescendancyList descendancy) + { + var children = new HashSet(); + + foreach (var filter in AllCatalogueFilters.Where(f => f.ExtractionInformation_ID == extractionInformation.ID)) + { + //add the filter as a child of the + children.Add(filter); + AddChildren(filter, descendancy.Add(filter)); + } + + AddToDictionaries(children, descendancy); + } + + private void AddChildren(ExtractionFilter filter, DescendancyList descendancy) + { + var children = new HashSet(); + var parameters = AllCatalogueParameters.Where(p => p.ExtractionFilter_ID == filter.ID).ToArray(); + var parameterSets = AllCatalogueValueSets.Where(vs => vs.ExtractionFilter_ID == filter.ID).ToArray(); + + filter.InjectKnown(parameterSets); + + foreach (var p in parameters) + children.Add(p); + + foreach (var set in parameterSets) + { + children.Add(set); + AddChildren(set, descendancy.Add(set), parameters); + } + + if (children.Any()) + AddToDictionaries(children, descendancy); + } + + private void AddChildren(ExtractionFilterParameterSet set, DescendancyList descendancy, + ExtractionFilterParameter[] filterParameters) + { + var children = new HashSet(); + + foreach (var setValue in AllCatalogueValueSetValues.Where(v => v.ExtractionFilterParameterSet_ID == set.ID)) + { + setValue.InjectKnown(filterParameters.SingleOrDefault(p => p.ID == setValue.ExtractionFilterParameter_ID)); + children.Add(setValue); + } + + AddToDictionaries(children, descendancy); + } + + private void AddChildren(CohortIdentificationConfiguration cic, DescendancyList descendancy) + { + var children = new HashSet(); + + //it has an associated query cache + if (cic.QueryCachingServer_ID != null) + children.Add(new QueryCacheUsedByCohortIdentificationNode(cic, + AllExternalServers.Single(s => s.ID == cic.QueryCachingServer_ID))); + + var parameters = AllAnyTableParameters.Where(p => p.IsReferenceTo(cic)).Cast().ToArray(); + foreach (var p in parameters) children.Add(p); + + //if it has a root container + if (cic.RootCohortAggregateContainer_ID != null) + { + var container = AllCohortAggregateContainers.Single(c => c.ID == cic.RootCohortAggregateContainer_ID); + AddChildren(container, descendancy.Add(container).SetBetterRouteExists()); + children.Add(container); + } + + //get the patient index tables + var joinableNode = new JoinableCollectionNode(cic, + AllJoinables.Where(j => j.CohortIdentificationConfiguration_ID == cic.ID).ToArray()); + AddChildren(joinableNode, descendancy.Add(joinableNode).SetBetterRouteExists()); + children.Add(joinableNode); + + AddToDictionaries(children, descendancy.SetBetterRouteExists()); + } + + private void AddChildren(JoinableCollectionNode joinablesNode, DescendancyList descendancy) + { + var children = new HashSet(); + + foreach (var joinable in joinablesNode.Joinables) + try + { + var agg = AllAggregateConfigurations.Single(ac => ac.ID == joinable.AggregateConfiguration_ID); + ForceAggregateNaming(agg, descendancy); + children.Add(agg); + + //it's no longer an orphan because it's in a known cic (as a patient index table) + OrphanAggregateConfigurations.Remove(agg); + + AddChildren(agg, descendancy.Add(agg)); + } + catch (Exception e) + { + throw new Exception( + $"JoinableCohortAggregateConfiguration (patient index table) object (ID={joinable.ID}) references AggregateConfiguration_ID {joinable.AggregateConfiguration_ID} but that AggregateConfiguration was not found", + e); + } + + AddToDictionaries(children, descendancy); + } + + private void AddChildren(CohortAggregateContainer container, DescendancyList descendancy) + { + //get subcontainers + var subcontainers = _cohortContainerManager.GetChildren(container).OfType().ToList(); + + //if there are subcontainers + foreach (var subcontainer in subcontainers) + AddChildren(subcontainer, descendancy.Add(subcontainer)); + + //get our configurations + var configurations = _cohortContainerManager.GetChildren(container).OfType().ToList(); + + //record the configurations children including full descendancy + foreach (var configuration in configurations) + { + ForceAggregateNaming(configuration, descendancy); + AddChildren(configuration, descendancy.Add(configuration)); + + //it's no longer an orphan because it's in a known cic + OrphanAggregateConfigurations.Remove(configuration); + } + + //all our children (containers and aggregates) + //children are all aggregates and containers at the current hierarchy level in order + var children = subcontainers.Union(configurations.Cast()).OrderBy(o => o.Order).ToList(); + + AddToDictionaries(new HashSet(children), descendancy); + } + + private void ForceAggregateNaming(AggregateConfiguration configuration, DescendancyList descendancy) + { + //configuration has the wrong name + if (!configuration.IsCohortIdentificationAggregate) + { + _errorsCheckNotifier.OnCheckPerformed(new CheckEventArgs( + $"Had to fix naming of configuration '{configuration}' because it didn't start with correct cic prefix", + CheckResult.Warning)); + descendancy.Parents.OfType().Single() + .EnsureNamingConvention(configuration); + configuration.SaveToDatabase(); + } + } + + private void AddChildren(TableInfoServerNode serverNode, DescendancyList descendancy) + { + //add empty hashset + var children = new HashSet(); + + var databases = + serverNode.Tables.GroupBy( + k => k.Database ?? TableInfoDatabaseNode.NullDatabaseNode, StringComparer.CurrentCultureIgnoreCase) + .Select(g => new TableInfoDatabaseNode(g.Key, serverNode, g)); + + foreach (var db in databases) + { + children.Add(db); + AddChildren(db, descendancy.Add(db)); + } + + //now we have recorded all the children add them with descendancy + AddToDictionaries(children, descendancy); + } + + private void AddChildren(TableInfoDatabaseNode dbNode, DescendancyList descendancy) + { + //add empty hashset + var children = new HashSet(); + + foreach (var t in dbNode.Tables) + { + //record the children of the table infos (mostly column infos) + children.Add(t); + + //the all servers node=>the TableInfoServerNode => the t + AddChildren(t, descendancy.Add(t)); + } + + //now we have recorded all the children add them with descendancy + AddToDictionaries(children, descendancy); + } + + private void AddChildren(TableInfo tableInfo, DescendancyList descendancy) + { + //add empty hashset + var children = new HashSet(); + + //if the table has an identifier dump listed + if (tableInfo.IdentifierDumpServer_ID != null) + { + //if there is a dump (e.g. for dilution and dumping - not appearing in the live table) + var server = AllExternalServers.Single(s => s.ID == tableInfo.IdentifierDumpServer_ID.Value); + + children.Add(new IdentifierDumpServerUsageNode(tableInfo, server)); + } + + //get the discarded columns in this table + var discardedCols = new HashSet(AllPreLoadDiscardedColumns.Where(c => c.TableInfo_ID == tableInfo.ID)); + + //tell the column who thier parent is so they don't need to look up the database + foreach (PreLoadDiscardedColumn discardedCol in discardedCols) + discardedCol.InjectKnown(tableInfo); + + //if there are discarded columns + if (discardedCols.Any()) + { + var identifierDumpNode = new PreLoadDiscardedColumnsNode(tableInfo); + + //record that the usage is a child of TableInfo + children.Add(identifierDumpNode); + + //record that the discarded columns are children of identifier dump usage node + AddToDictionaries(discardedCols, descendancy.Add(identifierDumpNode)); + } + + //if it is a table valued function + if (tableInfo.IsTableValuedFunction) + { + //that has parameters + var parameters = tableInfo.GetAllParameters(); + + foreach (var p in parameters) children.Add(p); + } + + //next add the column infos + if (TableInfosToColumnInfos.TryGetValue(tableInfo.ID, out var result)) + foreach (var c in result) + { + children.Add(c); + c.InjectKnown(tableInfo); + AddChildren(c, descendancy.Add(c).SetBetterRouteExists()); + } + + //finally add any credentials objects + if (AllDataAccessCredentialUsages.TryGetValue(tableInfo, out var nodes)) + foreach (var node in nodes) + children.Add(node); + + //now we have recorded all the children add them with descendancy via the TableInfo descendancy + AddToDictionaries(children, descendancy); + } + + private void AddChildren(ColumnInfo columnInfo, DescendancyList descendancy) + { + var lookups = AllLookups.Where(l => l.Description_ID == columnInfo.ID).ToArray(); + var joinInfos = AllJoinInfos.Where(j => j.PrimaryKey_ID == columnInfo.ID); + + var children = new HashSet(); + + foreach (var l in lookups) + children.Add(l); + + foreach (var j in joinInfos) + children.Add(j); + + if (children.Any()) + AddToDictionaries(children, descendancy); + } + + protected void AddToDictionaries(HashSet children, DescendancyList list) + { + if (list.IsEmpty) + throw new ArgumentException("DescendancyList cannot be empty", nameof(list)); + + //document that the last parent has these as children + var parent = list.Last(); + + _childDictionary.AddOrUpdate(parent, + children, (p, s) => children); + + //now document the entire parent order to reach each child object i.e. 'Root=>Grandparent=>Parent' is how you get to 'Child' + foreach (var o in children) + _descendancyDictionary.AddOrUpdate(o, list, (k, v) => HandleDescendancyCollision(k, v, list)); + + + foreach (var masquerader in children.OfType()) + { + var key = masquerader.MasqueradingAs(); + + if (!AllMasqueraders.ContainsKey(key)) + AllMasqueraders.AddOrUpdate(key, new HashSet(), (o, set) => set); + + lock (AllMasqueraders) + { + AllMasqueraders[key].Add(masquerader); + } + } + } + + private static DescendancyList HandleDescendancyCollision(object key, DescendancyList oldRoute, + DescendancyList newRoute) + { + //if the new route is the best best + if (newRoute.NewBestRoute && !oldRoute.NewBestRoute) + return newRoute; + + // If the new one is marked BetterRouteExists just throw away the new one + return newRoute.BetterRouteExists ? oldRoute : newRoute; + // If in doubt use the newest one + } + + private HashSet GetAllObjects() + { + //anything which has children or is a child of someone else (distinct because HashSet) + return new HashSet(_childDictionary.SelectMany(kvp => kvp.Value).Union(_childDictionary.Keys)); + } + + public virtual object[] GetChildren(object model) + { + lock (WriteLock) + { + //if we have a record of any children in the child dictionary for the parent model object + if (_childDictionary.TryGetValue(model, out var cached)) + return cached.OrderBy(static o => o.ToString()).ToArray(); + + return model switch + { + //if they want the children of a Pipeline (which we don't track) just serve the components + Pipeline p => p.PipelineComponents.ToArray(), + //if they want the children of a PipelineComponent (which we don't track) just serve the arguments + PipelineComponent pc => pc.PipelineComponentArguments.ToArray(), + _ => Array.Empty() + }; + } + } + + public IEnumerable GetAllObjects(Type type, bool unwrapMasqueraders) + { + lock (WriteLock) + { + //things that are a match on Type but not IMasqueradeAs + var exactMatches = GetAllSearchables().Keys.Where(t => t is not IMasqueradeAs).Where(type.IsInstanceOfType); + + //Union the unwrapped masqueraders + return unwrapMasqueraders + ? exactMatches.Union( + AllMasqueraders + .Select(kvp => kvp.Key) + .OfType() + .Where(type.IsInstanceOfType)) + .Distinct() + : exactMatches; + } + } + + public DescendancyList GetDescendancyListIfAnyFor(object model) + { + lock (WriteLock) + { + return _descendancyDictionary.GetValueOrDefault(model); + } + } + + + public object GetRootObjectOrSelf(object objectToEmphasise) + { + lock (WriteLock) + { + var descendancy = GetDescendancyListIfAnyFor(objectToEmphasise); + + return descendancy != null && descendancy.Parents.Any() ? descendancy.Parents[0] : objectToEmphasise; + } + } + + + public virtual Dictionary GetAllSearchables() + { + lock (WriteLock) + { + var toReturn = new Dictionary(); + + foreach (var kvp in _descendancyDictionary.Where(kvp => kvp.Key is IMapsDirectlyToDatabaseTable)) + toReturn.Add((IMapsDirectlyToDatabaseTable)kvp.Key, kvp.Value); + + return toReturn; + } + } + + public IEnumerable GetAllChildrenRecursively(object o) + { + lock (WriteLock) + { + var toReturn = new List(); + + foreach (var child in GetChildren(o)) + { + toReturn.Add(child); + toReturn.AddRange(GetAllChildrenRecursively(child)); + } + + return toReturn; + } + } + + /// + /// Asks all plugins to provide the child objects for every object we have found so far. This method is recursive, call it with null the first time to use all objects. It will then + /// call itself with all the new objects that were sent back by the plugin (so that new objects found can still have children). + /// + /// + protected void GetPluginChildren(HashSet objectsToAskAbout = null) + { + lock (WriteLock) + { + var newObjectsFound = new HashSet(); + + var sw = new Stopwatch(); + + var providers = _pluginChildProviders.Except(_blockedPlugins).ToArray(); + + //for every object found so far + if (providers.Any()) + foreach (var o in objectsToAskAbout ?? GetAllObjects()) + //for every plugin loaded (that is not forbidlisted) + foreach (var plugin in providers) + //ask about the children + try + { + sw.Restart(); + //otherwise ask plugin what its children are + var pluginChildren = plugin.GetChildren(o); + + //if the plugin takes too long to respond we need to stop + if (sw.ElapsedMilliseconds > 1000) + { + _blockedPlugins.Add(plugin); + throw new Exception( + $"Plugin '{plugin}' was forbidlisted for taking too long to respond to GetChildren(o) where o was a '{o.GetType().Name}' ('{o}')"); + } + + //it has children + if (pluginChildren != null && pluginChildren.Any()) + { + //get the descendancy of the parent + var parentDescendancy = GetDescendancyListIfAnyFor(o); + var newDescendancy = parentDescendancy == null + ? new DescendancyList(new[] { o }) + : //if the parent is a root level object start a new descendancy list from it + parentDescendancy + .Add(o); //otherwise keep going down, returns a new DescendancyList so doesn't corrupt the dictionary one + newDescendancy = + parentDescendancy + .Add(o); //otherwise keep going down, returns a new DescendancyList so doesn't corrupt the dictionary one + + //record that + foreach (var pluginChild in pluginChildren) + { + //if the parent didn't have any children before + if (!_childDictionary.ContainsKey(o)) + _childDictionary.AddOrUpdate(o, new HashSet(), + (o1, set) => set); //it does now + + + //add us to the parent objects child collection + _childDictionary[o].Add(pluginChild); + + //add to the child collection of the parent object kvp.Key + _descendancyDictionary.AddOrUpdate(pluginChild, newDescendancy, + (s, e) => newDescendancy); + + //we have found a new object so we must ask other plugins about it (chances are a plugin will have a whole tree of sub objects) + newObjectsFound.Add(pluginChild); + } + } + } + catch (Exception e) + { + _errorsCheckNotifier.OnCheckPerformed(new CheckEventArgs(e.Message, CheckResult.Fail, e)); + } + + if (newObjectsFound.Any()) + GetPluginChildren(newObjectsFound); + } + } + + public IEnumerable GetMasqueradersOf(object o) + { + lock (WriteLock) + { + return AllMasqueraders.TryGetValue(o, out var result) ? result : Array.Empty(); + } + } + + protected T[] GetAllObjects(IRepository repository) where T : IMapsDirectlyToDatabaseTable + { + lock (WriteLock) + { + return repository.GetAllObjects(); + } + } + + + protected void AddToReturnSearchablesWithNoDecendancy( + Dictionary toReturn, + IEnumerable toAdd) + { + lock (WriteLock) + { + foreach (var m in toAdd) + toReturn.Add(m, null); + } + } + + public virtual void UpdateTo(ICoreChildProvider other) + { + ArgumentNullException.ThrowIfNull(other); + + if (other is not CatalogueChildProvider otherCat) + throw new NotSupportedException( + $"Did not know how to UpdateTo ICoreChildProvider of type {other.GetType().Name}"); + + AllLoadMetadatas = otherCat.AllLoadMetadatas; + AllProcessTasks = otherCat.AllProcessTasks; + AllProcessTasksArguments = otherCat.AllProcessTasksArguments; + AllLoadProgresses = otherCat.AllLoadProgresses; + AllCacheProgresses = otherCat.AllCacheProgresses; + AllPermissionWindows = otherCat.AllPermissionWindows; + AllCatalogues = otherCat.AllCatalogues; + AllCataloguesDictionary = otherCat.AllCataloguesDictionary; + AllSupportingDocuments = otherCat.AllSupportingDocuments; + AllSupportingSQL = otherCat.AllSupportingSQL; + _childDictionary = otherCat._childDictionary; + _descendancyDictionary = otherCat._descendancyDictionary; + _catalogueToCatalogueItems = otherCat._catalogueToCatalogueItems; + AllCatalogueItemsDictionary = otherCat.AllCatalogueItemsDictionary; + _allColumnInfos = otherCat._allColumnInfos; + AllAggregateConfigurations = otherCat.AllAggregateConfigurations; + AllAggregateDimensions = otherCat.AllAggregateDimensions; + AllAggregateContinuousDateAxis = otherCat.AllAggregateContinuousDateAxis; + AllRDMPRemotesNode = otherCat.AllRDMPRemotesNode; + AllRemoteRDMPs = otherCat.AllRemoteRDMPs; + AllDashboardsNode = otherCat.AllDashboardsNode; + AllDashboards = otherCat.AllDashboards; + AllObjectSharingNode = otherCat.AllObjectSharingNode; + AllImports = otherCat.AllImports; + AllExports = otherCat.AllExports; + AllStandardRegexesNode = otherCat.AllStandardRegexesNode; + AllPipelinesNode = otherCat.AllPipelinesNode; + OtherPipelinesNode = otherCat.OtherPipelinesNode; + AllPipelines = otherCat.AllPipelines; + AllPipelineComponents = otherCat.AllPipelineComponents; + AllPipelineComponentsArguments = otherCat.AllPipelineComponentsArguments; + AllStandardRegexes = otherCat.AllStandardRegexes; + AllANOTablesNode = otherCat.AllANOTablesNode; + AllANOTables = otherCat.AllANOTables; + AllExternalServers = otherCat.AllExternalServers; + AllServers = otherCat.AllServers; + AllTableInfos = otherCat.AllTableInfos; + AllDataAccessCredentialsNode = otherCat.AllDataAccessCredentialsNode; + AllExternalServersNode = otherCat.AllExternalServersNode; + AllServersNode = otherCat.AllServersNode; + AllDataAccessCredentials = otherCat.AllDataAccessCredentials; + AllDataAccessCredentialUsages = otherCat.AllDataAccessCredentialUsages; + TableInfosToColumnInfos = otherCat.TableInfosToColumnInfos; + AllColumnInfos = otherCat.AllColumnInfos; + AllPreLoadDiscardedColumns = otherCat.AllPreLoadDiscardedColumns; + AllLookups = otherCat.AllLookups; + AllJoinInfos = otherCat.AllJoinInfos; + AllAnyTableParameters = otherCat.AllAnyTableParameters; + AllMasqueraders = otherCat.AllMasqueraders; + AllExtractionInformationsDictionary = otherCat.AllExtractionInformationsDictionary; + _pluginChildProviders = otherCat._pluginChildProviders; + AllPermissionWindowsNode = otherCat.AllPermissionWindowsNode; + LoadMetadataRootFolder = otherCat.LoadMetadataRootFolder; + CatalogueRootFolder = otherCat.CatalogueRootFolder; + CohortIdentificationConfigurationRootFolder = otherCat.CohortIdentificationConfigurationRootFolder; + AllConnectionStringKeywordsNode = otherCat.AllConnectionStringKeywordsNode; + AllConnectionStringKeywords = otherCat.AllConnectionStringKeywords; + AllAggregateContainersDictionary = otherCat.AllAggregateContainersDictionary; + AllAggregateFilters = otherCat.AllAggregateFilters; + AllAggregateFilterParameters = otherCat.AllAggregateFilterParameters; + AllCohortIdentificationConfigurations = otherCat.AllCohortIdentificationConfigurations; + AllCohortAggregateContainers = otherCat.AllCohortAggregateContainers; + AllJoinables = otherCat.AllJoinables; + AllJoinUses = otherCat.AllJoinUses; + AllGovernanceNode = otherCat.AllGovernanceNode; + AllGovernancePeriods = otherCat.AllGovernancePeriods; + AllGovernanceDocuments = otherCat.AllGovernanceDocuments; + GovernanceCoverage = otherCat.GovernanceCoverage; + AllJoinableCohortAggregateConfigurationUse = otherCat.AllJoinableCohortAggregateConfigurationUse; + AllPluginsNode = otherCat.AllPluginsNode; + PipelineUseCases = otherCat.PipelineUseCases; + OrphanAggregateConfigurationsNode = otherCat.OrphanAggregateConfigurationsNode; + TemplateAggregateConfigurationsNode = otherCat.TemplateAggregateConfigurationsNode; + AllCatalogueParameters = otherCat.AllCatalogueParameters; + AllCatalogueValueSets = otherCat.AllCatalogueValueSets; + AllCatalogueValueSetValues = otherCat.AllCatalogueValueSetValues; + OrphanAggregateConfigurations = otherCat.OrphanAggregateConfigurations; + } + + public virtual bool SelectiveRefresh(IMapsDirectlyToDatabaseTable databaseEntity) + { + ProgressStopwatch.Restart(); + + return databaseEntity switch + { + AggregateFilterParameter afp => SelectiveRefresh(afp.AggregateFilter), + AggregateFilter af => SelectiveRefresh(af), + AggregateFilterContainer afc => SelectiveRefresh(afc), + CohortAggregateContainer cac => SelectiveRefresh(cac), + ExtractionInformation ei => SelectiveRefresh(ei), + CatalogueItem ci => SelectiveRefresh(ci), + _ => false + }; + } + + + public bool SelectiveRefresh(CatalogueItem ci) + { + var descendancy = GetDescendancyListIfAnyFor(ci.Catalogue); + if (descendancy == null) return false; + + FetchCatalogueItems(); + FetchExtractionInformations(); + AddChildren(ci.Catalogue, descendancy.Add(ci.Catalogue)); + return true; + } + + public bool SelectiveRefresh(ExtractionInformation ei) + { + var descendancy = GetDescendancyListIfAnyFor(ei); + var cata = descendancy?.Parents.OfType().LastOrDefault() ?? ei.CatalogueItem.Catalogue; + + if (cata == null) + return false; + + var cataDescendancy = GetDescendancyListIfAnyFor(cata); + + if (cataDescendancy == null) + return false; + + FetchCatalogueItems(); + + foreach (var ci in AllCatalogueItems.Where(ci => ci.ID == ei.CatalogueItem_ID)) ci.ClearAllInjections(); + + // property changes or deleting the ExtractionInformation + FetchExtractionInformations(); + + // refresh the Catalogue + AddChildren(cata, cataDescendancy.Add(cata)); + return true; + } + + public bool SelectiveRefresh(CohortAggregateContainer container) + { + var parentContainer = container.GetParentContainerIfAny(); + if (parentContainer != null) + { + var descendancy = GetDescendancyListIfAnyFor(parentContainer); + + if (descendancy != null) + { + BuildAggregateConfigurations(); + + BuildCohortCohortAggregateContainers(); + AddChildren(parentContainer, descendancy.Add(parentContainer)); + return true; + } + } + + var cic = container.GetCohortIdentificationConfiguration(); + + if (cic != null) + { + var descendancy = GetDescendancyListIfAnyFor(cic); + + if (descendancy != null) + { + BuildAggregateConfigurations(); + BuildCohortCohortAggregateContainers(); + AddChildren(cic, descendancy.Add(cic)); + return true; + } + } + + return false; + } + + public bool SelectiveRefresh(AggregateFilter f) + { + var knownContainer = GetDescendancyListIfAnyFor(f.FilterContainer); + if (knownContainer == null) return false; + + BuildAggregateFilterContainers(); + AddChildren((AggregateFilterContainer)f.FilterContainer, knownContainer.Add(f.FilterContainer)); + return true; + } - public bool SelectiveRefresh(AggregateFilterContainer container) - { - var aggregate = container.GetAggregate(); + public bool SelectiveRefresh(AggregateFilterContainer container) + { + var aggregate = container.GetAggregate(); - if (aggregate == null) return false; + if (aggregate == null) return false; - var descendancy = GetDescendancyListIfAnyFor(aggregate); + var descendancy = GetDescendancyListIfAnyFor(aggregate); - if (descendancy == null) return false; + if (descendancy == null) return false; - // update just in case we became a root filter for someone - aggregate.RevertToDatabaseState(); + // update just in case we became a root filter for someone + aggregate.RevertToDatabaseState(); - BuildAggregateFilterContainers(); + BuildAggregateFilterContainers(); - AddChildren(aggregate, descendancy.Add(aggregate)); - return true; - } + AddChildren(aggregate, descendancy.Add(aggregate)); + return true; + } } \ No newline at end of file