From ba990553e9e7fd7dbed874122dd879b06ccdf3cd Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 14 Aug 2024 11:03:12 +0100 Subject: [PATCH 001/106] interim --- ...ommandAddNewRegexRedactionConfiguration.cs | 44 +++++++ ...mmandIdentifyRegexRedactionsInCatalogue.cs | 77 ++++++++++++ ...CommandPerformRegexRedactionOnCatalogue.cs | 112 ++++++++++++++++++ .../IRegexRedactionConfiguration.cs | 20 ++++ .../RegexRedaction/RegexRedaction.cs | 12 ++ .../RegexRedactionConfiguration.cs | 72 +++++++++++ .../up/086_AddRegexRedactionConfiguration.sql | 19 +++ Rdmp.Core/Rdmp.Core.csproj | 2 + .../ExecuteCommandRegexRedaction.cs | 34 ++++++ Rdmp.UI/Menus/CatalogueMenu.cs | 8 +- SharedAssemblyInfo.cs | 6 +- 11 files changed, 400 insertions(+), 6 deletions(-) create mode 100644 Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewRegexRedactionConfiguration.cs create mode 100644 Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandIdentifyRegexRedactionsInCatalogue.cs create mode 100644 Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs create mode 100644 Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedactionConfiguration.cs create mode 100644 Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedaction.cs create mode 100644 Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionConfiguration.cs create mode 100644 Rdmp.Core/Databases/CatalogueDatabase/up/086_AddRegexRedactionConfiguration.sql create mode 100644 Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandRegexRedaction.cs diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewRegexRedactionConfiguration.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewRegexRedactionConfiguration.cs new file mode 100644 index 0000000000..0e285edc90 --- /dev/null +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewRegexRedactionConfiguration.cs @@ -0,0 +1,44 @@ +using Rdmp.Core.Icons.IconProvision; +using Rdmp.Core.ReusableLibraryCode.Icons.IconProvision; +using SixLabors.ImageSharp.PixelFormats; +using SixLabors.ImageSharp; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Rdmp.Core.Curation.DataHelper.RegexRedaction; +using System.Text.RegularExpressions; +using Rdmp.Core.Curation.Data; + +namespace Rdmp.Core.CommandExecution.AtomicCommands; + +public class ExecuteCommandAddNewRegexRedactionConfiguration : BasicCommandExecution, IAtomicCommand +{ + + private string _name; + private string _redactionPattern; + private string _redactionString; + private string _description; + private IBasicActivateItems _activator; + + + public ExecuteCommandAddNewRegexRedactionConfiguration(IBasicActivateItems activator, [DemandsInitialization("Name")]string name, [DemandsInitialization("pattern")] string redactionPattern, [DemandsInitialization("redaction")] string redactionString, string description=null) : base(activator) + { + _activator = activator; + _name = name; + _redactionPattern = redactionPattern; + _redactionString = redactionString; + _description = description; + } + + public override void Execute() + { + base.Execute(); + var config = new RegexRedactionConfiguration(_activator.RepositoryLocator.CatalogueRepository, _name, new Regex(_redactionPattern), _redactionString,_description); + config.SaveToDatabase(); + } + + public override Image GetImage(IIconProvider iconProvider) => + iconProvider.GetImage(RDMPConcept.StandardRegex, OverlayKind.Add); +} diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandIdentifyRegexRedactionsInCatalogue.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandIdentifyRegexRedactionsInCatalogue.cs new file mode 100644 index 0000000000..c44aa8a3b6 --- /dev/null +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandIdentifyRegexRedactionsInCatalogue.cs @@ -0,0 +1,77 @@ +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.DataHelper.RegexRedaction; +using Rdmp.Core.QueryBuilding; +using Rdmp.Core.Repositories; +using Rdmp.Core.ReusableLibraryCode.DataAccess; +using System; +using System.Collections.Generic; +using System.Data; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; + +namespace Rdmp.Core.CommandExecution.AtomicCommands; + +public class ExecuteCommandIdentifyRegexRedactionsInCatalogue : BasicCommandExecution, IAtomicCommand +{ + private readonly IBasicActivateItems _activator; + private readonly ICatalogue _catalogue; + private readonly RegexRedactionConfiguration _redactionConfiguration; + private readonly List _columns; + + //public ExecuteCommandIdentifyRegexRedactionsInCatalogue(IBasicActivateItems activator, ICatalogue catalogue, RegexRedactionConfiguration redactionConfiguration, List columns=null) + //{ + // _activator = activator; + // _catalogue = catalogue; + // _redactionConfiguration = redactionConfiguration; + // _columns = columns; + //} + + + public ExecuteCommandIdentifyRegexRedactionsInCatalogue(IBasicActivateItems activator) + { + _activator = activator; + _catalogue = _activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Name", "Biochemistry").First(); + _redactionConfiguration = _activator.RepositoryLocator.CatalogueRepository.GetAllObjects().First(); + _columns = _catalogue.CatalogueItems.Select(ci => ci.ColumnInfo).ToList(); + + } + + public override void Execute() + { + base.Execute(); + var memoryRepo = new MemoryCatalogueRepository(); + foreach (var catalogueItem in _catalogue.CatalogueItems.Where(ci => !ci.ColumnInfo.IsPrimaryKey && _columns.Contains(ci.ColumnInfo))) + { + var columnName = catalogueItem.ColumnInfo.Name; + var table = catalogueItem.ColumnInfo.TableInfo.Name; + var server = _catalogue.GetDistinctLiveDatabaseServer(DataAccessContext.InternalDataProcessing, false); + var pkColumns = _catalogue.CatalogueItems.Select(x => x.ColumnInfo).Where(x => x.IsPrimaryKey); + if (pkColumns.Where(pkc => pkc.Name.Contains(table)).Any()) + { + var qb = new QueryBuilder(null, null, null); + qb.AddColumn(new ColumnInfoToIColumn(memoryRepo, catalogueItem.ColumnInfo)); + qb.AddColumnRange(pkColumns.Select(pk => new ColumnInfoToIColumn(memoryRepo, pk)).ToArray()); + var sql = qb.SQL; + var dt = new DataTable(); + dt.BeginLoadData(); + using (var cmd = server.GetCommand(sql, server.GetConnection())) + { + using var da = server.GetDataAdapter(cmd); + da.Fill(dt); + } + DataTable _regexMatches = new(); + _regexMatches.BeginLoadData(); + _regexMatches = dt.AsEnumerable().Where(row => Regex.IsMatch(row[catalogueItem.ColumnInfo.GetRuntimeName()].ToString(), _redactionConfiguration.RegexPattern)).CopyToDataTable(); + _regexMatches.EndLoadData(); + dt.EndLoadData(); + } + else + { + //Unable to find any primary keys + throw new Exception($"Unable to identify any primary keys in table '{table}'. Redactions cannot be performed on tables without primary keys"); + } + } + } +} diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs new file mode 100644 index 0000000000..2d75c5610b --- /dev/null +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs @@ -0,0 +1,112 @@ +using FAnsi; +using FAnsi.Discovery; +using FAnsi.Discovery.QuerySyntax; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.DataHelper.RegexRedaction; +using Rdmp.Core.QueryBuilding; +using Rdmp.Core.Repositories; +using Rdmp.Core.ReusableLibraryCode.DataAccess; +using System; +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; +using System.Data; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; + +namespace Rdmp.Core.CommandExecution.AtomicCommands; + +public class ExecuteCommandPerformRegexRedactionOnCatalogue : BasicCommandExecution, IAtomicCommand +{ + private readonly IBasicActivateItems _activator; + private readonly ICatalogue _catalogue; + private readonly RegexRedactionConfiguration _redactionConfiguration; + private readonly List _columns; + private readonly DiscoveredServer _server; + + + + //public ExecuteCommandPerformRegexRedactionOnCatalogue(IBasicActivateItems activator, ICatalogue catalogue, RegexRedactionConfiguration redactionConfiguration, List columns) { + // _activator = activator; + // _catalogue = catalogue; + // _redactionConfiguration = redactionConfiguration; + // _columns = columns; + //} + + public ExecuteCommandPerformRegexRedactionOnCatalogue(IBasicActivateItems activator) + { + _activator = activator; + _catalogue = _activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Name", "Biochemistry").First(); + _redactionConfiguration = _activator.RepositoryLocator.CatalogueRepository.GetAllObjects().First(); + _columns = _catalogue.CatalogueItems.Select(ci => ci.ColumnInfo).ToList(); + _server = _catalogue.GetDistinctLiveDatabaseServer(DataAccessContext.InternalDataProcessing, false); + } + + private void Redact(DiscoveredTable table, ColumnInfo column, List PKColumns, DataRow match, DataColumnCollection columns) + { + var updateHelper = _server.GetQuerySyntaxHelper().UpdateHelper; + var typeTranslator = _server.GetQuerySyntaxHelper().TypeTranslater; + var redactedValue = "TODO"; + + var sqlLines = new List + { + new CustomLine($"t1.{column.GetRuntimeName()} = '{redactedValue}'", QueryComponent.SET) + }; + foreach (var pk in PKColumns) + { + //ColumnInfo + var matchValue = $"'{match[pk.GetRuntimeName()]}'"; + //doesn;t work for datePKS + + sqlLines.Add(new CustomLine($"t1.{pk.GetRuntimeName()} = {matchValue}", QueryComponent.WHERE)); + sqlLines.Add(new CustomLine(string.Format("t1.{0} = t2.{0}", pk.GetRuntimeName()), QueryComponent.JoinInfoJoin)); + + } + var sql = updateHelper.BuildUpdate(table, table, sqlLines); + Console.WriteLine(sql); + } + + public override void Execute() + { + base.Execute(); + var memoryRepo = new MemoryCatalogueRepository(); + foreach (var catalogueItem in _catalogue.CatalogueItems.Where(ci => !ci.ColumnInfo.IsPrimaryKey && _columns.Contains(ci.ColumnInfo))) + { + var columnName = catalogueItem.ColumnInfo.Name; + var table = catalogueItem.ColumnInfo.TableInfo.Name; + DiscoveredDatabase discoveredDb = _server.ExpectDatabase(catalogueItem.ColumnInfo.TableInfo.Database); + DiscoveredTable discoveredTable = discoveredDb.ExpectTable(catalogueItem.ColumnInfo.TableInfo.GetRuntimeName()); + var pkColumns = _catalogue.CatalogueItems.Select(x => x.ColumnInfo).Where(x => x.IsPrimaryKey).ToList(); + if (pkColumns.Where(pkc => pkc.Name.Contains(table)).Any()) + { + var qb = new QueryBuilder(null, null, null); + qb.AddColumn(new ColumnInfoToIColumn(memoryRepo, catalogueItem.ColumnInfo)); + qb.AddColumnRange(pkColumns.Select(pk => new ColumnInfoToIColumn(memoryRepo, pk)).ToArray()); + var sql = qb.SQL; + var dt = new DataTable(); + dt.BeginLoadData(); + using (var cmd = _server.GetCommand(sql, _server.GetConnection())) + { + using var da = _server.GetDataAdapter(cmd); + da.Fill(dt); + } + DataTable _regexMatches = new(); + _regexMatches.BeginLoadData(); + _regexMatches = dt.AsEnumerable().Where(row => Regex.IsMatch(row[catalogueItem.ColumnInfo.GetRuntimeName()].ToString(), _redactionConfiguration.RegexPattern)).CopyToDataTable(); + _regexMatches.EndLoadData(); + dt.EndLoadData(); + + foreach (DataRow row in _regexMatches.Rows) + { + Redact(discoveredTable, catalogueItem.ColumnInfo, pkColumns, row, _regexMatches.Columns); + } + } + else + { + //Unable to find any primary keys + throw new Exception($"Unable to identify any primary keys in table '{table}'. Redactions cannot be performed on tables without primary keys"); + } + } + } +} diff --git a/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedactionConfiguration.cs b/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedactionConfiguration.cs new file mode 100644 index 0000000000..d5d6fe0124 --- /dev/null +++ b/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedactionConfiguration.cs @@ -0,0 +1,20 @@ +using Rdmp.Core.MapsDirectlyToDatabaseTable; +using Rdmp.Core.Repositories; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.DataHelper.RegexRedaction +{ + public interface IRegexRedactionConfiguration: IMapsDirectlyToDatabaseTable + { + ICatalogueRepository CatalogueRepository { get; } + + string Name { get; } + string Description { get; } + string RegexPattern { get; } + string RedactionString { get; } + } +} diff --git a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedaction.cs b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedaction.cs new file mode 100644 index 0000000000..28ab02b46b --- /dev/null +++ b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedaction.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.DataHelper.RegexRedaction +{ + internal class RegexRedaction + { + } +} diff --git a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionConfiguration.cs b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionConfiguration.cs new file mode 100644 index 0000000000..832a4ab962 --- /dev/null +++ b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionConfiguration.cs @@ -0,0 +1,72 @@ +using Rdmp.Core.Curation.Data; +using Rdmp.Core.MapsDirectlyToDatabaseTable.Attributes; +using System.Text.RegularExpressions; +using Rdmp.Core.ReusableLibraryCode.Annotations; +using Rdmp.Core.Repositories; +using System.Collections.Generic; +using System.Data.Common; +using Rdmp.Core.MapsDirectlyToDatabaseTable; + +namespace Rdmp.Core.Curation.DataHelper.RegexRedaction; + +public class RegexRedactionConfiguration: DatabaseEntity, IRegexRedactionConfiguration +{ + + + private string _regexPattern; + private string _redactionString; + private string _name; + private string _description; + + [Unique] + [NotNull] + public string Name + { + get => _name; + set => SetField(ref _name, value); + } + + public string Description + { + get => _description; + set => SetField(ref _description, value); + } + + [NotNull] + public string RegexPattern + { + get => _regexPattern; + set => SetField(ref _regexPattern, value.ToString()); + } + + [NotNull] + public string RedactionString + { + get => _redactionString; + set => SetField(ref _redactionString, value); + } + + + public RegexRedactionConfiguration() { } + + public RegexRedactionConfiguration(ICatalogueRepository repository,string name, Regex regexPattern, string redactionString,string description=null) + { + repository.InsertAndHydrate(this, new Dictionary + { + { "Name", name }, + { "RegexPattern", regexPattern.ToString() }, + {"RedactionString", redactionString }, + {"Description",description } + }); + } + + internal RegexRedactionConfiguration(ICatalogueRepository repository, DbDataReader r): base(repository,r) + { + Name = r["Name"].ToString(); + Description = r["Description"].ToString(); + RedactionString= r["RedactionString"].ToString(); + RegexPattern = r["RegexPattern"].ToString(); + } + + +} diff --git a/Rdmp.Core/Databases/CatalogueDatabase/up/086_AddRegexRedactionConfiguration.sql b/Rdmp.Core/Databases/CatalogueDatabase/up/086_AddRegexRedactionConfiguration.sql new file mode 100644 index 0000000000..8750571157 --- /dev/null +++ b/Rdmp.Core/Databases/CatalogueDatabase/up/086_AddRegexRedactionConfiguration.sql @@ -0,0 +1,19 @@ +--Version: 8.2.4 +--Description: Add configuration for regex redaction + + +if not exists (select 1 from sys.tables where name = 'RegexRedactionConfiguration') +BEGIN +CREATE TABLE [dbo].RegexRedactionConfiguration( + [ID] [int] IDENTITY(1,1) NOT NULL, + Name [nvarchar](250) NOT NULL, + Description [nvarchar](250), + RegexPattern [nvarchar](250) NOT NULL, + RedactionString [nvarchar](250) NOT NULL, +CONSTRAINT [PK_RegexRedactionConfiguration] PRIMARY KEY CLUSTERED +( + [ID] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +END +GO diff --git a/Rdmp.Core/Rdmp.Core.csproj b/Rdmp.Core/Rdmp.Core.csproj index c7c2c97d2c..fcbb11b2e3 100644 --- a/Rdmp.Core/Rdmp.Core.csproj +++ b/Rdmp.Core/Rdmp.Core.csproj @@ -128,6 +128,7 @@ + @@ -253,6 +254,7 @@ + diff --git a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandRegexRedaction.cs b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandRegexRedaction.cs new file mode 100644 index 0000000000..937e853771 --- /dev/null +++ b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandRegexRedaction.cs @@ -0,0 +1,34 @@ +using Rdmp.Core.CommandExecution.AtomicCommands; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Icons.IconProvision; +using Rdmp.Core.ReusableLibraryCode.Icons.IconProvision; +using Rdmp.UI.ExtractionUIs; +using Rdmp.UI.ItemActivation; +using SixLabors.ImageSharp.PixelFormats; +using SixLabors.ImageSharp; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.UI.CommandExecution.AtomicCommands; + +public class ExecuteCommandRegexRedaction: BasicUICommandExecution, IAtomicCommand +{ + private readonly Catalogue _catalogue; + + public ExecuteCommandRegexRedaction(IActivateItems activator, Catalogue catalogue) : base(activator) + { + _catalogue = catalogue; + } + + public override Image GetImage(IIconProvider iconProvider) => iconProvider.GetImage(RDMPConcept.StandardRegex); + + public override void Execute() + { + base.Execute(); + + //Activator.Activate(_catalogue); + } +} diff --git a/Rdmp.UI/Menus/CatalogueMenu.cs b/Rdmp.UI/Menus/CatalogueMenu.cs index 2322bb95c3..e08d71dd8e 100644 --- a/Rdmp.UI/Menus/CatalogueMenu.cs +++ b/Rdmp.UI/Menus/CatalogueMenu.cs @@ -4,15 +4,15 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . -using System; -using System.Linq; -using System.Windows.Forms; using Rdmp.Core.CommandExecution; using Rdmp.Core.CommandExecution.AtomicCommands; using Rdmp.Core.CommandExecution.AtomicCommands.Sharing; using Rdmp.Core.Curation.Data; using Rdmp.UI.CommandExecution.AtomicCommands; using Rdmp.UI.Menus.MenuItems; +using System; +using System.Linq; +using System.Windows.Forms; namespace Rdmp.UI.Menus; @@ -73,6 +73,8 @@ public CatalogueMenu(RDMPContextMenuStripArgs args, Catalogue catalogue) : base( { SuggestedCategory = CatalogueItems, Weight = -99.047f }); Add(new ExecuteCommandReOrderColumns(_activator, catalogue) { SuggestedCategory = CatalogueItems, Weight = -99.046f }); + Add(new ExecuteCommandRegexRedaction(_activator, catalogue) + { SuggestedCategory = CatalogueItems, Weight = -99.046f, OverrideCommandName="Perform Regex Redaction" }); Add(new ExecuteCommandGuessAssociatedColumns(_activator, catalogue, null) { SuggestedCategory = CatalogueItems, Weight = -99.045f, PromptForPartialMatching = true }); Add(new ExecuteCommandChangeExtractionCategory(_activator, diff --git a/SharedAssemblyInfo.cs b/SharedAssemblyInfo.cs index c13a1f52e1..c9960b0bb3 100644 --- a/SharedAssemblyInfo.cs +++ b/SharedAssemblyInfo.cs @@ -10,6 +10,6 @@ [assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] -[assembly: AssemblyVersion("8.2.3")] -[assembly: AssemblyFileVersion("8.2.3")] -[assembly: AssemblyInformationalVersion("8.2.3")] \ No newline at end of file +[assembly: AssemblyVersion("8.2.4")] +[assembly: AssemblyFileVersion("8.2.4")] +[assembly: AssemblyInformationalVersion("8.2.4")] \ No newline at end of file From f0509d7139bc3d83d61b920f6bd7fcd98c41efdf Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 15 Aug 2024 08:12:44 +0100 Subject: [PATCH 002/106] interim --- .../ExecuteCommandPerformRegexRedactionOnCatalogue.cs | 6 ++++++ Tools/rdmp/Databases.yaml | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs index 2d75c5610b..4f0346511e 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs @@ -57,6 +57,12 @@ private void Redact(DiscoveredTable table, ColumnInfo column, List P { //ColumnInfo var matchValue = $"'{match[pk.GetRuntimeName()]}'"; + var dataType = pk.Data_type; + //if(dataType == "datetime2") + //{ + // var date = new DateTime(match[pk.GetRuntimeName()].ToString()); + //} + //doesn;t work for datePKS sqlLines.Add(new CustomLine($"t1.{pk.GetRuntimeName()} = {matchValue}", QueryComponent.WHERE)); diff --git a/Tools/rdmp/Databases.yaml b/Tools/rdmp/Databases.yaml index 434fc12411..c688f62761 100644 --- a/Tools/rdmp/Databases.yaml +++ b/Tools/rdmp/Databases.yaml @@ -1,2 +1,2 @@ -CatalogueConnectionString: Server=(localdb)\MSSQLLocalDB;Database=TEST_Catalogue;Trusted_Connection=True;TrustServerCertificate=true; -DataExportConnectionString: Server=(localdb)\MSSQLLocalDB;Database=TEST_DataExport;Trusted_Connection=True;TrustServerCertificate=true; +CatalogueConnectionString: Server=(localdb)\MSSQLLocalDB;Database=RDMP_Catalogue;Trusted_Connection=True;TrustServerCertificate=true; +DataExportConnectionString: Server=(localdb)\MSSQLLocalDB;Database=RDMP_DataExport;Trusted_Connection=True;TrustServerCertificate=true; From 2cea9027825ca1ba4977f4c38bba0c1854057958 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 19 Aug 2024 16:03:36 +0100 Subject: [PATCH 003/106] redact and add some tests --- ...ndPerformRegexRedactionOnCatalogueTests.cs | 320 ++++++++++++++++++ ...CommandPerformRegexRedactionOnCatalogue.cs | 111 ++++-- .../RegexRedaction/RegexRedaction.cs | 60 +++- .../up/086_AddRegexRedactionConfiguration.sql | 19 ++ 4 files changed, 475 insertions(+), 35 deletions(-) create mode 100644 Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs new file mode 100644 index 0000000000..47d46df03f --- /dev/null +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs @@ -0,0 +1,320 @@ +using FAnsi; +using NUnit.Framework; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Tests.Common; +using Rdmp.Core.CommandExecution.AtomicCommands; +using Rdmp.Core.Curation.DataHelper.RegexRedaction; +using System.Text.RegularExpressions; +using Rdmp.Core.CommandLine.Interactive; +using Rdmp.Core.ReusableLibraryCode.Checks; +using MongoDB.Driver.Core.Servers; +using System.Data; + +namespace Rdmp.Core.Tests.CommandExecution +{ + public class ExecuteCommandPerformRegexRedactionOnCatalogueTests : DatabaseTests + { + + [Test] + public void Redaction_BasicRedactionTest() + { + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + const string sql = @" + CREATE TABLE [RedactionTest]( + [DischargeDate] [datetime] NOT NULL, + [Condition1] [varchar](400) NOT NULL, + [Condition2] [varchar](400) NOT NULL + CONSTRAINT [PK_DLCTest] PRIMARY KEY CLUSTERED + ( + [DischargeDate] ASC, + [Condition1] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] + ) ON [PRIMARY] + INSERT [RedactionTest]([DischargeDate],[Condition1],[Condition2]) + VALUES (CAST(0x000001B300000000 AS DateTime),N'1',N'1234TEST1234') + "; + var server = db.Server; + using (var con = server.GetConnection()) + { + con.Open(); + server.GetCommand(sql, con).ExecuteNonQuery(); + } + var table = db.ExpectTable("RedactionTest"); + var catalogue = new Catalogue(CatalogueRepository, "RedactionTest"); + var importer = new TableInfoImporter(CatalogueRepository, table); + importer.DoImport(out var _tableInfo, out var _columnInfos); + foreach (var columnInfo in _columnInfos) + { + var ci = new CatalogueItem(CatalogueRepository, catalogue, columnInfo.GetRuntimeName()); + ci.SaveToDatabase(); + var ei = new ExtractionInformation(CatalogueRepository, ci, columnInfo, ""); + ei.SaveToDatabase(); + } + var activator = new ConsoleInputManager(RepositoryLocator, ThrowImmediatelyCheckNotifier.Quiet); + var regexConfiguration = new RegexRedactionConfiguration(CatalogueRepository, "TestReplacer", new Regex("TEST"), "GG", "Replace TEST with GG"); + regexConfiguration.SaveToDatabase(); + var cmd = new ExecuteCommandPerformRegexRedactionOnCatalogue(activator, catalogue, regexConfiguration, _columnInfos.Where(c => c.GetRuntimeName() == "Condition2").ToList()); + Assert.DoesNotThrow(() => cmd.Execute()); + var retrieveSQL = @"select [Condition2] from [RedactionTest]"; + var dt = new DataTable(); + dt.BeginLoadData(); + using (var fetchCmd = server.GetCommand(retrieveSQL, server.GetConnection())) + { + using var da = server.GetDataAdapter(fetchCmd); + da.Fill(dt); + } + Assert.That(dt.Rows.Count, Is.EqualTo(1)); + Assert.That(dt.Rows[0].ItemArray[0], Is.EqualTo("12341234")); + } + + + [Test] + public void Redaction_OddStringLength() + { + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + const string sql = @" + CREATE TABLE [RedactionTest]( + [DischargeDate] [datetime] NOT NULL, + [Condition1] [varchar](400) NOT NULL, + [Condition2] [varchar](400) NOT NULL + CONSTRAINT [PK_DLCTest] PRIMARY KEY CLUSTERED + ( + [DischargeDate] ASC, + [Condition1] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] + ) ON [PRIMARY] + INSERT [RedactionTest]([DischargeDate],[Condition1],[Condition2]) + VALUES (CAST(0x000001B300000000 AS DateTime),N'1',N'1234TESTT1234') + "; + var server = db.Server; + using (var con = server.GetConnection()) + { + con.Open(); + server.GetCommand(sql, con).ExecuteNonQuery(); + } + var table = db.ExpectTable("RedactionTest"); + var catalogue = new Catalogue(CatalogueRepository, "RedactionTest"); + var importer = new TableInfoImporter(CatalogueRepository, table); + importer.DoImport(out var _tableInfo, out var _columnInfos); + foreach (var columnInfo in _columnInfos) + { + var ci = new CatalogueItem(CatalogueRepository, catalogue, columnInfo.GetRuntimeName()); + ci.SaveToDatabase(); + var ei = new ExtractionInformation(CatalogueRepository, ci, columnInfo, ""); + ei.SaveToDatabase(); + } + var activator = new ConsoleInputManager(RepositoryLocator, ThrowImmediatelyCheckNotifier.Quiet); + var regexConfiguration = new RegexRedactionConfiguration(CatalogueRepository, "TestReplacer", new Regex("TESTT"), "GG", "Replace TEST with GG"); + regexConfiguration.SaveToDatabase(); + var cmd = new ExecuteCommandPerformRegexRedactionOnCatalogue(activator, catalogue, regexConfiguration, _columnInfos.Where(c => c.GetRuntimeName() == "Condition2").ToList()); + Assert.DoesNotThrow(() => cmd.Execute()); + var retrieveSQL = @"select [Condition2] from [RedactionTest]"; + var dt = new DataTable(); + dt.BeginLoadData(); + using (var fetchCmd = server.GetCommand(retrieveSQL, server.GetConnection())) + { + using var da = server.GetDataAdapter(fetchCmd); + da.Fill(dt); + } + Assert.That(dt.Rows.Count, Is.EqualTo(1)); + Assert.That(dt.Rows[0].ItemArray[0], Is.EqualTo("1234>1234")); + } + + [Test] + public void Redaction_RedactionTooLong() + { + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + const string sql = @" + CREATE TABLE [RedactionTest]( + [DischargeDate] [datetime] NOT NULL, + [Condition1] [varchar](400) NOT NULL, + [Condition2] [varchar](400) NOT NULL + CONSTRAINT [PK_DLCTest] PRIMARY KEY CLUSTERED + ( + [DischargeDate] ASC, + [Condition1] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] + ) ON [PRIMARY] + INSERT [RedactionTest]([DischargeDate],[Condition1],[Condition2]) + VALUES (CAST(0x000001B300000000 AS DateTime),N'1',N'1234TESTT1234') + "; + var server = db.Server; + using (var con = server.GetConnection()) + { + con.Open(); + server.GetCommand(sql, con).ExecuteNonQuery(); + } + var table = db.ExpectTable("RedactionTest"); + var catalogue = new Catalogue(CatalogueRepository, "RedactionTest"); + var importer = new TableInfoImporter(CatalogueRepository, table); + importer.DoImport(out var _tableInfo, out var _columnInfos); + foreach (var columnInfo in _columnInfos) + { + var ci = new CatalogueItem(CatalogueRepository, catalogue, columnInfo.GetRuntimeName()); + ci.SaveToDatabase(); + var ei = new ExtractionInformation(CatalogueRepository, ci, columnInfo, ""); + ei.SaveToDatabase(); + } + var activator = new ConsoleInputManager(RepositoryLocator, ThrowImmediatelyCheckNotifier.Quiet); + var regexConfiguration = new RegexRedactionConfiguration(CatalogueRepository, "TestReplacer", new Regex("TEST"), "FARTOOLONG", "Replace TEST with GG"); + regexConfiguration.SaveToDatabase(); + var cmd = new ExecuteCommandPerformRegexRedactionOnCatalogue(activator, catalogue, regexConfiguration, _columnInfos.Where(c => c.GetRuntimeName() == "Condition2").ToList()); + var ex = Assert.Throws(() => cmd.Execute()); + Assert.That(ex.Message, Is.EqualTo("Redaction string 'FARTOOLONG' is longer than found match 'TEST'.")); + } + + [Test] + public void Redaction_RedactAPK() + { + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + const string sql = @" + CREATE TABLE [RedactionTest]( + [DischargeDate] [datetime] NOT NULL, + [Condition1] [varchar](400) NOT NULL, + [Condition2] [varchar](400) NOT NULL + CONSTRAINT [PK_DLCTest] PRIMARY KEY CLUSTERED + ( + [DischargeDate] ASC, + [Condition1] ASC, + [Condition2] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] + ) ON [PRIMARY] + INSERT [RedactionTest]([DischargeDate],[Condition1],[Condition2]) + VALUES (CAST(0x000001B300000000 AS DateTime),N'1',N'1234TEST1234') + "; + var server = db.Server; + using (var con = server.GetConnection()) + { + con.Open(); + server.GetCommand(sql, con).ExecuteNonQuery(); + } + var table = db.ExpectTable("RedactionTest"); + var catalogue = new Catalogue(CatalogueRepository, "RedactionTest"); + var importer = new TableInfoImporter(CatalogueRepository, table); + importer.DoImport(out var _tableInfo, out var _columnInfos); + foreach (var columnInfo in _columnInfos) + { + var ci = new CatalogueItem(CatalogueRepository, catalogue, columnInfo.GetRuntimeName()); + ci.SaveToDatabase(); + var ei = new ExtractionInformation(CatalogueRepository, ci, columnInfo, ""); + ei.SaveToDatabase(); + } + var activator = new ConsoleInputManager(RepositoryLocator, ThrowImmediatelyCheckNotifier.Quiet); + var regexConfiguration = new RegexRedactionConfiguration(CatalogueRepository, "TestReplacer", new Regex("TEST"), "GG", "Replace TEST with GG"); + regexConfiguration.SaveToDatabase(); + var cmd = new ExecuteCommandPerformRegexRedactionOnCatalogue(activator, catalogue, regexConfiguration, _columnInfos.Where(c => c.GetRuntimeName() == "Condition2").ToList()); + Assert.DoesNotThrow(() => cmd.Execute()); + var retrieveSQL = @"select [Condition2] from [RedactionTest]"; + var dt = new DataTable(); + dt.BeginLoadData(); + using (var fetchCmd = server.GetCommand(retrieveSQL, server.GetConnection())) + { + using var da = server.GetDataAdapter(fetchCmd); + da.Fill(dt); + } + Assert.That(dt.Rows.Count, Is.EqualTo(1)); + Assert.That(dt.Rows[0].ItemArray[0], Is.EqualTo("1234TEST1234")); + } + + [Test] + public void Redaction_NoPKS() + { + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + const string sql = @" + CREATE TABLE [RedactionTest]( + [DischargeDate] [datetime] NOT NULL, + [Condition1] [varchar](400) NOT NULL, + [Condition2] [varchar](400) NOT NULL + ) + INSERT [RedactionTest]([DischargeDate],[Condition1],[Condition2]) + VALUES (CAST(0x000001B300000000 AS DateTime),N'1',N'1234TEST1234') + "; + var server = db.Server; + using (var con = server.GetConnection()) + { + con.Open(); + server.GetCommand(sql, con).ExecuteNonQuery(); + } + var table = db.ExpectTable("RedactionTest"); + var catalogue = new Catalogue(CatalogueRepository, "RedactionTest"); + var importer = new TableInfoImporter(CatalogueRepository, table); + importer.DoImport(out var _tableInfo, out var _columnInfos); + foreach (var columnInfo in _columnInfos) + { + var ci = new CatalogueItem(CatalogueRepository, catalogue, columnInfo.GetRuntimeName()); + ci.SaveToDatabase(); + var ei = new ExtractionInformation(CatalogueRepository, ci, columnInfo, ""); + ei.SaveToDatabase(); + } + var activator = new ConsoleInputManager(RepositoryLocator, ThrowImmediatelyCheckNotifier.Quiet); + var regexConfiguration = new RegexRedactionConfiguration(CatalogueRepository, "TestReplacer", new Regex("TEST"), "GG", "Replace TEST with GG"); + regexConfiguration.SaveToDatabase(); + var cmd = new ExecuteCommandPerformRegexRedactionOnCatalogue(activator, catalogue, regexConfiguration, _columnInfos.Where(c => c.GetRuntimeName() == "Condition2").ToList()); + var ex = Assert.Throws(() => cmd.Execute()); + Assert.That(ex.Message, Is.EqualTo("Unable to identify any primary keys in table '[TEST_ScratchArea].[dbo].[RedactionTest]'. Redactions cannot be performed on tables without primary keys")); + + } + + [Test] + public void Redaction_MultipleInOneCell() + { + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + const string sql = @" + CREATE TABLE [RedactionTest]( + [DischargeDate] [datetime] NOT NULL, + [Condition1] [varchar](400) NOT NULL, + [Condition2] [varchar](400) NOT NULL + CONSTRAINT [PK_DLCTest] PRIMARY KEY CLUSTERED + ( + [DischargeDate] ASC, + [Condition1] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] + ) ON [PRIMARY] + INSERT [RedactionTest]([DischargeDate],[Condition1],[Condition2]) + VALUES (CAST(0x000001B300000000 AS DateTime),N'1',N'1234TEST1234TEST1234') + "; + var server = db.Server; + using (var con = server.GetConnection()) + { + con.Open(); + server.GetCommand(sql, con).ExecuteNonQuery(); + } + var table = db.ExpectTable("RedactionTest"); + var catalogue = new Catalogue(CatalogueRepository, "RedactionTest"); + var importer = new TableInfoImporter(CatalogueRepository, table); + importer.DoImport(out var _tableInfo, out var _columnInfos); + foreach (var columnInfo in _columnInfos) + { + var ci = new CatalogueItem(CatalogueRepository, catalogue, columnInfo.GetRuntimeName()); + ci.SaveToDatabase(); + var ei = new ExtractionInformation(CatalogueRepository, ci, columnInfo, ""); + ei.SaveToDatabase(); + } + var activator = new ConsoleInputManager(RepositoryLocator, ThrowImmediatelyCheckNotifier.Quiet); + var regexConfiguration = new RegexRedactionConfiguration(CatalogueRepository, "TestReplacer", new Regex("TEST"), "DB", "Replace TEST with GG"); + regexConfiguration.SaveToDatabase(); + var cmd = new ExecuteCommandPerformRegexRedactionOnCatalogue(activator, catalogue, regexConfiguration, _columnInfos.Where(c => c.GetRuntimeName() == "Condition2").ToList()); + Assert.DoesNotThrow(() => cmd.Execute()); + var retrieveSQL = @"select [Condition2] from [RedactionTest]"; + var dt = new DataTable(); + dt.BeginLoadData(); + using (var fetchCmd = server.GetCommand(retrieveSQL, server.GetConnection())) + { + using var da = server.GetDataAdapter(fetchCmd); + da.Fill(dt); + } + Assert.That(dt.Rows.Count, Is.EqualTo(1)); + Assert.That(dt.Rows[0].ItemArray[0], Is.EqualTo("123412341234")); + var redactions = CatalogueRepository.GetAllObjectsWhere("ReplacementValue", ""); + Assert.That(redactions.Length, Is.EqualTo(2)); + Assert.That(redactions[0].startingIndex, Is.EqualTo(4)); + Assert.That(redactions[1].startingIndex, Is.EqualTo(12)); + } + } +} diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs index 4f0346511e..7f391615a4 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs @@ -1,15 +1,19 @@ -using FAnsi; +using CommandLine; +using FAnsi; using FAnsi.Discovery; using FAnsi.Discovery.QuerySyntax; +using NPOI.SS.Formula.Functions; using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.DataHelper.RegexRedaction; using Rdmp.Core.QueryBuilding; using Rdmp.Core.Repositories; using Rdmp.Core.ReusableLibraryCode.DataAccess; +using SixLabors.ImageSharp.Drawing; using System; using System.Collections.Generic; using System.ComponentModel.DataAnnotations; using System.Data; +using System.Globalization; using System.Linq; using System.Text; using System.Text.RegularExpressions; @@ -24,53 +28,95 @@ public class ExecuteCommandPerformRegexRedactionOnCatalogue : BasicCommandExecut private readonly RegexRedactionConfiguration _redactionConfiguration; private readonly List _columns; private readonly DiscoveredServer _server; + private DiscoveredColumn[] _discoveredPKColumns; + public ExecuteCommandPerformRegexRedactionOnCatalogue(IBasicActivateItems activator, ICatalogue catalogue, RegexRedactionConfiguration redactionConfiguration, List columns) + { + _activator = activator; + _catalogue = catalogue; + _redactionConfiguration = redactionConfiguration; + _columns = columns; + _server = _catalogue.GetDistinctLiveDatabaseServer(DataAccessContext.InternalDataProcessing, false); - //public ExecuteCommandPerformRegexRedactionOnCatalogue(IBasicActivateItems activator, ICatalogue catalogue, RegexRedactionConfiguration redactionConfiguration, List columns) { + } + + //public ExecuteCommandPerformRegexRedactionOnCatalogue(IBasicActivateItems activator) + //{ // _activator = activator; - // _catalogue = catalogue; - // _redactionConfiguration = redactionConfiguration; - // _columns = columns; + // _catalogue = _activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Name", "Biochemistry").First(); + // _redactionConfiguration = _activator.RepositoryLocator.CatalogueRepository.GetAllObjects().First(); + // _columns = _catalogue.CatalogueItems.Select(ci => ci.ColumnInfo).Where(c => c.Name == "[RDMP_ExampleData].[dbo].[Biochemistry].[SampleType]").ToList(); + // _server = _catalogue.GetDistinctLiveDatabaseServer(DataAccessContext.InternalDataProcessing, false); //} - public ExecuteCommandPerformRegexRedactionOnCatalogue(IBasicActivateItems activator) + private string GetRedactionValue(string value) { - _activator = activator; - _catalogue = _activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Name", "Biochemistry").First(); - _redactionConfiguration = _activator.RepositoryLocator.CatalogueRepository.GetAllObjects().First(); - _columns = _catalogue.CatalogueItems.Select(ci => ci.ColumnInfo).ToList(); - _server = _catalogue.GetDistinctLiveDatabaseServer(DataAccessContext.InternalDataProcessing, false); + var matches = Regex.Matches(value, _redactionConfiguration.RegexPattern); + foreach (var match in matches) + { + var foundMatch = match.ToString(); + var startingIndex = value.IndexOf(foundMatch); + string replacementValue = _redactionConfiguration.RedactionString; + + if (foundMatch.Length < _redactionConfiguration.RedactionString.Length) + { + throw new Exception($"Redaction string '{_redactionConfiguration.RedactionString}' is longer than found match '{foundMatch}'."); + } + var paddingValue = '>'; + while (foundMatch.Length > replacementValue.Length) + { + if (paddingValue == '>') + { + replacementValue += paddingValue; + } + else + { + replacementValue = paddingValue + replacementValue; + } + paddingValue = paddingValue == '>' ? '<' : '>'; + } + value = value.Substring(0, startingIndex) + replacementValue + value.Substring(startingIndex + foundMatch.Length); + + //create redaction + var redaction = new RegexRedaction(_activator.RepositoryLocator.CatalogueRepository, _redactionConfiguration.ID, startingIndex, foundMatch, replacementValue); + redaction.SaveToDatabase(); + } + + return value; } - private void Redact(DiscoveredTable table, ColumnInfo column, List PKColumns, DataRow match, DataColumnCollection columns) + private void Redact(DiscoveredTable table, ColumnInfo column, DataRow match, DataColumnCollection columns) { var updateHelper = _server.GetQuerySyntaxHelper().UpdateHelper; - var typeTranslator = _server.GetQuerySyntaxHelper().TypeTranslater; - var redactedValue = "TODO"; + var index = columns.Cast().Select(dc => dc.ColumnName).ToList().IndexOf(column.GetRuntimeName()); + var redactedValue = GetRedactionValue(match[index].ToString()); var sqlLines = new List { new CustomLine($"t1.{column.GetRuntimeName()} = '{redactedValue}'", QueryComponent.SET) }; - foreach (var pk in PKColumns) + foreach (var pk in _discoveredPKColumns) { - //ColumnInfo var matchValue = $"'{match[pk.GetRuntimeName()]}'"; - var dataType = pk.Data_type; - //if(dataType == "datetime2") - //{ - // var date = new DateTime(match[pk.GetRuntimeName()].ToString()); - //} - - //doesn;t work for datePKS + if (pk.DataType.SQLType == "datetime2" || pk.DataType.SQLType == "datetime" ) + { + var x = DateTime.Parse(match[pk.GetRuntimeName()].ToString()); + var format = "yyyy-MM-dd HH:mm:ss:fff"; + matchValue = $"'{x.ToString(format)}'"; + } sqlLines.Add(new CustomLine($"t1.{pk.GetRuntimeName()} = {matchValue}", QueryComponent.WHERE)); sqlLines.Add(new CustomLine(string.Format("t1.{0} = t2.{0}", pk.GetRuntimeName()), QueryComponent.JoinInfoJoin)); } var sql = updateHelper.BuildUpdate(table, table, sqlLines); - Console.WriteLine(sql); + var conn = _server.GetConnection(); + using (var cmd = _server.GetCommand(sql, conn)) + { + conn.Open(); + cmd.ExecuteNonQuery(); + } } public override void Execute() @@ -81,14 +127,19 @@ public override void Execute() { var columnName = catalogueItem.ColumnInfo.Name; var table = catalogueItem.ColumnInfo.TableInfo.Name; - DiscoveredDatabase discoveredDb = _server.ExpectDatabase(catalogueItem.ColumnInfo.TableInfo.Database); - DiscoveredTable discoveredTable = discoveredDb.ExpectTable(catalogueItem.ColumnInfo.TableInfo.GetRuntimeName()); - var pkColumns = _catalogue.CatalogueItems.Select(x => x.ColumnInfo).Where(x => x.IsPrimaryKey).ToList(); - if (pkColumns.Where(pkc => pkc.Name.Contains(table)).Any()) + DiscoveredTable discoveredTable = catalogueItem.ColumnInfo.TableInfo.Discover(DataAccessContext.InternalDataProcessing); + DiscoveredColumn[] discoveredColumns = discoveredTable.DiscoverColumns(); + _discoveredPKColumns = discoveredColumns.Where(c => c.IsPrimaryKey).ToArray(); + if (_discoveredPKColumns.Any()) { + var cataloguePKs = _catalogue.CatalogueItems.Where(c => c.ColumnInfo.IsPrimaryKey); var qb = new QueryBuilder(null, null, null); qb.AddColumn(new ColumnInfoToIColumn(memoryRepo, catalogueItem.ColumnInfo)); - qb.AddColumnRange(pkColumns.Select(pk => new ColumnInfoToIColumn(memoryRepo, pk)).ToArray()); + foreach (var pk in _discoveredPKColumns) + { + var cataloguePk = cataloguePKs.FirstOrDefault(c => c.ColumnInfo.GetRuntimeName() == pk.GetRuntimeName()); + qb.AddColumn(new ColumnInfoToIColumn(memoryRepo, cataloguePk.ColumnInfo)); + } var sql = qb.SQL; var dt = new DataTable(); dt.BeginLoadData(); @@ -105,7 +156,7 @@ public override void Execute() foreach (DataRow row in _regexMatches.Rows) { - Redact(discoveredTable, catalogueItem.ColumnInfo, pkColumns, row, _regexMatches.Columns); + Redact(discoveredTable, catalogueItem.ColumnInfo, row, _regexMatches.Columns); } } else diff --git a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedaction.cs b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedaction.cs index 28ab02b46b..c515480bdf 100644 --- a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedaction.cs +++ b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedaction.cs @@ -1,12 +1,62 @@ -using System; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Repositories; +using System; using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; +using System.Data.Common; namespace Rdmp.Core.Curation.DataHelper.RegexRedaction { - internal class RegexRedaction + public class RegexRedaction : DatabaseEntity, IRegexRedaction + { + private int _redactionConfigurationID; + private int _startingIndex; + private string _redactedValue; + private string _replacementValue; + + + public int RedactionConfiguration_ID + { + get => _redactionConfigurationID; + set => SetField(ref _redactionConfigurationID, value); + } + + public string RedactedValue + { + get => _redactedValue; + set => SetField(ref _redactedValue, value.ToString()); + } + + public string ReplacementValue + { + get => _replacementValue; + set => SetField(ref _replacementValue, value.ToString()); + } + + public int startingIndex + { + get => _startingIndex; + set => SetField(ref _startingIndex, value); + } + + public RegexRedaction() { } + + public RegexRedaction(ICatalogueRepository repository, int redactionConfigurationID, int startingIndex, string redactionValue, string replacementValue) + { + repository.InsertAndHydrate(this, new Dictionary { + {"RedactionConfiguration_ID",redactionConfigurationID}, + {"StartingIndex",startingIndex}, + {"RedactedValue", redactionValue }, + {"ReplacementValue", replacementValue } + }); + } + + internal RegexRedaction(ICatalogueRepository repository, DbDataReader r) : base(repository, r) + { + _redactionConfigurationID = Int32.Parse(r["RedactionConfiguration_ID"].ToString()); + _startingIndex = Int32.Parse(r["StartingIndex"].ToString()); + _replacementValue = r["ReplacementValue"].ToString(); + _redactedValue = r["RedactedValue"].ToString(); + } } } diff --git a/Rdmp.Core/Databases/CatalogueDatabase/up/086_AddRegexRedactionConfiguration.sql b/Rdmp.Core/Databases/CatalogueDatabase/up/086_AddRegexRedactionConfiguration.sql index 8750571157..0e103a0f8c 100644 --- a/Rdmp.Core/Databases/CatalogueDatabase/up/086_AddRegexRedactionConfiguration.sql +++ b/Rdmp.Core/Databases/CatalogueDatabase/up/086_AddRegexRedactionConfiguration.sql @@ -17,3 +17,22 @@ CONSTRAINT [PK_RegexRedactionConfiguration] PRIMARY KEY CLUSTERED ) ON [PRIMARY] END GO + + +if not exists (select 1 from sys.tables where name = 'RegexRedaction') +BEGIN +CREATE TABLE [dbo].RegexRedaction( + [ID] [int] IDENTITY(1,1) NOT NULL, + RedactionConfiguration_ID [int] NOT NULL, + StartingIndex [int] NOT NULL, + RedactedValue [nvarchar](250), + ReplacementValue [nvarchar](250) NOT NULL, + FOREIGN KEY (RedactionConfiguration_ID) REFERENCES RegexRedactionConfiguration(ID), +CONSTRAINT [PK_RegexRedaction] PRIMARY KEY CLUSTERED +( + [ID] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +END +GO + From 093dffbcade114f5209969f7d27a271ce31bbc54 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 20 Aug 2024 08:17:16 +0100 Subject: [PATCH 004/106] add missing interface --- .../RegexRedaction/IRegexRedaction.cs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedaction.cs diff --git a/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedaction.cs b/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedaction.cs new file mode 100644 index 0000000000..bc0e46631f --- /dev/null +++ b/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedaction.cs @@ -0,0 +1,18 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.DataHelper.RegexRedaction +{ + public interface IRegexRedaction + { + + int RedactionConfiguration_ID { get; protected set; } + string RedactedValue { get; protected set; } + string ReplacementValue { get; protected set; } + + int startingIndex { get; protected set; } + } +} From 43a3a12cc9e27fade7a3dcc3d271a3643c10f5d4 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 20 Aug 2024 11:32:59 +0100 Subject: [PATCH 005/106] add regex redaction keys --- ...ndPerformRegexRedactionOnCatalogueTests.cs | 59 +++++++++- ...CommandPerformRegexRedactionOnCatalogue.cs | 38 +++---- ...andRestoreRegexRedactedValueInCatalogue.cs | 105 ++++++++++++++++++ .../RegexRedaction/IRegexRedaction.cs | 4 +- .../RegexRedaction/IRegexRedactionKey.cs | 17 +++ .../RegexRedaction/RegexRedaction.cs | 27 ++++- .../RegexRedaction/RegexRedactionKey.cs | 43 +++++++ .../up/086_AddRegexRedactionConfiguration.sql | 19 ++++ 8 files changed, 285 insertions(+), 27 deletions(-) create mode 100644 Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreRegexRedactedValueInCatalogue.cs create mode 100644 Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedactionKey.cs create mode 100644 Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionKey.cs diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs index 47d46df03f..f59ac91fdf 100644 --- a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs @@ -313,8 +313,63 @@ INSERT [RedactionTest]([DischargeDate],[Condition1],[Condition2]) Assert.That(dt.Rows[0].ItemArray[0], Is.EqualTo("123412341234")); var redactions = CatalogueRepository.GetAllObjectsWhere("ReplacementValue", ""); Assert.That(redactions.Length, Is.EqualTo(2)); - Assert.That(redactions[0].startingIndex, Is.EqualTo(4)); - Assert.That(redactions[1].startingIndex, Is.EqualTo(12)); + Assert.That(redactions[0].StartingIndex, Is.EqualTo(4)); + Assert.That(redactions[1].StartingIndex, Is.EqualTo(12)); + } + + [Test] + public void Redaction_DummyTest() + { + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + const string sql = @" + CREATE TABLE [RedactionTest]( + [DischargeDate] [datetime] NOT NULL, + [Condition1] [varchar](400) NOT NULL, + [Condition2] [varchar](400) NOT NULL + CONSTRAINT [PK_DLCTest] PRIMARY KEY CLUSTERED + ( + [DischargeDate] ASC, + [Condition1] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] + ) ON [PRIMARY] + INSERT [RedactionTest]([DischargeDate],[Condition1],[Condition2]) + VALUES (CAST(0x000001B300000000 AS DateTime),N'1',N'1234TEST1234') + "; + var server = db.Server; + using (var con = server.GetConnection()) + { + con.Open(); + server.GetCommand(sql, con).ExecuteNonQuery(); + } + var table = db.ExpectTable("RedactionTest"); + var catalogue = new Catalogue(CatalogueRepository, "RedactionTest"); + var importer = new TableInfoImporter(CatalogueRepository, table); + importer.DoImport(out var _tableInfo, out var _columnInfos); + foreach (var columnInfo in _columnInfos) + { + var ci = new CatalogueItem(CatalogueRepository, catalogue, columnInfo.GetRuntimeName()); + ci.SaveToDatabase(); + var ei = new ExtractionInformation(CatalogueRepository, ci, columnInfo, ""); + ei.SaveToDatabase(); + } + var activator = new ConsoleInputManager(RepositoryLocator, ThrowImmediatelyCheckNotifier.Quiet); + var regexConfiguration = new RegexRedactionConfiguration(CatalogueRepository, "TestReplacer", new Regex("TEST"), "GG", "Replace TEST with GG"); + regexConfiguration.SaveToDatabase(); + var cmd = new ExecuteCommandPerformRegexRedactionOnCatalogue(activator, catalogue, regexConfiguration, _columnInfos.Where(c => c.GetRuntimeName() == "Condition2").ToList()); + Assert.DoesNotThrow(() => cmd.Execute()); + var retrieveSQL = @"select [Condition2] from [RedactionTest]"; + var dt = new DataTable(); + dt.BeginLoadData(); + using (var fetchCmd = server.GetCommand(retrieveSQL, server.GetConnection())) + { + using var da = server.GetDataAdapter(fetchCmd); + da.Fill(dt); + } + Assert.That(dt.Rows.Count, Is.EqualTo(1)); + Assert.That(dt.Rows[0].ItemArray[0], Is.EqualTo("12341234")); + var redaction = CatalogueRepository.GetAllObjects().Last(); + var revert = new ExecuteCommandRestoreRegexRedactedValueInCatalogue(activator, redaction); + revert.Execute(); } } } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs index 7f391615a4..f22f50682f 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs @@ -2,22 +2,17 @@ using FAnsi; using FAnsi.Discovery; using FAnsi.Discovery.QuerySyntax; -using NPOI.SS.Formula.Functions; using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.DataHelper.RegexRedaction; using Rdmp.Core.QueryBuilding; using Rdmp.Core.Repositories; using Rdmp.Core.ReusableLibraryCode.DataAccess; -using SixLabors.ImageSharp.Drawing; using System; using System.Collections.Generic; -using System.ComponentModel.DataAnnotations; using System.Data; -using System.Globalization; using System.Linq; -using System.Text; using System.Text.RegularExpressions; -using System.Threading.Tasks; +using static System.Linq.Enumerable; namespace Rdmp.Core.CommandExecution.AtomicCommands; @@ -29,6 +24,7 @@ public class ExecuteCommandPerformRegexRedactionOnCatalogue : BasicCommandExecut private readonly List _columns; private readonly DiscoveredServer _server; private DiscoveredColumn[] _discoveredPKColumns; + private List _cataloguePKs; public ExecuteCommandPerformRegexRedactionOnCatalogue(IBasicActivateItems activator, ICatalogue catalogue, RegexRedactionConfiguration redactionConfiguration, List columns) @@ -41,16 +37,7 @@ public ExecuteCommandPerformRegexRedactionOnCatalogue(IBasicActivateItems activa } - //public ExecuteCommandPerformRegexRedactionOnCatalogue(IBasicActivateItems activator) - //{ - // _activator = activator; - // _catalogue = _activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Name", "Biochemistry").First(); - // _redactionConfiguration = _activator.RepositoryLocator.CatalogueRepository.GetAllObjects().First(); - // _columns = _catalogue.CatalogueItems.Select(ci => ci.ColumnInfo).Where(c => c.Name == "[RDMP_ExampleData].[dbo].[Biochemistry].[SampleType]").ToList(); - // _server = _catalogue.GetDistinctLiveDatabaseServer(DataAccessContext.InternalDataProcessing, false); - //} - - private string GetRedactionValue(string value) + private string GetRedactionValue(string value, ColumnInfo column, Dictionary pkLookup) { var matches = Regex.Matches(value, _redactionConfiguration.RegexPattern); foreach (var match in matches) @@ -79,7 +66,8 @@ private string GetRedactionValue(string value) value = value.Substring(0, startingIndex) + replacementValue + value.Substring(startingIndex + foundMatch.Length); //create redaction - var redaction = new RegexRedaction(_activator.RepositoryLocator.CatalogueRepository, _redactionConfiguration.ID, startingIndex, foundMatch, replacementValue); + int columnID = column.ID; + var redaction = new RegexRedaction(_activator.RepositoryLocator.CatalogueRepository, _redactionConfiguration.ID, startingIndex, foundMatch, replacementValue,columnID,pkLookup); redaction.SaveToDatabase(); } @@ -90,7 +78,17 @@ private void Redact(DiscoveredTable table, ColumnInfo column, DataRow match, Dat { var updateHelper = _server.GetQuerySyntaxHelper().UpdateHelper; var index = columns.Cast().Select(dc => dc.ColumnName).ToList().IndexOf(column.GetRuntimeName()); - var redactedValue = GetRedactionValue(match[index].ToString()); + + Dictionary pkLookup = new(); + foreach(int i in Range(0,columns.Count).Where(n => n != index)) + { + //these are all the PK values + var name = columns.Cast().ToList()[i].ColumnName; + var pkCatalogueItem = _cataloguePKs.Where(c => c.Name == name).First(); + pkLookup.Add(pkCatalogueItem.ColumnInfo, match[i].ToString()); + } + + var redactedValue = GetRedactionValue(match[index].ToString(),column,pkLookup); var sqlLines = new List { @@ -132,12 +130,12 @@ public override void Execute() _discoveredPKColumns = discoveredColumns.Where(c => c.IsPrimaryKey).ToArray(); if (_discoveredPKColumns.Any()) { - var cataloguePKs = _catalogue.CatalogueItems.Where(c => c.ColumnInfo.IsPrimaryKey); + _cataloguePKs = _catalogue.CatalogueItems.Where(c => c.ColumnInfo.IsPrimaryKey).ToList(); var qb = new QueryBuilder(null, null, null); qb.AddColumn(new ColumnInfoToIColumn(memoryRepo, catalogueItem.ColumnInfo)); foreach (var pk in _discoveredPKColumns) { - var cataloguePk = cataloguePKs.FirstOrDefault(c => c.ColumnInfo.GetRuntimeName() == pk.GetRuntimeName()); + var cataloguePk = _cataloguePKs.FirstOrDefault(c => c.ColumnInfo.GetRuntimeName() == pk.GetRuntimeName()); qb.AddColumn(new ColumnInfoToIColumn(memoryRepo, cataloguePk.ColumnInfo)); } var sql = qb.SQL; diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreRegexRedactedValueInCatalogue.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreRegexRedactedValueInCatalogue.cs new file mode 100644 index 0000000000..325cf946b3 --- /dev/null +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreRegexRedactedValueInCatalogue.cs @@ -0,0 +1,105 @@ +using FAnsi.Discovery; +using FAnsi.Discovery.QuerySyntax; +using MongoDB.Driver.Core.Servers; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.DataHelper.RegexRedaction; +using Rdmp.Core.QueryBuilding; +using Rdmp.Core.Repositories; +using Rdmp.Core.ReusableLibraryCode.DataAccess; +using System; +using System.Collections.Generic; +using System.Data; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.CommandExecution.AtomicCommands +{ + public class ExecuteCommandRestoreRegexRedactedValueInCatalogue : BasicCommandExecution, IAtomicCommand + { + + private readonly RegexRedaction _redaction; + private readonly IBasicActivateItems _activator; + + public ExecuteCommandRestoreRegexRedactedValueInCatalogue(IBasicActivateItems activator, RegexRedaction redaction) + { + _activator = activator; + _redaction = redaction; + } + + + public override void Execute() + { + base.Execute(); + var memoryRepo = new MemoryCatalogueRepository(); + var columnInfo = _activator.RepositoryLocator.CatalogueRepository.GetObjectByID(_redaction.ColumnInfo_ID); + var pks = _redaction.RedactionKeys.Select(pk => _activator.RepositoryLocator.CatalogueRepository.GetObjectByID(pk.ColumnInfo_ID)); + var catalogue = columnInfo.CatalogueItems.FirstOrDefault().Catalogue; + var server = catalogue.GetDistinctLiveDatabaseServer(DataAccessContext.InternalDataProcessing, false); + DiscoveredTable discoveredTable = columnInfo.TableInfo.Discover(DataAccessContext.InternalDataProcessing); + DiscoveredColumn[] discoveredColumns = discoveredTable.DiscoverColumns(); + var qb = new QueryBuilder(null, null, null); + qb.AddColumn(new ColumnInfoToIColumn(memoryRepo, columnInfo)); + foreach (var rk in _redaction.RedactionKeys) + { + var pkColumnInfo = _activator.RepositoryLocator.CatalogueRepository.GetObjectByID(rk.ColumnInfo_ID); + var matchValue = $"'{rk.Value}'"; + if (pkColumnInfo.Data_type == "datetime2" || pkColumnInfo.Data_type == "datetime") + { + var x = DateTime.Parse(rk.Value); + var format = "yyyy-MM-dd HH:mm:ss:fff"; + matchValue = $"'{x.ToString(format)}'"; + } + qb.AddCustomLine($"{pkColumnInfo.Name} = {matchValue}", QueryComponent.WHERE); + } + + var sql = qb.SQL; + var dt = new DataTable(); + dt.BeginLoadData(); + using (var cmd = server.GetCommand(sql, server.GetConnection())) + { + using var da = server.GetDataAdapter(cmd); + da.Fill(dt); + } + if (dt.Rows.Count > 1) + { + throw new Exception("More than 1 matching redaction for this configuration. Something has gone wrong..."); + } + string newValue = dt.Rows[0][0].ToString(); + if (newValue.IndexOf(_redaction.ReplacementValue, _redaction.StartingIndex, _redaction.ReplacementValue.Length) == _redaction.StartingIndex) + { + newValue = newValue.Remove(_redaction.StartingIndex, _redaction.ReplacementValue.Length).Insert(_redaction.StartingIndex, _redaction.RedactedValue); + var updateHelper = server.GetQuerySyntaxHelper().UpdateHelper; + var sqlLines = new List + { + new CustomLine($"t1.{columnInfo.GetRuntimeName()} = '{newValue}'", QueryComponent.SET) + }; + foreach (var rk in _redaction.RedactionKeys) + { + var pkColumnInfo = _activator.RepositoryLocator.CatalogueRepository.GetObjectByID(rk.ColumnInfo_ID); + var matchValue = $"'{rk.Value}'"; + if (pkColumnInfo.Data_type == "datetime2" || pkColumnInfo.Data_type == "datetime") + { + var x = DateTime.Parse(rk.Value); + var format = "yyyy-MM-dd HH:mm:ss:fff"; + matchValue = $"'{x.ToString(format)}'"; + } + sqlLines.Add(new CustomLine($"t1.{pkColumnInfo.GetRuntimeName()} = {matchValue}", QueryComponent.WHERE)); + sqlLines.Add(new CustomLine(string.Format("t1.{0} = t2.{0}", pkColumnInfo.GetRuntimeName()), QueryComponent.JoinInfoJoin)); + } + var updateSql = updateHelper.BuildUpdate(discoveredTable, discoveredTable, sqlLines); + var conn = server.GetConnection(); + using (var cmd = server.GetCommand(sql, conn)) + { + conn.Open(); + cmd.ExecuteNonQuery(); + } + _redaction.DeleteInDatabase(); + } + else + { + throw new Exception("Original redaction cannot be replaced"); + } + } + } +} diff --git a/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedaction.cs b/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedaction.cs index bc0e46631f..c9ece2cdec 100644 --- a/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedaction.cs +++ b/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedaction.cs @@ -13,6 +13,8 @@ public interface IRegexRedaction string RedactedValue { get; protected set; } string ReplacementValue { get; protected set; } - int startingIndex { get; protected set; } + int StartingIndex { get; protected set; } + + int ColumnInfo_ID { get; protected set; } } } diff --git a/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedactionKey.cs b/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedactionKey.cs new file mode 100644 index 0000000000..5d201f4e3b --- /dev/null +++ b/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedactionKey.cs @@ -0,0 +1,17 @@ +using Rdmp.Core.Curation.Data; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.DataHelper.RegexRedaction +{ + public interface IRegexRedactionKey + { + + int RegexRedaction_ID { get; protected set; } + int ColumnInfo_ID { get; protected set; } + string Value { get; protected set; } + } +} diff --git a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedaction.cs b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedaction.cs index c515480bdf..e66c9153bf 100644 --- a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedaction.cs +++ b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedaction.cs @@ -1,8 +1,10 @@ using Rdmp.Core.Curation.Data; +using Rdmp.Core.MapsDirectlyToDatabaseTable; using Rdmp.Core.Repositories; using System; using System.Collections.Generic; using System.Data.Common; +using System.Linq; namespace Rdmp.Core.Curation.DataHelper.RegexRedaction { @@ -13,7 +15,7 @@ public class RegexRedaction : DatabaseEntity, IRegexRedaction private int _startingIndex; private string _redactedValue; private string _replacementValue; - + private int _columnInfoID; public int RedactionConfiguration_ID { @@ -33,22 +35,38 @@ public string ReplacementValue set => SetField(ref _replacementValue, value.ToString()); } - public int startingIndex + public int StartingIndex { get => _startingIndex; set => SetField(ref _startingIndex, value); } + public int ColumnInfo_ID + { + get => _columnInfoID; + set => SetField(ref _columnInfoID, value); + } + + [NoMappingToDatabase] + public List RedactionKeys => [.. CatalogueRepository.GetAllObjectsWhere("RegexRedaction_ID", this.ID)]; + + public RegexRedaction() { } - public RegexRedaction(ICatalogueRepository repository, int redactionConfigurationID, int startingIndex, string redactionValue, string replacementValue) + public RegexRedaction(ICatalogueRepository repository, int redactionConfigurationID, int startingIndex, string redactionValue, string replacementValue, int columnInfoID, Dictionary pkValues) { repository.InsertAndHydrate(this, new Dictionary { {"RedactionConfiguration_ID",redactionConfigurationID}, {"StartingIndex",startingIndex}, {"RedactedValue", redactionValue }, - {"ReplacementValue", replacementValue } + {"ReplacementValue", replacementValue }, + {"ColumnInfo_ID", columnInfoID } }); + foreach (var tuple in pkValues) + { + var key = new RegexRedactionKey(repository, this, tuple.Key, tuple.Value); + key.SaveToDatabase(); + } } internal RegexRedaction(ICatalogueRepository repository, DbDataReader r) : base(repository, r) @@ -57,6 +75,7 @@ internal RegexRedaction(ICatalogueRepository repository, DbDataReader r) : base( _startingIndex = Int32.Parse(r["StartingIndex"].ToString()); _replacementValue = r["ReplacementValue"].ToString(); _redactedValue = r["RedactedValue"].ToString(); + _columnInfoID = Int32.Parse(r["ColumnInfo_ID"].ToString()); } } } diff --git a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionKey.cs b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionKey.cs new file mode 100644 index 0000000000..ec1e6f9561 --- /dev/null +++ b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionKey.cs @@ -0,0 +1,43 @@ +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Repositories; +using System; +using System.Collections.Generic; +using System.Data.Common; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.DataHelper.RegexRedaction +{ + public class RegexRedactionKey : DatabaseEntity, IRegexRedactionKey + { + private int _redaction; + private int _columnInfo; + private string _value; + + public int RegexRedaction_ID { get => _redaction; set => SetField(ref _redaction, value); } + public int ColumnInfo_ID { get => _columnInfo; set => SetField(ref _columnInfo, value); } + + public string Value { get => _value; set => SetField(ref _value, value); } + + public RegexRedactionKey() { } + + public RegexRedactionKey(ICatalogueRepository repository, RegexRedaction redaction, ColumnInfo pkColumn, string value) + { + repository.InsertAndHydrate(this, new Dictionary + { + {"RegexRedaction_ID", redaction.ID }, + {"ColumnInfo_ID", pkColumn.ID }, + {"Value", value }, + }); + } + + internal RegexRedactionKey(ICatalogueRepository repository, DbDataReader r) : base(repository, r) + { + _redaction = Int32.Parse(r["RegexRedaction_ID"].ToString()); + _columnInfo = Int32.Parse(r["ColumnInfo_ID"].ToString()); + _value = r["Value"].ToString(); + } + + } +} diff --git a/Rdmp.Core/Databases/CatalogueDatabase/up/086_AddRegexRedactionConfiguration.sql b/Rdmp.Core/Databases/CatalogueDatabase/up/086_AddRegexRedactionConfiguration.sql index 0e103a0f8c..6377b4be41 100644 --- a/Rdmp.Core/Databases/CatalogueDatabase/up/086_AddRegexRedactionConfiguration.sql +++ b/Rdmp.Core/Databases/CatalogueDatabase/up/086_AddRegexRedactionConfiguration.sql @@ -24,10 +24,12 @@ BEGIN CREATE TABLE [dbo].RegexRedaction( [ID] [int] IDENTITY(1,1) NOT NULL, RedactionConfiguration_ID [int] NOT NULL, + ColumnInfo_ID [int] NOT NULL, StartingIndex [int] NOT NULL, RedactedValue [nvarchar](250), ReplacementValue [nvarchar](250) NOT NULL, FOREIGN KEY (RedactionConfiguration_ID) REFERENCES RegexRedactionConfiguration(ID), + FOREIGN KEY (ColumnInfo_ID) REFERENCES ColumnInfo(ID) ON DELETE CASCADE, CONSTRAINT [PK_RegexRedaction] PRIMARY KEY CLUSTERED ( [ID] ASC @@ -36,3 +38,20 @@ CONSTRAINT [PK_RegexRedaction] PRIMARY KEY CLUSTERED END GO +if not exists (select 1 from sys.tables where name = 'RegexRedactionKey') +BEGIN +CREATE TABLE [dbo].RegexRedactionKey( + [ID] [int] IDENTITY(1,1) NOT NULL, + RegexRedaction_ID [int] NOT NULL, + ColumnInfo_ID [int] NOT NULL, + Value [nvarchar](max), + FOREIGN KEY (RegexRedaction_ID) REFERENCES RegexRedaction(ID) ON DELETE CASCADE, + FOREIGN KEY (ColumnInfo_ID) REFERENCES ColumnInfo(ID), +CONSTRAINT [PK_RegexRedactionKey] PRIMARY KEY CLUSTERED +( + [ID] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +END +GO + From b534ddba975f2c36799d93f6d56c7e3c07c47f7f Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 20 Aug 2024 11:41:58 +0100 Subject: [PATCH 006/106] add helper --- .../RegexRedaction/RegexRedactionHelper.cs | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs diff --git a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs new file mode 100644 index 0000000000..35889ff15f --- /dev/null +++ b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs @@ -0,0 +1,25 @@ +using Rdmp.Core.Curation.Data; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.DataHelper.RegexRedaction +{ + public static class RegexRedactionHelper + { + + public static string ConvertPotentialDateTimeObject(string value, string currentColumnType) + { + var matchValue = $"'{value}'"; + if (currentColumnType == "datetime2" || currentColumnType == "datetime") + { + var x = DateTime.Parse(value); + var format = "yyyy-MM-dd HH:mm:ss:fff"; + matchValue = $"'{x.ToString(format)}'"; + } + return matchValue; + } + } +} From 7b72d457a746de130b40c8bcc6c469ab06d45e98 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 20 Aug 2024 11:44:51 +0100 Subject: [PATCH 007/106] add concept --- ...uteCommandPerformRegexRedactionOnCatalogue.cs | 14 ++++---------- ...ommandRestoreRegexRedactedValueInCatalogue.cs | 16 ++-------------- Rdmp.Core/Icons/IconProvision/RDMPConcept.cs | 5 ++++- 3 files changed, 10 insertions(+), 25 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs index f22f50682f..547fe7bf91 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs @@ -67,7 +67,7 @@ private string GetRedactionValue(string value, ColumnInfo column, Dictionary().Select(dc => dc.ColumnName).ToList().IndexOf(column.GetRuntimeName()); Dictionary pkLookup = new(); - foreach(int i in Range(0,columns.Count).Where(n => n != index)) + foreach (int i in Range(0, columns.Count).Where(n => n != index)) { //these are all the PK values var name = columns.Cast().ToList()[i].ColumnName; @@ -88,7 +88,7 @@ private void Redact(DiscoveredTable table, ColumnInfo column, DataRow match, Dat pkLookup.Add(pkCatalogueItem.ColumnInfo, match[i].ToString()); } - var redactedValue = GetRedactionValue(match[index].ToString(),column,pkLookup); + var redactedValue = GetRedactionValue(match[index].ToString(), column, pkLookup); var sqlLines = new List { @@ -96,13 +96,7 @@ private void Redact(DiscoveredTable table, ColumnInfo column, DataRow match, Dat }; foreach (var pk in _discoveredPKColumns) { - var matchValue = $"'{match[pk.GetRuntimeName()]}'"; - if (pk.DataType.SQLType == "datetime2" || pk.DataType.SQLType == "datetime" ) - { - var x = DateTime.Parse(match[pk.GetRuntimeName()].ToString()); - var format = "yyyy-MM-dd HH:mm:ss:fff"; - matchValue = $"'{x.ToString(format)}'"; - } + var matchValue = RegexRedactionHelper.ConvertPotentialDateTimeObject(match[pk.GetRuntimeName()].ToString(), pk.DataType.SQLType); sqlLines.Add(new CustomLine($"t1.{pk.GetRuntimeName()} = {matchValue}", QueryComponent.WHERE)); sqlLines.Add(new CustomLine(string.Format("t1.{0} = t2.{0}", pk.GetRuntimeName()), QueryComponent.JoinInfoJoin)); diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreRegexRedactedValueInCatalogue.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreRegexRedactedValueInCatalogue.cs index 325cf946b3..3533402640 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreRegexRedactedValueInCatalogue.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreRegexRedactedValueInCatalogue.cs @@ -43,13 +43,7 @@ public override void Execute() foreach (var rk in _redaction.RedactionKeys) { var pkColumnInfo = _activator.RepositoryLocator.CatalogueRepository.GetObjectByID(rk.ColumnInfo_ID); - var matchValue = $"'{rk.Value}'"; - if (pkColumnInfo.Data_type == "datetime2" || pkColumnInfo.Data_type == "datetime") - { - var x = DateTime.Parse(rk.Value); - var format = "yyyy-MM-dd HH:mm:ss:fff"; - matchValue = $"'{x.ToString(format)}'"; - } + var matchValue = RegexRedactionHelper.ConvertPotentialDateTimeObject(rk.Value, pkColumnInfo.Data_type); qb.AddCustomLine($"{pkColumnInfo.Name} = {matchValue}", QueryComponent.WHERE); } @@ -77,13 +71,7 @@ public override void Execute() foreach (var rk in _redaction.RedactionKeys) { var pkColumnInfo = _activator.RepositoryLocator.CatalogueRepository.GetObjectByID(rk.ColumnInfo_ID); - var matchValue = $"'{rk.Value}'"; - if (pkColumnInfo.Data_type == "datetime2" || pkColumnInfo.Data_type == "datetime") - { - var x = DateTime.Parse(rk.Value); - var format = "yyyy-MM-dd HH:mm:ss:fff"; - matchValue = $"'{x.ToString(format)}'"; - } + var matchValue = RegexRedactionHelper.ConvertPotentialDateTimeObject(rk.Value, pkColumnInfo.Data_type); sqlLines.Add(new CustomLine($"t1.{pkColumnInfo.GetRuntimeName()} = {matchValue}", QueryComponent.WHERE)); sqlLines.Add(new CustomLine(string.Format("t1.{0} = t2.{0}", pkColumnInfo.GetRuntimeName()), QueryComponent.JoinInfoJoin)); } diff --git a/Rdmp.Core/Icons/IconProvision/RDMPConcept.cs b/Rdmp.Core/Icons/IconProvision/RDMPConcept.cs index 31ff8cefcf..8600117d46 100644 --- a/Rdmp.Core/Icons/IconProvision/RDMPConcept.cs +++ b/Rdmp.Core/Icons/IconProvision/RDMPConcept.cs @@ -206,5 +206,8 @@ public enum RDMPConcept Dataset, LoadMetadataCatalogueLinkage, Setting, - TicketingSystemReleaseStatus + TicketingSystemReleaseStatus, + RegexRedaction, + RegexRedactionConfiguration, + RegexRedactionKey } \ No newline at end of file From 46de2937be404df71b86aa936640e3281b3ceeb0 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 20 Aug 2024 11:46:25 +0100 Subject: [PATCH 008/106] update interfaces --- .../Curation/DataHelper/RegexRedaction/IRegexRedaction.cs | 5 +++-- .../Curation/DataHelper/RegexRedaction/IRegexRedactionKey.cs | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedaction.cs b/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedaction.cs index c9ece2cdec..fe16cff926 100644 --- a/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedaction.cs +++ b/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedaction.cs @@ -1,4 +1,5 @@ -using System; +using Rdmp.Core.MapsDirectlyToDatabaseTable; +using System; using System.Collections.Generic; using System.Linq; using System.Text; @@ -6,7 +7,7 @@ namespace Rdmp.Core.Curation.DataHelper.RegexRedaction { - public interface IRegexRedaction + public interface IRegexRedaction: IMapsDirectlyToDatabaseTable { int RedactionConfiguration_ID { get; protected set; } diff --git a/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedactionKey.cs b/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedactionKey.cs index 5d201f4e3b..f66c0942b8 100644 --- a/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedactionKey.cs +++ b/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedactionKey.cs @@ -1,4 +1,5 @@ using Rdmp.Core.Curation.Data; +using Rdmp.Core.MapsDirectlyToDatabaseTable; using System; using System.Collections.Generic; using System.Linq; @@ -7,7 +8,7 @@ namespace Rdmp.Core.Curation.DataHelper.RegexRedaction { - public interface IRegexRedactionKey + public interface IRegexRedactionKey: IMapsDirectlyToDatabaseTable { int RegexRedaction_ID { get; protected set; } From c73b38d90aa77d8a8a9c943a5fe3a427d80c92c7 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 20 Aug 2024 13:18:02 +0100 Subject: [PATCH 009/106] add icons and fix restore --- ...ndPerformRegexRedactionOnCatalogueTests.cs | 55 ------------------- ...andRestoreRegexRedactedValueInCatalogue.cs | 3 +- .../Icons/IconProvision/CatalogueIcons.resx | 11 +++- 3 files changed, 12 insertions(+), 57 deletions(-) diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs index f59ac91fdf..905048c29c 100644 --- a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs @@ -316,60 +316,5 @@ INSERT [RedactionTest]([DischargeDate],[Condition1],[Condition2]) Assert.That(redactions[0].StartingIndex, Is.EqualTo(4)); Assert.That(redactions[1].StartingIndex, Is.EqualTo(12)); } - - [Test] - public void Redaction_DummyTest() - { - var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); - const string sql = @" - CREATE TABLE [RedactionTest]( - [DischargeDate] [datetime] NOT NULL, - [Condition1] [varchar](400) NOT NULL, - [Condition2] [varchar](400) NOT NULL - CONSTRAINT [PK_DLCTest] PRIMARY KEY CLUSTERED - ( - [DischargeDate] ASC, - [Condition1] ASC - )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] - ) ON [PRIMARY] - INSERT [RedactionTest]([DischargeDate],[Condition1],[Condition2]) - VALUES (CAST(0x000001B300000000 AS DateTime),N'1',N'1234TEST1234') - "; - var server = db.Server; - using (var con = server.GetConnection()) - { - con.Open(); - server.GetCommand(sql, con).ExecuteNonQuery(); - } - var table = db.ExpectTable("RedactionTest"); - var catalogue = new Catalogue(CatalogueRepository, "RedactionTest"); - var importer = new TableInfoImporter(CatalogueRepository, table); - importer.DoImport(out var _tableInfo, out var _columnInfos); - foreach (var columnInfo in _columnInfos) - { - var ci = new CatalogueItem(CatalogueRepository, catalogue, columnInfo.GetRuntimeName()); - ci.SaveToDatabase(); - var ei = new ExtractionInformation(CatalogueRepository, ci, columnInfo, ""); - ei.SaveToDatabase(); - } - var activator = new ConsoleInputManager(RepositoryLocator, ThrowImmediatelyCheckNotifier.Quiet); - var regexConfiguration = new RegexRedactionConfiguration(CatalogueRepository, "TestReplacer", new Regex("TEST"), "GG", "Replace TEST with GG"); - regexConfiguration.SaveToDatabase(); - var cmd = new ExecuteCommandPerformRegexRedactionOnCatalogue(activator, catalogue, regexConfiguration, _columnInfos.Where(c => c.GetRuntimeName() == "Condition2").ToList()); - Assert.DoesNotThrow(() => cmd.Execute()); - var retrieveSQL = @"select [Condition2] from [RedactionTest]"; - var dt = new DataTable(); - dt.BeginLoadData(); - using (var fetchCmd = server.GetCommand(retrieveSQL, server.GetConnection())) - { - using var da = server.GetDataAdapter(fetchCmd); - da.Fill(dt); - } - Assert.That(dt.Rows.Count, Is.EqualTo(1)); - Assert.That(dt.Rows[0].ItemArray[0], Is.EqualTo("12341234")); - var redaction = CatalogueRepository.GetAllObjects().Last(); - var revert = new ExecuteCommandRestoreRegexRedactedValueInCatalogue(activator, redaction); - revert.Execute(); - } } } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreRegexRedactedValueInCatalogue.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreRegexRedactedValueInCatalogue.cs index 3533402640..9fb303bca4 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreRegexRedactedValueInCatalogue.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreRegexRedactedValueInCatalogue.cs @@ -73,11 +73,12 @@ public override void Execute() var pkColumnInfo = _activator.RepositoryLocator.CatalogueRepository.GetObjectByID(rk.ColumnInfo_ID); var matchValue = RegexRedactionHelper.ConvertPotentialDateTimeObject(rk.Value, pkColumnInfo.Data_type); sqlLines.Add(new CustomLine($"t1.{pkColumnInfo.GetRuntimeName()} = {matchValue}", QueryComponent.WHERE)); + //todo add original value for column into where sqlLines.Add(new CustomLine(string.Format("t1.{0} = t2.{0}", pkColumnInfo.GetRuntimeName()), QueryComponent.JoinInfoJoin)); } var updateSql = updateHelper.BuildUpdate(discoveredTable, discoveredTable, sqlLines); var conn = server.GetConnection(); - using (var cmd = server.GetCommand(sql, conn)) + using (var cmd = server.GetCommand(updateSql, conn)) { conn.Open(); cmd.ExecuteNonQuery(); diff --git a/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx b/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx index f33525f753..fa09fce753 100644 --- a/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx +++ b/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx @@ -763,4 +763,13 @@ ..\TicketingSystemReleaseStatus.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - + + ..\StandardRegex.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\StandardRegex1.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\StandardRegex2.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + \ No newline at end of file From ac0cb2a3b5344968baf5417b09f50443700a0653 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 20 Aug 2024 13:18:27 +0100 Subject: [PATCH 010/106] add missing files --- ...storeRegexRedactedValueInCatalogueTests.cs | 88 ++++++++++++++++++ Rdmp.Core/Icons/StandardRegex1.png | Bin 0 -> 267 bytes Rdmp.Core/Icons/StandardRegex2.png | Bin 0 -> 267 bytes 3 files changed, 88 insertions(+) create mode 100644 Rdmp.Core.Tests/CommandExecution/ExecuteCommandRestoreRegexRedactedValueInCatalogueTests.cs create mode 100644 Rdmp.Core/Icons/StandardRegex1.png create mode 100644 Rdmp.Core/Icons/StandardRegex2.png diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandRestoreRegexRedactedValueInCatalogueTests.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandRestoreRegexRedactedValueInCatalogueTests.cs new file mode 100644 index 0000000000..d366d1e167 --- /dev/null +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandRestoreRegexRedactedValueInCatalogueTests.cs @@ -0,0 +1,88 @@ +using FAnsi; +using NUnit.Framework; +using Rdmp.Core.CommandExecution.AtomicCommands; +using Rdmp.Core.CommandLine.Interactive; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.DataHelper.RegexRedaction; +using Rdmp.Core.Curation; +using Rdmp.Core.ReusableLibraryCode.Checks; +using System; +using System.Collections.Generic; +using System.Data; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using Tests.Common; + +namespace Rdmp.Core.Tests.CommandExecution; + +public class ExecuteCommandRestoreRegexRedactedValueInCatalogueTests: DatabaseTests +{ + [Test] + public void RedactionRestore_BasicTest() + { + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + const string sql = @" + CREATE TABLE [RedactionTest]( + [DischargeDate] [datetime] NOT NULL, + [Condition1] [varchar](400) NOT NULL, + [Condition2] [varchar](400) NOT NULL + CONSTRAINT [PK_DLCTest] PRIMARY KEY CLUSTERED + ( + [DischargeDate] ASC, + [Condition1] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] + ) ON [PRIMARY] + INSERT [RedactionTest]([DischargeDate],[Condition1],[Condition2]) + VALUES (CAST(0x000001B300000000 AS DateTime),N'1',N'1234TEST1234') + "; + var server = db.Server; + using (var con = server.GetConnection()) + { + con.Open(); + server.GetCommand(sql, con).ExecuteNonQuery(); + } + var table = db.ExpectTable("RedactionTest"); + var catalogue = new Catalogue(CatalogueRepository, "RedactionTest"); + var importer = new TableInfoImporter(CatalogueRepository, table); + importer.DoImport(out var _tableInfo, out var _columnInfos); + foreach (var columnInfo in _columnInfos) + { + var ci = new CatalogueItem(CatalogueRepository, catalogue, columnInfo.GetRuntimeName()); + ci.SaveToDatabase(); + var ei = new ExtractionInformation(CatalogueRepository, ci, columnInfo, ""); + ei.SaveToDatabase(); + } + var activator = new ConsoleInputManager(RepositoryLocator, ThrowImmediatelyCheckNotifier.Quiet); + var regexConfiguration = new RegexRedactionConfiguration(CatalogueRepository, "TestReplacer", new Regex("TEST"), "GG", "Replace TEST with GG"); + regexConfiguration.SaveToDatabase(); + var cmd = new ExecuteCommandPerformRegexRedactionOnCatalogue(activator, catalogue, regexConfiguration, _columnInfos.Where(c => c.GetRuntimeName() == "Condition2").ToList()); + Assert.DoesNotThrow(() => cmd.Execute()); + var retrieveSQL = @"select [Condition2] from [RedactionTest]"; + var dt = new DataTable(); + dt.BeginLoadData(); + using (var fetchCmd = server.GetCommand(retrieveSQL, server.GetConnection())) + { + using var da = server.GetDataAdapter(fetchCmd); + da.Fill(dt); + } + Assert.That(dt.Rows.Count, Is.EqualTo(1)); + Assert.That(dt.Rows[0].ItemArray[0], Is.EqualTo("12341234")); + var redaction = CatalogueRepository.GetAllObjects().Last(); + var revert = new ExecuteCommandRestoreRegexRedactedValueInCatalogue(activator, redaction); + Assert.DoesNotThrow(() => revert.Execute()); + var foundRedaction = CatalogueRepository.GetAllObjectsWhere("RedactionConfiguration_ID", regexConfiguration.ID).ToList(); + Assert.That(foundRedaction.Count, Is.EqualTo(0)); + dt = new DataTable(); + dt.BeginLoadData(); + using (var fetchCmd = server.GetCommand(retrieveSQL, server.GetConnection())) + { + using var da = server.GetDataAdapter(fetchCmd); + da.Fill(dt); + } + Assert.That(dt.Rows.Count, Is.EqualTo(1)); + Assert.That(dt.Rows[0].ItemArray[0], Is.EqualTo("1234TEST1234")); + } + +} diff --git a/Rdmp.Core/Icons/StandardRegex1.png b/Rdmp.Core/Icons/StandardRegex1.png new file mode 100644 index 0000000000000000000000000000000000000000..e4ae71daedd18329e7b3da3d9314aac2f7be7ed5 GIT binary patch literal 267 zcmeAS@N?(olHy`uVBq!ia0vp^!XV7S1|*9D%+3HQmUKs7M+SzC{oH>NS%G}c0*}aI z1_r*pAk66gd%{|vpk#?_L`iUdT1k0gQ7S`0VrE{6US4X6f{C7io}rmb=K8NdMID|l zjv*HQdwUM@HW=_QTeJFqi|?^yvdM`I3%b=*AyXtZO2y^bDsStwyZa6V03n^J};TM(`}3JQt|f_)$IHp1Kq^n>FVdQ I&MBb@0L{i-;{X5v literal 0 HcmV?d00001 diff --git a/Rdmp.Core/Icons/StandardRegex2.png b/Rdmp.Core/Icons/StandardRegex2.png new file mode 100644 index 0000000000000000000000000000000000000000..e4ae71daedd18329e7b3da3d9314aac2f7be7ed5 GIT binary patch literal 267 zcmeAS@N?(olHy`uVBq!ia0vp^!XV7S1|*9D%+3HQmUKs7M+SzC{oH>NS%G}c0*}aI z1_r*pAk66gd%{|vpk#?_L`iUdT1k0gQ7S`0VrE{6US4X6f{C7io}rmb=K8NdMID|l zjv*HQdwUM@HW=_QTeJFqi|?^yvdM`I3%b=*AyXtZO2y^bDsStwyZa6V03n^J};TM(`}3JQt|f_)$IHp1Kq^n>FVdQ I&MBb@0L{i-;{X5v literal 0 HcmV?d00001 From a98f2b17da1b7f0b6e8313b43433e8cb4578f93e Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 20 Aug 2024 14:34:30 +0100 Subject: [PATCH 011/106] add docs --- .../RegexRedaction/IRegexRedaction.cs | 32 ++--- .../IRegexRedactionConfiguration.cs | 27 ++-- .../RegexRedaction/IRegexRedactionKey.cs | 28 ++-- .../RegexRedaction/RegexRedaction.cs | 126 ++++++++++-------- .../RegexRedactionConfiguration.cs | 12 +- .../RegexRedaction/RegexRedactionKey.cs | 62 +++++---- 6 files changed, 159 insertions(+), 128 deletions(-) diff --git a/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedaction.cs b/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedaction.cs index fe16cff926..1763b44f95 100644 --- a/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedaction.cs +++ b/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedaction.cs @@ -1,21 +1,23 @@ -using Rdmp.Core.MapsDirectlyToDatabaseTable; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . -namespace Rdmp.Core.Curation.DataHelper.RegexRedaction +using Rdmp.Core.MapsDirectlyToDatabaseTable; +namespace Rdmp.Core.Curation.DataHelper.RegexRedaction; + +/// +/// Stores an instance of a regex redaction +/// +public interface IRegexRedaction: IMapsDirectlyToDatabaseTable { - public interface IRegexRedaction: IMapsDirectlyToDatabaseTable - { - int RedactionConfiguration_ID { get; protected set; } - string RedactedValue { get; protected set; } - string ReplacementValue { get; protected set; } + int RedactionConfiguration_ID { get; protected set; } + string RedactedValue { get; protected set; } + string ReplacementValue { get; protected set; } - int StartingIndex { get; protected set; } + int StartingIndex { get; protected set; } - int ColumnInfo_ID { get; protected set; } - } + int ColumnInfo_ID { get; protected set; } } diff --git a/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedactionConfiguration.cs b/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedactionConfiguration.cs index d5d6fe0124..c0ce09d102 100644 --- a/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedactionConfiguration.cs +++ b/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedactionConfiguration.cs @@ -1,4 +1,10 @@ -using Rdmp.Core.MapsDirectlyToDatabaseTable; +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using Rdmp.Core.MapsDirectlyToDatabaseTable; using Rdmp.Core.Repositories; using System; using System.Collections.Generic; @@ -6,15 +12,16 @@ using System.Text; using System.Threading.Tasks; -namespace Rdmp.Core.Curation.DataHelper.RegexRedaction +namespace Rdmp.Core.Curation.DataHelper.RegexRedaction; +/// +/// Stores the configuration to run regex redactions on data +/// +public interface IRegexRedactionConfiguration: IMapsDirectlyToDatabaseTable { - public interface IRegexRedactionConfiguration: IMapsDirectlyToDatabaseTable - { - ICatalogueRepository CatalogueRepository { get; } + ICatalogueRepository CatalogueRepository { get; } - string Name { get; } - string Description { get; } - string RegexPattern { get; } - string RedactionString { get; } - } + string Name { get; } + string Description { get; } + string RegexPattern { get; } + string RedactionString { get; } } diff --git a/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedactionKey.cs b/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedactionKey.cs index f66c0942b8..d8fed11442 100644 --- a/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedactionKey.cs +++ b/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedactionKey.cs @@ -1,18 +1,20 @@ -using Rdmp.Core.Curation.Data; +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + using Rdmp.Core.MapsDirectlyToDatabaseTable; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -namespace Rdmp.Core.Curation.DataHelper.RegexRedaction +namespace Rdmp.Core.Curation.DataHelper.RegexRedaction; + +/// +/// Stores the PKs for a redaction +/// +public interface IRegexRedactionKey: IMapsDirectlyToDatabaseTable { - public interface IRegexRedactionKey: IMapsDirectlyToDatabaseTable - { - int RegexRedaction_ID { get; protected set; } - int ColumnInfo_ID { get; protected set; } - string Value { get; protected set; } - } + int RegexRedaction_ID { get; protected set; } + int ColumnInfo_ID { get; protected set; } + string Value { get; protected set; } } diff --git a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedaction.cs b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedaction.cs index e66c9153bf..5d5025efb6 100644 --- a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedaction.cs +++ b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedaction.cs @@ -1,81 +1,89 @@ -using Rdmp.Core.Curation.Data; +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using Rdmp.Core.Curation.Data; using Rdmp.Core.MapsDirectlyToDatabaseTable; using Rdmp.Core.Repositories; using System; using System.Collections.Generic; using System.Data.Common; -using System.Linq; -namespace Rdmp.Core.Curation.DataHelper.RegexRedaction +namespace Rdmp.Core.Curation.DataHelper.RegexRedaction; + + +/// +/// Stores a redaction +/// +public class RegexRedaction : DatabaseEntity, IRegexRedaction + { - public class RegexRedaction : DatabaseEntity, IRegexRedaction + private int _redactionConfigurationID; + private int _startingIndex; + private string _redactedValue; + private string _replacementValue; + private int _columnInfoID; + public int RedactionConfiguration_ID { - private int _redactionConfigurationID; - private int _startingIndex; - private string _redactedValue; - private string _replacementValue; - private int _columnInfoID; - - public int RedactionConfiguration_ID - { - get => _redactionConfigurationID; - set => SetField(ref _redactionConfigurationID, value); - } + get => _redactionConfigurationID; + set => SetField(ref _redactionConfigurationID, value); + } - public string RedactedValue - { - get => _redactedValue; - set => SetField(ref _redactedValue, value.ToString()); - } + public string RedactedValue + { + get => _redactedValue; + set => SetField(ref _redactedValue, value.ToString()); + } - public string ReplacementValue - { - get => _replacementValue; - set => SetField(ref _replacementValue, value.ToString()); - } + public string ReplacementValue + { + get => _replacementValue; + set => SetField(ref _replacementValue, value.ToString()); + } - public int StartingIndex - { - get => _startingIndex; - set => SetField(ref _startingIndex, value); - } + public int StartingIndex + { + get => _startingIndex; + set => SetField(ref _startingIndex, value); + } - public int ColumnInfo_ID - { - get => _columnInfoID; - set => SetField(ref _columnInfoID, value); - } + public int ColumnInfo_ID + { + get => _columnInfoID; + set => SetField(ref _columnInfoID, value); + } - [NoMappingToDatabase] - public List RedactionKeys => [.. CatalogueRepository.GetAllObjectsWhere("RegexRedaction_ID", this.ID)]; + [NoMappingToDatabase] + public List RedactionKeys => [.. CatalogueRepository.GetAllObjectsWhere("RegexRedaction_ID", this.ID)]; - public RegexRedaction() { } + public RegexRedaction() { } - public RegexRedaction(ICatalogueRepository repository, int redactionConfigurationID, int startingIndex, string redactionValue, string replacementValue, int columnInfoID, Dictionary pkValues) + public RegexRedaction(ICatalogueRepository repository, int redactionConfigurationID, int startingIndex, string redactionValue, string replacementValue, int columnInfoID, Dictionary pkValues) + { + repository.InsertAndHydrate(this, new Dictionary { + {"RedactionConfiguration_ID",redactionConfigurationID}, + {"StartingIndex",startingIndex}, + {"RedactedValue", redactionValue }, + {"ReplacementValue", replacementValue }, + {"ColumnInfo_ID", columnInfoID } + }); + foreach (var tuple in pkValues) { - repository.InsertAndHydrate(this, new Dictionary { - {"RedactionConfiguration_ID",redactionConfigurationID}, - {"StartingIndex",startingIndex}, - {"RedactedValue", redactionValue }, - {"ReplacementValue", replacementValue }, - {"ColumnInfo_ID", columnInfoID } - }); - foreach (var tuple in pkValues) - { - var key = new RegexRedactionKey(repository, this, tuple.Key, tuple.Value); - key.SaveToDatabase(); - } + var key = new RegexRedactionKey(repository, this, tuple.Key, tuple.Value); + key.SaveToDatabase(); } + } - internal RegexRedaction(ICatalogueRepository repository, DbDataReader r) : base(repository, r) - { - _redactionConfigurationID = Int32.Parse(r["RedactionConfiguration_ID"].ToString()); - _startingIndex = Int32.Parse(r["StartingIndex"].ToString()); - _replacementValue = r["ReplacementValue"].ToString(); - _redactedValue = r["RedactedValue"].ToString(); - _columnInfoID = Int32.Parse(r["ColumnInfo_ID"].ToString()); - } + internal RegexRedaction(ICatalogueRepository repository, DbDataReader r) : base(repository, r) + { + _redactionConfigurationID = Int32.Parse(r["RedactionConfiguration_ID"].ToString()); + _startingIndex = Int32.Parse(r["StartingIndex"].ToString()); + _replacementValue = r["ReplacementValue"].ToString(); + _redactedValue = r["RedactedValue"].ToString(); + _columnInfoID = Int32.Parse(r["ColumnInfo_ID"].ToString()); } } diff --git a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionConfiguration.cs b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionConfiguration.cs index 832a4ab962..c3c9fa50db 100644 --- a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionConfiguration.cs +++ b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionConfiguration.cs @@ -1,14 +1,22 @@ -using Rdmp.Core.Curation.Data; +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using Rdmp.Core.Curation.Data; using Rdmp.Core.MapsDirectlyToDatabaseTable.Attributes; using System.Text.RegularExpressions; using Rdmp.Core.ReusableLibraryCode.Annotations; using Rdmp.Core.Repositories; using System.Collections.Generic; using System.Data.Common; -using Rdmp.Core.MapsDirectlyToDatabaseTable; namespace Rdmp.Core.Curation.DataHelper.RegexRedaction; +/// +/// Stores the configuration used to perform regex redactions +/// public class RegexRedactionConfiguration: DatabaseEntity, IRegexRedactionConfiguration { diff --git a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionKey.cs b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionKey.cs index ec1e6f9561..540517a550 100644 --- a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionKey.cs +++ b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionKey.cs @@ -1,43 +1,47 @@ -using Rdmp.Core.Curation.Data; +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . +using Rdmp.Core.Curation.Data; using Rdmp.Core.Repositories; using System; using System.Collections.Generic; using System.Data.Common; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -namespace Rdmp.Core.Curation.DataHelper.RegexRedaction +namespace Rdmp.Core.Curation.DataHelper.RegexRedaction; + +/// +/// Stores the Pks of an data entry that has a redaction +/// +public class RegexRedactionKey : DatabaseEntity, IRegexRedactionKey { - public class RegexRedactionKey : DatabaseEntity, IRegexRedactionKey - { - private int _redaction; - private int _columnInfo; - private string _value; + private int _redaction; + private int _columnInfo; + private string _value; - public int RegexRedaction_ID { get => _redaction; set => SetField(ref _redaction, value); } - public int ColumnInfo_ID { get => _columnInfo; set => SetField(ref _columnInfo, value); } + public int RegexRedaction_ID { get => _redaction; set => SetField(ref _redaction, value); } + public int ColumnInfo_ID { get => _columnInfo; set => SetField(ref _columnInfo, value); } - public string Value { get => _value; set => SetField(ref _value, value); } + public string Value { get => _value; set => SetField(ref _value, value); } - public RegexRedactionKey() { } + public RegexRedactionKey() { } - public RegexRedactionKey(ICatalogueRepository repository, RegexRedaction redaction, ColumnInfo pkColumn, string value) - { - repository.InsertAndHydrate(this, new Dictionary - { - {"RegexRedaction_ID", redaction.ID }, - {"ColumnInfo_ID", pkColumn.ID }, - {"Value", value }, - }); - } - - internal RegexRedactionKey(ICatalogueRepository repository, DbDataReader r) : base(repository, r) + public RegexRedactionKey(ICatalogueRepository repository, RegexRedaction redaction, ColumnInfo pkColumn, string value) + { + repository.InsertAndHydrate(this, new Dictionary { - _redaction = Int32.Parse(r["RegexRedaction_ID"].ToString()); - _columnInfo = Int32.Parse(r["ColumnInfo_ID"].ToString()); - _value = r["Value"].ToString(); - } + {"RegexRedaction_ID", redaction.ID }, + {"ColumnInfo_ID", pkColumn.ID }, + {"Value", value }, + }); + } + internal RegexRedactionKey(ICatalogueRepository repository, DbDataReader r) : base(repository, r) + { + _redaction = Int32.Parse(r["RegexRedaction_ID"].ToString()); + _columnInfo = Int32.Parse(r["ColumnInfo_ID"].ToString()); + _value = r["Value"].ToString(); } + } From d214aaeaff27b1bdcd8073f2e542219d466b71df Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 20 Aug 2024 15:38:04 +0100 Subject: [PATCH 012/106] add regions --- .../Curation/DataHelper/RegexRedaction/RegexRedaction.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedaction.cs b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedaction.cs index 5d5025efb6..649be54f80 100644 --- a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedaction.cs +++ b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedaction.cs @@ -26,6 +26,8 @@ public class RegexRedaction : DatabaseEntity, IRegexRedaction private string _replacementValue; private int _columnInfoID; + #region Database Properties + public int RedactionConfiguration_ID { get => _redactionConfigurationID; @@ -55,10 +57,12 @@ public int ColumnInfo_ID get => _columnInfoID; set => SetField(ref _columnInfoID, value); } + #endregion + #region Relationships [NoMappingToDatabase] public List RedactionKeys => [.. CatalogueRepository.GetAllObjectsWhere("RegexRedaction_ID", this.ID)]; - + #endregion public RegexRedaction() { } From 77118cfc207da6f3cee6a68f13b586ac0c457ede Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 20 Aug 2024 16:07:43 +0100 Subject: [PATCH 013/106] add ignore --- Rdmp.UI.Tests/DesignPatternTests/RunUITests.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Rdmp.UI.Tests/DesignPatternTests/RunUITests.cs b/Rdmp.UI.Tests/DesignPatternTests/RunUITests.cs index 182055a69e..d0463360d8 100644 --- a/Rdmp.UI.Tests/DesignPatternTests/RunUITests.cs +++ b/Rdmp.UI.Tests/DesignPatternTests/RunUITests.cs @@ -74,7 +74,8 @@ private List allowedToBeIncompatible typeof(ExecuteCommandImportFilterDescriptionsFromShare), typeof(ExecuteCommandSetArgument), typeof(ExecuteCommandAddToSession), - typeof(ExecuteCommandDeletePlugin) + typeof(ExecuteCommandDeletePlugin), + typeof(ExecuteCommandPerformRegexRedactionOnCatalogue) }); [Test] From 64120db4c15a46ecbcebdc0bd3200f7dc181532c Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 21 Aug 2024 09:56:25 +0100 Subject: [PATCH 014/106] add speed test --- ...ndPerformRegexRedactionOnCatalogueTests.cs | 86 ++++++++- ...storeRegexRedactedValueInCatalogueTests.cs | 175 ++++++++++++++++++ ...andRestoreRegexRedactedValueInCatalogue.cs | 2 +- .../RegexRedaction/IRegexRedaction.cs | 3 +- .../RegexRedaction/IRegexRedactionKey.cs | 3 +- 5 files changed, 265 insertions(+), 4 deletions(-) diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs index 905048c29c..1266d216e1 100644 --- a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs @@ -15,6 +15,9 @@ using Rdmp.Core.ReusableLibraryCode.Checks; using MongoDB.Driver.Core.Servers; using System.Data; +using System.Diagnostics; +using static MongoDB.Driver.WriteConcern; +using FAnsi.Discovery; namespace Rdmp.Core.Tests.CommandExecution { @@ -257,7 +260,7 @@ INSERT [RedactionTest]([DischargeDate],[Condition1],[Condition2]) regexConfiguration.SaveToDatabase(); var cmd = new ExecuteCommandPerformRegexRedactionOnCatalogue(activator, catalogue, regexConfiguration, _columnInfos.Where(c => c.GetRuntimeName() == "Condition2").ToList()); var ex = Assert.Throws(() => cmd.Execute()); - Assert.That(ex.Message, Is.EqualTo("Unable to identify any primary keys in table '[TEST_ScratchArea].[dbo].[RedactionTest]'. Redactions cannot be performed on tables without primary keys")); + Assert.That(ex.Message, Is.EqualTo($"Unable to identify any primary keys in table '{db.GetWrappedName()}.[dbo].[RedactionTest]'. Redactions cannot be performed on tables without primary keys")); } @@ -316,5 +319,86 @@ INSERT [RedactionTest]([DischargeDate],[Condition1],[Condition2]) Assert.That(redactions[0].StartingIndex, Is.EqualTo(4)); Assert.That(redactions[1].StartingIndex, Is.EqualTo(12)); } + + + private string GetInsert(int i) + { + return @$" + ({i}, N'{i}', N'1234TEST1234')"; + } + + private void InsertIntoDB(DiscoveredDatabase db, int start) + { + var insertText = string.Join(',', Enumerable.Range(start, 1000).ToArray().Select(i => GetInsert(i))); + string sql = $@" + INSERT[RedactionTest]([DischargeDate], [Condition1], [Condition2]) + VALUES + {insertText} + "; + var server = db.Server; + using (var con = server.GetConnection()) + { + con.Open(); + server.GetCommand(sql, con).ExecuteNonQuery(); + } + } + + [Test] + public void Redaction_BulkTest() + { + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + var server = db.Server; + string sql = $@" + CREATE TABLE [RedactionTest]( + [DischargeDate] [int] NOT NULL, + [Condition1] [varchar](400) NOT NULL, + [Condition2] [varchar](400) NOT NULL + CONSTRAINT [PK_DLCTest] PRIMARY KEY CLUSTERED + ( + [DischargeDate] ASC, + [Condition1] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] + ) ON [PRIMARY] + "; + using (var con = server.GetConnection()) + { + con.Open(); + server.GetCommand(sql, con).ExecuteNonQuery(); + } + var rowCount = 10000; + foreach(var i in Enumerable.Range(0,rowCount / 1000)) { + InsertIntoDB(db, i * 1000); + } + var table = db.ExpectTable("RedactionTest"); + var catalogue = new Catalogue(CatalogueRepository, "RedactionTest"); + var importer = new TableInfoImporter(CatalogueRepository, table); + importer.DoImport(out var _tableInfo, out var _columnInfos); + foreach (var columnInfo in _columnInfos) + { + var ci = new CatalogueItem(CatalogueRepository, catalogue, columnInfo.GetRuntimeName()); + ci.SaveToDatabase(); + var ei = new ExtractionInformation(CatalogueRepository, ci, columnInfo, ""); + ei.SaveToDatabase(); + } + var activator = new ConsoleInputManager(RepositoryLocator, ThrowImmediatelyCheckNotifier.Quiet); + var regexConfiguration = new RegexRedactionConfiguration(CatalogueRepository, "TestReplacer", new Regex("TEST"), "GG", "Replace TEST with GG"); + regexConfiguration.SaveToDatabase(); + var cmd = new ExecuteCommandPerformRegexRedactionOnCatalogue(activator, catalogue, regexConfiguration, _columnInfos.Where(c => c.GetRuntimeName() == "Condition2").ToList()); + var timer = Stopwatch.StartNew(); + Assert.DoesNotThrow(() => cmd.Execute()); + timer.Stop(); + var x = timer.ElapsedMilliseconds / 1000; + Assert.That(timer.ElapsedMilliseconds / 1000, Is.EqualTo(100));// currently ~100persecond + var retrieveSQL = @"select [Condition2] from [RedactionTest]"; + var dt = new DataTable(); + dt.BeginLoadData(); + using (var fetchCmd = server.GetCommand(retrieveSQL, server.GetConnection())) + { + using var da = server.GetDataAdapter(fetchCmd); + da.Fill(dt); + } + Assert.That(dt.Rows.Count, Is.EqualTo(rowCount)); + Assert.That(dt.Rows[0].ItemArray[0], Is.EqualTo("12341234")); + } } } diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandRestoreRegexRedactedValueInCatalogueTests.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandRestoreRegexRedactedValueInCatalogueTests.cs index d366d1e167..69d3e457d8 100644 --- a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandRestoreRegexRedactedValueInCatalogueTests.cs +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandRestoreRegexRedactedValueInCatalogueTests.cs @@ -14,6 +14,7 @@ using System.Text.RegularExpressions; using System.Threading.Tasks; using Tests.Common; +using Rdmp.Core.Validation.Constraints.Secondary.Predictor; namespace Rdmp.Core.Tests.CommandExecution; @@ -84,5 +85,179 @@ INSERT [RedactionTest]([DischargeDate],[Condition1],[Condition2]) Assert.That(dt.Rows.Count, Is.EqualTo(1)); Assert.That(dt.Rows[0].ItemArray[0], Is.EqualTo("1234TEST1234")); } + [Test] + public void RedactionRestore_RestoreTwice() + { + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + const string sql = @" + CREATE TABLE [RedactionTest]( + [DischargeDate] [datetime] NOT NULL, + [Condition1] [varchar](400) NOT NULL, + [Condition2] [varchar](400) NOT NULL + CONSTRAINT [PK_DLCTest] PRIMARY KEY CLUSTERED + ( + [DischargeDate] ASC, + [Condition1] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] + ) ON [PRIMARY] + INSERT [RedactionTest]([DischargeDate],[Condition1],[Condition2]) + VALUES (CAST(0x000001B300000000 AS DateTime),N'1',N'1234TEST1234') + "; + var server = db.Server; + using (var con = server.GetConnection()) + { + con.Open(); + server.GetCommand(sql, con).ExecuteNonQuery(); + } + var table = db.ExpectTable("RedactionTest"); + var catalogue = new Catalogue(CatalogueRepository, "RedactionTest"); + var importer = new TableInfoImporter(CatalogueRepository, table); + importer.DoImport(out var _tableInfo, out var _columnInfos); + foreach (var columnInfo in _columnInfos) + { + var ci = new CatalogueItem(CatalogueRepository, catalogue, columnInfo.GetRuntimeName()); + ci.SaveToDatabase(); + var ei = new ExtractionInformation(CatalogueRepository, ci, columnInfo, ""); + ei.SaveToDatabase(); + } + var activator = new ConsoleInputManager(RepositoryLocator, ThrowImmediatelyCheckNotifier.Quiet); + var regexConfiguration = new RegexRedactionConfiguration(CatalogueRepository, "TestReplacer", new Regex("TEST"), "GG", "Replace TEST with GG"); + regexConfiguration.SaveToDatabase(); + var cmd = new ExecuteCommandPerformRegexRedactionOnCatalogue(activator, catalogue, regexConfiguration, _columnInfos.Where(c => c.GetRuntimeName() == "Condition2").ToList()); + Assert.DoesNotThrow(() => cmd.Execute()); + var retrieveSQL = @"select [Condition2] from [RedactionTest]"; + var dt = new DataTable(); + dt.BeginLoadData(); + using (var fetchCmd = server.GetCommand(retrieveSQL, server.GetConnection())) + { + using var da = server.GetDataAdapter(fetchCmd); + da.Fill(dt); + } + Assert.That(dt.Rows.Count, Is.EqualTo(1)); + Assert.That(dt.Rows[0].ItemArray[0], Is.EqualTo("12341234")); + var redaction = CatalogueRepository.GetAllObjects().Last(); + var revert = new ExecuteCommandRestoreRegexRedactedValueInCatalogue(activator, redaction); + Assert.DoesNotThrow(() => revert.Execute()); + var foundRedaction = CatalogueRepository.GetAllObjectsWhere("RedactionConfiguration_ID", regexConfiguration.ID).ToList(); + Assert.That(foundRedaction.Count, Is.EqualTo(0)); + dt = new DataTable(); + dt.BeginLoadData(); + using (var fetchCmd = server.GetCommand(retrieveSQL, server.GetConnection())) + { + using var da = server.GetDataAdapter(fetchCmd); + da.Fill(dt); + } + Assert.That(dt.Rows.Count, Is.EqualTo(1)); + Assert.That(dt.Rows[0].ItemArray[0], Is.EqualTo("1234TEST1234")); + Assert.DoesNotThrow(() => cmd.Execute()); + dt = new DataTable(); + dt.BeginLoadData(); + using (var fetchCmd = server.GetCommand(retrieveSQL, server.GetConnection())) + { + using var da = server.GetDataAdapter(fetchCmd); + da.Fill(dt); + } + Assert.That(dt.Rows.Count, Is.EqualTo(1)); + Assert.That(dt.Rows[0].ItemArray[0], Is.EqualTo("12341234")); + redaction = CatalogueRepository.GetAllObjects().Last(); + revert = new ExecuteCommandRestoreRegexRedactedValueInCatalogue(activator, redaction); + Assert.DoesNotThrow(() => revert.Execute()); + foundRedaction = CatalogueRepository.GetAllObjectsWhere("RedactionConfiguration_ID", regexConfiguration.ID).ToList(); + Assert.That(foundRedaction.Count, Is.EqualTo(0)); + dt = new DataTable(); + dt.BeginLoadData(); + using (var fetchCmd = server.GetCommand(retrieveSQL, server.GetConnection())) + { + using var da = server.GetDataAdapter(fetchCmd); + da.Fill(dt); + } + Assert.That(dt.Rows.Count, Is.EqualTo(1)); + Assert.That(dt.Rows[0].ItemArray[0], Is.EqualTo("1234TEST1234")); + } + + //restore second in string + [Test] + public void RedactionRestore_MultipleInOneCell() + { + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + const string sql = @" + CREATE TABLE [RedactionTest]( + [DischargeDate] [datetime] NOT NULL, + [Condition1] [varchar](400) NOT NULL, + [Condition2] [varchar](400) NOT NULL + CONSTRAINT [PK_DLCTest] PRIMARY KEY CLUSTERED + ( + [DischargeDate] ASC, + [Condition1] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] + ) ON [PRIMARY] + INSERT [RedactionTest]([DischargeDate],[Condition1],[Condition2]) + VALUES (CAST(0x000001B300000000 AS DateTime),N'1',N'1234TEST1234TEST1234') + "; + var server = db.Server; + using (var con = server.GetConnection()) + { + con.Open(); + server.GetCommand(sql, con).ExecuteNonQuery(); + } + var table = db.ExpectTable("RedactionTest"); + var catalogue = new Catalogue(CatalogueRepository, "RedactionTest"); + var importer = new TableInfoImporter(CatalogueRepository, table); + importer.DoImport(out var _tableInfo, out var _columnInfos); + foreach (var columnInfo in _columnInfos) + { + var ci = new CatalogueItem(CatalogueRepository, catalogue, columnInfo.GetRuntimeName()); + ci.SaveToDatabase(); + var ei = new ExtractionInformation(CatalogueRepository, ci, columnInfo, ""); + ei.SaveToDatabase(); + } + var activator = new ConsoleInputManager(RepositoryLocator, ThrowImmediatelyCheckNotifier.Quiet); + var regexConfiguration = new RegexRedactionConfiguration(CatalogueRepository, "TestReplacer", new Regex("TEST"), "DB", "Replace TEST with GG"); + regexConfiguration.SaveToDatabase(); + var cmd = new ExecuteCommandPerformRegexRedactionOnCatalogue(activator, catalogue, regexConfiguration, _columnInfos.Where(c => c.GetRuntimeName() == "Condition2").ToList()); + Assert.DoesNotThrow(() => cmd.Execute()); + var retrieveSQL = @"select [Condition2] from [RedactionTest]"; + var dt = new DataTable(); + dt.BeginLoadData(); + using (var fetchCmd = server.GetCommand(retrieveSQL, server.GetConnection())) + { + using var da = server.GetDataAdapter(fetchCmd); + da.Fill(dt); + } + Assert.That(dt.Rows.Count, Is.EqualTo(1)); + Assert.That(dt.Rows[0].ItemArray[0], Is.EqualTo("123412341234")); + var redactions = CatalogueRepository.GetAllObjectsWhere("ReplacementValue", ""); + Assert.That(redactions.Length, Is.EqualTo(2)); + Assert.That(redactions[0].StartingIndex, Is.EqualTo(4)); + Assert.That(redactions[1].StartingIndex, Is.EqualTo(12)); + //resore the second one + var revert = new ExecuteCommandRestoreRegexRedactedValueInCatalogue(activator, redactions[1]); + Assert.DoesNotThrow(() => revert.Execute()); + var foundRedaction = CatalogueRepository.GetAllObjectsWhere("RedactionConfiguration_ID", regexConfiguration.ID).ToList(); + Assert.That(foundRedaction.Count, Is.EqualTo(1)); + dt = new DataTable(); + dt.BeginLoadData(); + using (var fetchCmd = server.GetCommand(retrieveSQL, server.GetConnection())) + { + using var da = server.GetDataAdapter(fetchCmd); + da.Fill(dt); + } + Assert.That(dt.Rows.Count, Is.EqualTo(1)); + Assert.That(dt.Rows[0].ItemArray[0], Is.EqualTo("12341234TEST1234")); + //resore the first one + revert = new ExecuteCommandRestoreRegexRedactedValueInCatalogue(activator, redactions[0]); + Assert.DoesNotThrow(() => revert.Execute()); + foundRedaction = CatalogueRepository.GetAllObjectsWhere("RedactionConfiguration_ID", regexConfiguration.ID).ToList(); + Assert.That(foundRedaction.Count, Is.EqualTo(0)); + dt = new DataTable(); + dt.BeginLoadData(); + using (var fetchCmd = server.GetCommand(retrieveSQL, server.GetConnection())) + { + using var da = server.GetDataAdapter(fetchCmd); + da.Fill(dt); + } + Assert.That(dt.Rows.Count, Is.EqualTo(1)); + Assert.That(dt.Rows[0].ItemArray[0], Is.EqualTo("1234TEST1234TEST1234")); + } } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreRegexRedactedValueInCatalogue.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreRegexRedactedValueInCatalogue.cs index 9fb303bca4..04c523fe1e 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreRegexRedactedValueInCatalogue.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreRegexRedactedValueInCatalogue.cs @@ -73,7 +73,7 @@ public override void Execute() var pkColumnInfo = _activator.RepositoryLocator.CatalogueRepository.GetObjectByID(rk.ColumnInfo_ID); var matchValue = RegexRedactionHelper.ConvertPotentialDateTimeObject(rk.Value, pkColumnInfo.Data_type); sqlLines.Add(new CustomLine($"t1.{pkColumnInfo.GetRuntimeName()} = {matchValue}", QueryComponent.WHERE)); - //todo add original value for column into where + sqlLines.Add(new CustomLine($"t1.{columnInfo.GetRuntimeName()} = '{dt.Rows[0][0]}'", QueryComponent.WHERE)); sqlLines.Add(new CustomLine(string.Format("t1.{0} = t2.{0}", pkColumnInfo.GetRuntimeName()), QueryComponent.JoinInfoJoin)); } var updateSql = updateHelper.BuildUpdate(discoveredTable, discoveredTable, sqlLines); diff --git a/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedaction.cs b/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedaction.cs index 1763b44f95..0a309b5cee 100644 --- a/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedaction.cs +++ b/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedaction.cs @@ -5,6 +5,7 @@ // You should have received a copy of the GNU General Public License along with RDMP. If not, see . using Rdmp.Core.MapsDirectlyToDatabaseTable; +using Rdmp.Core.Repositories; namespace Rdmp.Core.Curation.DataHelper.RegexRedaction; /// @@ -12,7 +13,7 @@ namespace Rdmp.Core.Curation.DataHelper.RegexRedaction; /// public interface IRegexRedaction: IMapsDirectlyToDatabaseTable { - + ICatalogueRepository CatalogueRepository { get; } int RedactionConfiguration_ID { get; protected set; } string RedactedValue { get; protected set; } string ReplacementValue { get; protected set; } diff --git a/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedactionKey.cs b/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedactionKey.cs index d8fed11442..e735a4909d 100644 --- a/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedactionKey.cs +++ b/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedactionKey.cs @@ -5,6 +5,7 @@ // You should have received a copy of the GNU General Public License along with RDMP. If not, see . using Rdmp.Core.MapsDirectlyToDatabaseTable; +using Rdmp.Core.Repositories; namespace Rdmp.Core.Curation.DataHelper.RegexRedaction; @@ -13,7 +14,7 @@ namespace Rdmp.Core.Curation.DataHelper.RegexRedaction; /// public interface IRegexRedactionKey: IMapsDirectlyToDatabaseTable { - + ICatalogueRepository CatalogueRepository { get; } int RegexRedaction_ID { get; protected set; } int ColumnInfo_ID { get; protected set; } string Value { get; protected set; } From a461974dd9b3734d06cb415e4bd7e2c9ca260990 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 21 Aug 2024 10:14:49 +0100 Subject: [PATCH 015/106] improve speed --- ...CommandPerformRegexRedactionOnCatalogue.cs | 24 +++++++------------ 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs index 547fe7bf91..186de50177 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs @@ -46,28 +46,20 @@ private string GetRedactionValue(string value, ColumnInfo column, Dictionary replacementValue.Length) + if (lengthDiff > 0) { - if (paddingValue == '>') - { - replacementValue += paddingValue; - } - else - { - replacementValue = paddingValue + replacementValue; - } - paddingValue = paddingValue == '>' ? '<' : '>'; + var start = (int)Math.Floor((float)(lengthDiff / 2)); + var end = (int)Math.Ceiling((float)(lengthDiff / 2)); + replacementValue = replacementValue.PadLeft(start, '<').PadRight(end,'>'); } - value = value.Substring(0, startingIndex) + replacementValue + value.Substring(startingIndex + foundMatch.Length); + value = value[..startingIndex] + replacementValue + value[(startingIndex + foundMatch.Length)..]; - //create redaction - int columnID = column.ID; - var redaction = new RegexRedaction(_activator.RepositoryLocator.CatalogueRepository, _redactionConfiguration.ID, startingIndex, foundMatch, replacementValue, columnID, pkLookup); + var redaction = new RegexRedaction(_activator.RepositoryLocator.CatalogueRepository, _redactionConfiguration.ID, startingIndex, foundMatch, replacementValue, column.ID, pkLookup); redaction.SaveToDatabase(); } From 6a3a8d3e72332a40591de4be8583720eb21fbde5 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 21 Aug 2024 10:37:15 +0100 Subject: [PATCH 016/106] improve speed --- ...CommandPerformRegexRedactionOnCatalogue.cs | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs index 186de50177..947412df50 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs @@ -47,7 +47,7 @@ private string GetRedactionValue(string value, ColumnInfo column, Dictionary'); + replacementValue = replacementValue.PadLeft(start, '<').PadRight(end, '>'); } value = value[..startingIndex] + replacementValue + value[(startingIndex + foundMatch.Length)..]; @@ -69,18 +69,26 @@ private string GetRedactionValue(string value, ColumnInfo column, Dictionary().Select(dc => dc.ColumnName).ToList().IndexOf(column.GetRuntimeName()); - + int columnIndex = -1; + var dcColumnIndexes = columns.Cast().Select((item, index) => + { + if (item.ColumnName == column.GetRuntimeName()) + { + columnIndex = index; + } + return index; + }); + var columnNames = columns.Cast().Select(c => c.ColumnName).ToList(); Dictionary pkLookup = new(); - foreach (int i in Range(0, columns.Count).Where(n => n != index)) + foreach (int i in dcColumnIndexes.Where(n => n != columnIndex)) { //these are all the PK values - var name = columns.Cast().ToList()[i].ColumnName; + var name = columnNames[i]; var pkCatalogueItem = _cataloguePKs.Where(c => c.Name == name).First(); pkLookup.Add(pkCatalogueItem.ColumnInfo, match[i].ToString()); } - var redactedValue = GetRedactionValue(match[index].ToString(), column, pkLookup); + var redactedValue = GetRedactionValue(match[columnIndex].ToString(), column, pkLookup); var sqlLines = new List { From 0c33c807f4ff92c336201853d859d94f7a931d22 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 21 Aug 2024 12:15:39 +0100 Subject: [PATCH 017/106] update padding --- .../ExecuteCommandPerformRegexRedactionOnCatalogue.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs index 947412df50..fba9285795 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs @@ -2,6 +2,7 @@ using FAnsi; using FAnsi.Discovery; using FAnsi.Discovery.QuerySyntax; +using NPOI.OpenXmlFormats; using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.DataHelper.RegexRedaction; using Rdmp.Core.QueryBuilding; @@ -10,6 +11,7 @@ using System; using System.Collections.Generic; using System.Data; +using System.Data.Common; using System.Linq; using System.Text.RegularExpressions; using static System.Linq.Enumerable; @@ -25,7 +27,7 @@ public class ExecuteCommandPerformRegexRedactionOnCatalogue : BasicCommandExecut private readonly DiscoveredServer _server; private DiscoveredColumn[] _discoveredPKColumns; private List _cataloguePKs; - + //private DbConnection _conn; public ExecuteCommandPerformRegexRedactionOnCatalogue(IBasicActivateItems activator, ICatalogue catalogue, RegexRedactionConfiguration redactionConfiguration, List columns) { @@ -55,7 +57,8 @@ private string GetRedactionValue(string value, ColumnInfo column, Dictionary'); + replacementValue = replacementValue.PadLeft(start + replacementValue.Length, '<'); + replacementValue = replacementValue.PadRight(end+replacementValue.Length, '>'); } value = value[..startingIndex] + replacementValue + value[(startingIndex + foundMatch.Length)..]; From c522448dabc3fd0dd1e678443ad2ba3296a3e8c6 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 21 Aug 2024 14:21:02 +0100 Subject: [PATCH 018/106] stress test --- ...ndPerformRegexRedactionOnCatalogueTests.cs | 117 +++++++++--------- ...CommandPerformRegexRedactionOnCatalogue.cs | 20 +-- 2 files changed, 70 insertions(+), 67 deletions(-) diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs index 1266d216e1..5fe64637ab 100644 --- a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs @@ -323,8 +323,9 @@ INSERT [RedactionTest]([DischargeDate],[Condition1],[Condition2]) private string GetInsert(int i) { + var text = i % 7 == 0 ? "TEST" : ""; return @$" - ({i}, N'{i}', N'1234TEST1234')"; + ({i}, N'{i}', N'1234{text}1234')"; } private void InsertIntoDB(DiscoveredDatabase db, int start) @@ -343,62 +344,62 @@ private void InsertIntoDB(DiscoveredDatabase db, int start) } } - [Test] - public void Redaction_BulkTest() - { - var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); - var server = db.Server; - string sql = $@" - CREATE TABLE [RedactionTest]( - [DischargeDate] [int] NOT NULL, - [Condition1] [varchar](400) NOT NULL, - [Condition2] [varchar](400) NOT NULL - CONSTRAINT [PK_DLCTest] PRIMARY KEY CLUSTERED - ( - [DischargeDate] ASC, - [Condition1] ASC - )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] - ) ON [PRIMARY] - "; - using (var con = server.GetConnection()) - { - con.Open(); - server.GetCommand(sql, con).ExecuteNonQuery(); - } - var rowCount = 10000; - foreach(var i in Enumerable.Range(0,rowCount / 1000)) { - InsertIntoDB(db, i * 1000); - } - var table = db.ExpectTable("RedactionTest"); - var catalogue = new Catalogue(CatalogueRepository, "RedactionTest"); - var importer = new TableInfoImporter(CatalogueRepository, table); - importer.DoImport(out var _tableInfo, out var _columnInfos); - foreach (var columnInfo in _columnInfos) - { - var ci = new CatalogueItem(CatalogueRepository, catalogue, columnInfo.GetRuntimeName()); - ci.SaveToDatabase(); - var ei = new ExtractionInformation(CatalogueRepository, ci, columnInfo, ""); - ei.SaveToDatabase(); - } - var activator = new ConsoleInputManager(RepositoryLocator, ThrowImmediatelyCheckNotifier.Quiet); - var regexConfiguration = new RegexRedactionConfiguration(CatalogueRepository, "TestReplacer", new Regex("TEST"), "GG", "Replace TEST with GG"); - regexConfiguration.SaveToDatabase(); - var cmd = new ExecuteCommandPerformRegexRedactionOnCatalogue(activator, catalogue, regexConfiguration, _columnInfos.Where(c => c.GetRuntimeName() == "Condition2").ToList()); - var timer = Stopwatch.StartNew(); - Assert.DoesNotThrow(() => cmd.Execute()); - timer.Stop(); - var x = timer.ElapsedMilliseconds / 1000; - Assert.That(timer.ElapsedMilliseconds / 1000, Is.EqualTo(100));// currently ~100persecond - var retrieveSQL = @"select [Condition2] from [RedactionTest]"; - var dt = new DataTable(); - dt.BeginLoadData(); - using (var fetchCmd = server.GetCommand(retrieveSQL, server.GetConnection())) - { - using var da = server.GetDataAdapter(fetchCmd); - da.Fill(dt); - } - Assert.That(dt.Rows.Count, Is.EqualTo(rowCount)); - Assert.That(dt.Rows[0].ItemArray[0], Is.EqualTo("12341234")); - } + //[Test] + //public void Redaction_BulkTest() + //{ + // var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + // var server = db.Server; + // string sql = $@" + // CREATE TABLE [RedactionTest]( + // [DischargeDate] [int] NOT NULL, + // [Condition1] [varchar](400) NOT NULL, + // [Condition2] [varchar](400) NOT NULL + // CONSTRAINT [PK_DLCTest] PRIMARY KEY CLUSTERED + // ( + // [DischargeDate] ASC, + // [Condition1] ASC + // )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] + // ) ON [PRIMARY] + // "; + // using (var con = server.GetConnection()) + // { + // con.Open(); + // server.GetCommand(sql, con).ExecuteNonQuery(); + // } + // var rowCount = 1000000; + // foreach(var i in Enumerable.Range(0,rowCount / 1000)) { + // InsertIntoDB(db, i * 1000); + // } + // var table = db.ExpectTable("RedactionTest"); + // var catalogue = new Catalogue(CatalogueRepository, "RedactionTest"); + // var importer = new TableInfoImporter(CatalogueRepository, table); + // importer.DoImport(out var _tableInfo, out var _columnInfos); + // foreach (var columnInfo in _columnInfos) + // { + // var ci = new CatalogueItem(CatalogueRepository, catalogue, columnInfo.GetRuntimeName()); + // ci.SaveToDatabase(); + // var ei = new ExtractionInformation(CatalogueRepository, ci, columnInfo, ""); + // ei.SaveToDatabase(); + // } + // var activator = new ConsoleInputManager(RepositoryLocator, ThrowImmediatelyCheckNotifier.Quiet); + // var regexConfiguration = new RegexRedactionConfiguration(CatalogueRepository, "TestReplacer", new Regex("TEST"), "GG", "Replace TEST with GG"); + // regexConfiguration.SaveToDatabase(); + // var cmd = new ExecuteCommandPerformRegexRedactionOnCatalogue(activator, catalogue, regexConfiguration, _columnInfos.Where(c => c.GetRuntimeName() == "Condition2").ToList()); + // var timer = Stopwatch.StartNew(); + // Assert.DoesNotThrow(() => cmd.Execute()); + // timer.Stop(); + // var x = timer.ElapsedMilliseconds / 1000; + // Assert.That(timer.ElapsedMilliseconds / 1000, Is.EqualTo(100));// currently ~100persecond + // var retrieveSQL = @"select [Condition2] from [RedactionTest]"; + // var dt = new DataTable(); + // dt.BeginLoadData(); + // using (var fetchCmd = server.GetCommand(retrieveSQL, server.GetConnection())) + // { + // using var da = server.GetDataAdapter(fetchCmd); + // da.Fill(dt); + // } + // Assert.That(dt.Rows.Count, Is.EqualTo(rowCount)); + // Assert.That(dt.Rows[0].ItemArray[0], Is.EqualTo("12341234")); + //} } } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs index fba9285795..c1210ed594 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs @@ -12,8 +12,10 @@ using System.Collections.Generic; using System.Data; using System.Data.Common; +using System.Diagnostics; using System.Linq; using System.Text.RegularExpressions; +using System.Threading.Tasks; using static System.Linq.Enumerable; namespace Rdmp.Core.CommandExecution.AtomicCommands; @@ -27,7 +29,7 @@ public class ExecuteCommandPerformRegexRedactionOnCatalogue : BasicCommandExecut private readonly DiscoveredServer _server; private DiscoveredColumn[] _discoveredPKColumns; private List _cataloguePKs; - //private DbConnection _conn; + private DbConnection _conn; public ExecuteCommandPerformRegexRedactionOnCatalogue(IBasicActivateItems activator, ICatalogue catalogue, RegexRedactionConfiguration redactionConfiguration, List columns) { @@ -48,17 +50,17 @@ private string GetRedactionValue(string value, ColumnInfo column, Dictionary 0) { - var start = (int)Math.Floor((float)(lengthDiff / 2)); - var end = (int)Math.Ceiling((float)(lengthDiff / 2)); + var start = (int)Math.Floor(lengthDiff / 2); + var end = (int)Math.Ceiling(lengthDiff / 2); replacementValue = replacementValue.PadLeft(start + replacementValue.Length, '<'); - replacementValue = replacementValue.PadRight(end+replacementValue.Length, '>'); + replacementValue = replacementValue.PadRight(end + replacementValue.Length, '>'); } value = value[..startingIndex] + replacementValue + value[(startingIndex + foundMatch.Length)..]; @@ -106,10 +108,8 @@ private void Redact(DiscoveredTable table, ColumnInfo column, DataRow match, Dat } var sql = updateHelper.BuildUpdate(table, table, sqlLines); - var conn = _server.GetConnection(); - using (var cmd = _server.GetCommand(sql, conn)) + using (var cmd = _server.GetCommand(sql, _conn)) { - conn.Open(); cmd.ExecuteNonQuery(); } } @@ -118,6 +118,8 @@ public override void Execute() { base.Execute(); var memoryRepo = new MemoryCatalogueRepository(); + _conn = _server.GetConnection(); + _conn.Open(); foreach (var catalogueItem in _catalogue.CatalogueItems.Where(ci => !ci.ColumnInfo.IsPrimaryKey && _columns.Contains(ci.ColumnInfo))) { var columnName = catalogueItem.ColumnInfo.Name; @@ -138,7 +140,7 @@ public override void Execute() var sql = qb.SQL; var dt = new DataTable(); dt.BeginLoadData(); - using (var cmd = _server.GetCommand(sql, _server.GetConnection())) + using (var cmd = _server.GetCommand(sql, _conn)) { using var da = _server.GetDataAdapter(cmd); da.Fill(dt); From 4d996dfb1a749b52fa1de00b75c9be7f00ce6d2b Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 21 Aug 2024 15:43:32 +0100 Subject: [PATCH 019/106] add test case --- .../DataHelper/RegexRedaction/RegexRedaction.cs | 8 ++++++++ Tests.Common/UnitTests.cs | 15 +++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedaction.cs b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedaction.cs index 649be54f80..08766de84c 100644 --- a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedaction.cs +++ b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedaction.cs @@ -6,7 +6,9 @@ using Rdmp.Core.Curation.Data; using Rdmp.Core.MapsDirectlyToDatabaseTable; +using Rdmp.Core.MapsDirectlyToDatabaseTable.Attributes; using Rdmp.Core.Repositories; +using Rdmp.Core.ReusableLibraryCode.Annotations; using System; using System.Collections.Generic; using System.Data.Common; @@ -34,18 +36,23 @@ public int RedactionConfiguration_ID set => SetField(ref _redactionConfigurationID, value); } + [UsefulProperty] + [NotNull] public string RedactedValue { get => _redactedValue; set => SetField(ref _redactedValue, value.ToString()); } + [UsefulProperty] + [NotNull] public string ReplacementValue { get => _replacementValue; set => SetField(ref _replacementValue, value.ToString()); } + [NotNull] public int StartingIndex { get => _startingIndex; @@ -64,6 +71,7 @@ public int ColumnInfo_ID public List RedactionKeys => [.. CatalogueRepository.GetAllObjectsWhere("RegexRedaction_ID", this.ID)]; #endregion + public RegexRedaction() { } public RegexRedaction(ICatalogueRepository repository, int redactionConfigurationID, int startingIndex, string redactionValue, string replacementValue, int columnInfoID, Dictionary pkValues) diff --git a/Tests.Common/UnitTests.cs b/Tests.Common/UnitTests.cs index d0f45c5724..4853068808 100644 --- a/Tests.Common/UnitTests.cs +++ b/Tests.Common/UnitTests.cs @@ -32,6 +32,7 @@ using Rdmp.Core.Curation.Data.Pipelines; using Rdmp.Core.Curation.Data.Remoting; using Rdmp.Core.Curation.Data.Spontaneous; +using Rdmp.Core.Curation.DataHelper.RegexRedaction; using Rdmp.Core.Databases; using Rdmp.Core.DataExport.Data; using Rdmp.Core.DataExport.DataRelease; @@ -591,6 +592,20 @@ public static T WhenIHaveA(MemoryDataExportRepository repository) where T : D return (T)(object)new Setting(repository.CatalogueRepository, "", ""); } + if(typeof(T) == typeof(RegexRedaction)) + { + return (T)(object)new RegexRedaction(repository.CatalogueRepository, 0, 0, "", "", 0, new Dictionary()); + } + if (typeof(T) == typeof(RegexRedactionConfiguration)) + { + return (T)(object)new RegexRedactionConfiguration(repository.CatalogueRepository,"name",new System.Text.RegularExpressions.Regex(".*"),"T"); + } + if (typeof(T) == typeof(RegexRedactionKey)) + { + return (T)(object)new RegexRedactionKey(repository.CatalogueRepository,WhenIHaveA(repository),WhenIHaveA(repository),"PK"); + } + + throw new TestCaseNotWrittenYetException(typeof(T)); } From fc1c30d17b07aea5d3bd3442fe6c816eb6e16788 Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 22 Aug 2024 07:55:47 +0100 Subject: [PATCH 020/106] add fk names --- Rdmp.Core/Curation/KeywordHelp.txt | 6 +++++- .../up/086_AddRegexRedactionConfiguration.sql | 8 ++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/Rdmp.Core/Curation/KeywordHelp.txt b/Rdmp.Core/Curation/KeywordHelp.txt index e9cc5a02bd..2e0da0114a 100644 --- a/Rdmp.Core/Curation/KeywordHelp.txt +++ b/Rdmp.Core/Curation/KeywordHelp.txt @@ -94,4 +94,8 @@ FK_RowState_Evaluation: Ensures that when a DQE Evaluation is deleted all child FK_PeriodicityState_Evaluation: Ensures that when a DQE Evaluation is deleted all child database objects are also deleted FK_DQEGraphAnnotation_Evaluation: Ensures that when a DQE Evaluation is deleted all child database objects are also deleted FK_Memento_Commit: Ensures that when a Commit is deleted all its child database objects are also deleted -UNIQUE_SettingKey: Ensures the uniqueness of key values in the setting table \ No newline at end of file +UNIQUE_SettingKey: Ensures the uniqueness of key values in the setting table +FK_Redaction_RedactionConfiguration_ID: Prevents you from deleting a redaction configuration if ut's in use +FK_Redaction_ColumnInfo_ID: Prevents redactions from becoming orphaned from their associated catalogue columns +FK_RedactionKey_Redaction_ID: Prevents redaction primary keys from being orphaned from the associated redaction +FK_RedactionKey_ColumnInfo_ID: Prevents redaction keys from becoming orphaned from their associated catalogue columns \ No newline at end of file diff --git a/Rdmp.Core/Databases/CatalogueDatabase/up/086_AddRegexRedactionConfiguration.sql b/Rdmp.Core/Databases/CatalogueDatabase/up/086_AddRegexRedactionConfiguration.sql index 6377b4be41..21b8cf216a 100644 --- a/Rdmp.Core/Databases/CatalogueDatabase/up/086_AddRegexRedactionConfiguration.sql +++ b/Rdmp.Core/Databases/CatalogueDatabase/up/086_AddRegexRedactionConfiguration.sql @@ -28,8 +28,8 @@ CREATE TABLE [dbo].RegexRedaction( StartingIndex [int] NOT NULL, RedactedValue [nvarchar](250), ReplacementValue [nvarchar](250) NOT NULL, - FOREIGN KEY (RedactionConfiguration_ID) REFERENCES RegexRedactionConfiguration(ID), - FOREIGN KEY (ColumnInfo_ID) REFERENCES ColumnInfo(ID) ON DELETE CASCADE, + CONSTRAINT FK_Redaction_RedactionConfiguration_ID FOREIGN KEY (RedactionConfiguration_ID) REFERENCES RegexRedactionConfiguration(ID), + CONSTRAINT FK_Redaction_ColumnInfo_ID FOREIGN KEY (ColumnInfo_ID) REFERENCES ColumnInfo(ID) ON DELETE CASCADE, CONSTRAINT [PK_RegexRedaction] PRIMARY KEY CLUSTERED ( [ID] ASC @@ -45,8 +45,8 @@ CREATE TABLE [dbo].RegexRedactionKey( RegexRedaction_ID [int] NOT NULL, ColumnInfo_ID [int] NOT NULL, Value [nvarchar](max), - FOREIGN KEY (RegexRedaction_ID) REFERENCES RegexRedaction(ID) ON DELETE CASCADE, - FOREIGN KEY (ColumnInfo_ID) REFERENCES ColumnInfo(ID), + CONSTRAINT FK_RedactionKey_Redaction_ID FOREIGN KEY (RegexRedaction_ID) REFERENCES RegexRedaction(ID) ON DELETE CASCADE, + CONSTRAINT FK_RedactionKey_ColumnInfo_ID FOREIGN KEY (ColumnInfo_ID) REFERENCES ColumnInfo(ID), CONSTRAINT [PK_RegexRedactionKey] PRIMARY KEY CLUSTERED ( [ID] ASC From 88b407087ae64d9fde87f75a2ddc4490d4b154ce Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 22 Aug 2024 09:31:13 +0100 Subject: [PATCH 021/106] faster data load --- ...ndPerformRegexRedactionOnCatalogueTests.cs | 49 ++++++++++++++++++- ...CommandPerformRegexRedactionOnCatalogue.cs | 32 ++++++------ 2 files changed, 66 insertions(+), 15 deletions(-) diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs index 5fe64637ab..b8dd6d8c9d 100644 --- a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs @@ -323,7 +323,7 @@ INSERT [RedactionTest]([DischargeDate],[Condition1],[Condition2]) private string GetInsert(int i) { - var text = i % 7 == 0 ? "TEST" : ""; + var text = i % 66 == 0 ? "TEST" : ""; return @$" ({i}, N'{i}', N'1234{text}1234')"; } @@ -344,6 +344,53 @@ private void InsertIntoDB(DiscoveredDatabase db, int start) } } + + [Test] + public void Redaction_SpeedTest() + { + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + var server = db.Server; + string sql = $@" + CREATE TABLE [RedactionTest]( + [DischargeDate] [int] NOT NULL, + [Condition1] [varchar](400) NOT NULL, + [Condition2] [varchar](400) NOT NULL + CONSTRAINT [PK_DLCTest] PRIMARY KEY CLUSTERED + ( + [DischargeDate] ASC, + [Condition1] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] + ) ON [PRIMARY] + "; + using (var con = server.GetConnection()) + { + con.Open(); + server.GetCommand(sql, con).ExecuteNonQuery(); + } + var rowCount = 1000; + foreach (var i in Enumerable.Range(0, rowCount / 1000)) + { + InsertIntoDB(db, i * 1000); + } + var table = db.ExpectTable("RedactionTest"); + var catalogue = new Catalogue(CatalogueRepository, "RedactionTest"); + var importer = new TableInfoImporter(CatalogueRepository, table); + importer.DoImport(out var _tableInfo, out var _columnInfos); + foreach (var columnInfo in _columnInfos) + { + var ci = new CatalogueItem(CatalogueRepository, catalogue, columnInfo.GetRuntimeName()); + ci.SaveToDatabase(); + var ei = new ExtractionInformation(CatalogueRepository, ci, columnInfo, ""); + ei.SaveToDatabase(); + } + var activator = new ConsoleInputManager(RepositoryLocator, ThrowImmediatelyCheckNotifier.Quiet); + var regexConfiguration = new RegexRedactionConfiguration(CatalogueRepository, "TestReplacer", new Regex("TEST"), "GG", "Replace TEST with GG"); + regexConfiguration.SaveToDatabase(); + var columns = new []{ catalogue.CatalogueItems.Where(c => c.Name == "Condition2").First().ColumnInfo}; + var cmd = new ExecuteCommandPerformRegexRedactionOnCatalogue(activator, catalogue, regexConfiguration, columns.ToList()); + cmd.Execute(); + } + //[Test] //public void Redaction_BulkTest() //{ diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs index c1210ed594..97091a7f35 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs @@ -110,7 +110,7 @@ private void Redact(DiscoveredTable table, ColumnInfo column, DataRow match, Dat var sql = updateHelper.BuildUpdate(table, table, sqlLines); using (var cmd = _server.GetCommand(sql, _conn)) { - cmd.ExecuteNonQuery(); + cmd.ExecuteNonQuery(); } } @@ -120,41 +120,43 @@ public override void Execute() var memoryRepo = new MemoryCatalogueRepository(); _conn = _server.GetConnection(); _conn.Open(); - foreach (var catalogueItem in _catalogue.CatalogueItems.Where(ci => !ci.ColumnInfo.IsPrimaryKey && _columns.Contains(ci.ColumnInfo))) + var timer = Stopwatch.StartNew(); + foreach (var columnInfo in _columns.Where(c => !c.IsPrimaryKey)) { - var columnName = catalogueItem.ColumnInfo.Name; - var table = catalogueItem.ColumnInfo.TableInfo.Name; - DiscoveredTable discoveredTable = catalogueItem.ColumnInfo.TableInfo.Discover(DataAccessContext.InternalDataProcessing); + + var columnName = columnInfo.Name; + var table = columnInfo.TableInfo.Name; + DiscoveredTable discoveredTable = columnInfo.TableInfo.Discover(DataAccessContext.InternalDataProcessing); DiscoveredColumn[] discoveredColumns = discoveredTable.DiscoverColumns(); _discoveredPKColumns = discoveredColumns.Where(c => c.IsPrimaryKey).ToArray(); if (_discoveredPKColumns.Any()) { _cataloguePKs = _catalogue.CatalogueItems.Where(c => c.ColumnInfo.IsPrimaryKey).ToList(); var qb = new QueryBuilder(null, null, null); - qb.AddColumn(new ColumnInfoToIColumn(memoryRepo, catalogueItem.ColumnInfo)); + qb.AddColumn(new ColumnInfoToIColumn(memoryRepo, columnInfo)); foreach (var pk in _discoveredPKColumns) { var cataloguePk = _cataloguePKs.FirstOrDefault(c => c.ColumnInfo.GetRuntimeName() == pk.GetRuntimeName()); qb.AddColumn(new ColumnInfoToIColumn(memoryRepo, cataloguePk.ColumnInfo)); } + qb.AddCustomLine($"{columnInfo.GetRuntimeName()} LIKE '%{_redactionConfiguration.RegexPattern}%'", QueryComponent.WHERE); var sql = qb.SQL; var dt = new DataTable(); dt.BeginLoadData(); + using (var cmd = _server.GetCommand(sql, _conn)) { using var da = _server.GetDataAdapter(cmd); da.Fill(dt); } - DataTable _regexMatches = new(); - _regexMatches.BeginLoadData(); - _regexMatches = dt.AsEnumerable().Where(row => Regex.IsMatch(row[catalogueItem.ColumnInfo.GetRuntimeName()].ToString(), _redactionConfiguration.RegexPattern)).CopyToDataTable(); - _regexMatches.EndLoadData(); - dt.EndLoadData(); - - foreach (DataRow row in _regexMatches.Rows) + timer.Stop(); + var y = timer.ElapsedMilliseconds; + timer.Start(); + foreach (DataRow row in dt.Rows) { - Redact(discoveredTable, catalogueItem.ColumnInfo, row, _regexMatches.Columns); + Redact(discoveredTable, columnInfo, row, dt.Columns); } + } else { @@ -162,5 +164,7 @@ public override void Execute() throw new Exception($"Unable to identify any primary keys in table '{table}'. Redactions cannot be performed on tables without primary keys"); } } + timer.Stop(); + var x = timer.ElapsedMilliseconds; } } From e983d5fb7d23bb3c11ef534c007d3e16398a1152 Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 22 Aug 2024 10:02:58 +0100 Subject: [PATCH 022/106] faster update --- ...ndPerformRegexRedactionOnCatalogueTests.cs | 2 +- ...CommandPerformRegexRedactionOnCatalogue.cs | 83 ++++++++++++++----- 2 files changed, 63 insertions(+), 22 deletions(-) diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs index b8dd6d8c9d..692310c835 100644 --- a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs @@ -367,7 +367,7 @@ [Condition1] ASC con.Open(); server.GetCommand(sql, con).ExecuteNonQuery(); } - var rowCount = 1000; + var rowCount = 1000000; foreach (var i in Enumerable.Range(0, rowCount / 1000)) { InsertIntoDB(db, i * 1000); diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs index 97091a7f35..0337e4e651 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs @@ -28,9 +28,12 @@ public class ExecuteCommandPerformRegexRedactionOnCatalogue : BasicCommandExecut private readonly List _columns; private readonly DiscoveredServer _server; private DiscoveredColumn[] _discoveredPKColumns; + private DiscoveredTable _discoveredTable; private List _cataloguePKs; private DbConnection _conn; + private DataTable redactionUpates = new(); + public ExecuteCommandPerformRegexRedactionOnCatalogue(IBasicActivateItems activator, ICatalogue catalogue, RegexRedactionConfiguration redactionConfiguration, List columns) { _activator = activator; @@ -71,9 +74,35 @@ private string GetRedactionValue(string value, ColumnInfo column, Dictionary + { + new CustomLine($"t1.{column.GetRuntimeName()} = t2.{column.GetRuntimeName()}", QueryComponent.SET) + }; + foreach (var pk in _discoveredPKColumns) + { + sqlLines.Add(new CustomLine($"t1.{pk.GetRuntimeName()} = t2.{pk.GetRuntimeName()}", QueryComponent.WHERE)); + sqlLines.Add(new CustomLine(string.Format("t1.{0} = t2.{0}", pk.GetRuntimeName()), QueryComponent.JoinInfoJoin)); + + } + var sql = updateHelper.BuildUpdate(_discoveredTable, redactionTable, sqlLines); + using (var cmd = _server.GetCommand(sql, _conn)) + { + cmd.ExecuteNonQuery(); + } + redactionTable.Drop(); + } + + private void Redact(DiscoveredTable table, ColumnInfo column, DataRow match, DataColumnCollection columns) + { + + + + //var updateHelper = _server.GetQuerySyntaxHelper().UpdateHelper; int columnIndex = -1; var dcColumnIndexes = columns.Cast().Select((item, index) => { @@ -84,6 +113,8 @@ private void Redact(DiscoveredTable table, ColumnInfo column, DataRow match, Dat return index; }); var columnNames = columns.Cast().Select(c => c.ColumnName).ToList(); + + //feel like we could do this PK stuff just once? Dictionary pkLookup = new(); foreach (int i in dcColumnIndexes.Where(n => n != columnIndex)) { @@ -94,24 +125,27 @@ private void Redact(DiscoveredTable table, ColumnInfo column, DataRow match, Dat } var redactedValue = GetRedactionValue(match[columnIndex].ToString(), column, pkLookup); + match[columnIndex] = redactedValue; - var sqlLines = new List - { - new CustomLine($"t1.{column.GetRuntimeName()} = '{redactedValue}'", QueryComponent.SET) - }; - foreach (var pk in _discoveredPKColumns) - { - var matchValue = RegexRedactionHelper.ConvertPotentialDateTimeObject(match[pk.GetRuntimeName()].ToString(), pk.DataType.SQLType); + //var sqlLines = new List + //{ + // new CustomLine($"t1.{column.GetRuntimeName()} = '{redactedValue}'", QueryComponent.SET) + //}; + //foreach (var pk in _discoveredPKColumns) + //{ + // var matchValue = RegexRedactionHelper.ConvertPotentialDateTimeObject(match[pk.GetRuntimeName()].ToString(), pk.DataType.SQLType); - sqlLines.Add(new CustomLine($"t1.{pk.GetRuntimeName()} = {matchValue}", QueryComponent.WHERE)); - sqlLines.Add(new CustomLine(string.Format("t1.{0} = t2.{0}", pk.GetRuntimeName()), QueryComponent.JoinInfoJoin)); + // sqlLines.Add(new CustomLine($"t1.{pk.GetRuntimeName()} = {matchValue}", QueryComponent.WHERE)); + // sqlLines.Add(new CustomLine(string.Format("t1.{0} = t2.{0}", pk.GetRuntimeName()), QueryComponent.JoinInfoJoin)); - } - var sql = updateHelper.BuildUpdate(table, table, sqlLines); - using (var cmd = _server.GetCommand(sql, _conn)) - { - cmd.ExecuteNonQuery(); - } + //} + //var sql = updateHelper.BuildUpdate(table, table, sqlLines); + //using (var cmd = _server.GetCommand(sql, _conn)) + //{ + // cmd.ExecuteNonQuery(); + //} + + redactionUpates.ImportRow(match); } public override void Execute() @@ -123,11 +157,11 @@ public override void Execute() var timer = Stopwatch.StartNew(); foreach (var columnInfo in _columns.Where(c => !c.IsPrimaryKey)) { - + var columnName = columnInfo.Name; var table = columnInfo.TableInfo.Name; - DiscoveredTable discoveredTable = columnInfo.TableInfo.Discover(DataAccessContext.InternalDataProcessing); - DiscoveredColumn[] discoveredColumns = discoveredTable.DiscoverColumns(); + _discoveredTable = columnInfo.TableInfo.Discover(DataAccessContext.InternalDataProcessing); + DiscoveredColumn[] discoveredColumns = _discoveredTable.DiscoverColumns(); _discoveredPKColumns = discoveredColumns.Where(c => c.IsPrimaryKey).ToArray(); if (_discoveredPKColumns.Any()) { @@ -143,7 +177,7 @@ public override void Execute() var sql = qb.SQL; var dt = new DataTable(); dt.BeginLoadData(); - + using (var cmd = _server.GetCommand(sql, _conn)) { using var da = _server.GetDataAdapter(cmd); @@ -152,9 +186,16 @@ public override void Execute() timer.Stop(); var y = timer.ElapsedMilliseconds; timer.Start(); + redactionUpates = dt.Clone(); + redactionUpates.BeginLoadData(); foreach (DataRow row in dt.Rows) { - Redact(discoveredTable, columnInfo, row, dt.Columns); + Redact(_discoveredTable, columnInfo, row, dt.Columns); + } + redactionUpates.EndLoadData(); + if (dt.Rows.Count > 0) + { + DoJoinUpdate(columnInfo); } } From d5cadff69645182030ccc957d05322ca37b3ed9f Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 22 Aug 2024 10:46:23 +0100 Subject: [PATCH 023/106] more timers --- ...ndPerformRegexRedactionOnCatalogueTests.cs | 2 +- ...CommandPerformRegexRedactionOnCatalogue.cs | 57 ++++--------------- 2 files changed, 12 insertions(+), 47 deletions(-) diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs index 692310c835..89826cc6dc 100644 --- a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs @@ -323,7 +323,7 @@ INSERT [RedactionTest]([DischargeDate],[Condition1],[Condition2]) private string GetInsert(int i) { - var text = i % 66 == 0 ? "TEST" : ""; + var text = i % 7 == 0 ? "TEST" : ""; return @$" ({i}, N'{i}', N'1234{text}1234')"; } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs index 0337e4e651..bfdbc3197d 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs @@ -97,54 +97,12 @@ private void DoJoinUpdate(ColumnInfo column) redactionTable.Drop(); } - private void Redact(DiscoveredTable table, ColumnInfo column, DataRow match, DataColumnCollection columns) + private void Redact(DiscoveredTable table, ColumnInfo column, DataRow match) { - - - - //var updateHelper = _server.GetQuerySyntaxHelper().UpdateHelper; - int columnIndex = -1; - var dcColumnIndexes = columns.Cast().Select((item, index) => - { - if (item.ColumnName == column.GetRuntimeName()) - { - columnIndex = index; - } - return index; - }); - var columnNames = columns.Cast().Select(c => c.ColumnName).ToList(); - - //feel like we could do this PK stuff just once? - Dictionary pkLookup = new(); - foreach (int i in dcColumnIndexes.Where(n => n != columnIndex)) - { - //these are all the PK values - var name = columnNames[i]; - var pkCatalogueItem = _cataloguePKs.Where(c => c.Name == name).First(); - pkLookup.Add(pkCatalogueItem.ColumnInfo, match[i].ToString()); - } - + int columnIndex = 0;// -1; + Dictionary pkLookup = Enumerable.Range(0, _cataloguePKs.Count).ToDictionary(i => _cataloguePKs[i].ColumnInfo, i => match[i + 1].ToString()); var redactedValue = GetRedactionValue(match[columnIndex].ToString(), column, pkLookup); match[columnIndex] = redactedValue; - - //var sqlLines = new List - //{ - // new CustomLine($"t1.{column.GetRuntimeName()} = '{redactedValue}'", QueryComponent.SET) - //}; - //foreach (var pk in _discoveredPKColumns) - //{ - // var matchValue = RegexRedactionHelper.ConvertPotentialDateTimeObject(match[pk.GetRuntimeName()].ToString(), pk.DataType.SQLType); - - // sqlLines.Add(new CustomLine($"t1.{pk.GetRuntimeName()} = {matchValue}", QueryComponent.WHERE)); - // sqlLines.Add(new CustomLine(string.Format("t1.{0} = t2.{0}", pk.GetRuntimeName()), QueryComponent.JoinInfoJoin)); - - //} - //var sql = updateHelper.BuildUpdate(table, table, sqlLines); - //using (var cmd = _server.GetCommand(sql, _conn)) - //{ - // cmd.ExecuteNonQuery(); - //} - redactionUpates.ImportRow(match); } @@ -186,17 +144,24 @@ public override void Execute() timer.Stop(); var y = timer.ElapsedMilliseconds; timer.Start(); + redactionUpates = dt.Clone(); redactionUpates.BeginLoadData(); foreach (DataRow row in dt.Rows) { - Redact(_discoveredTable, columnInfo, row, dt.Columns); + Redact(_discoveredTable, columnInfo, row); } redactionUpates.EndLoadData(); + timer.Stop(); + Console.WriteLine(timer.ElapsedMilliseconds); + timer.Start(); if (dt.Rows.Count > 0) { DoJoinUpdate(columnInfo); } + timer.Stop(); + Console.WriteLine(timer.ElapsedMilliseconds); + timer.Start(); } else From 5cdc35a93cea17eb738e0eec49e6c302872fa68c Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 22 Aug 2024 13:32:38 +0100 Subject: [PATCH 024/106] super fast redaction --- ...CommandPerformRegexRedactionOnCatalogue.cs | 63 +++++++++++++++++-- 1 file changed, 59 insertions(+), 4 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs index bfdbc3197d..b3e55adac8 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs @@ -2,9 +2,11 @@ using FAnsi; using FAnsi.Discovery; using FAnsi.Discovery.QuerySyntax; +using MongoDB.Driver; using NPOI.OpenXmlFormats; using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.DataHelper.RegexRedaction; +using Rdmp.Core.MapsDirectlyToDatabaseTable; using Rdmp.Core.QueryBuilding; using Rdmp.Core.Repositories; using Rdmp.Core.ReusableLibraryCode.DataAccess; @@ -31,6 +33,8 @@ public class ExecuteCommandPerformRegexRedactionOnCatalogue : BasicCommandExecut private DiscoveredTable _discoveredTable; private List _cataloguePKs; private DbConnection _conn; + private DataTable redactionsToSaveTable = new(); + private DataTable redactionUpates = new(); @@ -41,9 +45,14 @@ public ExecuteCommandPerformRegexRedactionOnCatalogue(IBasicActivateItems activa _redactionConfiguration = redactionConfiguration; _columns = columns; _server = _catalogue.GetDistinctLiveDatabaseServer(DataAccessContext.InternalDataProcessing, false); - + redactionsToSaveTable.Columns.Add("RedactionConfiguration_ID"); + redactionsToSaveTable.Columns.Add("ColumnInfo_ID"); + redactionsToSaveTable.Columns.Add("startingIndex"); + redactionsToSaveTable.Columns.Add("ReplacementValue"); + redactionsToSaveTable.Columns.Add("RedactedValue"); } + private string GetRedactionValue(string value, ColumnInfo column, Dictionary pkLookup) { var matches = Regex.Matches(value, _redactionConfiguration.RegexPattern); @@ -66,9 +75,9 @@ private string GetRedactionValue(string value, ColumnInfo column, Dictionary'); } value = value[..startingIndex] + replacementValue + value[(startingIndex + foundMatch.Length)..]; - - var redaction = new RegexRedaction(_activator.RepositoryLocator.CatalogueRepository, _redactionConfiguration.ID, startingIndex, foundMatch, replacementValue, column.ID, pkLookup); - redaction.SaveToDatabase(); + redactionsToSaveTable.Rows.Add([_redactionConfiguration.ID, column.ID, startingIndex, replacementValue, foundMatch]); + //var redaction = new RegexRedaction(_activator.RepositoryLocator.CatalogueRepository, _redactionConfiguration.ID, startingIndex, foundMatch, replacementValue, column.ID, pkLookup); + //redaction.SaveToDatabase(); } return value; @@ -90,6 +99,8 @@ private void DoJoinUpdate(ColumnInfo column) } var sql = updateHelper.BuildUpdate(_discoveredTable, redactionTable, sqlLines); + _conn = _server.GetConnection(); + _conn.Open(); using (var cmd = _server.GetCommand(sql, _conn)) { cmd.ExecuteNonQuery(); @@ -106,6 +117,49 @@ private void Redact(DiscoveredTable table, ColumnInfo column, DataRow match) redactionUpates.ImportRow(match); } + + + private void SaveRedactions() + { + if (redactionsToSaveTable.Rows.Count > 1000) + { + var read = 0; + while(read < redactionsToSaveTable.Rows.Count) + { + var rows = redactionsToSaveTable.AsEnumerable().Skip(read).Take(1000); + var insertValue = string.Join( + @", + ", rows.Select(r => $"({r[0]},{r[1]},{r[2]},'{r[3]}','{r[4]}')")); + var sql = @$" + INSERT INTO RegexRedaction(RedactionConfiguration_ID,ColumnInfo_ID,startingIndex,ReplacementValue,RedactedValue) + VALUES + {insertValue} + "; + (_activator.RepositoryLocator.CatalogueRepository as TableRepository).Insert( + sql, new Dictionary() + ); + read = read + 1000; + } + //dt drop + } + else + { + + var insertValue = string.Join( + @", + ", redactionsToSaveTable.AsEnumerable().Select(r => $"({r[0]},{r[1]},{r[2]},'{r[3]}','{r[4]}')")); + var sql = @$" + INSERT INTO RegexRedaction(RedactionConfiguration_ID,ColumnInfo_ID,startingIndex,ReplacementValue,RedactedValue) + VALUES + {insertValue} + "; + (_activator.RepositoryLocator.CatalogueRepository as TableRepository).Insert( + sql, new Dictionary() + ); + } + + } + public override void Execute() { base.Execute(); @@ -152,6 +206,7 @@ public override void Execute() Redact(_discoveredTable, columnInfo, row); } redactionUpates.EndLoadData(); + SaveRedactions(); timer.Stop(); Console.WriteLine(timer.ElapsedMilliseconds); timer.Start(); From a1f89dbf424865c0c602dd07be5f3e209c07f750 Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 22 Aug 2024 15:45:53 +0100 Subject: [PATCH 025/106] tidy up code --- ...CommandPerformRegexRedactionOnCatalogue.cs | 52 +++++++------------ 1 file changed, 19 insertions(+), 33 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs index b3e55adac8..4bd47120f6 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs @@ -1,9 +1,6 @@ -using CommandLine; -using FAnsi; -using FAnsi.Discovery; +using FAnsi.Discovery; using FAnsi.Discovery.QuerySyntax; using MongoDB.Driver; -using NPOI.OpenXmlFormats; using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.DataHelper.RegexRedaction; using Rdmp.Core.MapsDirectlyToDatabaseTable; @@ -17,7 +14,6 @@ using System.Diagnostics; using System.Linq; using System.Text.RegularExpressions; -using System.Threading.Tasks; using static System.Linq.Enumerable; namespace Rdmp.Core.CommandExecution.AtomicCommands; @@ -32,8 +28,7 @@ public class ExecuteCommandPerformRegexRedactionOnCatalogue : BasicCommandExecut private DiscoveredColumn[] _discoveredPKColumns; private DiscoveredTable _discoveredTable; private List _cataloguePKs; - private DbConnection _conn; - private DataTable redactionsToSaveTable = new(); + private readonly DataTable redactionsToSaveTable = new(); private DataTable redactionUpates = new(); @@ -76,8 +71,6 @@ private string GetRedactionValue(string value, ColumnInfo column, Dictionary pkLookup = Enumerable.Range(0, _cataloguePKs.Count).ToDictionary(i => _cataloguePKs[i].ColumnInfo, i => match[i + 1].ToString()); - var redactedValue = GetRedactionValue(match[columnIndex].ToString(), column, pkLookup); - match[columnIndex] = redactedValue; + var redactedValue = GetRedactionValue(match[0].ToString(), column, pkLookup); + match[0] = redactedValue; redactionUpates.ImportRow(match); } @@ -136,11 +129,10 @@ INSERT INTO RegexRedaction(RedactionConfiguration_ID,ColumnInfo_ID,startingIndex {insertValue} "; (_activator.RepositoryLocator.CatalogueRepository as TableRepository).Insert( - sql, new Dictionary() + sql, [] ); - read = read + 1000; + read += 1000; } - //dt drop } else { @@ -154,7 +146,7 @@ INSERT INTO RegexRedaction(RedactionConfiguration_ID,ColumnInfo_ID,startingIndex {insertValue} "; (_activator.RepositoryLocator.CatalogueRepository as TableRepository).Insert( - sql, new Dictionary() + sql, [] ); } @@ -164,8 +156,7 @@ public override void Execute() { base.Execute(); var memoryRepo = new MemoryCatalogueRepository(); - _conn = _server.GetConnection(); - _conn.Open(); + var timer = Stopwatch.StartNew(); foreach (var columnInfo in _columns.Where(c => !c.IsPrimaryKey)) { @@ -175,7 +166,7 @@ public override void Execute() _discoveredTable = columnInfo.TableInfo.Discover(DataAccessContext.InternalDataProcessing); DiscoveredColumn[] discoveredColumns = _discoveredTable.DiscoverColumns(); _discoveredPKColumns = discoveredColumns.Where(c => c.IsPrimaryKey).ToArray(); - if (_discoveredPKColumns.Any()) + if (_discoveredPKColumns.Length != 0) { _cataloguePKs = _catalogue.CatalogueItems.Where(c => c.ColumnInfo.IsPrimaryKey).ToList(); var qb = new QueryBuilder(null, null, null); @@ -189,27 +180,23 @@ public override void Execute() var sql = qb.SQL; var dt = new DataTable(); dt.BeginLoadData(); - - using (var cmd = _server.GetCommand(sql, _conn)) + var conn = _server.GetConnection(); + conn.Open(); + using (var cmd = _server.GetCommand(sql, conn)) { using var da = _server.GetDataAdapter(cmd); da.Fill(dt); } - timer.Stop(); - var y = timer.ElapsedMilliseconds; - timer.Start(); + conn.Close(); redactionUpates = dt.Clone(); redactionUpates.BeginLoadData(); foreach (DataRow row in dt.Rows) { - Redact(_discoveredTable, columnInfo, row); + Redact(columnInfo, row); } redactionUpates.EndLoadData(); SaveRedactions(); - timer.Stop(); - Console.WriteLine(timer.ElapsedMilliseconds); - timer.Start(); if (dt.Rows.Count > 0) { DoJoinUpdate(columnInfo); @@ -221,7 +208,6 @@ public override void Execute() } else { - //Unable to find any primary keys throw new Exception($"Unable to identify any primary keys in table '{table}'. Redactions cannot be performed on tables without primary keys"); } } From 0963aa30ecbf7090744e704c6e55527cbfeaee62 Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 23 Aug 2024 10:43:07 +0100 Subject: [PATCH 026/106] woking keys --- ...ndPerformRegexRedactionOnCatalogueTests.cs | 2 +- ...storeRegexRedactedValueInCatalogueTests.cs | 1 + ...CommandPerformRegexRedactionOnCatalogue.cs | 102 +++++++++++++----- 3 files changed, 75 insertions(+), 30 deletions(-) diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs index 89826cc6dc..ca9ef2bb39 100644 --- a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs @@ -367,7 +367,7 @@ [Condition1] ASC con.Open(); server.GetCommand(sql, con).ExecuteNonQuery(); } - var rowCount = 1000000; + var rowCount = 15000; foreach (var i in Enumerable.Range(0, rowCount / 1000)) { InsertIntoDB(db, i * 1000); diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandRestoreRegexRedactedValueInCatalogueTests.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandRestoreRegexRedactedValueInCatalogueTests.cs index 69d3e457d8..9791b82393 100644 --- a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandRestoreRegexRedactedValueInCatalogueTests.cs +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandRestoreRegexRedactedValueInCatalogueTests.cs @@ -149,6 +149,7 @@ INSERT [RedactionTest]([DischargeDate],[Condition1],[Condition2]) } Assert.That(dt.Rows.Count, Is.EqualTo(1)); Assert.That(dt.Rows[0].ItemArray[0], Is.EqualTo("1234TEST1234")); + cmd = new ExecuteCommandPerformRegexRedactionOnCatalogue(activator, catalogue, regexConfiguration, _columnInfos.Where(c => c.GetRuntimeName() == "Condition2").ToList()); Assert.DoesNotThrow(() => cmd.Execute()); dt = new DataTable(); dt.BeginLoadData(); diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs index 4bd47120f6..d15be52ddc 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs @@ -14,6 +14,7 @@ using System.Diagnostics; using System.Linq; using System.Text.RegularExpressions; +using Terminal.Gui; using static System.Linq.Enumerable; namespace Rdmp.Core.CommandExecution.AtomicCommands; @@ -29,7 +30,7 @@ public class ExecuteCommandPerformRegexRedactionOnCatalogue : BasicCommandExecut private DiscoveredTable _discoveredTable; private List _cataloguePKs; private readonly DataTable redactionsToSaveTable = new(); - + private readonly DataTable pksToSave = new(); private DataTable redactionUpates = new(); @@ -45,12 +46,17 @@ public ExecuteCommandPerformRegexRedactionOnCatalogue(IBasicActivateItems activa redactionsToSaveTable.Columns.Add("startingIndex"); redactionsToSaveTable.Columns.Add("ReplacementValue"); redactionsToSaveTable.Columns.Add("RedactedValue"); + pksToSave.Columns.Add("redactionsToSaveTable_Index"); + pksToSave.Columns.Add("ColumnInfo_ID"); + pksToSave.Columns.Add("Value"); } - private string GetRedactionValue(string value, ColumnInfo column, Dictionary pkLookup) + private string GetRedactionValue(string value, ColumnInfo column, DataRow m) { + Dictionary pkLookup = Enumerable.Range(0, _cataloguePKs.Count).ToDictionary(i => _cataloguePKs[i].ColumnInfo, i => m[i + 1].ToString()); var matches = Regex.Matches(value, _redactionConfiguration.RegexPattern); + var offset = 0; foreach (var match in matches) { var foundMatch = match.ToString(); @@ -71,6 +77,11 @@ private string GetRedactionValue(string value, ColumnInfo column, Dictionary pkLookup = Enumerable.Range(0, _cataloguePKs.Count).ToDictionary(i => _cataloguePKs[i].ColumnInfo, i => match[i + 1].ToString()); - var redactedValue = GetRedactionValue(match[0].ToString(), column, pkLookup); + + var redactedValue = GetRedactionValue(match[0].ToString(), column, match); match[0] = redactedValue; redactionUpates.ImportRow(match); } + private void DoInsertForRedactionKey(IEnumerable pks, DataTable ids) + { + + var x = ids.Rows.Count; + var y = pks.Count(); + var pkValues = string.Join( + @", + ", pks.Select(r => $"({ids.Rows[int.Parse(r[0].ToString())][0]},{r[1]},'{r[2]}')")); + var pksql = $@" + INSERT INTO RegexRedactionKey(RegexRedaction_ID,ColumnInfo_ID,Value) + VALUES + {pkValues} + "; + (_activator.RepositoryLocator.CatalogueRepository as TableRepository).Insert(pksql, null); + } + + + private void DoInsertForRedactions(IEnumerable rows, DataTable dt) + { + var insertValue = string.Join( + @", + ", rows.Select(r => $"({r[0]},{r[1]},{r[2]},'{r[3]}','{r[4]}')")); + var sql = @$" + DECLARE @output TABLE (id int) + INSERT INTO RegexRedaction(RedactionConfiguration_ID,ColumnInfo_ID,startingIndex,ReplacementValue,RedactedValue) OUTPUT inserted.ID INTO @output + VALUES + {insertValue}; + SELECT * from @output + "; + var conn = (_activator.RepositoryLocator.CatalogueRepository as TableRepository).GetConnection(); + using (var cmd = _server.GetCommand(sql, conn.Connection)) + { + using var da = _server.GetDataAdapter(cmd); + da.Fill(dt); + } + conn.Connection.Close(); + } private void SaveRedactions() { + DataTable dt = new(); if (redactionsToSaveTable.Rows.Count > 1000) { var read = 0; - while(read < redactionsToSaveTable.Rows.Count) + while (read < redactionsToSaveTable.Rows.Count) { var rows = redactionsToSaveTable.AsEnumerable().Skip(read).Take(1000); - var insertValue = string.Join( - @", - ", rows.Select(r => $"({r[0]},{r[1]},{r[2]},'{r[3]}','{r[4]}')")); - var sql = @$" - INSERT INTO RegexRedaction(RedactionConfiguration_ID,ColumnInfo_ID,startingIndex,ReplacementValue,RedactedValue) - VALUES - {insertValue} - "; - (_activator.RepositoryLocator.CatalogueRepository as TableRepository).Insert( - sql, [] - ); + var pkRows = pksToSave.AsEnumerable().Where(r => int.Parse(r[0].ToString()) >= read && int.Parse(r[0].ToString()) < (read + 1000)); + DoInsertForRedactions(rows, dt); read += 1000; } } else { - - var insertValue = string.Join( - @", - ", redactionsToSaveTable.AsEnumerable().Select(r => $"({r[0]},{r[1]},{r[2]},'{r[3]}','{r[4]}')")); - var sql = @$" - INSERT INTO RegexRedaction(RedactionConfiguration_ID,ColumnInfo_ID,startingIndex,ReplacementValue,RedactedValue) - VALUES - {insertValue} - "; - (_activator.RepositoryLocator.CatalogueRepository as TableRepository).Insert( - sql, [] - ); + DoInsertForRedactions(redactionsToSaveTable.AsEnumerable(), dt); + } + if (pksToSave.Rows.Count > 1000) + { + var read = 0; + while (read < pksToSave.Rows.Count) + { + DoInsertForRedactionKey(pksToSave.AsEnumerable().Skip(read).Take(1000), dt); + read += 1000; + } + } + else + { + var x = dt.Rows.Count; + var y = pksToSave.Rows.Count; + DoInsertForRedactionKey(pksToSave.AsEnumerable(), dt); } } @@ -156,7 +200,7 @@ public override void Execute() { base.Execute(); var memoryRepo = new MemoryCatalogueRepository(); - + var timer = Stopwatch.StartNew(); foreach (var columnInfo in _columns.Where(c => !c.IsPrimaryKey)) { From c3e49957b15195bea5e8fe3bd0593a21d79e0abc Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 23 Aug 2024 16:32:10 +0100 Subject: [PATCH 027/106] improved --- ...ndPerformRegexRedactionOnCatalogueTests.cs | 9 +- ...CommandPerformRegexRedactionOnCatalogue.cs | 191 +++++++++++++----- 2 files changed, 145 insertions(+), 55 deletions(-) diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs index ca9ef2bb39..422d88ec60 100644 --- a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs @@ -40,7 +40,9 @@ [Condition1] ASC )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] ) ON [PRIMARY] INSERT [RedactionTest]([DischargeDate],[Condition1],[Condition2]) - VALUES (CAST(0x000001B300000000 AS DateTime),N'1',N'1234TEST1234') + VALUES + (CAST(0x000001B300000000 AS DateTime),N'1',N'1234TEST1234'), + (CAST(0x000001B400000000 AS DateTime),N'2',N'1234TEST1234') "; var server = db.Server; using (var con = server.GetConnection()) @@ -72,8 +74,9 @@ INSERT [RedactionTest]([DischargeDate],[Condition1],[Condition2]) using var da = server.GetDataAdapter(fetchCmd); da.Fill(dt); } - Assert.That(dt.Rows.Count, Is.EqualTo(1)); + Assert.That(dt.Rows.Count, Is.EqualTo(2)); Assert.That(dt.Rows[0].ItemArray[0], Is.EqualTo("12341234")); + Assert.That(dt.Rows[1].ItemArray[0], Is.EqualTo("12341234")); } @@ -367,7 +370,7 @@ [Condition1] ASC con.Open(); server.GetCommand(sql, con).ExecuteNonQuery(); } - var rowCount = 15000; + var rowCount = 5000000; foreach (var i in Enumerable.Range(0, rowCount / 1000)) { InsertIntoDB(db, i * 1000); diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs index d15be52ddc..779d697eb8 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs @@ -1,5 +1,6 @@ using FAnsi.Discovery; using FAnsi.Discovery.QuerySyntax; +using Microsoft.Data.SqlClient; using MongoDB.Driver; using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.DataHelper.RegexRedaction; @@ -54,6 +55,7 @@ public ExecuteCommandPerformRegexRedactionOnCatalogue(IBasicActivateItems activa private string GetRedactionValue(string value, ColumnInfo column, DataRow m) { + Dictionary pkLookup = Enumerable.Range(0, _cataloguePKs.Count).ToDictionary(i => _cataloguePKs[i].ColumnInfo, i => m[i + 1].ToString()); var matches = Regex.Matches(value, _redactionConfiguration.RegexPattern); var offset = 0; @@ -121,79 +123,156 @@ private void Redact(ColumnInfo column, DataRow match) redactionUpates.ImportRow(match); } - private void DoInsertForRedactionKey(IEnumerable pks, DataTable ids) - { + //private void DoInsertForRedactionKey(IEnumerable pks, DataTable ids) + //{ + // var pkValues = string.Join( + // @", + // ", pks.Select(r => $"({r[0]},{r[1]},'{r[2]}')")); + // var pksql = $@" + // INSERT INTO RegexRedactionKey(RegexRedaction_ID,ColumnInfo_ID,Value) + // VALUES + // {pkValues} + // "; + // (_activator.RepositoryLocator.CatalogueRepository as TableRepository).Insert(pksql, null); + //} - var x = ids.Rows.Count; - var y = pks.Count(); - var pkValues = string.Join( - @", - ", pks.Select(r => $"({ids.Rows[int.Parse(r[0].ToString())][0]},{r[1]},'{r[2]}')")); - var pksql = $@" - INSERT INTO RegexRedactionKey(RegexRedaction_ID,ColumnInfo_ID,Value) - VALUES - {pkValues} - "; - (_activator.RepositoryLocator.CatalogueRepository as TableRepository).Insert(pksql, null); - } + //private void DoInsertForRedactions(IEnumerable rows, DataTable dt) + //{ + // var insertValue = string.Join( + // @", + // ", rows.Select(r => $"({r[0]},{r[1]},{r[2]},'{r[3]}','{r[4]}')")); + // var sql = @$" + // DECLARE @output TABLE (id int) + // INSERT INTO RegexRedaction(RedactionConfiguration_ID,ColumnInfo_ID,startingIndex,ReplacementValue,RedactedValue) OUTPUT inserted.ID INTO @output + // VALUES + // {insertValue}; + // SELECT * from @output + // "; + // var conn = (_activator.RepositoryLocator.CatalogueRepository as TableRepository).GetConnection(); + // using (var cmd = _server.GetCommand(sql, conn.Connection)) + // { + // using var da = _server.GetDataAdapter(cmd); + // da.Fill(dt); + // } + // conn.Connection.Close(); + //} - private void DoInsertForRedactions(IEnumerable rows, DataTable dt) + private void DoInsert(IEnumerable redactionRows, IEnumerable pkRows, int offset) { - var insertValue = string.Join( + string redactionString = string.Join( @", - ", rows.Select(r => $"({r[0]},{r[1]},{r[2]},'{r[3]}','{r[4]}')")); - var sql = @$" - DECLARE @output TABLE (id int) - INSERT INTO RegexRedaction(RedactionConfiguration_ID,ColumnInfo_ID,startingIndex,ReplacementValue,RedactedValue) OUTPUT inserted.ID INTO @output - VALUES - {insertValue}; - SELECT * from @output + ", redactionRows.Select(r => $"({r[0]},{r[1]},{r[2]},'{r[3]}','{r[4]}')")); + string keysString = string.Join( + @", + ", pkRows.Select(r => $"({r[0]},{r[1]},'{r[2]}')")); + + var sql = $@" + DECLARE @output TABLE (id int, inc int IDENTITY(1,1)) + INSERT INTO RegexRedaction(RedactionConfiguration_ID,ColumnInfo_ID,startingIndex,ReplacementValue,RedactedValue) OUTPUT inserted.id INTO @output + VALUES + {redactionString}; + DECLARE @redactionKeys TABLE (id int IDENTITY(1,1),RegexRedaction_ID int,ColumnInfo_ID int , Value varchar(max)); + INSERT INTO @redactionKeys(RegexRedaction_ID,ColumnInfo_ID,Value) + VALUES + {keysString}; + UPDATE t1 + SET t1.RegexRedaction_ID = t2.id + FROM @redactionKeys as t1 + INNER JOIN @output AS t2 + ON (t1.RegexRedaction_ID -{offset} +1) = t2.inc + WHERE (t1.RegexRedaction_ID -{offset} +1) = t2.inc; + INSERT INTO RegexRedactionKey + select RegexRedaction_ID,ColumnInfo_ID,Value FROM @redactionKeys; + select * from RegexRedactionKey "; - var conn = (_activator.RepositoryLocator.CatalogueRepository as TableRepository).GetConnection(); - using (var cmd = _server.GetCommand(sql, conn.Connection)) + try + { + (_activator.RepositoryLocator.CatalogueRepository as TableRepository).Insert(sql, null); + } + catch (SqlException ex) { - using var da = _server.GetDataAdapter(cmd); - da.Fill(dt); + Console.WriteLine(ex.Message); } - conn.Connection.Close(); + //DECLARE @output TABLE(id int, inc int IDENTITY(1, 1)) + // INSERT INTO Test_Catalogue.dbo.RegexRedaction(RedactionConfiguration_ID, ColumnInfo_ID, startingIndex, ReplacementValue, RedactedValue) OUTPUT inserted.id INTO @output + // VALUES + // (253, 854, 4, '', 'TEST'); + //DECLARE @redactionKeys TABLE(id int IDENTITY(1, 1), RegexRedaction_ID int, ColumnInfo_ID int, Value varchar(max)); + //INSERT INTO @redactionKeys(RegexRedaction_ID, ColumnInfo_ID, Value) + // VALUES + // (0, 854, '12/03/1901 00:00:00'), + // (0, 855, '1') + + // UPDATE t1 + + // SET t1.RegexRedaction_ID = t2.id + + // FROM @redactionKeys as t1 + + // INNER JOIN @output AS t2 + // ON t1.RegexRedaction_ID = t2.inc - 1 + + // WHERE t1.RegexRedaction_ID = t2.inc - 1; + //INSERT INTO Test_Catalogue.dbo.RegexRedactionKey + //select RegexRedaction_ID,ColumnInfo_ID,Value FROM @redactionKeys } private void SaveRedactions() { - DataTable dt = new(); - if (redactionsToSaveTable.Rows.Count > 1000) + var max = 1000; + //there will be an equal or greater number of PKs, so use that and take as many redactions as we can + if(pksToSave.Rows.Count > max) { var read = 0; - while (read < redactionsToSaveTable.Rows.Count) + var minColIndex = 0; + while(read < pksToSave.Rows.Count) { - var rows = redactionsToSaveTable.AsEnumerable().Skip(read).Take(1000); - var pkRows = pksToSave.AsEnumerable().Where(r => int.Parse(r[0].ToString()) >= read && int.Parse(r[0].ToString()) < (read + 1000)); - DoInsertForRedactions(rows, dt); - read += 1000; + var pkRows = pksToSave.AsEnumerable().Skip(read).Take(max); + var colIndex = int.Parse(pkRows.Last()[0].ToString()); + var redactionRows = redactionsToSaveTable.AsEnumerable().Skip(minColIndex).Take(colIndex - minColIndex+1); + DoInsert(redactionRows.AsEnumerable(), pkRows.AsEnumerable(), minColIndex); + minColIndex = colIndex; + read += max; } } else { - DoInsertForRedactions(redactionsToSaveTable.AsEnumerable(), dt); - } - if (pksToSave.Rows.Count > 1000) - { - var read = 0; - while (read < pksToSave.Rows.Count) - { - DoInsertForRedactionKey(pksToSave.AsEnumerable().Skip(read).Take(1000), dt); - read += 1000; - } - } - else - { - var x = dt.Rows.Count; - var y = pksToSave.Rows.Count; - DoInsertForRedactionKey(pksToSave.AsEnumerable(), dt); + DoInsert(redactionsToSaveTable.AsEnumerable(), pksToSave.AsEnumerable(),0); } + //DataTable dt = new(); + //if (redactionsToSaveTable.Rows.Count > 1000) + //{ + // var read = 0; + // while (read < redactionsToSaveTable.Rows.Count) + // { + // var rows = redactionsToSaveTable.AsEnumerable().Skip(read).Take(1000); + // var pkRows = pksToSave.AsEnumerable().Where(r => int.Parse(r[0].ToString()) >= read && int.Parse(r[0].ToString()) < (read + 1000)); + // DoInsertForRedactions(rows, dt); + // read += 1000; + // } + //} + //else + //{ + // DoInsertForRedactions(redactionsToSaveTable.AsEnumerable(), dt); + //} + //replace pk column[0] with valur at row indxo of dt + //if (pksToSave.Rows.Count > 1000) + //{ + // var read = 0; + // while (read < pksToSave.Rows.Count) + // { + // DoInsertForRedactionKey(pksToSave.AsEnumerable().Skip(read).Take(1000), dt); + // read += 1000; + // } + //} + //else + //{ + // DoInsertForRedactionKey(pksToSave.AsEnumerable(), dt); + //} + } public override void Execute() @@ -232,7 +311,9 @@ public override void Execute() da.Fill(dt); } conn.Close(); - + timer.Stop(); + Console.WriteLine(timer.ElapsedMilliseconds); + timer.Start(); redactionUpates = dt.Clone(); redactionUpates.BeginLoadData(); foreach (DataRow row in dt.Rows) @@ -240,7 +321,13 @@ public override void Execute() Redact(columnInfo, row); } redactionUpates.EndLoadData(); + timer.Stop(); + Console.WriteLine(timer.ElapsedMilliseconds); + timer.Start(); SaveRedactions(); + timer.Stop(); + Console.WriteLine(timer.ElapsedMilliseconds); + timer.Start(); if (dt.Rows.Count > 0) { DoJoinUpdate(columnInfo); From 03a1d4d7c9ab6b39f5a949da2db997d30cd8a47e Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 2 Sep 2024 08:58:11 +0100 Subject: [PATCH 028/106] add limiting --- ...ndPerformRegexRedactionOnCatalogueTests.cs | 55 +++++++++++ ...CommandPerformRegexRedactionOnCatalogue.cs | 97 ++----------------- 2 files changed, 62 insertions(+), 90 deletions(-) diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs index 422d88ec60..4a8d75b7fc 100644 --- a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs @@ -79,6 +79,61 @@ INSERT [RedactionTest]([DischargeDate],[Condition1],[Condition2]) Assert.That(dt.Rows[1].ItemArray[0], Is.EqualTo("12341234")); } + [Test] + public void Redaction_BasicRedactionTestWithLimit() + { + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + const string sql = @" + CREATE TABLE [RedactionTest]( + [DischargeDate] [datetime] NOT NULL, + [Condition1] [varchar](400) NOT NULL, + [Condition2] [varchar](400) NOT NULL + CONSTRAINT [PK_DLCTest] PRIMARY KEY CLUSTERED + ( + [DischargeDate] ASC, + [Condition1] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] + ) ON [PRIMARY] + INSERT [RedactionTest]([DischargeDate],[Condition1],[Condition2]) + VALUES + (CAST(0x000001B300000000 AS DateTime),N'1',N'1234TEST1234'), + (CAST(0x000001B400000000 AS DateTime),N'2',N'1234TEST1234') + "; + var server = db.Server; + using (var con = server.GetConnection()) + { + con.Open(); + server.GetCommand(sql, con).ExecuteNonQuery(); + } + var table = db.ExpectTable("RedactionTest"); + var catalogue = new Catalogue(CatalogueRepository, "RedactionTest"); + var importer = new TableInfoImporter(CatalogueRepository, table); + importer.DoImport(out var _tableInfo, out var _columnInfos); + foreach (var columnInfo in _columnInfos) + { + var ci = new CatalogueItem(CatalogueRepository, catalogue, columnInfo.GetRuntimeName()); + ci.SaveToDatabase(); + var ei = new ExtractionInformation(CatalogueRepository, ci, columnInfo, ""); + ei.SaveToDatabase(); + } + var activator = new ConsoleInputManager(RepositoryLocator, ThrowImmediatelyCheckNotifier.Quiet); + var regexConfiguration = new RegexRedactionConfiguration(CatalogueRepository, "TestReplacer", new Regex("TEST"), "GG", "Replace TEST with GG"); + regexConfiguration.SaveToDatabase(); + var cmd = new ExecuteCommandPerformRegexRedactionOnCatalogue(activator, catalogue, regexConfiguration, _columnInfos.Where(c => c.GetRuntimeName() == "Condition2").ToList(),1); + Assert.DoesNotThrow(() => cmd.Execute()); + var retrieveSQL = @"select [Condition2] from [RedactionTest]"; + var dt = new DataTable(); + dt.BeginLoadData(); + using (var fetchCmd = server.GetCommand(retrieveSQL, server.GetConnection())) + { + using var da = server.GetDataAdapter(fetchCmd); + da.Fill(dt); + } + Assert.That(dt.Rows.Count, Is.EqualTo(2)); + Assert.That(dt.Rows[0].ItemArray[0], Is.EqualTo("12341234")); + Assert.That(dt.Rows[1].ItemArray[0], Is.EqualTo("12341234")); + } + [Test] public void Redaction_OddStringLength() diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs index 779d697eb8..66021d15ef 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs @@ -32,16 +32,17 @@ public class ExecuteCommandPerformRegexRedactionOnCatalogue : BasicCommandExecut private List _cataloguePKs; private readonly DataTable redactionsToSaveTable = new(); private readonly DataTable pksToSave = new(); - + private readonly int? _readLimit; private DataTable redactionUpates = new(); - public ExecuteCommandPerformRegexRedactionOnCatalogue(IBasicActivateItems activator, ICatalogue catalogue, RegexRedactionConfiguration redactionConfiguration, List columns) + public ExecuteCommandPerformRegexRedactionOnCatalogue(IBasicActivateItems activator, ICatalogue catalogue, RegexRedactionConfiguration redactionConfiguration, List columns, int? readLimit=null) { _activator = activator; _catalogue = catalogue; _redactionConfiguration = redactionConfiguration; _columns = columns; _server = _catalogue.GetDistinctLiveDatabaseServer(DataAccessContext.InternalDataProcessing, false); + _readLimit = readLimit; redactionsToSaveTable.Columns.Add("RedactionConfiguration_ID"); redactionsToSaveTable.Columns.Add("ColumnInfo_ID"); redactionsToSaveTable.Columns.Add("startingIndex"); @@ -123,41 +124,6 @@ private void Redact(ColumnInfo column, DataRow match) redactionUpates.ImportRow(match); } - //private void DoInsertForRedactionKey(IEnumerable pks, DataTable ids) - //{ - // var pkValues = string.Join( - // @", - // ", pks.Select(r => $"({r[0]},{r[1]},'{r[2]}')")); - // var pksql = $@" - // INSERT INTO RegexRedactionKey(RegexRedaction_ID,ColumnInfo_ID,Value) - // VALUES - // {pkValues} - // "; - // (_activator.RepositoryLocator.CatalogueRepository as TableRepository).Insert(pksql, null); - //} - - - //private void DoInsertForRedactions(IEnumerable rows, DataTable dt) - //{ - // var insertValue = string.Join( - // @", - // ", rows.Select(r => $"({r[0]},{r[1]},{r[2]},'{r[3]}','{r[4]}')")); - // var sql = @$" - // DECLARE @output TABLE (id int) - // INSERT INTO RegexRedaction(RedactionConfiguration_ID,ColumnInfo_ID,startingIndex,ReplacementValue,RedactedValue) OUTPUT inserted.ID INTO @output - // VALUES - // {insertValue}; - // SELECT * from @output - // "; - // var conn = (_activator.RepositoryLocator.CatalogueRepository as TableRepository).GetConnection(); - // using (var cmd = _server.GetCommand(sql, conn.Connection)) - // { - // using var da = _server.GetDataAdapter(cmd); - // da.Fill(dt); - // } - // conn.Connection.Close(); - //} - private void DoInsert(IEnumerable redactionRows, IEnumerable pkRows, int offset) { string redactionString = string.Join( @@ -194,28 +160,6 @@ INSERT INTO RegexRedactionKey { Console.WriteLine(ex.Message); } - //DECLARE @output TABLE(id int, inc int IDENTITY(1, 1)) - // INSERT INTO Test_Catalogue.dbo.RegexRedaction(RedactionConfiguration_ID, ColumnInfo_ID, startingIndex, ReplacementValue, RedactedValue) OUTPUT inserted.id INTO @output - // VALUES - // (253, 854, 4, '', 'TEST'); - //DECLARE @redactionKeys TABLE(id int IDENTITY(1, 1), RegexRedaction_ID int, ColumnInfo_ID int, Value varchar(max)); - //INSERT INTO @redactionKeys(RegexRedaction_ID, ColumnInfo_ID, Value) - // VALUES - // (0, 854, '12/03/1901 00:00:00'), - // (0, 855, '1') - - // UPDATE t1 - - // SET t1.RegexRedaction_ID = t2.id - - // FROM @redactionKeys as t1 - - // INNER JOIN @output AS t2 - // ON t1.RegexRedaction_ID = t2.inc - 1 - - // WHERE t1.RegexRedaction_ID = t2.inc - 1; - //INSERT INTO Test_Catalogue.dbo.RegexRedactionKey - //select RegexRedaction_ID,ColumnInfo_ID,Value FROM @redactionKeys } @@ -242,37 +186,6 @@ private void SaveRedactions() DoInsert(redactionsToSaveTable.AsEnumerable(), pksToSave.AsEnumerable(),0); } - //DataTable dt = new(); - //if (redactionsToSaveTable.Rows.Count > 1000) - //{ - // var read = 0; - // while (read < redactionsToSaveTable.Rows.Count) - // { - // var rows = redactionsToSaveTable.AsEnumerable().Skip(read).Take(1000); - // var pkRows = pksToSave.AsEnumerable().Where(r => int.Parse(r[0].ToString()) >= read && int.Parse(r[0].ToString()) < (read + 1000)); - // DoInsertForRedactions(rows, dt); - // read += 1000; - // } - //} - //else - //{ - // DoInsertForRedactions(redactionsToSaveTable.AsEnumerable(), dt); - //} - //replace pk column[0] with valur at row indxo of dt - //if (pksToSave.Rows.Count > 1000) - //{ - // var read = 0; - // while (read < pksToSave.Rows.Count) - // { - // DoInsertForRedactionKey(pksToSave.AsEnumerable().Skip(read).Take(1000), dt); - // read += 1000; - // } - //} - //else - //{ - // DoInsertForRedactionKey(pksToSave.AsEnumerable(), dt); - //} - } public override void Execute() @@ -300,6 +213,10 @@ public override void Execute() qb.AddColumn(new ColumnInfoToIColumn(memoryRepo, cataloguePk.ColumnInfo)); } qb.AddCustomLine($"{columnInfo.GetRuntimeName()} LIKE '%{_redactionConfiguration.RegexPattern}%'", QueryComponent.WHERE); + if(_readLimit is not null) + { + qb.TopX = (int)_readLimit; + } var sql = qb.SQL; var dt = new DataTable(); dt.BeginLoadData(); From c93603986031f8911ff2272396a43f5d2ed006a8 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 3 Sep 2024 10:57:13 +0100 Subject: [PATCH 029/106] shared helper --- ...ndPerformRegexRedactionOnCatalogueTests.cs | 2 +- ...mmandIdentifyRegexRedactionsInCatalogue.cs | 63 ++--- ...CommandPerformRegexRedactionOnCatalogue.cs | 141 +---------- Rdmp.Core/Curation/Data/DataLoad/Argument.cs | 3 +- .../RegexRedactionConfiguration.cs | 1 + .../RegexRedaction/RegexRedactionHelper.cs | 142 ++++++++++- .../MatchingTablesMutilatorWithDataLoadJob.cs | 112 +++++++++ .../Mutilators/RegexRedactionMutilator.cs | 177 ++++++++++++++ .../ExecuteCommandRegexRedaction.cs | 5 +- Rdmp.UI/Rdmp.UI.csproj | 3 + .../RedactCatalogueUI.Designer.cs | 227 ++++++++++++++++++ Rdmp.UI/SimpleDialogs/RedactCatalogueUI.cs | 102 ++++++++ Rdmp.UI/SimpleDialogs/RedactCatalogueUI.resx | 120 +++++++++ 13 files changed, 929 insertions(+), 169 deletions(-) create mode 100644 Rdmp.Core/DataLoad/Modules/Mutilators/MatchingTablesMutilatorWithDataLoadJob.cs create mode 100644 Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs create mode 100644 Rdmp.UI/SimpleDialogs/RedactCatalogueUI.Designer.cs create mode 100644 Rdmp.UI/SimpleDialogs/RedactCatalogueUI.cs create mode 100644 Rdmp.UI/SimpleDialogs/RedactCatalogueUI.resx diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs index 4a8d75b7fc..baf338cfc8 100644 --- a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs @@ -131,7 +131,7 @@ INSERT [RedactionTest]([DischargeDate],[Condition1],[Condition2]) } Assert.That(dt.Rows.Count, Is.EqualTo(2)); Assert.That(dt.Rows[0].ItemArray[0], Is.EqualTo("12341234")); - Assert.That(dt.Rows[1].ItemArray[0], Is.EqualTo("12341234")); + Assert.That(dt.Rows[1].ItemArray[0], Is.EqualTo("1234TEST1234")); } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandIdentifyRegexRedactionsInCatalogue.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandIdentifyRegexRedactionsInCatalogue.cs index c44aa8a3b6..d5b714dd93 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandIdentifyRegexRedactionsInCatalogue.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandIdentifyRegexRedactionsInCatalogue.cs @@ -1,4 +1,7 @@ -using Rdmp.Core.Curation.Data; +using FAnsi.Discovery; +using FAnsi.Discovery.QuerySyntax; +using MongoDB.Driver.Core.Servers; +using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.DataHelper.RegexRedaction; using Rdmp.Core.QueryBuilding; using Rdmp.Core.Repositories; @@ -19,40 +22,45 @@ public class ExecuteCommandIdentifyRegexRedactionsInCatalogue : BasicCommandExec private readonly ICatalogue _catalogue; private readonly RegexRedactionConfiguration _redactionConfiguration; private readonly List _columns; + private readonly int? _readLimit; + public DataTable results; - //public ExecuteCommandIdentifyRegexRedactionsInCatalogue(IBasicActivateItems activator, ICatalogue catalogue, RegexRedactionConfiguration redactionConfiguration, List columns=null) - //{ - // _activator = activator; - // _catalogue = catalogue; - // _redactionConfiguration = redactionConfiguration; - // _columns = columns; - //} - - - public ExecuteCommandIdentifyRegexRedactionsInCatalogue(IBasicActivateItems activator) + public ExecuteCommandIdentifyRegexRedactionsInCatalogue(IBasicActivateItems activator, ICatalogue catalogue, RegexRedactionConfiguration redactionConfiguration, List columns = null, int? readLimit = null) { _activator = activator; - _catalogue = _activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Name", "Biochemistry").First(); - _redactionConfiguration = _activator.RepositoryLocator.CatalogueRepository.GetAllObjects().First(); - _columns = _catalogue.CatalogueItems.Select(ci => ci.ColumnInfo).ToList(); - + _catalogue = catalogue; + _redactionConfiguration = redactionConfiguration; + _columns = columns; + _readLimit = readLimit; } public override void Execute() { base.Execute(); var memoryRepo = new MemoryCatalogueRepository(); - foreach (var catalogueItem in _catalogue.CatalogueItems.Where(ci => !ci.ColumnInfo.IsPrimaryKey && _columns.Contains(ci.ColumnInfo))) + foreach (var columnInfo in _columns.Where(c => !c.IsPrimaryKey)) { - var columnName = catalogueItem.ColumnInfo.Name; - var table = catalogueItem.ColumnInfo.TableInfo.Name; var server = _catalogue.GetDistinctLiveDatabaseServer(DataAccessContext.InternalDataProcessing, false); - var pkColumns = _catalogue.CatalogueItems.Select(x => x.ColumnInfo).Where(x => x.IsPrimaryKey); - if (pkColumns.Where(pkc => pkc.Name.Contains(table)).Any()) + var columnName = columnInfo.Name; + var table = columnInfo.TableInfo.Name; + var _discoveredTable = columnInfo.TableInfo.Discover(DataAccessContext.InternalDataProcessing); + DiscoveredColumn[] discoveredColumns = _discoveredTable.DiscoverColumns(); + var _discoveredPKColumns = discoveredColumns.Where(c => c.IsPrimaryKey).ToArray(); + if (_discoveredPKColumns.Length != 0) { + var _cataloguePKs = _catalogue.CatalogueItems.Where(c => c.ColumnInfo.IsPrimaryKey).ToList(); var qb = new QueryBuilder(null, null, null); - qb.AddColumn(new ColumnInfoToIColumn(memoryRepo, catalogueItem.ColumnInfo)); - qb.AddColumnRange(pkColumns.Select(pk => new ColumnInfoToIColumn(memoryRepo, pk)).ToArray()); + qb.AddColumn(new ColumnInfoToIColumn(memoryRepo, columnInfo)); + foreach (var pk in _discoveredPKColumns) + { + var cataloguePk = _cataloguePKs.FirstOrDefault(c => c.ColumnInfo.GetRuntimeName() == pk.GetRuntimeName()); + qb.AddColumn(new ColumnInfoToIColumn(memoryRepo, cataloguePk.ColumnInfo)); + } + qb.AddCustomLine($"{columnInfo.GetRuntimeName()} LIKE '%{_redactionConfiguration.RegexPattern}%'", QueryComponent.WHERE); + if (_readLimit is not null) + { + qb.TopX = (int)_readLimit; + } var sql = qb.SQL; var dt = new DataTable(); dt.BeginLoadData(); @@ -61,11 +69,12 @@ public override void Execute() using var da = server.GetDataAdapter(cmd); da.Fill(dt); } - DataTable _regexMatches = new(); - _regexMatches.BeginLoadData(); - _regexMatches = dt.AsEnumerable().Where(row => Regex.IsMatch(row[catalogueItem.ColumnInfo.GetRuntimeName()].ToString(), _redactionConfiguration.RegexPattern)).CopyToDataTable(); - _regexMatches.EndLoadData(); - dt.EndLoadData(); + dt.Columns.Add("Proposed Redaction"); + foreach(DataRow row in dt.Rows) + { + row["Proposed Redaction"] = "TODO"; + } + results = dt; } else { diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs index 66021d15ef..d9dafff9c7 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs @@ -53,141 +53,6 @@ public ExecuteCommandPerformRegexRedactionOnCatalogue(IBasicActivateItems activa pksToSave.Columns.Add("Value"); } - - private string GetRedactionValue(string value, ColumnInfo column, DataRow m) - { - - Dictionary pkLookup = Enumerable.Range(0, _cataloguePKs.Count).ToDictionary(i => _cataloguePKs[i].ColumnInfo, i => m[i + 1].ToString()); - var matches = Regex.Matches(value, _redactionConfiguration.RegexPattern); - var offset = 0; - foreach (var match in matches) - { - var foundMatch = match.ToString(); - var startingIndex = value.IndexOf(foundMatch); - string replacementValue = _redactionConfiguration.RedactionString; - - var lengthDiff = (float)foundMatch.Length - replacementValue.Length; - if (lengthDiff < 1) - { - throw new Exception($"Redaction string '{_redactionConfiguration.RedactionString}' is longer than found match '{foundMatch}'."); - } - if (lengthDiff > 0) - { - var start = (int)Math.Floor(lengthDiff / 2); - var end = (int)Math.Ceiling(lengthDiff / 2); - replacementValue = replacementValue.PadLeft(start + replacementValue.Length, '<'); - replacementValue = replacementValue.PadRight(end + replacementValue.Length, '>'); - } - value = value[..startingIndex] + replacementValue + value[(startingIndex + foundMatch.Length)..]; - redactionsToSaveTable.Rows.Add([_redactionConfiguration.ID, column.ID, startingIndex, replacementValue, foundMatch]); - foreach (var pk in pkLookup) - { - pksToSave.Rows.Add([redactionUpates.Rows.Count + offset, pk.Key.ID, pk.Value]); - } - offset++; - } - - return value; - } - - private void DoJoinUpdate(ColumnInfo column) - { - var redactionTable = _discoveredTable.Database.CreateTable("TEMP_RedactionUpdates", redactionUpates); - var updateHelper = _server.GetQuerySyntaxHelper().UpdateHelper; - - var sqlLines = new List - { - new CustomLine($"t1.{column.GetRuntimeName()} = t2.{column.GetRuntimeName()}", QueryComponent.SET) - }; - foreach (var pk in _discoveredPKColumns) - { - sqlLines.Add(new CustomLine($"t1.{pk.GetRuntimeName()} = t2.{pk.GetRuntimeName()}", QueryComponent.WHERE)); - sqlLines.Add(new CustomLine(string.Format("t1.{0} = t2.{0}", pk.GetRuntimeName()), QueryComponent.JoinInfoJoin)); - - } - var sql = updateHelper.BuildUpdate(_discoveredTable, redactionTable, sqlLines); - var conn = _server.GetConnection(); - conn.Open(); - using (var cmd = _server.GetCommand(sql, conn)) - { - cmd.ExecuteNonQuery(); - } - conn.Close(); - redactionTable.Drop(); - } - - private void Redact(ColumnInfo column, DataRow match) - { - - var redactedValue = GetRedactionValue(match[0].ToString(), column, match); - match[0] = redactedValue; - redactionUpates.ImportRow(match); - } - - private void DoInsert(IEnumerable redactionRows, IEnumerable pkRows, int offset) - { - string redactionString = string.Join( - @", - ", redactionRows.Select(r => $"({r[0]},{r[1]},{r[2]},'{r[3]}','{r[4]}')")); - string keysString = string.Join( - @", - ", pkRows.Select(r => $"({r[0]},{r[1]},'{r[2]}')")); - - var sql = $@" - DECLARE @output TABLE (id int, inc int IDENTITY(1,1)) - INSERT INTO RegexRedaction(RedactionConfiguration_ID,ColumnInfo_ID,startingIndex,ReplacementValue,RedactedValue) OUTPUT inserted.id INTO @output - VALUES - {redactionString}; - DECLARE @redactionKeys TABLE (id int IDENTITY(1,1),RegexRedaction_ID int,ColumnInfo_ID int , Value varchar(max)); - INSERT INTO @redactionKeys(RegexRedaction_ID,ColumnInfo_ID,Value) - VALUES - {keysString}; - UPDATE t1 - SET t1.RegexRedaction_ID = t2.id - FROM @redactionKeys as t1 - INNER JOIN @output AS t2 - ON (t1.RegexRedaction_ID -{offset} +1) = t2.inc - WHERE (t1.RegexRedaction_ID -{offset} +1) = t2.inc; - INSERT INTO RegexRedactionKey - select RegexRedaction_ID,ColumnInfo_ID,Value FROM @redactionKeys; - select * from RegexRedactionKey - "; - try - { - (_activator.RepositoryLocator.CatalogueRepository as TableRepository).Insert(sql, null); - } - catch (SqlException ex) - { - Console.WriteLine(ex.Message); - } - } - - - private void SaveRedactions() - { - var max = 1000; - //there will be an equal or greater number of PKs, so use that and take as many redactions as we can - if(pksToSave.Rows.Count > max) - { - var read = 0; - var minColIndex = 0; - while(read < pksToSave.Rows.Count) - { - var pkRows = pksToSave.AsEnumerable().Skip(read).Take(max); - var colIndex = int.Parse(pkRows.Last()[0].ToString()); - var redactionRows = redactionsToSaveTable.AsEnumerable().Skip(minColIndex).Take(colIndex - minColIndex+1); - DoInsert(redactionRows.AsEnumerable(), pkRows.AsEnumerable(), minColIndex); - minColIndex = colIndex; - read += max; - } - } - else - { - DoInsert(redactionsToSaveTable.AsEnumerable(), pksToSave.AsEnumerable(),0); - } - - } - public override void Execute() { base.Execute(); @@ -235,19 +100,19 @@ public override void Execute() redactionUpates.BeginLoadData(); foreach (DataRow row in dt.Rows) { - Redact(columnInfo, row); + RegexRedactionHelper.Redact(columnInfo, row, _cataloguePKs, _redactionConfiguration, redactionsToSaveTable, pksToSave, redactionUpates); } redactionUpates.EndLoadData(); timer.Stop(); Console.WriteLine(timer.ElapsedMilliseconds); timer.Start(); - SaveRedactions(); + RegexRedactionHelper.SaveRedactions(_activator,pksToSave,redactionsToSaveTable); timer.Stop(); Console.WriteLine(timer.ElapsedMilliseconds); timer.Start(); if (dt.Rows.Count > 0) { - DoJoinUpdate(columnInfo); + RegexRedactionHelper.DoJoinUpdate(columnInfo,_discoveredTable,_server,redactionUpates,_discoveredPKColumns); } timer.Stop(); Console.WriteLine(timer.ElapsedMilliseconds); diff --git a/Rdmp.Core/Curation/Data/DataLoad/Argument.cs b/Rdmp.Core/Curation/Data/DataLoad/Argument.cs index 141f960991..1b174663ff 100644 --- a/Rdmp.Core/Curation/Data/DataLoad/Argument.cs +++ b/Rdmp.Core/Curation/Data/DataLoad/Argument.cs @@ -18,6 +18,7 @@ using Rdmp.Core.Curation.Data.Cohort; using Rdmp.Core.Curation.Data.Pipelines; using Rdmp.Core.Curation.Data.Remoting; +using Rdmp.Core.Curation.DataHelper.RegexRedaction; using Rdmp.Core.MapsDirectlyToDatabaseTable; using Rdmp.Core.MapsDirectlyToDatabaseTable.Attributes; using Rdmp.Core.Repositories; @@ -62,7 +63,7 @@ public abstract class Argument : DatabaseEntity, IArgument typeof(CacheProgress), typeof(ExternalDatabaseServer), typeof(StandardRegex), typeof(CohortIdentificationConfiguration), typeof(RemoteRDMP), typeof(Catalogue), typeof(CatalogueItem), - typeof(DataAccessCredentials), + typeof(DataAccessCredentials), typeof(RegexRedactionConfiguration), //wierd special cases typeof(ICustomUIDrivenClass), typeof(EncryptedString), diff --git a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionConfiguration.cs b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionConfiguration.cs index c3c9fa50db..bc1c441639 100644 --- a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionConfiguration.cs +++ b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionConfiguration.cs @@ -11,6 +11,7 @@ using Rdmp.Core.Repositories; using System.Collections.Generic; using System.Data.Common; +using Rdmp.Core.MapsDirectlyToDatabaseTable; namespace Rdmp.Core.Curation.DataHelper.RegexRedaction; diff --git a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs index 35889ff15f..d19228e0bd 100644 --- a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs +++ b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs @@ -1,8 +1,16 @@ -using Rdmp.Core.Curation.Data; +using FAnsi.Discovery.QuerySyntax; +using FAnsi.Discovery; +using Microsoft.Data.SqlClient; +using MongoDB.Driver.Core.Servers; +using Rdmp.Core.CommandExecution; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.MapsDirectlyToDatabaseTable; using System; using System.Collections.Generic; +using System.Data; using System.Linq; using System.Text; +using System.Text.RegularExpressions; using System.Threading.Tasks; namespace Rdmp.Core.Curation.DataHelper.RegexRedaction @@ -21,5 +29,137 @@ public static string ConvertPotentialDateTimeObject(string value, string current } return matchValue; } + + public static string GetRedactionValue(string value, ColumnInfo column, DataRow m, List _cataloguePKs,RegexRedactionConfiguration _redactionConfiguration, DataTable redactionsToSaveTable, DataTable pksToSave, DataTable redactionUpates) + { + + Dictionary pkLookup = Enumerable.Range(0, _cataloguePKs.Count).ToDictionary(i => _cataloguePKs[i].ColumnInfo, i => m[i + 1].ToString()); + var matches = Regex.Matches(value, _redactionConfiguration.RegexPattern); + var offset = 0; + foreach (var match in matches) + { + var foundMatch = match.ToString(); + var startingIndex = value.IndexOf(foundMatch); + string replacementValue = _redactionConfiguration.RedactionString; + + var lengthDiff = (float)foundMatch.Length - replacementValue.Length; + if (lengthDiff < 1) + { + throw new Exception($"Redaction string '{_redactionConfiguration.RedactionString}' is longer than found match '{foundMatch}'."); + } + if (lengthDiff > 0) + { + var start = (int)Math.Floor(lengthDiff / 2); + var end = (int)Math.Ceiling(lengthDiff / 2); + replacementValue = replacementValue.PadLeft(start + replacementValue.Length, '<'); + replacementValue = replacementValue.PadRight(end + replacementValue.Length, '>'); + } + value = value[..startingIndex] + replacementValue + value[(startingIndex + foundMatch.Length)..]; + redactionsToSaveTable.Rows.Add([_redactionConfiguration.ID, column.ID, startingIndex, replacementValue, foundMatch]); + foreach (var pk in pkLookup) + { + pksToSave.Rows.Add([redactionUpates.Rows.Count + offset, pk.Key.ID, pk.Value]); + } + offset++; + } + + return value; + } + + public static void Redact(ColumnInfo column, DataRow match, List _cataloguePKs, RegexRedactionConfiguration _redactionConfiguration, DataTable redactionsToSaveTable, DataTable pksToSave, DataTable redactionUpates) + { + + var redactedValue = GetRedactionValue(match[0].ToString(), column, match, _cataloguePKs, _redactionConfiguration, redactionsToSaveTable, pksToSave, redactionUpates); + match[0] = redactedValue; + redactionUpates.ImportRow(match); + } + + public static void SaveRedactions(IBasicActivateItems _activator, DataTable pksToSave, DataTable redactionsToSaveTable) + { + var max = 1000; + //there will be an equal or greater number of PKs, so use that and take as many redactions as we can + if (pksToSave.Rows.Count > max) + { + var read = 0; + var minColIndex = 0; + while (read < pksToSave.Rows.Count) + { + var pkRows = pksToSave.AsEnumerable().Skip(read).Take(max); + var colIndex = int.Parse(pkRows.Last()[0].ToString()); + var redactionRows = redactionsToSaveTable.AsEnumerable().Skip(minColIndex).Take(colIndex - minColIndex + 1); + DoInsert(_activator, redactionRows.AsEnumerable(), pkRows.AsEnumerable(), minColIndex); + minColIndex = colIndex; + read += max; + } + } + else + { + DoInsert(_activator, redactionsToSaveTable.AsEnumerable(), pksToSave.AsEnumerable(), 0); + } + } + + public static void DoInsert(IBasicActivateItems _activator, IEnumerable redactionRows, IEnumerable pkRows, int offset) + { + string redactionString = string.Join( + @", + ", redactionRows.Select(r => $"({r[0]},{r[1]},{r[2]},'{r[3]}','{r[4]}')")); + string keysString = string.Join( + @", + ", pkRows.Select(r => $"({r[0]},{r[1]},'{r[2]}')")); + + var sql = $@" + DECLARE @output TABLE (id int, inc int IDENTITY(1,1)) + INSERT INTO RegexRedaction(RedactionConfiguration_ID,ColumnInfo_ID,startingIndex,ReplacementValue,RedactedValue) OUTPUT inserted.id INTO @output + VALUES + {redactionString}; + DECLARE @redactionKeys TABLE (id int IDENTITY(1,1),RegexRedaction_ID int,ColumnInfo_ID int , Value varchar(max)); + INSERT INTO @redactionKeys(RegexRedaction_ID,ColumnInfo_ID,Value) + VALUES + {keysString}; + UPDATE t1 + SET t1.RegexRedaction_ID = t2.id + FROM @redactionKeys as t1 + INNER JOIN @output AS t2 + ON (t1.RegexRedaction_ID -{offset} +1) = t2.inc + WHERE (t1.RegexRedaction_ID -{offset} +1) = t2.inc; + INSERT INTO RegexRedactionKey + select RegexRedaction_ID,ColumnInfo_ID,Value FROM @redactionKeys; + select * from RegexRedactionKey + "; + try + { + (_activator.RepositoryLocator.CatalogueRepository as TableRepository).Insert(sql, null); + } + catch (SqlException ex) + { + Console.WriteLine(ex.Message); + } + } + + public static void DoJoinUpdate(ColumnInfo column, DiscoveredTable _discoveredTable, DiscoveredServer _server, DataTable redactionUpates, DiscoveredColumn[] _discoveredPKColumns) + { + var redactionTable = _discoveredTable.Database.CreateTable("TEMP_RedactionUpdates", redactionUpates); + var updateHelper = _server.GetQuerySyntaxHelper().UpdateHelper; + + var sqlLines = new List + { + new CustomLine($"t1.{column.GetRuntimeName()} = t2.{column.GetRuntimeName()}", QueryComponent.SET) + }; + foreach (var pk in _discoveredPKColumns) + { + sqlLines.Add(new CustomLine($"t1.{pk.GetRuntimeName()} = t2.{pk.GetRuntimeName()}", QueryComponent.WHERE)); + sqlLines.Add(new CustomLine(string.Format("t1.{0} = t2.{0}", pk.GetRuntimeName()), QueryComponent.JoinInfoJoin)); + + } + var sql = updateHelper.BuildUpdate(_discoveredTable, redactionTable, sqlLines); + var conn = _server.GetConnection(); + conn.Open(); + using (var cmd = _server.GetCommand(sql, conn)) + { + cmd.ExecuteNonQuery(); + } + conn.Close(); + redactionTable.Drop(); + } } } diff --git a/Rdmp.Core/DataLoad/Modules/Mutilators/MatchingTablesMutilatorWithDataLoadJob.cs b/Rdmp.Core/DataLoad/Modules/Mutilators/MatchingTablesMutilatorWithDataLoadJob.cs new file mode 100644 index 0000000000..1b7ef6bfa9 --- /dev/null +++ b/Rdmp.Core/DataLoad/Modules/Mutilators/MatchingTablesMutilatorWithDataLoadJob.cs @@ -0,0 +1,112 @@ +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using System; +using System.Diagnostics; +using System.Linq; +using System.Text.RegularExpressions; +using FAnsi.Discovery; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.DataLoad; +using Rdmp.Core.DataLoad.Engine.Job; +using Rdmp.Core.DataLoad.Engine.Mutilators; +using Rdmp.Core.ReusableLibraryCode.Checks; +using Rdmp.Core.ReusableLibraryCode.Progress; + +namespace Rdmp.Core.DataLoad.Modules.Mutilators; + +public abstract class MatchingTablesMutilatorWithDataLoadJob : IPluginMutilateDataTables +{ + private readonly LoadStage[] _allowedStages; + + [DemandsInitialization( + "All tables matching this pattern which have a TableInfo defined in the load will be affected by this mutilation", + DefaultValue = ".*")] + public Regex TableRegexPattern { get; set; } + + [DemandsInitialization( + "Overrides TableRegexPattern. If this is set then the tables chosen will be mutilated instead")] + public TableInfo[] OnlyTables { get; set; } + + [DemandsInitialization("How long to allow for each command to execute in seconds", DefaultValue = 600)] + public int Timeout { get; set; } + + protected DiscoveredDatabase DbInfo; + private LoadStage _loadStage; + + protected MatchingTablesMutilatorWithDataLoadJob(params LoadStage[] allowedStages) + { + _allowedStages = allowedStages; + } + + public virtual void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventsListener) + { + } + + public void Initialize(DiscoveredDatabase dbInfo, LoadStage loadStage) + { + if (_allowedStages != null && !_allowedStages.Contains(loadStage)) + throw new NotSupportedException($"Mutilation {GetType()} is not allowed at stage {loadStage}"); + + _loadStage = loadStage; + DbInfo = dbInfo; + } + + public ExitCodeType Mutilate(IDataLoadJob job) + { + if (TableRegexPattern != null) + TableRegexPattern = new Regex(TableRegexPattern.ToString(), RegexOptions.IgnoreCase); + + foreach (var tableInfo in job.RegularTablesToLoad) + if (OnlyTables != null && OnlyTables.Any()) + { + if (OnlyTables.Contains(tableInfo)) + FireMutilate(tableInfo, job); + } + else if (TableRegexPattern == null) + { + throw new Exception("You must specify either TableRegexPattern or OnlyTables"); + } + else if (TableRegexPattern.IsMatch(tableInfo.GetRuntimeName())) + { + FireMutilate(tableInfo, job); + } + + return ExitCodeType.Success; + } + + private void FireMutilate(ITableInfo tableInfo, IDataLoadJob job) + { + var tbl = DbInfo.ExpectTable(tableInfo.GetRuntimeName(_loadStage, job.Configuration.DatabaseNamer)); + + if (!tbl.Exists()) + { + job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error, + $"Expected table {tbl} did not exist in RAW")); + } + else + { + job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, + $"About to run {GetType()} mutilation on table {tbl}")); + var sw = new Stopwatch(); + sw.Start(); + MutilateTable(job, tableInfo, tbl); + sw.Stop(); + job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, + $"{GetType()} mutilation on table {tbl} completed after {sw.ElapsedMilliseconds} ms")); + } + } + + protected abstract void MutilateTable(IDataLoadJob job, ITableInfo tableInfo, DiscoveredTable table); + + public virtual void Check(ICheckNotifier notifier) + { + if (TableRegexPattern == null && (OnlyTables == null || OnlyTables.Length == 0)) + notifier.OnCheckPerformed(new CheckEventArgs( + "You must specify either a regex pattern (TableRegexPattern) or set OnlyTables for identifying tables which need to be processed", + CheckResult.Fail)); + } +} \ No newline at end of file diff --git a/Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs b/Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs new file mode 100644 index 0000000000..581263784f --- /dev/null +++ b/Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs @@ -0,0 +1,177 @@ +using FAnsi.Discovery; +using FAnsi.Discovery.QuerySyntax.Update; +using FAnsi.Discovery.QuerySyntax; +using MongoDB.Driver.Core.Servers; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.DataLoad; +using Rdmp.Core.Curation.DataHelper.RegexRedaction; +using Rdmp.Core.DataLoad.Engine.Job; +using Rdmp.Core.DataLoad.Engine.Mutilators; +using Rdmp.Core.ReusableLibraryCode.Checks; +using Rdmp.Core.ReusableLibraryCode.Progress; +using System; +using System.Collections.Generic; +using System.Data; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using Rdmp.Core.QueryBuilding; +using Rdmp.Core.Repositories; +using Rdmp.Core.CommandExecution; + +namespace Rdmp.Core.DataLoad.Modules.Mutilators; + +public class RegexRedactionMutilator : MatchingTablesMutilatorWithDataLoadJob +{ + [DemandsInitialization("the regex redaction configuration to use")] + public RegexRedactionConfiguration redactionConfiguration { get; set; } + + [DemandsInitialization( + "All Columns matching this pattern which have a ColumnInfo defined in the load will be affected by this mutilation", + DefaultValue = ".*")] + public Regex ColumnRegexPattern { get; set; } + + [DemandsInitialization( + "Overrides ColumnRegexPattern. If this is set then the columns chosen will be mutilated instead")] + public ColumnInfo[] OnlyColumns { get; set; } + + private DiscoveredColumn[] _discoveredPKColumns; + + + public RegexRedactionMutilator() : base([LoadStage.AdjustRaw, LoadStage.AdjustStaging]) { } + + private void Redact(IDataLoadJob job, DiscoveredTable table, DataRow row, DiscoveredColumn[] columns, int index) + { + var currentValue = row[index].ToString(); + var matches = Regex.Matches(currentValue, redactionConfiguration.RegexPattern); + var catalogues = job.LoadMetadata.GetAllCatalogues(); + foreach (var match in matches) + { + var foundMatch = match.ToString(); + var startingIndex = currentValue.IndexOf(foundMatch); + string replacementValue = redactionConfiguration.RedactionString; + var lengthDiff = (float)foundMatch.Length - replacementValue.Length; + if (lengthDiff < 1) + { + throw new Exception($"Redaction string '{redactionConfiguration.RedactionString}' is longer than found match '{foundMatch}'."); + } + if (lengthDiff > 0) + { + var start = (int)Math.Floor(lengthDiff / 2); + var end = (int)Math.Ceiling(lengthDiff / 2); + replacementValue = replacementValue.PadLeft(start + replacementValue.Length, '<'); + replacementValue = replacementValue.PadRight(end + replacementValue.Length, '>'); + } + currentValue = currentValue[..startingIndex] + replacementValue + currentValue[(startingIndex + foundMatch.Length)..]; + } + var sqlLines = new List + { + new CustomLine($"t1.{columns[index].GetRuntimeName()} = '{currentValue}'", QueryComponent.SET) + }; + foreach (var pk in _discoveredPKColumns) + { + var matchValue = RegexRedactionHelper.ConvertPotentialDateTimeObject(row[pk.GetRuntimeName()].ToString(), pk.DataType.SQLType); + + sqlLines.Add(new CustomLine($"t1.{pk.GetRuntimeName()} = {matchValue}", QueryComponent.WHERE)); + sqlLines.Add(new CustomLine(string.Format("t1.{0} = t2.{0}", pk.GetRuntimeName()), QueryComponent.JoinInfoJoin)); + + } + var _server = table.Database.Server; + var updateHelper = _server.GetQuerySyntaxHelper().UpdateHelper; + var sql = updateHelper.BuildUpdate(table, table, sqlLines); + var conn = _server.GetConnection(); + using (var cmd = _server.GetCommand(sql, conn)) + { + cmd.ExecuteNonQuery(); + } + + + } + + private bool ColumnMatches(DiscoveredColumn column) + { + if (OnlyColumns.Length > 0) + { + return OnlyColumns.Select(c => c.GetRuntimeName()).Contains(column.GetRuntimeName()); + } + if (ColumnRegexPattern != null) + { + ColumnRegexPattern = new Regex(ColumnRegexPattern.ToString(), RegexOptions.IgnoreCase); + return ColumnRegexPattern.IsMatch(column.GetRuntimeName()); + } + return false; + } + + protected override void MutilateTable(IDataLoadJob job, ITableInfo tableInfo, DiscoveredTable table) + { + DataTable redactionsToSaveTable = new(); + DataTable pksToSave = new(); + redactionsToSaveTable.Columns.Add("RedactionConfiguration_ID"); + redactionsToSaveTable.Columns.Add("ColumnInfo_ID"); + redactionsToSaveTable.Columns.Add("startingIndex"); + redactionsToSaveTable.Columns.Add("ReplacementValue"); + redactionsToSaveTable.Columns.Add("RedactedValue"); + pksToSave.Columns.Add("redactionsToSaveTable_Index"); + pksToSave.Columns.Add("ColumnInfo_ID"); + pksToSave.Columns.Add("Value"); + + var columns = table.DiscoverColumns(); + + var relatedCatalogues = tableInfo.GetAllRelatedCatalogues(); + var cataloguePks = relatedCatalogues.SelectMany(c => c.CatalogueItems).Where(ci => ci.ColumnInfo.IsPrimaryKey).ToList(); + _discoveredPKColumns = columns.Where(c => cataloguePks.Select(cpk=>cpk.Name).Contains(c.GetRuntimeName())).ToArray(); //will have to figure out the pks + //TODO should error/fail if there are no promary keys + //do we tidy up if it is a replacement? + var pkColumnInfos = cataloguePks.Select(c => c.ColumnInfo); + var memoryRepo = new MemoryCatalogueRepository(); + foreach (var column in columns.Where(c => !pkColumnInfos.Select(c => c.GetRuntimeName()).Contains(c.GetRuntimeName()))) + { + if (ColumnMatches(column)) + { + var pkSeparator = pkColumnInfos.Count() > 0 ? "," : ""; + var sql = @$" + SELECT {column.GetRuntimeName()} {pkSeparator} {string.Join(", ", pkColumnInfos.Select(c => c.GetRuntimeName()))} + FROM {table.GetRuntimeName()} + WHERE {column.GetRuntimeName()} LIKE '%{redactionConfiguration.RegexPattern}%' + "; + var dt = new DataTable(); + dt.BeginLoadData(); + var conn = table.Database.Server.GetConnection(); + conn.Open(); + using (var cmd = table.Database.Server.GetCommand(sql, conn)) + { + using var da = table.Database.Server.GetDataAdapter(cmd); + da.Fill(dt); + } + dt.EndLoadData(); + var redactionUpates = dt.Clone(); + var columnInfo = relatedCatalogues.SelectMany(c => c.CatalogueItems).ToArray().Select(ci => ci.ColumnInfo).Where(ci => ci.GetRuntimeName() == column.GetRuntimeName()).First(); + foreach (DataRow row in dt.Rows) + { + RegexRedactionHelper.Redact(columnInfo, row, cataloguePks, redactionConfiguration, redactionsToSaveTable, pksToSave, redactionUpates); + } + RegexRedactionHelper.SaveRedactions(new ThrowImmediatelyActivator(job.RepositoryLocator), pksToSave, redactionsToSaveTable); + RegexRedactionHelper.DoJoinUpdate(columnInfo, table, table.Database.Server, redactionUpates, _discoveredPKColumns); + + } + } + //if (!columns.Any(c => c.IsPrimaryKey)) + //{ + // throw new Exception($"Table '{tableInfo}' has no IsPrimaryKey columns"); + //} + //_discoveredPKColumns = columns.Where(c => c.IsPrimaryKey).ToArray(); + //var dt = table.GetDataTable(); + //var columnNames = dt.Columns.Cast().Select(c => c.ColumnName).ToList(); + //foreach (DataRow row in dt.AsEnumerable()) + //{ + // foreach (var column in columns.Select((value, index) => new { value, index }).Where(c => !c.value.IsPrimaryKey)) + // { + // if (Regex.IsMatch(row[column.index].ToString(), redactionConfiguration.RegexPattern)) + // { + // Redact(job, table, row, columns, column.index); + // } + // } + //} + } +} diff --git a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandRegexRedaction.cs b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandRegexRedaction.cs index 937e853771..9bf1697773 100644 --- a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandRegexRedaction.cs +++ b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandRegexRedaction.cs @@ -11,15 +11,18 @@ using System.Linq; using System.Text; using System.Threading.Tasks; +using Rdmp.UI.SimpleDialogs; namespace Rdmp.UI.CommandExecution.AtomicCommands; public class ExecuteCommandRegexRedaction: BasicUICommandExecution, IAtomicCommand { private readonly Catalogue _catalogue; + private readonly IActivateItems _activator; public ExecuteCommandRegexRedaction(IActivateItems activator, Catalogue catalogue) : base(activator) { + _activator = activator; _catalogue = catalogue; } @@ -29,6 +32,6 @@ public override void Execute() { base.Execute(); - //Activator.Activate(_catalogue); + Activator.Activate(_catalogue); } } diff --git a/Rdmp.UI/Rdmp.UI.csproj b/Rdmp.UI/Rdmp.UI.csproj index ad5df71f36..f09049bec8 100644 --- a/Rdmp.UI/Rdmp.UI.csproj +++ b/Rdmp.UI/Rdmp.UI.csproj @@ -187,6 +187,9 @@ ServerDatabaseTableSelector.cs + + UserControl + Form diff --git a/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.Designer.cs b/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.Designer.cs new file mode 100644 index 0000000000..ab5cf1dfdd --- /dev/null +++ b/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.Designer.cs @@ -0,0 +1,227 @@ +using BrightIdeasSoftware; +using System.Text.RegularExpressions; + +namespace Rdmp.UI.SimpleDialogs; + +partial class RedactCatalogueUI +{ + /// + /// Required designer variable. + /// + private System.ComponentModel.IContainer components = null; + + /// + /// Clean up any resources being used. + /// + /// true if managed resources should be disposed; otherwise, false. + protected override void Dispose(bool disposing) + { + if (disposing && (components != null)) + { + components.Dispose(); + } + base.Dispose(disposing); + } + + #region Component Designer generated code + + /// + /// Required method for Designer support - do not modify + /// the contents of this method with the code editor. + /// + private void InitializeComponent() + { + panel1 = new System.Windows.Forms.Panel(); + label4 = new System.Windows.Forms.Label(); + tbMaxCount = new System.Windows.Forms.TextBox(); + btnRedact = new System.Windows.Forms.Button(); + btnIdentify = new System.Windows.Forms.Button(); + label3 = new System.Windows.Forms.Label(); + checkedListBox1 = new System.Windows.Forms.CheckedListBox(); + comboBox1 = new System.Windows.Forms.ComboBox(); + label2 = new System.Windows.Forms.Label(); + tbCatalogueName = new System.Windows.Forms.TextBox(); + label1 = new System.Windows.Forms.Label(); + folv = new BrightIdeasSoftware.FastObjectListView(); + folvFoundValue = new OLVColumn(); + folvReplacmentValue = new OLVColumn(); + folvRedactButton = new OLVColumn(); + panel1.SuspendLayout(); + ((System.ComponentModel.ISupportInitialize)folv).BeginInit(); + SuspendLayout(); + // + // panel1 + // + panel1.Controls.Add(folv); + panel1.Controls.Add(label4); + panel1.Controls.Add(tbMaxCount); + panel1.Controls.Add(btnRedact); + panel1.Controls.Add(btnIdentify); + panel1.Controls.Add(label3); + panel1.Controls.Add(checkedListBox1); + panel1.Controls.Add(comboBox1); + panel1.Controls.Add(label2); + panel1.Controls.Add(tbCatalogueName); + panel1.Controls.Add(label1); + panel1.Dock = System.Windows.Forms.DockStyle.Fill; + panel1.Location = new System.Drawing.Point(0, 0); + panel1.Margin = new System.Windows.Forms.Padding(4, 3, 4, 3); + panel1.Name = "panel1"; + panel1.Size = new System.Drawing.Size(628, 220); + panel1.TabIndex = 18; + // + // label4 + // + label4.AutoSize = true; + label4.Location = new System.Drawing.Point(509, 176); + label4.Name = "label4"; + label4.Size = new System.Drawing.Size(33, 15); + label4.TabIndex = 9; + label4.Text = "Max:"; + // + // tbMaxCount + // + tbMaxCount.Location = new System.Drawing.Point(548, 172); + tbMaxCount.Name = "tbMaxCount"; + tbMaxCount.Size = new System.Drawing.Size(77, 23); + tbMaxCount.TabIndex = 8; + // + // btnRedact + // + btnRedact.Location = new System.Drawing.Point(373, 172); + btnRedact.Name = "btnRedact"; + btnRedact.Size = new System.Drawing.Size(113, 23); + btnRedact.TabIndex = 7; + btnRedact.Text = "Redact All"; + btnRedact.UseVisualStyleBackColor = true; + btnRedact.Click += btnRedact_Click; + // + // btnIdentify + // + btnIdentify.Location = new System.Drawing.Point(223, 172); + btnIdentify.Name = "btnIdentify"; + btnIdentify.Size = new System.Drawing.Size(144, 23); + btnIdentify.TabIndex = 6; + btnIdentify.Text = "Identify Redactions"; + btnIdentify.UseVisualStyleBackColor = true; + btnIdentify.Click += btnIdentify_Click; + // + // label3 + // + label3.AutoSize = true; + label3.Location = new System.Drawing.Point(22, 83); + label3.Name = "label3"; + label3.Size = new System.Drawing.Size(102, 15); + label3.TabIndex = 5; + label3.Text = "Colums To Redact"; + // + // checkedListBox1 + // + checkedListBox1.FormattingEnabled = true; + checkedListBox1.Location = new System.Drawing.Point(13, 101); + checkedListBox1.Name = "checkedListBox1"; + checkedListBox1.Size = new System.Drawing.Size(189, 94); + checkedListBox1.TabIndex = 4; + // + // comboBox1 + // + comboBox1.FormattingEnabled = true; + comboBox1.Location = new System.Drawing.Point(82, 44); + comboBox1.Name = "comboBox1"; + comboBox1.Size = new System.Drawing.Size(300, 23); + comboBox1.TabIndex = 3; + // + // label2 + // + label2.AutoSize = true; + label2.Location = new System.Drawing.Point(35, 47); + label2.Name = "label2"; + label2.Size = new System.Drawing.Size(42, 15); + label2.TabIndex = 2; + label2.Text = "Regex:"; + // + // tbCatalogueName + // + tbCatalogueName.Enabled = false; + tbCatalogueName.Location = new System.Drawing.Point(83, 8); + tbCatalogueName.Name = "tbCatalogueName"; + tbCatalogueName.Size = new System.Drawing.Size(299, 23); + tbCatalogueName.TabIndex = 1; + // + // label1 + // + label1.AutoSize = true; + label1.Location = new System.Drawing.Point(13, 11); + label1.Name = "label1"; + label1.Size = new System.Drawing.Size(64, 15); + label1.TabIndex = 0; + label1.Text = "Catalogue:"; + // + // folv + // + folv.Location = new System.Drawing.Point(13, 201); + folv.Name = "folv"; + folv.ShowGroups = false; + folv.Size = new System.Drawing.Size(612, 287); + folv.TabIndex = 10; + folv.View = System.Windows.Forms.View.Details; + folv.VirtualMode = true; + folv.AllColumns.Add(folvFoundValue); + folv.AllColumns.Add(folvReplacmentValue); + folv.AllColumns.Add(folvRedactButton); + // + // folvFoundValue + // + folvFoundValue.AspectName = "FoundValue"; + folvFoundValue.FillsFreeSpace = true; + folvFoundValue.MinimumWidth = 100; + folvFoundValue.Text = "FoundValue"; + folvFoundValue.Width = 100; + // + // folvReplacmentValue + // + folvReplacmentValue.AspectName = "RedactionValue"; + folvReplacmentValue.FillsFreeSpace = true; + folvReplacmentValue.MinimumWidth = 100; + folvReplacmentValue.Text = "Redaction Value"; + folvReplacmentValue.Width = 100; + // + // folvRedactButton + // + folvRedactButton.AspectName = ""; + folvRedactButton.FillsFreeSpace = true; + folvRedactButton.MinimumWidth = 100; + folvRedactButton.Text = "Redact"; + folvRedactButton.Width = 100; + // + // RedactCatalogueUI + // + AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); + AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; + Controls.Add(panel1); + Margin = new System.Windows.Forms.Padding(4, 3, 4, 3); + Name = "RedactCatalogueUI"; + Size = new System.Drawing.Size(628, 220); + panel1.ResumeLayout(false); + panel1.PerformLayout(); + ((System.ComponentModel.ISupportInitialize)folv).EndInit(); + ResumeLayout(false); + } + + #endregion + private System.Windows.Forms.Panel panel1; + private System.Windows.Forms.TextBox tbCatalogueName; + private System.Windows.Forms.Label label1; + private System.Windows.Forms.Label label3; + private System.Windows.Forms.CheckedListBox checkedListBox1; + private System.Windows.Forms.ComboBox comboBox1; + private System.Windows.Forms.Label label2; + private System.Windows.Forms.Label label4; + private System.Windows.Forms.TextBox tbMaxCount; + private System.Windows.Forms.Button btnRedact; + private System.Windows.Forms.Button btnIdentify; + private BrightIdeasSoftware.FastObjectListView folv; + private OLVColumn folvFoundValue; + private OLVColumn folvReplacmentValue; + private OLVColumn folvRedactButton; +} diff --git a/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.cs b/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.cs new file mode 100644 index 0000000000..af464b0ec9 --- /dev/null +++ b/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.cs @@ -0,0 +1,102 @@ +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Data; +using System.Linq; +using System.Windows.Forms; +using NPOI.SS.Formula.Functions; +using Rdmp.Core; +using Rdmp.Core.CommandExecution; +using Rdmp.Core.CommandExecution.AtomicCommands; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.Governance; +using Rdmp.Core.Curation.Data.Remoting; +using Rdmp.Core.Curation.DataHelper.RegexRedaction; +using Rdmp.UI.ItemActivation; +using Rdmp.UI.Rules; +using Rdmp.UI.SimpleControls; +using Rdmp.UI.TestsAndSetup.ServicePropogation; + + +namespace Rdmp.UI.SimpleDialogs; + +/// +/// Lets you change the settings for a RedactCatalogue which is a set of web credentials / url to reach another RDMP instance across the network/internet via a web service. +/// +public partial class RedactCatalogueUI : RedactCatalogueUI_Design +{ + private IActivateItems _activator; + private Catalogue _catalogue; + public RedactCatalogueUI() + { + InitializeComponent(); + AssociatedCollection = RDMPCollection.Tables; + } + + public override void SetDatabaseObject(IActivateItems activator, Catalogue databaseObject) + { + base.SetDatabaseObject(activator, databaseObject); + Bind(tbCatalogueName, "Text", "Name", c => c.Name); + + _activator = activator; + _catalogue = databaseObject; + var regexConfigurations = _activator.RepositoryLocator.CatalogueRepository.GetAllObjects().ToArray(); + comboBox1.Items.AddRange(regexConfigurations); + comboBox1.DisplayMember = "Name"; + + var nonPKColumns = databaseObject.CatalogueItems.Where(c => !c.ColumnInfo.IsPrimaryKey).ToArray(); + checkedListBox1.Items.AddRange(nonPKColumns); + } + + + private void btnIdentify_Click(object sender, EventArgs e) + { + int? max = string.IsNullOrWhiteSpace(tbMaxCount.Text) ? null : int.Parse(tbMaxCount.Text); + var columns = new List(); + foreach (object item in checkedListBox1.CheckedItems) + { + columns.Add(((CatalogueItem)item).ColumnInfo); + } + var config = comboBox1.SelectedItem as RegexRedactionConfiguration; + var cmd = new ExecuteCommandIdentifyRegexRedactionsInCatalogue(_activator, _catalogue, config, columns, max); + cmd.Execute(); + var x = cmd.results; + foreach(DataRow row in cmd.results.Rows) + { + object o = new + { + FoundValue = row.ItemArray[0], + RedactionValue = row.ItemArray[1] + }; + folv.AddObject(o); + } + } + + private void btnRedact_Click(object sender, EventArgs e) + { + if (_activator.YesNo("Are you sure?", "TODO")) + { + int? max = string.IsNullOrWhiteSpace(tbMaxCount.Text) ? null : int.Parse(tbMaxCount.Text); + var columns = new List(); + foreach (object item in checkedListBox1.CheckedItems) + { + columns.Add(((CatalogueItem)item).ColumnInfo); + } + var cmd = new ExecuteCommandPerformRegexRedactionOnCatalogue(_activator, _catalogue, (RegexRedactionConfiguration)comboBox1.SelectedValue, columns, max); + cmd.Execute(); + //TODO some sort up update to let the user know how many we updated? + } + } + +} + +[TypeDescriptionProvider(typeof(AbstractControlDescriptionProvider))] +public abstract class RedactCatalogueUI_Design : RDMPSingleDatabaseObjectControl +{ +} \ No newline at end of file diff --git a/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.resx b/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.resx new file mode 100644 index 0000000000..af32865ec1 --- /dev/null +++ b/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.resx @@ -0,0 +1,120 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + \ No newline at end of file From af711a76adfdc76140946cc2646db895aca62a80 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 3 Sep 2024 14:30:14 +0100 Subject: [PATCH 030/106] shared --- ...ndPerformRegexRedactionOnCatalogueTests.cs | 2 +- ...CommandPerformRegexRedactionOnCatalogue.cs | 13 ++- .../RegexRedaction/RegexRedactionHelper.cs | 66 +++--------- .../Mutilators/RegexRedactionMutilator.cs | 101 ++++++------------ 4 files changed, 57 insertions(+), 125 deletions(-) diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs index baf338cfc8..3693fb0c9e 100644 --- a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs @@ -425,7 +425,7 @@ [Condition1] ASC con.Open(); server.GetCommand(sql, con).ExecuteNonQuery(); } - var rowCount = 5000000; + var rowCount = 1000000; foreach (var i in Enumerable.Range(0, rowCount / 1000)) { InsertIntoDB(db, i * 1000); diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs index d9dafff9c7..856956d6e8 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs @@ -48,7 +48,7 @@ public ExecuteCommandPerformRegexRedactionOnCatalogue(IBasicActivateItems activa redactionsToSaveTable.Columns.Add("startingIndex"); redactionsToSaveTable.Columns.Add("ReplacementValue"); redactionsToSaveTable.Columns.Add("RedactedValue"); - pksToSave.Columns.Add("redactionsToSaveTable_Index"); + pksToSave.Columns.Add("RegexRedaction_ID"); pksToSave.Columns.Add("ColumnInfo_ID"); pksToSave.Columns.Add("Value"); } @@ -106,7 +106,16 @@ public override void Execute() timer.Stop(); Console.WriteLine(timer.ElapsedMilliseconds); timer.Start(); - RegexRedactionHelper.SaveRedactions(_activator,pksToSave,redactionsToSaveTable); + pksToSave.Columns.Add("ID", typeof(int)); + for (int i = 0; i < dt.Rows.Count; i++) + { + pksToSave.Rows[i]["ID"] = i + 1; + } + var t1 = _discoveredTable.Database.CreateTable("pksToSave_Temp", pksToSave); + var t2 = _discoveredTable.Database.CreateTable("redactionsToSaveTable_Temp", redactionsToSaveTable); + RegexRedactionHelper.SaveRedactions(_activator.RepositoryLocator.CatalogueRepository,t1,t2); + t1.Drop(); + t2.Drop(); timer.Stop(); Console.WriteLine(timer.ElapsedMilliseconds); timer.Start(); diff --git a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs index d19228e0bd..458f922788 100644 --- a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs +++ b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs @@ -12,6 +12,8 @@ using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; +using MongoDB.Driver; +using Rdmp.Core.Repositories; namespace Rdmp.Core.Curation.DataHelper.RegexRedaction { @@ -74,61 +76,23 @@ public static void Redact(ColumnInfo column, DataRow match, List redactionUpates.ImportRow(match); } - public static void SaveRedactions(IBasicActivateItems _activator, DataTable pksToSave, DataTable redactionsToSaveTable) + public static void SaveRedactions(ICatalogueRepository catalogueRepo, DiscoveredTable pksToSave, DiscoveredTable redactionsToSaveTable) { - var max = 1000; - //there will be an equal or greater number of PKs, so use that and take as many redactions as we can - if (pksToSave.Rows.Count > max) - { - var read = 0; - var minColIndex = 0; - while (read < pksToSave.Rows.Count) - { - var pkRows = pksToSave.AsEnumerable().Skip(read).Take(max); - var colIndex = int.Parse(pkRows.Last()[0].ToString()); - var redactionRows = redactionsToSaveTable.AsEnumerable().Skip(minColIndex).Take(colIndex - minColIndex + 1); - DoInsert(_activator, redactionRows.AsEnumerable(), pkRows.AsEnumerable(), minColIndex); - minColIndex = colIndex; - read += max; - } - } - else - { - DoInsert(_activator, redactionsToSaveTable.AsEnumerable(), pksToSave.AsEnumerable(), 0); - } - } - - public static void DoInsert(IBasicActivateItems _activator, IEnumerable redactionRows, IEnumerable pkRows, int offset) - { - string redactionString = string.Join( - @", - ", redactionRows.Select(r => $"({r[0]},{r[1]},{r[2]},'{r[3]}','{r[4]}')")); - string keysString = string.Join( - @", - ", pkRows.Select(r => $"({r[0]},{r[1]},'{r[2]}')")); - var sql = $@" - DECLARE @output TABLE (id int, inc int IDENTITY(1,1)) - INSERT INTO RegexRedaction(RedactionConfiguration_ID,ColumnInfo_ID,startingIndex,ReplacementValue,RedactedValue) OUTPUT inserted.id INTO @output - VALUES - {redactionString}; - DECLARE @redactionKeys TABLE (id int IDENTITY(1,1),RegexRedaction_ID int,ColumnInfo_ID int , Value varchar(max)); - INSERT INTO @redactionKeys(RegexRedaction_ID,ColumnInfo_ID,Value) - VALUES - {keysString}; - UPDATE t1 - SET t1.RegexRedaction_ID = t2.id - FROM @redactionKeys as t1 - INNER JOIN @output AS t2 - ON (t1.RegexRedaction_ID -{offset} +1) = t2.inc - WHERE (t1.RegexRedaction_ID -{offset} +1) = t2.inc; - INSERT INTO RegexRedactionKey - select RegexRedaction_ID,ColumnInfo_ID,Value FROM @redactionKeys; - select * from RegexRedactionKey - "; + INSERT INTO RegexRedaction(RedactionConfiguration_ID,ColumnInfo_ID,startingIndex,ReplacementValue,RedactedValue) + SELECT RedactionConfiguration_ID,ColumnInfo_ID,startingIndex,ReplacementValue,RedactedValue FROM {redactionsToSaveTable.GetFullyQualifiedName()}; + UPDATE t1 + SET t1.RegexRedaction_ID = t2.id + FROM {pksToSave.GetFullyQualifiedName()} as t1 + INNER JOIN {redactionsToSaveTable.GetFullyQualifiedName()} AS t2 + ON (t1.RegexRedaction_ID +1) = t2.ID + WHERE (t1.RegexRedaction_ID +1) = t2.ID; + INSERT INTO RegexRedactionKey + select RegexRedaction_ID,ColumnInfo_ID,Value FROM {pksToSave.GetFullyQualifiedName()}; + "; try { - (_activator.RepositoryLocator.CatalogueRepository as TableRepository).Insert(sql, null); + (catalogueRepo as TableRepository).Insert(sql, null); } catch (SqlException ex) { diff --git a/Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs b/Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs index 581263784f..9474beb5ff 100644 --- a/Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs +++ b/Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs @@ -19,6 +19,7 @@ using Rdmp.Core.QueryBuilding; using Rdmp.Core.Repositories; using Rdmp.Core.CommandExecution; +using System.Diagnostics; namespace Rdmp.Core.DataLoad.Modules.Mutilators; @@ -41,54 +42,6 @@ public class RegexRedactionMutilator : MatchingTablesMutilatorWithDataLoadJob public RegexRedactionMutilator() : base([LoadStage.AdjustRaw, LoadStage.AdjustStaging]) { } - private void Redact(IDataLoadJob job, DiscoveredTable table, DataRow row, DiscoveredColumn[] columns, int index) - { - var currentValue = row[index].ToString(); - var matches = Regex.Matches(currentValue, redactionConfiguration.RegexPattern); - var catalogues = job.LoadMetadata.GetAllCatalogues(); - foreach (var match in matches) - { - var foundMatch = match.ToString(); - var startingIndex = currentValue.IndexOf(foundMatch); - string replacementValue = redactionConfiguration.RedactionString; - var lengthDiff = (float)foundMatch.Length - replacementValue.Length; - if (lengthDiff < 1) - { - throw new Exception($"Redaction string '{redactionConfiguration.RedactionString}' is longer than found match '{foundMatch}'."); - } - if (lengthDiff > 0) - { - var start = (int)Math.Floor(lengthDiff / 2); - var end = (int)Math.Ceiling(lengthDiff / 2); - replacementValue = replacementValue.PadLeft(start + replacementValue.Length, '<'); - replacementValue = replacementValue.PadRight(end + replacementValue.Length, '>'); - } - currentValue = currentValue[..startingIndex] + replacementValue + currentValue[(startingIndex + foundMatch.Length)..]; - } - var sqlLines = new List - { - new CustomLine($"t1.{columns[index].GetRuntimeName()} = '{currentValue}'", QueryComponent.SET) - }; - foreach (var pk in _discoveredPKColumns) - { - var matchValue = RegexRedactionHelper.ConvertPotentialDateTimeObject(row[pk.GetRuntimeName()].ToString(), pk.DataType.SQLType); - - sqlLines.Add(new CustomLine($"t1.{pk.GetRuntimeName()} = {matchValue}", QueryComponent.WHERE)); - sqlLines.Add(new CustomLine(string.Format("t1.{0} = t2.{0}", pk.GetRuntimeName()), QueryComponent.JoinInfoJoin)); - - } - var _server = table.Database.Server; - var updateHelper = _server.GetQuerySyntaxHelper().UpdateHelper; - var sql = updateHelper.BuildUpdate(table, table, sqlLines); - var conn = _server.GetConnection(); - using (var cmd = _server.GetCommand(sql, conn)) - { - cmd.ExecuteNonQuery(); - } - - - } - private bool ColumnMatches(DiscoveredColumn column) { if (OnlyColumns.Length > 0) @@ -105,6 +58,7 @@ private bool ColumnMatches(DiscoveredColumn column) protected override void MutilateTable(IDataLoadJob job, ITableInfo tableInfo, DiscoveredTable table) { + var timer = Stopwatch.StartNew(); DataTable redactionsToSaveTable = new(); DataTable pksToSave = new(); redactionsToSaveTable.Columns.Add("RedactionConfiguration_ID"); @@ -112,7 +66,7 @@ protected override void MutilateTable(IDataLoadJob job, ITableInfo tableInfo, Di redactionsToSaveTable.Columns.Add("startingIndex"); redactionsToSaveTable.Columns.Add("ReplacementValue"); redactionsToSaveTable.Columns.Add("RedactedValue"); - pksToSave.Columns.Add("redactionsToSaveTable_Index"); + pksToSave.Columns.Add("RegexRedaction_ID"); pksToSave.Columns.Add("ColumnInfo_ID"); pksToSave.Columns.Add("Value"); @@ -120,9 +74,11 @@ protected override void MutilateTable(IDataLoadJob job, ITableInfo tableInfo, Di var relatedCatalogues = tableInfo.GetAllRelatedCatalogues(); var cataloguePks = relatedCatalogues.SelectMany(c => c.CatalogueItems).Where(ci => ci.ColumnInfo.IsPrimaryKey).ToList(); - _discoveredPKColumns = columns.Where(c => cataloguePks.Select(cpk=>cpk.Name).Contains(c.GetRuntimeName())).ToArray(); //will have to figure out the pks - //TODO should error/fail if there are no promary keys - //do we tidy up if it is a replacement? + _discoveredPKColumns = columns.Where(c => cataloguePks.Select(cpk=>cpk.Name).Contains(c.GetRuntimeName())).ToArray(); + if(_discoveredPKColumns.Length == 0) + { + job.OnNotify(this,new NotifyEventArgs(ProgressEventType.Warning,"No Primary Keys found. Redaction cannot be perfomed without a promary key.")); + } var pkColumnInfos = cataloguePks.Select(c => c.ColumnInfo); var memoryRepo = new MemoryCatalogueRepository(); foreach (var column in columns.Where(c => !pkColumnInfos.Select(c => c.GetRuntimeName()).Contains(c.GetRuntimeName()))) @@ -151,27 +107,30 @@ protected override void MutilateTable(IDataLoadJob job, ITableInfo tableInfo, Di { RegexRedactionHelper.Redact(columnInfo, row, cataloguePks, redactionConfiguration, redactionsToSaveTable, pksToSave, redactionUpates); } - RegexRedactionHelper.SaveRedactions(new ThrowImmediatelyActivator(job.RepositoryLocator), pksToSave, redactionsToSaveTable); - RegexRedactionHelper.DoJoinUpdate(columnInfo, table, table.Database.Server, redactionUpates, _discoveredPKColumns); + job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"Regex Redactions generated redactions in {timer.ElapsedMilliseconds}ms and found {dt.Rows.Count} redactions.")); + pksToSave.Columns.Add("ID", typeof(int)); + for (int i = 0; i < dt.Rows.Count; i++) + { + pksToSave.Rows[i]["ID"] = i + 1; + } + redactionsToSaveTable.Columns.Add("ID", typeof(int)); + for (int i = 0; i < dt.Rows.Count; i++) + { + redactionsToSaveTable.Rows[i]["ID"] = i + 1; + } + job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"Creating Temporary tables")); + var t1 = table.Database.CreateTable("pksToSave_Temp", pksToSave); + var t2 = table.Database.CreateTable("redactionsToSaveTable_Temp", redactionsToSaveTable); + job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"Saving Redactions")); + RegexRedactionHelper.SaveRedactions(job.RepositoryLocator.CatalogueRepository, t1, t2); + t1.Drop(); + t2.Drop(); + job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"Performing join update")); + var b = timer.ElapsedMilliseconds; + RegexRedactionHelper.DoJoinUpdate(columnInfo, table, table.Database.Server, redactionUpates, _discoveredPKColumns); + job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"Regex Redactions tool {timer.ElapsedMilliseconds}ms and found {dt.Rows.Count} redactions.")); } } - //if (!columns.Any(c => c.IsPrimaryKey)) - //{ - // throw new Exception($"Table '{tableInfo}' has no IsPrimaryKey columns"); - //} - //_discoveredPKColumns = columns.Where(c => c.IsPrimaryKey).ToArray(); - //var dt = table.GetDataTable(); - //var columnNames = dt.Columns.Cast().Select(c => c.ColumnName).ToList(); - //foreach (DataRow row in dt.AsEnumerable()) - //{ - // foreach (var column in columns.Select((value, index) => new { value, index }).Where(c => !c.value.IsPrimaryKey)) - // { - // if (Regex.IsMatch(row[column.index].ToString(), redactionConfiguration.RegexPattern)) - // { - // Redact(job, table, row, columns, column.index); - // } - // } - //} } } From 6c59fa67af880c8e5d062206ecdbdf90d05c1394 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 4 Sep 2024 08:04:37 +0100 Subject: [PATCH 031/106] interim --- ...cuteCommandPerformRegexRedactionOnCatalogue.cs | 2 +- .../RegexRedaction/RegexRedactionHelper.cs | 15 +++++++++------ .../Modules/Mutilators/RegexRedactionMutilator.cs | 8 +++++--- .../TableRepository.cs | 8 ++++++++ 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs index 856956d6e8..34eb54bd6c 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs @@ -113,7 +113,7 @@ public override void Execute() } var t1 = _discoveredTable.Database.CreateTable("pksToSave_Temp", pksToSave); var t2 = _discoveredTable.Database.CreateTable("redactionsToSaveTable_Temp", redactionsToSaveTable); - RegexRedactionHelper.SaveRedactions(_activator.RepositoryLocator.CatalogueRepository,t1,t2); + RegexRedactionHelper.SaveRedactions(_activator.RepositoryLocator.CatalogueRepository,t1,t2,_server); t1.Drop(); t2.Drop(); timer.Stop(); diff --git a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs index 458f922788..48acb2c29e 100644 --- a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs +++ b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs @@ -32,7 +32,7 @@ public static string ConvertPotentialDateTimeObject(string value, string current return matchValue; } - public static string GetRedactionValue(string value, ColumnInfo column, DataRow m, List _cataloguePKs,RegexRedactionConfiguration _redactionConfiguration, DataTable redactionsToSaveTable, DataTable pksToSave, DataTable redactionUpates) + public static string GetRedactionValue(string value, ColumnInfo column, DataRow m, List _cataloguePKs, RegexRedactionConfiguration _redactionConfiguration, DataTable redactionsToSaveTable, DataTable pksToSave, DataTable redactionUpates) { Dictionary pkLookup = Enumerable.Range(0, _cataloguePKs.Count).ToDictionary(i => _cataloguePKs[i].ColumnInfo, i => m[i + 1].ToString()); @@ -76,23 +76,25 @@ public static void Redact(ColumnInfo column, DataRow match, List redactionUpates.ImportRow(match); } - public static void SaveRedactions(ICatalogueRepository catalogueRepo, DiscoveredTable pksToSave, DiscoveredTable redactionsToSaveTable) + public static void SaveRedactions(ICatalogueRepository catalogueRepo, DiscoveredTable pksToSave, DiscoveredTable redactionsToSaveTable, DiscoveredServer _server, int timeout = 30000) { + //TODO tghe update isnt working... + //THE IDS are bungled - redactionsToSaveTable don't have the correct values var sql = $@" INSERT INTO RegexRedaction(RedactionConfiguration_ID,ColumnInfo_ID,startingIndex,ReplacementValue,RedactedValue) SELECT RedactionConfiguration_ID,ColumnInfo_ID,startingIndex,ReplacementValue,RedactedValue FROM {redactionsToSaveTable.GetFullyQualifiedName()}; UPDATE t1 - SET t1.RegexRedaction_ID = t2.id + SET t1.RegexRedaction_ID = t2.ID FROM {pksToSave.GetFullyQualifiedName()} as t1 INNER JOIN {redactionsToSaveTable.GetFullyQualifiedName()} AS t2 ON (t1.RegexRedaction_ID +1) = t2.ID WHERE (t1.RegexRedaction_ID +1) = t2.ID; - INSERT INTO RegexRedactionKey + INSERT INTO RegexRedactionKey(RegexRedaction_ID,ColumnInfo_ID,Value) select RegexRedaction_ID,ColumnInfo_ID,Value FROM {pksToSave.GetFullyQualifiedName()}; "; try { - (catalogueRepo as TableRepository).Insert(sql, null); + (catalogueRepo as TableRepository).Insert(sql, null,timeout); } catch (SqlException ex) { @@ -100,7 +102,7 @@ INSERT INTO RegexRedactionKey } } - public static void DoJoinUpdate(ColumnInfo column, DiscoveredTable _discoveredTable, DiscoveredServer _server, DataTable redactionUpates, DiscoveredColumn[] _discoveredPKColumns) + public static void DoJoinUpdate(ColumnInfo column, DiscoveredTable _discoveredTable, DiscoveredServer _server, DataTable redactionUpates, DiscoveredColumn[] _discoveredPKColumns, int timeout = 30000) { var redactionTable = _discoveredTable.Database.CreateTable("TEMP_RedactionUpdates", redactionUpates); var updateHelper = _server.GetQuerySyntaxHelper().UpdateHelper; @@ -120,6 +122,7 @@ public static void DoJoinUpdate(ColumnInfo column, DiscoveredTable _discoveredTa conn.Open(); using (var cmd = _server.GetCommand(sql, conn)) { + cmd.CommandTimeout = timeout; cmd.ExecuteNonQuery(); } conn.Close(); diff --git a/Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs b/Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs index 9474beb5ff..5cdb9a7b82 100644 --- a/Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs +++ b/Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs @@ -20,6 +20,7 @@ using Rdmp.Core.Repositories; using Rdmp.Core.CommandExecution; using System.Diagnostics; +using Rdmp.Core.ReusableLibraryCode.DataAccess; namespace Rdmp.Core.DataLoad.Modules.Mutilators; @@ -117,18 +118,19 @@ protected override void MutilateTable(IDataLoadJob job, ITableInfo tableInfo, Di redactionsToSaveTable.Columns.Add("ID", typeof(int)); for (int i = 0; i < dt.Rows.Count; i++) { - redactionsToSaveTable.Rows[i]["ID"] = i + 1; + redactionsToSaveTable.Rows[i]["ID"] = i + 1 ; } job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"Creating Temporary tables")); var t1 = table.Database.CreateTable("pksToSave_Temp", pksToSave); var t2 = table.Database.CreateTable("redactionsToSaveTable_Temp", redactionsToSaveTable); job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"Saving Redactions")); - RegexRedactionHelper.SaveRedactions(job.RepositoryLocator.CatalogueRepository, t1, t2); + var _server = relatedCatalogues.First().GetDistinctLiveDatabaseServer(DataAccessContext.InternalDataProcessing, false); + RegexRedactionHelper.SaveRedactions(job.RepositoryLocator.CatalogueRepository, t1, t2, _server, Timeout*1000); t1.Drop(); t2.Drop(); job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"Performing join update")); var b = timer.ElapsedMilliseconds; - RegexRedactionHelper.DoJoinUpdate(columnInfo, table, table.Database.Server, redactionUpates, _discoveredPKColumns); + RegexRedactionHelper.DoJoinUpdate(columnInfo, table, table.Database.Server, redactionUpates, _discoveredPKColumns, Timeout * 1000); job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"Regex Redactions tool {timer.ElapsedMilliseconds}ms and found {dt.Rows.Count} redactions.")); } } diff --git a/Rdmp.Core/MapsDirectlyToDatabaseTable/TableRepository.cs b/Rdmp.Core/MapsDirectlyToDatabaseTable/TableRepository.cs index 5bf06aa420..185cd8ec08 100644 --- a/Rdmp.Core/MapsDirectlyToDatabaseTable/TableRepository.cs +++ b/Rdmp.Core/MapsDirectlyToDatabaseTable/TableRepository.cs @@ -23,6 +23,7 @@ using Rdmp.Core.MapsDirectlyToDatabaseTable.Versioning; using Rdmp.Core.ReusableLibraryCode; using Rdmp.Core.ReusableLibraryCode.DataAccess; +using static Terminal.Gui.MainLoop; namespace Rdmp.Core.MapsDirectlyToDatabaseTable; @@ -809,9 +810,16 @@ public void SaveSpecificPropertyOnlyToDatabase(IMapsDirectlyToDatabaseTable enti } public int Insert(string sql, Dictionary parameters) + { + return Insert(sql, parameters, 30000); + } + + + public int Insert(string sql, Dictionary parameters,int timeout) { using var opener = GetConnection(); using var cmd = PrepareCommand(sql, parameters, opener.Connection, opener.Transaction); + cmd.CommandTimeout = timeout; return cmd.ExecuteNonQuery(); } From d4690b33bee7b3df8462156ab866382f80871c5b Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 4 Sep 2024 10:42:43 +0100 Subject: [PATCH 032/106] working --- .../RegexRedaction/RegexRedactionHelper.cs | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs index 48acb2c29e..de27643fff 100644 --- a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs +++ b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs @@ -14,6 +14,7 @@ using System.Threading.Tasks; using MongoDB.Driver; using Rdmp.Core.Repositories; +using NPOI.SS.Formula.Functions; namespace Rdmp.Core.Curation.DataHelper.RegexRedaction { @@ -81,20 +82,22 @@ public static void SaveRedactions(ICatalogueRepository catalogueRepo, Discovered //TODO tghe update isnt working... //THE IDS are bungled - redactionsToSaveTable don't have the correct values var sql = $@" - INSERT INTO RegexRedaction(RedactionConfiguration_ID,ColumnInfo_ID,startingIndex,ReplacementValue,RedactedValue) + DECLARE @output TABLE (id int, inc int IDENTITY(1,1)) + INSERT INTO RegexRedaction(RedactionConfiguration_ID,ColumnInfo_ID,startingIndex,ReplacementValue,RedactedValue) OUTPUT inserted.id INTO @output SELECT RedactionConfiguration_ID,ColumnInfo_ID,startingIndex,ReplacementValue,RedactedValue FROM {redactionsToSaveTable.GetFullyQualifiedName()}; - UPDATE t1 - SET t1.RegexRedaction_ID = t2.ID - FROM {pksToSave.GetFullyQualifiedName()} as t1 - INNER JOIN {redactionsToSaveTable.GetFullyQualifiedName()} AS t2 - ON (t1.RegexRedaction_ID +1) = t2.ID - WHERE (t1.RegexRedaction_ID +1) = t2.ID; - INSERT INTO RegexRedactionKey(RegexRedaction_ID,ColumnInfo_ID,Value) + + update t2 + set t2.RegexRedaction_ID = t1.id + from @output as t1 + inner join{pksToSave.GetFullyQualifiedName()} as t2 + on t1.inc = t2.RegexRedaction_ID+1; + + INSERT INTO RegexRedactionKey(RegexRedaction_ID,ColumnInfo_ID,Value) select RegexRedaction_ID,ColumnInfo_ID,Value FROM {pksToSave.GetFullyQualifiedName()}; "; try { - (catalogueRepo as TableRepository).Insert(sql, null,timeout); + (catalogueRepo as TableRepository).Insert(sql, null, timeout); } catch (SqlException ex) { From 70ec963940421636c4d446b2a68a74f00de0a059 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 4 Sep 2024 11:42:48 +0100 Subject: [PATCH 033/106] add summary --- .../DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs | 4 +++- Rdmp.UI/SimpleDialogs/RedactCatalogueUI.cs | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs b/Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs index 5cdb9a7b82..db6c9f6e48 100644 --- a/Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs +++ b/Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs @@ -23,7 +23,9 @@ using Rdmp.Core.ReusableLibraryCode.DataAccess; namespace Rdmp.Core.DataLoad.Modules.Mutilators; - +/// +/// Redacts an incoming data table +/// public class RegexRedactionMutilator : MatchingTablesMutilatorWithDataLoadJob { [DemandsInitialization("the regex redaction configuration to use")] diff --git a/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.cs b/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.cs index af464b0ec9..21337f9ef7 100644 --- a/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.cs +++ b/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.cs @@ -27,7 +27,7 @@ namespace Rdmp.UI.SimpleDialogs; /// -/// Lets you change the settings for a RedactCatalogue which is a set of web credentials / url to reach another RDMP instance across the network/internet via a web service. +/// Lets you check for and redact entries in a catalogue /// public partial class RedactCatalogueUI : RedactCatalogueUI_Design { From 1142a28ed37b07dbb6c07f34bceefc77059ce68f Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 5 Sep 2024 09:42:06 +0100 Subject: [PATCH 034/106] restore tests working --- ...CommandPerformRegexRedactionOnCatalogue.cs | 25 +++---- ...andRestoreRegexRedactedValueInCatalogue.cs | 1 + .../RegexRedaction/RegexRedactionHelper.cs | 73 ++++++++++++++----- .../Mutilators/RegexRedactionMutilator.cs | 51 ++++--------- 4 files changed, 81 insertions(+), 69 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs index 34eb54bd6c..1d8f7254f4 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs @@ -30,12 +30,12 @@ public class ExecuteCommandPerformRegexRedactionOnCatalogue : BasicCommandExecut private DiscoveredColumn[] _discoveredPKColumns; private DiscoveredTable _discoveredTable; private List _cataloguePKs; - private readonly DataTable redactionsToSaveTable = new(); - private readonly DataTable pksToSave = new(); + private readonly DataTable redactionsToSaveTable; + private readonly DataTable pksToSave; private readonly int? _readLimit; private DataTable redactionUpates = new(); - public ExecuteCommandPerformRegexRedactionOnCatalogue(IBasicActivateItems activator, ICatalogue catalogue, RegexRedactionConfiguration redactionConfiguration, List columns, int? readLimit=null) + public ExecuteCommandPerformRegexRedactionOnCatalogue(IBasicActivateItems activator, ICatalogue catalogue, RegexRedactionConfiguration redactionConfiguration, List columns, int? readLimit = null) { _activator = activator; _catalogue = catalogue; @@ -43,14 +43,8 @@ public ExecuteCommandPerformRegexRedactionOnCatalogue(IBasicActivateItems activa _columns = columns; _server = _catalogue.GetDistinctLiveDatabaseServer(DataAccessContext.InternalDataProcessing, false); _readLimit = readLimit; - redactionsToSaveTable.Columns.Add("RedactionConfiguration_ID"); - redactionsToSaveTable.Columns.Add("ColumnInfo_ID"); - redactionsToSaveTable.Columns.Add("startingIndex"); - redactionsToSaveTable.Columns.Add("ReplacementValue"); - redactionsToSaveTable.Columns.Add("RedactedValue"); - pksToSave.Columns.Add("RegexRedaction_ID"); - pksToSave.Columns.Add("ColumnInfo_ID"); - pksToSave.Columns.Add("Value"); + redactionsToSaveTable = RegexRedactionHelper.GenerateRedactionsDataTable(); + pksToSave = RegexRedactionHelper.GeneratePKDataTable(); } public override void Execute() @@ -78,7 +72,7 @@ public override void Execute() qb.AddColumn(new ColumnInfoToIColumn(memoryRepo, cataloguePk.ColumnInfo)); } qb.AddCustomLine($"{columnInfo.GetRuntimeName()} LIKE '%{_redactionConfiguration.RegexPattern}%'", QueryComponent.WHERE); - if(_readLimit is not null) + if (_readLimit is not null) { qb.TopX = (int)_readLimit; } @@ -106,14 +100,13 @@ public override void Execute() timer.Stop(); Console.WriteLine(timer.ElapsedMilliseconds); timer.Start(); - pksToSave.Columns.Add("ID", typeof(int)); - for (int i = 0; i < dt.Rows.Count; i++) + for (int i = 0; i < pksToSave.Rows.Count; i++) { pksToSave.Rows[i]["ID"] = i + 1; } var t1 = _discoveredTable.Database.CreateTable("pksToSave_Temp", pksToSave); var t2 = _discoveredTable.Database.CreateTable("redactionsToSaveTable_Temp", redactionsToSaveTable); - RegexRedactionHelper.SaveRedactions(_activator.RepositoryLocator.CatalogueRepository,t1,t2,_server); + RegexRedactionHelper.SaveRedactions(_activator.RepositoryLocator.CatalogueRepository, t1, t2, _server); t1.Drop(); t2.Drop(); timer.Stop(); @@ -121,7 +114,7 @@ public override void Execute() timer.Start(); if (dt.Rows.Count > 0) { - RegexRedactionHelper.DoJoinUpdate(columnInfo,_discoveredTable,_server,redactionUpates,_discoveredPKColumns); + RegexRedactionHelper.DoJoinUpdate(columnInfo, _discoveredTable, _server, redactionUpates, _discoveredPKColumns); } timer.Stop(); Console.WriteLine(timer.ElapsedMilliseconds); diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreRegexRedactedValueInCatalogue.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreRegexRedactedValueInCatalogue.cs index 04c523fe1e..572389e634 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreRegexRedactedValueInCatalogue.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreRegexRedactedValueInCatalogue.cs @@ -78,6 +78,7 @@ public override void Execute() } var updateSql = updateHelper.BuildUpdate(discoveredTable, discoveredTable, sqlLines); var conn = server.GetConnection(); + //TODO this is borked using (var cmd = server.GetCommand(updateSql, conn)) { conn.Open(); diff --git a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs index de27643fff..a9c029c90a 100644 --- a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs +++ b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs @@ -15,12 +15,52 @@ using MongoDB.Driver; using Rdmp.Core.Repositories; using NPOI.SS.Formula.Functions; +using Rdmp.Core.QueryBuilding; namespace Rdmp.Core.Curation.DataHelper.RegexRedaction { public static class RegexRedactionHelper { + //public static QueryBuilder SQLLikeQuery(ColumnInfo columnInfo, CatalogueItem[] pkCatalogueItems, RegexRedactionConfiguration regexConfiguration,int? readLimit) + //{ + // var memoryRepo = new MemoryCatalogueRepository(); + // var qb = new QueryBuilder(null, null, null); + // qb.AddColumn(new ColumnInfoToIColumn(memoryRepo, columnInfo)); + // foreach(var pk in pkCatalogueItems) + // { + // qb.AddColumn(new ColumnInfoToIColumn(memoryRepo, pk.ColumnInfo)); + // } + // qb.AddCustomLine($"{columnInfo.GetRuntimeName()} LIKE '%{regexConfiguration.RegexPattern}%'", QueryComponent.WHERE); + // if (readLimit is not null) + // { + // qb.TopX = (int)readLimit; + // } + + // return qb; + //} + + public static DataTable GenerateRedactionsDataTable() + { + DataTable redactionsToSaveTable = new DataTable(); + redactionsToSaveTable.Columns.Add("RedactionConfiguration_ID"); + redactionsToSaveTable.Columns.Add("ColumnInfo_ID"); + redactionsToSaveTable.Columns.Add("startingIndex"); + redactionsToSaveTable.Columns.Add("ReplacementValue"); + redactionsToSaveTable.Columns.Add("RedactedValue"); + return redactionsToSaveTable; + } + + public static DataTable GeneratePKDataTable() + { + DataTable pkDataTable = new DataTable(); + pkDataTable.Columns.Add("RegexRedaction_ID"); + pkDataTable.Columns.Add("ColumnInfo_ID"); + pkDataTable.Columns.Add("Value"); + pkDataTable.Columns.Add("ID", typeof(int)); + return pkDataTable; + } + public static string ConvertPotentialDateTimeObject(string value, string currentColumnType) { var matchValue = $"'{value}'"; @@ -79,30 +119,29 @@ public static void Redact(ColumnInfo column, DataRow match, List public static void SaveRedactions(ICatalogueRepository catalogueRepo, DiscoveredTable pksToSave, DiscoveredTable redactionsToSaveTable, DiscoveredServer _server, int timeout = 30000) { - //TODO tghe update isnt working... - //THE IDS are bungled - redactionsToSaveTable don't have the correct values + //the update isn't working? and do we need the +1? + var sql = $@" - DECLARE @output TABLE (id int, inc int IDENTITY(1,1)) - INSERT INTO RegexRedaction(RedactionConfiguration_ID,ColumnInfo_ID,startingIndex,ReplacementValue,RedactedValue) OUTPUT inserted.id INTO @output + DECLARE @output TABLE (id1 int, inc int IDENTITY(1,1)) + INSERT INTO RegexRedaction(RedactionConfiguration_ID,ColumnInfo_ID,startingIndex,ReplacementValue,RedactedValue) OUTPUT inserted.id as id1 INTO @output SELECT RedactionConfiguration_ID,ColumnInfo_ID,startingIndex,ReplacementValue,RedactedValue FROM {redactionsToSaveTable.GetFullyQualifiedName()}; + + DECLARE @IDMATCHER TABLE (RegexRedaction_ID int,ColumnInfo_ID int ,Value varchar(max),ID int , id1 int , inc int) + insert into @IDMATCHER(RegexRedaction_ID, ColumnInfo_ID,Value,ID,id1,inc) + select RegexRedaction_ID, ColumnInfo_ID,Value,ID,id1,inc + FROM {pksToSave.GetFullyQualifiedName()} as t1 + JOIN @output as t2 ON t1.RegexRedaction_ID+1 = t2.inc + where t1.RegexRedaction_ID+1 = t2.inc; - update t2 - set t2.RegexRedaction_ID = t1.id - from @output as t1 - inner join{pksToSave.GetFullyQualifiedName()} as t2 - on t1.inc = t2.RegexRedaction_ID+1; + update @IDMATCHER + set RegexRedaction_ID = id1 + where RegexRedaction_ID+1 = inc; INSERT INTO RegexRedactionKey(RegexRedaction_ID,ColumnInfo_ID,Value) - select RegexRedaction_ID,ColumnInfo_ID,Value FROM {pksToSave.GetFullyQualifiedName()}; + select RegexRedaction_ID,ColumnInfo_ID,Value FROM @IDMATCHER; "; - try - { + (catalogueRepo as TableRepository).Insert(sql, null, timeout); - } - catch (SqlException ex) - { - Console.WriteLine(ex.Message); - } } public static void DoJoinUpdate(ColumnInfo column, DiscoveredTable _discoveredTable, DiscoveredServer _server, DataTable redactionUpates, DiscoveredColumn[] _discoveredPKColumns, int timeout = 30000) diff --git a/Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs b/Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs index db6c9f6e48..e73e052777 100644 --- a/Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs +++ b/Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs @@ -1,25 +1,13 @@ using FAnsi.Discovery; -using FAnsi.Discovery.QuerySyntax.Update; -using FAnsi.Discovery.QuerySyntax; -using MongoDB.Driver.Core.Servers; using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.Data.DataLoad; using Rdmp.Core.Curation.DataHelper.RegexRedaction; using Rdmp.Core.DataLoad.Engine.Job; -using Rdmp.Core.DataLoad.Engine.Mutilators; -using Rdmp.Core.ReusableLibraryCode.Checks; using Rdmp.Core.ReusableLibraryCode.Progress; using System; -using System.Collections.Generic; using System.Data; using System.Linq; -using System.Text; using System.Text.RegularExpressions; -using System.Threading.Tasks; -using Rdmp.Core.QueryBuilding; -using Rdmp.Core.Repositories; -using Rdmp.Core.CommandExecution; -using System.Diagnostics; using Rdmp.Core.ReusableLibraryCode.DataAccess; namespace Rdmp.Core.DataLoad.Modules.Mutilators; @@ -47,7 +35,7 @@ public RegexRedactionMutilator() : base([LoadStage.AdjustRaw, LoadStage.AdjustSt private bool ColumnMatches(DiscoveredColumn column) { - if (OnlyColumns.Length > 0) + if (OnlyColumns is not null && OnlyColumns.Length > 0) { return OnlyColumns.Select(c => c.GetRuntimeName()).Contains(column.GetRuntimeName()); } @@ -61,17 +49,8 @@ private bool ColumnMatches(DiscoveredColumn column) protected override void MutilateTable(IDataLoadJob job, ITableInfo tableInfo, DiscoveredTable table) { - var timer = Stopwatch.StartNew(); - DataTable redactionsToSaveTable = new(); - DataTable pksToSave = new(); - redactionsToSaveTable.Columns.Add("RedactionConfiguration_ID"); - redactionsToSaveTable.Columns.Add("ColumnInfo_ID"); - redactionsToSaveTable.Columns.Add("startingIndex"); - redactionsToSaveTable.Columns.Add("ReplacementValue"); - redactionsToSaveTable.Columns.Add("RedactedValue"); - pksToSave.Columns.Add("RegexRedaction_ID"); - pksToSave.Columns.Add("ColumnInfo_ID"); - pksToSave.Columns.Add("Value"); + DataTable redactionsToSaveTable = RegexRedactionHelper.GenerateRedactionsDataTable(); + DataTable pksToSave = RegexRedactionHelper.GeneratePKDataTable(); var columns = table.DiscoverColumns(); @@ -80,10 +59,11 @@ protected override void MutilateTable(IDataLoadJob job, ITableInfo tableInfo, Di _discoveredPKColumns = columns.Where(c => cataloguePks.Select(cpk=>cpk.Name).Contains(c.GetRuntimeName())).ToArray(); if(_discoveredPKColumns.Length == 0) { - job.OnNotify(this,new NotifyEventArgs(ProgressEventType.Warning,"No Primary Keys found. Redaction cannot be perfomed without a promary key.")); + job.OnNotify(this,new NotifyEventArgs(ProgressEventType.Warning,"No Primary Keys found. Redaction cannot be perfomed without a primary key.")); + //Don't want to fail the data load, but just let the user know + return; } var pkColumnInfos = cataloguePks.Select(c => c.ColumnInfo); - var memoryRepo = new MemoryCatalogueRepository(); foreach (var column in columns.Where(c => !pkColumnInfos.Select(c => c.GetRuntimeName()).Contains(c.GetRuntimeName()))) { if (ColumnMatches(column)) @@ -100,28 +80,28 @@ protected override void MutilateTable(IDataLoadJob job, ITableInfo tableInfo, Di conn.Open(); using (var cmd = table.Database.Server.GetCommand(sql, conn)) { + cmd.CommandTimeout = Timeout * 1000; using var da = table.Database.Server.GetDataAdapter(cmd); da.Fill(dt); } dt.EndLoadData(); var redactionUpates = dt.Clone(); - var columnInfo = relatedCatalogues.SelectMany(c => c.CatalogueItems).ToArray().Select(ci => ci.ColumnInfo).Where(ci => ci.GetRuntimeName() == column.GetRuntimeName()).First(); + var columnInfo = relatedCatalogues.SelectMany(c => c.CatalogueItems).ToArray().Select(ci => ci.ColumnInfo).Where(ci => ci.GetRuntimeName() == column.GetRuntimeName()).FirstOrDefault(); + if(columnInfo is null) + { + job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error, "Unable to find the related column info")); + return; + } foreach (DataRow row in dt.Rows) { RegexRedactionHelper.Redact(columnInfo, row, cataloguePks, redactionConfiguration, redactionsToSaveTable, pksToSave, redactionUpates); } - job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"Regex Redactions generated redactions in {timer.ElapsedMilliseconds}ms and found {dt.Rows.Count} redactions.")); + job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"Regex Redaction mutilator found {dt.Rows.Count} redactions.")); - pksToSave.Columns.Add("ID", typeof(int)); for (int i = 0; i < dt.Rows.Count; i++) { pksToSave.Rows[i]["ID"] = i + 1; } - redactionsToSaveTable.Columns.Add("ID", typeof(int)); - for (int i = 0; i < dt.Rows.Count; i++) - { - redactionsToSaveTable.Rows[i]["ID"] = i + 1 ; - } job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"Creating Temporary tables")); var t1 = table.Database.CreateTable("pksToSave_Temp", pksToSave); var t2 = table.Database.CreateTable("redactionsToSaveTable_Temp", redactionsToSaveTable); @@ -131,9 +111,8 @@ protected override void MutilateTable(IDataLoadJob job, ITableInfo tableInfo, Di t1.Drop(); t2.Drop(); job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"Performing join update")); - var b = timer.ElapsedMilliseconds; RegexRedactionHelper.DoJoinUpdate(columnInfo, table, table.Database.Server, redactionUpates, _discoveredPKColumns, Timeout * 1000); - job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"Regex Redactions tool {timer.ElapsedMilliseconds}ms and found {dt.Rows.Count} redactions.")); + job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"Regex Redactions tool found {dt.Rows.Count} redactions.")); } } } From de4877131ac6e6acfe7db68369294938096e90c7 Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 6 Sep 2024 08:47:51 +0100 Subject: [PATCH 035/106] add tests --- .../RegexRedactionMutilatorTests.cs | 727 ++++++++++++++++++ .../RegexRedactionConfiguration.cs | 16 +- .../Mutilators/RegexRedactionMutilator.cs | 27 +- 3 files changed, 756 insertions(+), 14 deletions(-) create mode 100644 Rdmp.Core.Tests/DataLoad/Modules/Mutilators/RegexRedactionMutilatorTests.cs diff --git a/Rdmp.Core.Tests/DataLoad/Modules/Mutilators/RegexRedactionMutilatorTests.cs b/Rdmp.Core.Tests/DataLoad/Modules/Mutilators/RegexRedactionMutilatorTests.cs new file mode 100644 index 0000000000..33cfc213e7 --- /dev/null +++ b/Rdmp.Core.Tests/DataLoad/Modules/Mutilators/RegexRedactionMutilatorTests.cs @@ -0,0 +1,727 @@ +using FAnsi; +using FAnsi.Discovery; +using NPOI.SS.Formula.Functions; +using NSubstitute; +using NUnit.Framework; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.DataLoad; +using Rdmp.Core.Curation.Data.Defaults; +using Rdmp.Core.Curation.DataHelper.RegexRedaction; +using Rdmp.Core.DataFlowPipeline; +using Rdmp.Core.DataLoad; +using Rdmp.Core.DataLoad.Engine.Checks.Checkers; +using Rdmp.Core.DataLoad.Engine.DatabaseManagement.EntityNaming; +using Rdmp.Core.DataLoad.Engine.Job; +using Rdmp.Core.DataLoad.Engine.LoadExecution; +using Rdmp.Core.DataLoad.Engine.LoadExecution.Components.Standard; +using Rdmp.Core.DataLoad.Engine.LoadProcess; +using Rdmp.Core.DataLoad.Modules.Attachers; +using Rdmp.Core.DataLoad.Modules.Mutilators; +using Rdmp.Core.Logging; +using Rdmp.Core.ReusableLibraryCode.Checks; +using Rdmp.Core.ReusableLibraryCode.Progress; +using Rdmp.Core.Tests.DataLoad.Engine.Integration; +using Rdmp.Core.Tests.DataLoad.Engine.Integration.CrossDatabaseTypeTests; +using System; +using System.Collections.Generic; +using System.Data; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using Tests.Common; +using TypeGuesser; + +namespace Rdmp.Core.Tests.DataLoad.Modules.Mutilators; + +internal class RegexRedactionMutilatorTests : DataLoadEngineTestsBase +{ + + [Test] + public void RedactionMutilator_BasicTest() + { + var defaults = CatalogueRepository; + var logServer = defaults.GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID); + var logManager = new LogManager(logServer); + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + + var raw = db.Server.ExpectDatabase($"{db.GetRuntimeName()}_RAW"); + if (raw.Exists()) + raw.Drop(); + + using var dt = new DataTable("MyTable"); + dt.Columns.Add("Name"); + dt.Columns.Add("DateOfBirth"); + dt.Columns.Add("FavouriteColour"); + dt.Rows.Add("Bob", "2001-01-01", "Pink"); + dt.Rows.Add("Frank", "2001-01-01", "Orange"); + + var nameCol = new DatabaseColumnRequest("Name", new DatabaseTypeRequest(typeof(string), 20), false) + { IsPrimaryKey = true }; + DiscoveredTable tbl = db.CreateTable("MyTable", dt, new[] + { + nameCol, + new DatabaseColumnRequest("DateOfBirth", new DatabaseTypeRequest(typeof(DateTime)), false) + { IsPrimaryKey = true } + }); + + Assert.That(tbl.GetRowCount(), Is.EqualTo(2)); + + //define a new load configuration + var lmd = new LoadMetadata(CatalogueRepository, "MyLoad"); + + lmd.IgnoreTrigger = true; + lmd.SaveToDatabase(); + var ti = Import(tbl, lmd, logManager); + + var projectDirectory = SetupLoadDirectory(lmd); + + CreateCSVProcessTask(lmd, ti, "*.csv"); + + + var regex = new RegexRedactionConfiguration(CatalogueRepository, "No Yelling", new Regex("Yella"), "FFF"); + regex.SaveToDatabase(); + + var pt = new ProcessTask(CatalogueRepository, lmd, LoadStage.AdjustRaw) + { + Path = typeof(RegexRedactionMutilator).FullName, + ProcessTaskType = ProcessTaskType.MutilateDataTable, + Name = $"Mutilate {ti.GetRuntimeName()}" + }; + pt.SaveToDatabase(); + + pt.CreateArgumentsForClassIfNotExists(); + pt.SetArgumentValue("ColumnRegexPattern", ".*"); + pt.SetArgumentValue("redactionConfiguration", regex); + pt.SetArgumentValue("TableRegexPattern", ".*"); + + pt.Check(ThrowImmediatelyCheckNotifier.Quiet); + + File.WriteAllText( + Path.Combine(projectDirectory.ForLoading.FullName, "LoadMe.csv"), + @"Name,DateOfBirth,FavouriteColour +MrMurder,2001-01-01,Yella"); + + var dbConfig = new HICDatabaseConfiguration(lmd, null); + + var job = new DataLoadJob(RepositoryLocator, "Go go go!", logManager, lmd, projectDirectory, + ThrowImmediatelyDataLoadEventListener.Quiet, dbConfig); + + var loadFactory = new HICDataLoadFactory( + lmd, + dbConfig, + new HICLoadConfigurationFlags(), + CatalogueRepository, + logManager + ); + + var exe = loadFactory.Create(ThrowImmediatelyDataLoadEventListener.Quiet); + + var exitCode = exe.Run( + job, + new GracefulCancellationToken()); + + var redactions = CatalogueRepository.GetAllObjects(); + Assert.That(redactions.Count(), Is.EqualTo(1)); + Assert.That(redactions[0].RedactionConfiguration_ID, Is.EqualTo(regex.ID)); + Assert.That(redactions[0].StartingIndex, Is.EqualTo(0)); + Assert.That(redactions[0].RedactedValue, Is.EqualTo("Yella")); + Assert.That(redactions[0].ReplacementValue, Is.EqualTo("")); + + var redactionKeys = CatalogueRepository.GetAllObjects(); + Assert.That(redactionKeys.Length, Is.EqualTo(2)); + Assert.That(redactionKeys[0].Value == "MrMurder"); + Assert.That(redactionKeys[1].Value == "01/01/2001 00:00:00"); + foreach (var r in redactions) + { + r.DeleteInDatabase(); + } + foreach (var r in redactionKeys) + { + r.DeleteInDatabase(); + } + } + + [Test] + public void RedactionMutilator_OddStringLength() + { + var defaults = CatalogueRepository; + var logServer = defaults.GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID); + var logManager = new LogManager(logServer); + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + + var raw = db.Server.ExpectDatabase($"{db.GetRuntimeName()}_RAW"); + if (raw.Exists()) + raw.Drop(); + + using var dt = new DataTable("MyTable"); + dt.Columns.Add("Name"); + dt.Columns.Add("DateOfBirth"); + dt.Columns.Add("FavouriteColour"); + dt.Rows.Add("Bob", "2001-01-01", "Pink"); + dt.Rows.Add("Frank", "2001-01-01", "Orange"); + + var nameCol = new DatabaseColumnRequest("Name", new DatabaseTypeRequest(typeof(string), 20), false) + { IsPrimaryKey = true }; + DiscoveredTable tbl = db.CreateTable("MyTable", dt, new[] + { + nameCol, + new DatabaseColumnRequest("DateOfBirth", new DatabaseTypeRequest(typeof(DateTime)), false) + { IsPrimaryKey = true } + }); + + Assert.That(tbl.GetRowCount(), Is.EqualTo(2)); + + //define a new load configuration + var lmd = new LoadMetadata(CatalogueRepository, "MyLoad"); + + lmd.IgnoreTrigger = true; + lmd.SaveToDatabase(); + var ti = Import(tbl, lmd, logManager); + + var projectDirectory = SetupLoadDirectory(lmd); + + CreateCSVProcessTask(lmd, ti, "*.csv"); + + + var regex = new RegexRedactionConfiguration(CatalogueRepository, "No Yelling", new Regex("Yell"), "FFF"); + regex.SaveToDatabase(); + + var pt = new ProcessTask(CatalogueRepository, lmd, LoadStage.AdjustRaw) + { + Path = typeof(RegexRedactionMutilator).FullName, + ProcessTaskType = ProcessTaskType.MutilateDataTable, + Name = $"Mutilate {ti.GetRuntimeName()}" + }; + pt.SaveToDatabase(); + + pt.CreateArgumentsForClassIfNotExists(); + pt.SetArgumentValue("ColumnRegexPattern", ".*"); + pt.SetArgumentValue("redactionConfiguration", regex); + pt.SetArgumentValue("TableRegexPattern", ".*"); + + pt.Check(ThrowImmediatelyCheckNotifier.Quiet); + + File.WriteAllText( + Path.Combine(projectDirectory.ForLoading.FullName, "LoadMe.csv"), + @"Name,DateOfBirth,FavouriteColour +MrMurder,2001-01-01,Yella"); + + var dbConfig = new HICDatabaseConfiguration(lmd, null); + + var job = new DataLoadJob(RepositoryLocator, "Go go go!", logManager, lmd, projectDirectory, + ThrowImmediatelyDataLoadEventListener.Quiet, dbConfig); + + var loadFactory = new HICDataLoadFactory( + lmd, + dbConfig, + new HICLoadConfigurationFlags(), + CatalogueRepository, + logManager + ); + + var exe = loadFactory.Create(ThrowImmediatelyDataLoadEventListener.Quiet); + + var exitCode = exe.Run( + job, + new GracefulCancellationToken()); + + var redactions = CatalogueRepository.GetAllObjects(); + Assert.That(redactions.Count(), Is.EqualTo(1)); + Assert.That(redactions[0].RedactionConfiguration_ID, Is.EqualTo(regex.ID)); + Assert.That(redactions[0].StartingIndex, Is.EqualTo(0)); + Assert.That(redactions[0].RedactedValue, Is.EqualTo("Yell")); + Assert.That(redactions[0].ReplacementValue, Is.EqualTo("FFF>")); + + var redactionKeys = CatalogueRepository.GetAllObjects(); + Assert.That(redactionKeys.Length, Is.EqualTo(2)); + Assert.That(redactionKeys[0].Value == "MrMurder"); + Assert.That(redactionKeys[1].Value == "01/01/2001 00:00:00"); + foreach (var r in redactions) + { + r.DeleteInDatabase(); + } + foreach (var r in redactionKeys) + { + r.DeleteInDatabase(); + } + } + + [Test] + public void RedactionMutilator_RedactionTooLong() + { + var defaults = CatalogueRepository; + var logServer = defaults.GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID); + var logManager = new LogManager(logServer); + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + + var raw = db.Server.ExpectDatabase($"{db.GetRuntimeName()}_RAW"); + if (raw.Exists()) + raw.Drop(); + + using var dt = new DataTable("MyTable"); + dt.Columns.Add("Name"); + dt.Columns.Add("DateOfBirth"); + dt.Columns.Add("FavouriteColour"); + dt.Rows.Add("Bob", "2001-01-01", "Pink"); + dt.Rows.Add("Frank", "2001-01-01", "Orange"); + + var nameCol = new DatabaseColumnRequest("Name", new DatabaseTypeRequest(typeof(string), 20), false) + { IsPrimaryKey = true }; + DiscoveredTable tbl = db.CreateTable("MyTable", dt, new[] + { + nameCol, + new DatabaseColumnRequest("DateOfBirth", new DatabaseTypeRequest(typeof(DateTime)), false) + { IsPrimaryKey = true } + }); + + Assert.That(tbl.GetRowCount(), Is.EqualTo(2)); + + //define a new load configuration + var lmd = new LoadMetadata(CatalogueRepository, "MyLoad"); + + lmd.IgnoreTrigger = true; + lmd.SaveToDatabase(); + var ti = Import(tbl, lmd, logManager); + + var projectDirectory = SetupLoadDirectory(lmd); + + CreateCSVProcessTask(lmd, ti, "*.csv"); + + + var regex = new RegexRedactionConfiguration(CatalogueRepository, "Too Long!", new Regex("Yella"), "FFFFFFFF"); + regex.SaveToDatabase(); + + var pt = new ProcessTask(CatalogueRepository, lmd, LoadStage.AdjustRaw) + { + Path = typeof(RegexRedactionMutilator).FullName, + ProcessTaskType = ProcessTaskType.MutilateDataTable, + Name = $"Mutilate {ti.GetRuntimeName()}" + }; + pt.SaveToDatabase(); + + pt.CreateArgumentsForClassIfNotExists(); + pt.SetArgumentValue("ColumnRegexPattern", ".*"); + pt.SetArgumentValue("redactionConfiguration", regex); + pt.SetArgumentValue("TableRegexPattern", ".*"); + + pt.Check(ThrowImmediatelyCheckNotifier.Quiet); + + File.WriteAllText( + Path.Combine(projectDirectory.ForLoading.FullName, "LoadMe.csv"), + @"Name,DateOfBirth,FavouriteColour +MrMurder,2001-01-01,Yella"); + + var dbConfig = new HICDatabaseConfiguration(lmd, null); + + var job = new DataLoadJob(RepositoryLocator, "Go go go!", logManager, lmd, projectDirectory, + ThrowImmediatelyDataLoadEventListener.Quiet, dbConfig); + + var loadFactory = new HICDataLoadFactory( + lmd, + dbConfig, + new HICLoadConfigurationFlags(), + CatalogueRepository, + logManager + ); + + var exe = loadFactory.Create(ThrowImmediatelyDataLoadEventListener.Quiet); + + var exitCode = exe.Run( + job, + new GracefulCancellationToken()); + + var redactions = CatalogueRepository.GetAllObjects(); + Assert.That(redactions.Count(), Is.EqualTo(0)); + + var redactionKeys = CatalogueRepository.GetAllObjects(); + Assert.That(redactionKeys.Length, Is.EqualTo(0)); + } + + [Test] + public void RedactionMutilator_RedactAPK() + { + var defaults = CatalogueRepository; + var logServer = defaults.GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID); + var logManager = new LogManager(logServer); + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + + var raw = db.Server.ExpectDatabase($"{db.GetRuntimeName()}_RAW"); + if (raw.Exists()) + raw.Drop(); + + using var dt = new DataTable("MyTable"); + dt.Columns.Add("Name"); + dt.Columns.Add("DateOfBirth"); + dt.Columns.Add("FavouriteColour"); + dt.Rows.Add("Bob", "2001-01-01", "Pink"); + dt.Rows.Add("Frank", "2001-01-01", "Orange"); + + var nameCol = new DatabaseColumnRequest("Name", new DatabaseTypeRequest(typeof(string), 20), false) + { IsPrimaryKey = true }; + DiscoveredTable tbl = db.CreateTable("MyTable", dt, new[] + { + nameCol, + new DatabaseColumnRequest("DateOfBirth", new DatabaseTypeRequest(typeof(DateTime)), false) + { IsPrimaryKey = true } + }); + + Assert.That(tbl.GetRowCount(), Is.EqualTo(2)); + + //define a new load configuration + var lmd = new LoadMetadata(CatalogueRepository, "MyLoad"); + + lmd.IgnoreTrigger = true; + lmd.SaveToDatabase(); + var ti = Import(tbl, lmd, logManager); + + var projectDirectory = SetupLoadDirectory(lmd); + + CreateCSVProcessTask(lmd, ti, "*.csv"); + + + var regex = new RegexRedactionConfiguration(CatalogueRepository, "No Yelling", new Regex("Yella"), "FFF"); + regex.SaveToDatabase(); + + var pt = new ProcessTask(CatalogueRepository, lmd, LoadStage.AdjustRaw) + { + Path = typeof(RegexRedactionMutilator).FullName, + ProcessTaskType = ProcessTaskType.MutilateDataTable, + Name = $"Mutilate {ti.GetRuntimeName()}" + }; + pt.SaveToDatabase(); + + pt.CreateArgumentsForClassIfNotExists(); + pt.SetArgumentValue("ColumnRegexPattern", "DateOfBirth"); + pt.SetArgumentValue("redactionConfiguration", regex); + pt.SetArgumentValue("TableRegexPattern", ".*"); + + pt.Check(ThrowImmediatelyCheckNotifier.Quiet); + + File.WriteAllText( + Path.Combine(projectDirectory.ForLoading.FullName, "LoadMe.csv"), + @"Name,DateOfBirth,FavouriteColour +MrMurder,2001-01-01,Yella"); + + var dbConfig = new HICDatabaseConfiguration(lmd, null); + + var job = new DataLoadJob(RepositoryLocator, "Go go go!", logManager, lmd, projectDirectory, + ThrowImmediatelyDataLoadEventListener.Quiet, dbConfig); + + var loadFactory = new HICDataLoadFactory( + lmd, + dbConfig, + new HICLoadConfigurationFlags(), + CatalogueRepository, + logManager + ); + + var exe = loadFactory.Create(ThrowImmediatelyDataLoadEventListener.Quiet); + + var exitCode = exe.Run( + job, + new GracefulCancellationToken()); + + var redactions = CatalogueRepository.GetAllObjects(); + Assert.That(redactions.Count(), Is.EqualTo(0)); + + var redactionKeys = CatalogueRepository.GetAllObjects(); + Assert.That(redactionKeys.Length, Is.EqualTo(0)); + } + + [Test] + public void RedactionMutilator_NoPKS() + { + var defaults = CatalogueRepository; + var logServer = defaults.GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID); + var logManager = new LogManager(logServer); + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + + var raw = db.Server.ExpectDatabase($"{db.GetRuntimeName()}_RAW"); + if (raw.Exists()) + raw.Drop(); + + using var dt = new DataTable("MyTable"); + dt.Columns.Add("Name"); + dt.Columns.Add("DateOfBirth"); + dt.Columns.Add("FavouriteColour"); + dt.Rows.Add("Bob", "2001-01-01", "Pink"); + dt.Rows.Add("Frank", "2001-01-01", "Orange"); + + var nameCol = new DatabaseColumnRequest("Name", new DatabaseTypeRequest(typeof(string), 20), false) + { IsPrimaryKey = true }; + DiscoveredTable tbl = db.CreateTable("MyTable", dt, new[] + { + nameCol, + new DatabaseColumnRequest("DateOfBirth", new DatabaseTypeRequest(typeof(DateTime)), false) + { IsPrimaryKey = false } + }); + + Assert.That(tbl.GetRowCount(), Is.EqualTo(2)); + + //define a new load configuration + var lmd = new LoadMetadata(CatalogueRepository, "MyLoad"); + + lmd.IgnoreTrigger = true; + lmd.SaveToDatabase(); + var ti = Import(tbl, lmd, logManager); + + var projectDirectory = SetupLoadDirectory(lmd); + + CreateCSVProcessTask(lmd, ti, "*.csv"); + + + var regex = new RegexRedactionConfiguration(CatalogueRepository, "No Yelling", new Regex("Yella"), "FFF"); + regex.SaveToDatabase(); + + var pt = new ProcessTask(CatalogueRepository, lmd, LoadStage.AdjustRaw) + { + Path = typeof(RegexRedactionMutilator).FullName, + ProcessTaskType = ProcessTaskType.MutilateDataTable, + Name = $"Mutilate {ti.GetRuntimeName()}" + }; + pt.SaveToDatabase(); + + pt.CreateArgumentsForClassIfNotExists(); + pt.SetArgumentValue("ColumnRegexPattern", "DateOfBirth"); + pt.SetArgumentValue("redactionConfiguration", regex); + pt.SetArgumentValue("TableRegexPattern", ".*"); + + pt.Check(ThrowImmediatelyCheckNotifier.Quiet); + + File.WriteAllText( + Path.Combine(projectDirectory.ForLoading.FullName, "LoadMe.csv"), + @"Name,DateOfBirth,FavouriteColour +MrMurder,2001-01-01,Yella"); + + var dbConfig = new HICDatabaseConfiguration(lmd, null); + + var job = new DataLoadJob(RepositoryLocator, "Go go go!", logManager, lmd, projectDirectory, + ThrowImmediatelyDataLoadEventListener.Quiet, dbConfig); + + var loadFactory = new HICDataLoadFactory( + lmd, + dbConfig, + new HICLoadConfigurationFlags(), + CatalogueRepository, + logManager + ); + + var exe = loadFactory.Create(ThrowImmediatelyDataLoadEventListener.Quiet); + + var exitCode = exe.Run( + job, + new GracefulCancellationToken()); + + var redactions = CatalogueRepository.GetAllObjects(); + Assert.That(redactions.Count(), Is.EqualTo(0)); + + var redactionKeys = CatalogueRepository.GetAllObjects(); + Assert.That(redactionKeys.Length, Is.EqualTo(0)); + } + + [Test] + public void RedactionMutilator_MultipleInOneCell() + { + var defaults = CatalogueRepository; + var logServer = defaults.GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID); + var logManager = new LogManager(logServer); + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + + var raw = db.Server.ExpectDatabase($"{db.GetRuntimeName()}_RAW"); + if (raw.Exists()) + raw.Drop(); + + using var dt = new DataTable("MyTable"); + dt.Columns.Add("Name"); + dt.Columns.Add("DateOfBirth"); + dt.Columns.Add("FavouriteColour"); + dt.Rows.Add("Bob", "2001-01-01", "PinkPinkPink"); + dt.Rows.Add("Frank", "2001-01-01", "OrangeOrangeOrange"); + + var nameCol = new DatabaseColumnRequest("Name", new DatabaseTypeRequest(typeof(string), 20), false) + { IsPrimaryKey = true }; + DiscoveredTable tbl = db.CreateTable("MyTable", dt, new[] + { + nameCol, + new DatabaseColumnRequest("DateOfBirth", new DatabaseTypeRequest(typeof(DateTime)), false) + { IsPrimaryKey = true } + }); + + Assert.That(tbl.GetRowCount(), Is.EqualTo(2)); + + //define a new load configuration + var lmd = new LoadMetadata(CatalogueRepository, "MyLoad"); + + lmd.IgnoreTrigger = true; + lmd.SaveToDatabase(); + var ti = Import(tbl, lmd, logManager); + + var projectDirectory = SetupLoadDirectory(lmd); + + CreateCSVProcessTask(lmd, ti, "*.csv"); + + + var regex = new RegexRedactionConfiguration(CatalogueRepository, "No Yelling", new Regex("Yella"), "FFF"); + regex.SaveToDatabase(); + + var pt = new ProcessTask(CatalogueRepository, lmd, LoadStage.AdjustRaw) + { + Path = typeof(RegexRedactionMutilator).FullName, + ProcessTaskType = ProcessTaskType.MutilateDataTable, + Name = $"Mutilate {ti.GetRuntimeName()}" + }; + pt.SaveToDatabase(); + + pt.CreateArgumentsForClassIfNotExists(); + pt.SetArgumentValue("ColumnRegexPattern", ".*"); + pt.SetArgumentValue("redactionConfiguration", regex); + pt.SetArgumentValue("TableRegexPattern", ".*"); + + pt.Check(ThrowImmediatelyCheckNotifier.Quiet); + + File.WriteAllText( + Path.Combine(projectDirectory.ForLoading.FullName, "LoadMe.csv"), + @"Name,DateOfBirth,FavouriteColour +MrMurder,2001-01-01,YellaUUUYella"); + + var dbConfig = new HICDatabaseConfiguration(lmd, null); + + var job = new DataLoadJob(RepositoryLocator, "Go go go!", logManager, lmd, projectDirectory, + ThrowImmediatelyDataLoadEventListener.Quiet, dbConfig); + + var loadFactory = new HICDataLoadFactory( + lmd, + dbConfig, + new HICLoadConfigurationFlags(), + CatalogueRepository, + logManager + ); + + var exe = loadFactory.Create(ThrowImmediatelyDataLoadEventListener.Quiet); + + var exitCode = exe.Run( + job, + new GracefulCancellationToken()); + + var redactions = CatalogueRepository.GetAllObjects(); + Assert.That(redactions.Count(), Is.EqualTo(2)); + Assert.That(redactions[0].RedactionConfiguration_ID, Is.EqualTo(regex.ID)); + Assert.That(redactions[0].StartingIndex, Is.EqualTo(0)); + Assert.That(redactions[0].RedactedValue, Is.EqualTo("Yella")); + Assert.That(redactions[0].ReplacementValue, Is.EqualTo("")); + Assert.That(redactions[1].RedactionConfiguration_ID, Is.EqualTo(regex.ID)); + Assert.That(redactions[1].StartingIndex, Is.EqualTo(8)); + Assert.That(redactions[1].RedactedValue, Is.EqualTo("Yella")); + Assert.That(redactions[1].ReplacementValue, Is.EqualTo("")); + + var redactionKeys = CatalogueRepository.GetAllObjects(); + Assert.That(redactionKeys.Length, Is.EqualTo(4)); + Assert.That(redactionKeys[0].Value == "MrMurder"); + Assert.That(redactionKeys[1].Value == "01/01/2001 00:00:00"); + Assert.That(redactionKeys[2].Value == "MrMurder"); + Assert.That(redactionKeys[3].Value == "01/01/2001 00:00:00"); + foreach (var r in redactions) + { + r.DeleteInDatabase(); + } + foreach (var r in redactionKeys) + { + r.DeleteInDatabase(); + } + } + + [Test] + public void RedactionMutilator_SpeedTest() + { + var defaults = CatalogueRepository; + var logServer = defaults.GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID); + var logManager = new LogManager(logServer); + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + + var raw = db.Server.ExpectDatabase($"{db.GetRuntimeName()}_RAW"); + if (raw.Exists()) + raw.Drop(); + + using var dt = new DataTable("MyTable"); + dt.Columns.Add("Name"); + dt.Columns.Add("DateOfBirth"); + dt.Columns.Add("FavouriteColour"); + dt.Rows.Add("Bob", "2001-01-01", "Pink"); + dt.Rows.Add("Frank", "2001-01-01", "Orange"); + + var nameCol = new DatabaseColumnRequest("Name", new DatabaseTypeRequest(typeof(string), 20), false) + { IsPrimaryKey = true }; + DiscoveredTable tbl = db.CreateTable("MyTable", dt, new[] + { + nameCol, + new DatabaseColumnRequest("DateOfBirth", new DatabaseTypeRequest(typeof(DateTime)), false) + { IsPrimaryKey = true } + }); + + Assert.That(tbl.GetRowCount(), Is.EqualTo(2)); + + //define a new load configuration + var lmd = new LoadMetadata(CatalogueRepository, "MyLoad"); + + lmd.IgnoreTrigger = true; + lmd.SaveToDatabase(); + var ti = Import(tbl, lmd, logManager); + + var projectDirectory = SetupLoadDirectory(lmd); + + CreateCSVProcessTask(lmd, ti, "*.csv"); + + + var regex = new RegexRedactionConfiguration(CatalogueRepository, "PM ONLY", new Regex("amber"), "XX"); + regex.SaveToDatabase(); + + var pt = new ProcessTask(CatalogueRepository, lmd, LoadStage.AdjustRaw) + { + Path = typeof(RegexRedactionMutilator).FullName, + ProcessTaskType = ProcessTaskType.MutilateDataTable, + Name = $"Mutilate {ti.GetRuntimeName()}" + }; + pt.SaveToDatabase(); + + pt.CreateArgumentsForClassIfNotExists(); + pt.SetArgumentValue("ColumnRegexPattern", ".*"); + pt.SetArgumentValue("redactionConfiguration", regex); + pt.SetArgumentValue("TableRegexPattern", ".*"); + + pt.Check(ThrowImmediatelyCheckNotifier.Quiet); + + File.WriteAllText( + Path.Combine(projectDirectory.ForLoading.FullName, "LoadMe.csv"), + @$"Name,DateOfBirth,FavouriteColour +MrMurder,2001-01-01,Yella +"); + + var data = Enumerable.Repeat("name,2001-01-01,amber", 1000000).ToArray(); + + + File.AppendAllLines(Path.Combine(projectDirectory.ForLoading.FullName, "LoadMe.csv"), data); + + + var dbConfig = new HICDatabaseConfiguration(lmd, null); + + var job = new DataLoadJob(RepositoryLocator, "Go go go!", logManager, lmd, projectDirectory, + ThrowImmediatelyDataLoadEventListener.Quiet, dbConfig); + + var loadFactory = new HICDataLoadFactory( + lmd, + dbConfig, + new HICLoadConfigurationFlags(), + CatalogueRepository, + logManager + ); + + var exe = loadFactory.Create(ThrowImmediatelyDataLoadEventListener.Quiet); + var exitCode = exe.Run( + job, + new GracefulCancellationToken()); + + } +} diff --git a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionConfiguration.cs b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionConfiguration.cs index bc1c441639..9d9fd609c4 100644 --- a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionConfiguration.cs +++ b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionConfiguration.cs @@ -18,7 +18,7 @@ namespace Rdmp.Core.Curation.DataHelper.RegexRedaction; /// /// Stores the configuration used to perform regex redactions /// -public class RegexRedactionConfiguration: DatabaseEntity, IRegexRedactionConfiguration +public class RegexRedactionConfiguration : DatabaseEntity, IRegexRedactionConfiguration { @@ -27,8 +27,16 @@ public class RegexRedactionConfiguration: DatabaseEntity, IRegexRedactionConfigu private string _name; private string _description; + + + public override string ToString() + { + return _name; + } + [Unique] [NotNull] + [UsefulProperty] public string Name { get => _name; @@ -58,7 +66,7 @@ public string RedactionString public RegexRedactionConfiguration() { } - public RegexRedactionConfiguration(ICatalogueRepository repository,string name, Regex regexPattern, string redactionString,string description=null) + public RegexRedactionConfiguration(ICatalogueRepository repository, string name, Regex regexPattern, string redactionString, string description = null) { repository.InsertAndHydrate(this, new Dictionary { @@ -69,11 +77,11 @@ public RegexRedactionConfiguration(ICatalogueRepository repository,string name, }); } - internal RegexRedactionConfiguration(ICatalogueRepository repository, DbDataReader r): base(repository,r) + internal RegexRedactionConfiguration(ICatalogueRepository repository, DbDataReader r) : base(repository, r) { Name = r["Name"].ToString(); Description = r["Description"].ToString(); - RedactionString= r["RedactionString"].ToString(); + RedactionString = r["RedactionString"].ToString(); RegexPattern = r["RegexPattern"].ToString(); } diff --git a/Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs b/Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs index e73e052777..d1b67f7989 100644 --- a/Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs +++ b/Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs @@ -30,7 +30,6 @@ public class RegexRedactionMutilator : MatchingTablesMutilatorWithDataLoadJob private DiscoveredColumn[] _discoveredPKColumns; - public RegexRedactionMutilator() : base([LoadStage.AdjustRaw, LoadStage.AdjustStaging]) { } private bool ColumnMatches(DiscoveredColumn column) @@ -53,13 +52,13 @@ protected override void MutilateTable(IDataLoadJob job, ITableInfo tableInfo, Di DataTable pksToSave = RegexRedactionHelper.GeneratePKDataTable(); var columns = table.DiscoverColumns(); - + var relatedCatalogues = tableInfo.GetAllRelatedCatalogues(); var cataloguePks = relatedCatalogues.SelectMany(c => c.CatalogueItems).Where(ci => ci.ColumnInfo.IsPrimaryKey).ToList(); - _discoveredPKColumns = columns.Where(c => cataloguePks.Select(cpk=>cpk.Name).Contains(c.GetRuntimeName())).ToArray(); - if(_discoveredPKColumns.Length == 0) + _discoveredPKColumns = columns.Where(c => cataloguePks.Select(cpk => cpk.Name).Contains(c.GetRuntimeName())).ToArray(); + if (_discoveredPKColumns.Length == 0) { - job.OnNotify(this,new NotifyEventArgs(ProgressEventType.Warning,"No Primary Keys found. Redaction cannot be perfomed without a primary key.")); + job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "No Primary Keys found. Redaction cannot be perfomed without a primary key.")); //Don't want to fail the data load, but just let the user know return; } @@ -87,18 +86,26 @@ protected override void MutilateTable(IDataLoadJob job, ITableInfo tableInfo, Di dt.EndLoadData(); var redactionUpates = dt.Clone(); var columnInfo = relatedCatalogues.SelectMany(c => c.CatalogueItems).ToArray().Select(ci => ci.ColumnInfo).Where(ci => ci.GetRuntimeName() == column.GetRuntimeName()).FirstOrDefault(); - if(columnInfo is null) + if (columnInfo is null) { job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error, "Unable to find the related column info")); return; } foreach (DataRow row in dt.Rows) { - RegexRedactionHelper.Redact(columnInfo, row, cataloguePks, redactionConfiguration, redactionsToSaveTable, pksToSave, redactionUpates); + try + { + RegexRedactionHelper.Redact(columnInfo, row, cataloguePks, redactionConfiguration, redactionsToSaveTable, pksToSave, redactionUpates); + } + catch (Exception e) + { + job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, $"{e.Message}")); + + } } job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"Regex Redaction mutilator found {dt.Rows.Count} redactions.")); - - for (int i = 0; i < dt.Rows.Count; i++) + if (redactionsToSaveTable.Rows.Count == 0) return; + for (int i = 0; i < pksToSave.Rows.Count; i++) { pksToSave.Rows[i]["ID"] = i + 1; } @@ -107,7 +114,7 @@ protected override void MutilateTable(IDataLoadJob job, ITableInfo tableInfo, Di var t2 = table.Database.CreateTable("redactionsToSaveTable_Temp", redactionsToSaveTable); job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"Saving Redactions")); var _server = relatedCatalogues.First().GetDistinctLiveDatabaseServer(DataAccessContext.InternalDataProcessing, false); - RegexRedactionHelper.SaveRedactions(job.RepositoryLocator.CatalogueRepository, t1, t2, _server, Timeout*1000); + RegexRedactionHelper.SaveRedactions(job.RepositoryLocator.CatalogueRepository, t1, t2, _server, Timeout * 1000); t1.Drop(); t2.Drop(); job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"Performing join update")); From 24c43b9804e76d8025ba23666898da8928141bc8 Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 6 Sep 2024 14:38:42 +0100 Subject: [PATCH 036/106] start og ui update --- .../TopBar/RDMPTaskBarUI.Designer.cs | 20 +- .../WindowManagement/TopBar/RDMPTaskBarUI.cs | 12 +- .../TopBar/RDMPTaskBarUI.resx | 2 +- .../WindowManagement/WindowManager.cs | 8 +- Rdmp.Core/Curation/Data/Dataset.cs | 2 +- .../IRegexRedactionConfiguration.cs | 8 +- .../RegexRedactionConfiguration.cs | 5 +- .../IconProvision/CatalogueIcons.Designer.cs | 1660 +++++++++++------ .../Icons/IconProvision/CatalogueIcons.resx | 6 + Rdmp.Core/Icons/IconProvision/RDMPConcept.cs | 1 + Rdmp.Core/Providers/CatalogueChildProvider.cs | 48 +- Rdmp.Core/Providers/ICoreChildProvider.cs | 4 + Rdmp.Core/Providers/Nodes/AllDatasetsNode.cs | 15 + .../AllRegexRedactionConfigurationsNode.cs | 14 + Rdmp.Core/RDMPCollection.cs | 3 +- ...=> ConfigurationsCollectionUI.Designer.cs} | 48 +- .../Collections/ConfigurationsCollectionUI.cs | 55 + ...I.resx => ConfigurationsCollectionUI.resx} | 0 Rdmp.UI/Collections/DatasetsCollectionUI.cs | 127 -- .../RedactCatalogueUI.Designer.cs | 91 +- Rdmp.UI/SimpleDialogs/RedactCatalogueUI.cs | 12 +- ...dactionConfigurationCreationUI.Designer.cs | 59 + .../RegexRedactionConfigurationCreationUI.cs | 22 + ...RegexRedactionConfigurationCreationUI.resx | 120 ++ Rdmp.UI/SimpleDialogs/SelectDialog`1.resx | 63 - Rdmp.UI/Theme/BackColorProvider.cs | 1 + 26 files changed, 1503 insertions(+), 903 deletions(-) create mode 100644 Rdmp.Core/Providers/Nodes/AllDatasetsNode.cs create mode 100644 Rdmp.Core/Providers/Nodes/AllRegexRedactionConfigurationsNode.cs rename Rdmp.UI/Collections/{DatasetsCollectionUI.Designer.cs => ConfigurationsCollectionUI.Designer.cs} (57%) create mode 100644 Rdmp.UI/Collections/ConfigurationsCollectionUI.cs rename Rdmp.UI/Collections/{DatasetsCollectionUI.resx => ConfigurationsCollectionUI.resx} (100%) delete mode 100644 Rdmp.UI/Collections/DatasetsCollectionUI.cs create mode 100644 Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationCreationUI.Designer.cs create mode 100644 Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationCreationUI.cs create mode 100644 Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationCreationUI.resx delete mode 100644 Rdmp.UI/SimpleDialogs/SelectDialog`1.resx diff --git a/Application/ResearchDataManagementPlatform/WindowManagement/TopBar/RDMPTaskBarUI.Designer.cs b/Application/ResearchDataManagementPlatform/WindowManagement/TopBar/RDMPTaskBarUI.Designer.cs index 5548b83c3c..1fe181c1e4 100644 --- a/Application/ResearchDataManagementPlatform/WindowManagement/TopBar/RDMPTaskBarUI.Designer.cs +++ b/Application/ResearchDataManagementPlatform/WindowManagement/TopBar/RDMPTaskBarUI.Designer.cs @@ -37,7 +37,7 @@ private void InitializeComponent() btnCohorts = new System.Windows.Forms.ToolStripButton(); btnDataExport = new System.Windows.Forms.ToolStripButton(); btnTables = new System.Windows.Forms.ToolStripButton(); - btnDataSets = new System.Windows.Forms.ToolStripButton(); + btnConfiguration = new System.Windows.Forms.ToolStripButton(); toolStripSeparator2 = new System.Windows.Forms.ToolStripSeparator(); toolStripSeparator = new System.Windows.Forms.ToolStripSeparator(); toolStrip1 = new System.Windows.Forms.ToolStrip(); @@ -105,14 +105,14 @@ private void InitializeComponent() btnTables.Text = "Tables (Advanced)"; btnTables.Click += ToolboxButtonClicked; // - // btnDataSets + // btnConfiguration // - btnDataSets.Image = (System.Drawing.Image)resources.GetObject("btnDataSets.Image"); - btnDataSets.ImageTransparentColor = System.Drawing.Color.Magenta; - btnDataSets.Name = "btnDataSets"; - btnDataSets.Size = new System.Drawing.Size(71, 22); - btnDataSets.Text = "Datasets"; - btnDataSets.Click += ToolboxButtonClicked; + btnConfiguration.Image = (System.Drawing.Image)resources.GetObject("btnConfiguration.Image"); + btnConfiguration.ImageTransparentColor = System.Drawing.Color.Magenta; + btnConfiguration.Name = "btnConfiguration"; + btnConfiguration.Size = new System.Drawing.Size(71, 22); + btnConfiguration.Text = "Configuration"; + btnConfiguration.Click += ToolboxButtonClicked; // // toolStripSeparator2 // @@ -126,7 +126,7 @@ private void InitializeComponent() // // toolStrip1 // - toolStrip1.Items.AddRange(new System.Windows.Forms.ToolStripItem[] { btnBack, btnForward, btnHome, toolStripSeparator1, btnFavourites, btnCatalogues, btnCohorts, btnSavedCohorts, btnDataExport, toolStripSeparator, btnTables, btnLoads, btnDataSets, toolStripSeparator2, toolStripLabel2, cbxLayouts, btnSaveWindowLayout, btnDeleteLayout, toolStripSeparator4, cbCommits, toolStripSeparator3 }); + toolStrip1.Items.AddRange(new System.Windows.Forms.ToolStripItem[] { btnBack, btnForward, btnHome, toolStripSeparator1, btnFavourites, btnCatalogues, btnCohorts, btnSavedCohorts, btnDataExport, toolStripSeparator, btnTables, btnLoads, btnConfiguration, toolStripSeparator2, toolStripLabel2, cbxLayouts, btnSaveWindowLayout, btnDeleteLayout, toolStripSeparator4, cbCommits, toolStripSeparator3 }); toolStrip1.Location = new System.Drawing.Point(0, 0); toolStrip1.Name = "toolStrip1"; toolStrip1.Size = new System.Drawing.Size(1539, 25); @@ -259,7 +259,7 @@ private void InitializeComponent() private System.Windows.Forms.ToolStripButton btnCohorts; private System.Windows.Forms.ToolStripButton btnDataExport; private System.Windows.Forms.ToolStripButton btnTables; - private System.Windows.Forms.ToolStripButton btnDataSets; + private System.Windows.Forms.ToolStripButton btnConfiguration; private System.Windows.Forms.ToolStripSeparator toolStripSeparator2; private System.Windows.Forms.ToolStripSeparator toolStripSeparator; private System.Windows.Forms.ToolStrip toolStrip1; diff --git a/Application/ResearchDataManagementPlatform/WindowManagement/TopBar/RDMPTaskBarUI.cs b/Application/ResearchDataManagementPlatform/WindowManagement/TopBar/RDMPTaskBarUI.cs index d79e7e9645..47e794744d 100644 --- a/Application/ResearchDataManagementPlatform/WindowManagement/TopBar/RDMPTaskBarUI.cs +++ b/Application/ResearchDataManagementPlatform/WindowManagement/TopBar/RDMPTaskBarUI.cs @@ -51,11 +51,11 @@ public RDMPTaskBarUI() btnTables.Image = CatalogueIcons.TableInfo.ImageToBitmap(); btnTables.BackgroundImage = BackColorProvider.GetBackgroundImage(btnTables.Size, RDMPCollection.Tables); - btnDataSets.Image = CatalogueIcons.Dataset.ImageToBitmap(); - btnDataSets.BackgroundImage = BackColorProvider.GetBackgroundImage(btnDataSets.Size, RDMPCollection.Datasets); + btnConfiguration.Image = FamFamFamIcons.pencil_go.ImageToBitmap(); + btnConfiguration.BackgroundImage = BackColorProvider.GetBackgroundImage(btnConfiguration.Size, RDMPCollection.Datasets); btnLoads.Image = CatalogueIcons.LoadMetadata.ImageToBitmap(); - btnLoads.BackgroundImage = BackColorProvider.GetBackgroundImage(btnDataSets.Size, RDMPCollection.DataLoad); + btnLoads.BackgroundImage = BackColorProvider.GetBackgroundImage(btnConfiguration.Size, RDMPCollection.DataLoad); btnFavourites.Image = CatalogueIcons.Favourite.ImageToBitmap(); btnDeleteLayout.Image = FamFamFamIcons.delete.ImageToBitmap(); @@ -111,7 +111,7 @@ private void SetupToolTipText() btnTables.ToolTipText = "Advanced features e.g. logging, credentials, dashboards etc"; btnLoads.ToolTipText = "Load configurations for reading data into your databases"; btnFavourites.ToolTipText = "Collection of all objects that you have favourited"; - btnDataSets.ToolTipText = "All external datasets that have been configured for use in RDMP"; + btnConfiguration.ToolTipText = "All external datasets that have been configured for use in RDMP"; } catch (Exception e) { @@ -190,8 +190,8 @@ private RDMPCollection ButtonToEnum(object button) collectionToToggle = RDMPCollection.SavedCohorts; else if (button == btnFavourites) collectionToToggle = RDMPCollection.Favourites; - else if (button == btnDataSets) - collectionToToggle = RDMPCollection.Datasets; + else if (button == btnConfiguration) + collectionToToggle = RDMPCollection.Configurations; else throw new ArgumentOutOfRangeException(nameof(button)); diff --git a/Application/ResearchDataManagementPlatform/WindowManagement/TopBar/RDMPTaskBarUI.resx b/Application/ResearchDataManagementPlatform/WindowManagement/TopBar/RDMPTaskBarUI.resx index 46a8e47b5d..7f145f50ec 100644 --- a/Application/ResearchDataManagementPlatform/WindowManagement/TopBar/RDMPTaskBarUI.resx +++ b/Application/ResearchDataManagementPlatform/WindowManagement/TopBar/RDMPTaskBarUI.resx @@ -178,7 +178,7 @@ TgDQASA1MVpwzwAAAABJRU5ErkJggg== - + iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8 YQUAAAAJcEhZcwAADsMAAA7DAcdvqGQAAAIDSURBVDhPpZLrS5NhGMb3j4SWh0oRQVExD4gonkDpg4hG diff --git a/Application/ResearchDataManagementPlatform/WindowManagement/WindowManager.cs b/Application/ResearchDataManagementPlatform/WindowManagement/WindowManager.cs index 406e858db8..08a6af5cca 100644 --- a/Application/ResearchDataManagementPlatform/WindowManagement/WindowManager.cs +++ b/Application/ResearchDataManagementPlatform/WindowManagement/WindowManager.cs @@ -148,10 +148,10 @@ public PersistableToolboxDockContent Create(RDMPCollection collectionToCreate, Image.Load(CatalogueIcons.Favourite)); break; - case RDMPCollection.Datasets: - collection = new DatasetsCollectionUI(); - toReturn = Show(RDMPCollection.Datasets, collection, "Datasets", - Image.Load(CatalogueIcons.Dataset)); + case RDMPCollection.Configurations: + collection = new ConfigurationsCollectionUI(); + toReturn = Show(RDMPCollection.Configurations, collection, "Configurations", + Image.Load(FamFamFamIcons.pencil_go)); break; default: throw new ArgumentOutOfRangeException(nameof(collectionToCreate)); diff --git a/Rdmp.Core/Curation/Data/Dataset.cs b/Rdmp.Core/Curation/Data/Dataset.cs index aac0df9eae..5623568db7 100644 --- a/Rdmp.Core/Curation/Data/Dataset.cs +++ b/Rdmp.Core/Curation/Data/Dataset.cs @@ -16,7 +16,7 @@ namespace Rdmp.Core.Curation.Data; /// -public sealed class Dataset : DatabaseEntity, IDataset, IHasFolder +public sealed class Dataset : DatabaseEntity, IDataset, IHasFolder { private string _name; private string _digitalObjectIdentifier; diff --git a/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedactionConfiguration.cs b/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedactionConfiguration.cs index c0ce09d102..a4f795b688 100644 --- a/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedactionConfiguration.cs +++ b/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedactionConfiguration.cs @@ -4,19 +4,15 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . +using Rdmp.Core.Curation.Data; using Rdmp.Core.MapsDirectlyToDatabaseTable; using Rdmp.Core.Repositories; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace Rdmp.Core.Curation.DataHelper.RegexRedaction; /// /// Stores the configuration to run regex redactions on data /// -public interface IRegexRedactionConfiguration: IMapsDirectlyToDatabaseTable +public interface IRegexRedactionConfiguration: IMapsDirectlyToDatabaseTable, IHasFolder { ICatalogueRepository CatalogueRepository { get; } diff --git a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionConfiguration.cs b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionConfiguration.cs index 9d9fd609c4..9b2e90f8c7 100644 --- a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionConfiguration.cs +++ b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionConfiguration.cs @@ -26,12 +26,13 @@ public class RegexRedactionConfiguration : DatabaseEntity, IRegexRedactionConfig private string _redactionString; private string _name; private string _description; + private string _folder; public override string ToString() { - return _name; + return $"{_name}({_regexPattern})"; } [Unique] @@ -62,7 +63,7 @@ public string RedactionString get => _redactionString; set => SetField(ref _redactionString, value); } - + public string Folder { get => _folder; set => SetField(ref _folder, value); } public RegexRedactionConfiguration() { } diff --git a/Rdmp.Core/Icons/IconProvision/CatalogueIcons.Designer.cs b/Rdmp.Core/Icons/IconProvision/CatalogueIcons.Designer.cs index bce3ee5236..2097dd4914 100644 --- a/Rdmp.Core/Icons/IconProvision/CatalogueIcons.Designer.cs +++ b/Rdmp.Core/Icons/IconProvision/CatalogueIcons.Designer.cs @@ -26,240 +26,287 @@ namespace Rdmp.Core.Icons.IconProvision [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "16.0.0.0")] [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] - public class CatalogueIcons { - + public class CatalogueIcons + { + private static global::System.Resources.ResourceManager resourceMan; - + private static global::System.Globalization.CultureInfo resourceCulture; - + [global::System.Diagnostics.CodeAnalysis.SuppressMessageAttribute("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] - internal CatalogueIcons() { + internal CatalogueIcons() + { } - + /// /// Returns the cached ResourceManager instance used by this class. /// [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] - public static global::System.Resources.ResourceManager ResourceManager { - get { - if (object.ReferenceEquals(resourceMan, null)) { + public static global::System.Resources.ResourceManager ResourceManager + { + get + { + if (object.ReferenceEquals(resourceMan, null)) + { global::System.Resources.ResourceManager temp = new global::System.Resources.ResourceManager("Rdmp.Core.Icons.IconProvision.CatalogueIcons", typeof(CatalogueIcons).Assembly); resourceMan = temp; } return resourceMan; } } - + /// /// Overrides the current thread's CurrentUICulture property for all /// resource lookups using this strongly typed resource class. /// [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] - public static global::System.Globalization.CultureInfo Culture { - get { + public static global::System.Globalization.CultureInfo Culture + { + get + { return resourceCulture; } - set { + set + { resourceCulture = value; } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] AggregateContinuousDateAxis { - get { + public static Byte[] AggregateContinuousDateAxis + { + get + { object obj = ResourceManager.GetObject("AggregateContinuousDateAxis", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] AggregateDimension { - get { + public static Byte[] AggregateDimension + { + get + { object obj = ResourceManager.GetObject("AggregateDimension", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] AggregateGraph { - get { + public static Byte[] AggregateGraph + { + get + { object obj = ResourceManager.GetObject("AggregateGraph", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] AggregatesNode { - get { + public static Byte[] AggregatesNode + { + get + { object obj = ResourceManager.GetObject("AggregatesNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] AggregateTopX { - get { + public static Byte[] AggregateTopX + { + get + { object obj = ResourceManager.GetObject("AggregateTopX", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] AllANOTablesNode { - get { + public static Byte[] AllANOTablesNode + { + get + { object obj = ResourceManager.GetObject("AllANOTablesNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] AllAutomationServerSlotsNode { - get { + public static Byte[] AllAutomationServerSlotsNode + { + get + { object obj = ResourceManager.GetObject("AllAutomationServerSlotsNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] AllCataloguesUsedByLoadMetadataNode { - get { + public static Byte[] AllCataloguesUsedByLoadMetadataNode + { + get + { object obj = ResourceManager.GetObject("AllCataloguesUsedByLoadMetadataNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] AllCohortsNode { - get { + public static Byte[] AllCohortsNode + { + get + { object obj = ResourceManager.GetObject("AllCohortsNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] AllConnectionStringKeywordsNode { - get { + public static Byte[] AllConnectionStringKeywordsNode + { + get + { object obj = ResourceManager.GetObject("AllConnectionStringKeywordsNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] AllDashboardsNode { - get { + public static Byte[] AllDashboardsNode + { + get + { object obj = ResourceManager.GetObject("AllDashboardsNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] AllDataAccessCredentialsNode { - get { + public static Byte[] AllDataAccessCredentialsNode + { + get + { object obj = ResourceManager.GetObject("AllDataAccessCredentialsNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] AllExternalServersNode { - get { + public static Byte[] AllExternalServersNode + { + get + { object obj = ResourceManager.GetObject("AllExternalServersNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] AllFreeCohortIdentificationConfigurationsNode { - get { + public static Byte[] AllFreeCohortIdentificationConfigurationsNode + { + get + { object obj = ResourceManager.GetObject("AllFreeCohortIdentificationConfigurationsNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] AllGovernanceNode { - get { + public static Byte[] AllGovernanceNode + { + get + { object obj = ResourceManager.GetObject("AllGovernanceNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] AllObjectImportsNode { - get { + public static Byte[] AllObjectImportsNode + { + get + { object obj = ResourceManager.GetObject("AllObjectImportsNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] AllObjectSharingNode { - get { + public static Byte[] AllObjectSharingNode + { + get + { object obj = ResourceManager.GetObject("AllObjectSharingNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] AllOrphanAggregateConfigurationsNode { - get { + public static Byte[] AllOrphanAggregateConfigurationsNode + { + get + { object obj = ResourceManager.GetObject("AllOrphanAggregateConfigurationsNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] AllPermissionWindowsNode { - get { + public static Byte[] AllPermissionWindowsNode + { + get + { object obj = ResourceManager.GetObject("AllPermissionWindowsNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// public static Byte[] AllTemplateAggregateConfigurationsNode { - get { + get + { object obj = ResourceManager.GetObject("AllTemplateAggregateConfigurationsNode", resourceCulture); return ((Byte[])(obj)); } @@ -267,208 +314,269 @@ public static Byte[] AllTemplateAggregateConfigurationsNode /// /// Looks up a localized resource of type Image. /// - public static Byte[] AllPipelinesNode { - get { + public static Byte[] AllPipelinesNode + { + get + { object obj = ResourceManager.GetObject("AllPipelinesNode", resourceCulture); return ((Byte[])(obj)); } } - /// /// Looks up a localized resource of type Image. /// - public static Byte[] AllPluginsNode { - get { + public static Byte[] AllRegexRedactionConfigurationsNode + { + get + { + object obj = ResourceManager.GetObject("AllRegexRedactionConfigurationsNode", resourceCulture); + return ((Byte[])(obj)); + } + } + public static Byte[] AllDatasetsNode + { + get + { + object obj = ResourceManager.GetObject("AllDatasetsNode", resourceCulture); + return ((Byte[])(obj)); + } + } + + /// + /// Looks up a localized resource of type Image. + /// + public static Byte[] AllPluginsNode + { + get + { object obj = ResourceManager.GetObject("AllPluginsNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] AllProcessTasksUsedByLoadMetadataNode { - get { + public static Byte[] AllProcessTasksUsedByLoadMetadataNode + { + get + { object obj = ResourceManager.GetObject("AllProcessTasksUsedByLoadMetadataNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] AllProjectCohortIdentificationConfigurationsNode { - get { + public static Byte[] AllProjectCohortIdentificationConfigurationsNode + { + get + { object obj = ResourceManager.GetObject("AllProjectCohortIdentificationConfigurationsNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] AllRDMPRemotesNode { - get { + public static Byte[] AllRDMPRemotesNode + { + get + { object obj = ResourceManager.GetObject("AllRDMPRemotesNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] AllServersNode { - get { + public static Byte[] AllServersNode + { + get + { object obj = ResourceManager.GetObject("AllServersNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] AllStandardRegexesNode { - get { + public static Byte[] AllStandardRegexesNode + { + get + { object obj = ResourceManager.GetObject("AllStandardRegexesNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ANOColumnInfo { - get { + public static Byte[] ANOColumnInfo + { + get + { object obj = ResourceManager.GetObject("ANOColumnInfo", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ANOTable { - get { + public static Byte[] ANOTable + { + get + { object obj = ResourceManager.GetObject("ANOTable", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] AutomateablePipeline { - get { + public static Byte[] AutomateablePipeline + { + get + { object obj = ResourceManager.GetObject("AutomateablePipeline", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] AutomationServiceException { - get { + public static Byte[] AutomationServiceException + { + get + { object obj = ResourceManager.GetObject("AutomationServiceException", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] AutomationServiceSlot { - get { + public static Byte[] AutomationServiceSlot + { + get + { object obj = ResourceManager.GetObject("AutomationServiceSlot", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] Backup { - get { + public static Byte[] Backup + { + get + { object obj = ResourceManager.GetObject("Backup", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] BigCohort { - get { + public static Byte[] BigCohort + { + get + { object obj = ResourceManager.GetObject("BigCohort", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] BigGraph { - get { + public static Byte[] BigGraph + { + get + { object obj = ResourceManager.GetObject("BigGraph", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] BigPatientIndexTable { - get { + public static Byte[] BigPatientIndexTable + { + get + { object obj = ResourceManager.GetObject("BigPatientIndexTable", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] CacheFetchFailure { - get { + public static Byte[] CacheFetchFailure + { + get + { object obj = ResourceManager.GetObject("CacheFetchFailure", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] CacheProgress { - get { + public static Byte[] CacheProgress + { + get + { object obj = ResourceManager.GetObject("CacheProgress", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] Catalogue { - get { + public static Byte[] Catalogue + { + get + { object obj = ResourceManager.GetObject("Catalogue", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] CatalogueFolder { - get { + public static Byte[] CatalogueFolder + { + get + { object obj = ResourceManager.GetObject("CatalogueFolder", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] CatalogueItem { - get { + public static Byte[] CatalogueItem + { + get + { object obj = ResourceManager.GetObject("CatalogueItem", resourceCulture); return ((Byte[])(obj)); } @@ -489,768 +597,922 @@ public static Byte[] CatalogueItemTransform /// /// Looks up a localized resource of type Image. /// - public static Byte[] CatalogueItemIssue { - get { + public static Byte[] CatalogueItemIssue + { + get + { object obj = ResourceManager.GetObject("CatalogueItemIssue", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] CatalogueItemsNode { - get { + public static Byte[] CatalogueItemsNode + { + get + { object obj = ResourceManager.GetObject("CatalogueItemsNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] CatalogueLookupsNode { - get { + public static Byte[] CatalogueLookupsNode + { + get + { object obj = ResourceManager.GetObject("CatalogueLookupsNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] Clipboard { - get { + public static Byte[] Clipboard + { + get + { object obj = ResourceManager.GetObject("Clipboard", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] CloneExtractionConfiguration { - get { + public static Byte[] CloneExtractionConfiguration + { + get + { object obj = ResourceManager.GetObject("CloneExtractionConfiguration", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] CohortAggregate { - get { + public static Byte[] CohortAggregate + { + get + { object obj = ResourceManager.GetObject("CohortAggregate", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] CohortAggregateContainer { - get { + public static Byte[] CohortAggregateContainer + { + get + { object obj = ResourceManager.GetObject("CohortAggregateContainer", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] CohortCustomColumn { - get { + public static Byte[] CohortCustomColumn + { + get + { object obj = ResourceManager.GetObject("CohortCustomColumn", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] CohortIdentificationConfiguration { - get { + public static Byte[] CohortIdentificationConfiguration + { + get + { object obj = ResourceManager.GetObject("CohortIdentificationConfiguration", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] collapseAllNodes { - get { + public static Byte[] collapseAllNodes + { + get + { object obj = ResourceManager.GetObject("collapseAllNodes", resourceCulture); return (Byte[])obj; } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] CollisionResolution { - get { + public static Byte[] CollisionResolution + { + get + { object obj = ResourceManager.GetObject("CollisionResolution", resourceCulture); return (Byte[])obj; } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ColumnInfo { - get { + public static Byte[] ColumnInfo + { + get + { object obj = ResourceManager.GetObject("ColumnInfo", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ConnectionStringKeyword { - get { + public static Byte[] ConnectionStringKeyword + { + get + { object obj = ResourceManager.GetObject("ConnectionStringKeyword", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] CumulativeExtractionResults { - get { + public static Byte[] CumulativeExtractionResults + { + get + { object obj = ResourceManager.GetObject("CumulativeExtractionResults", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] DashboardControl { - get { + public static Byte[] DashboardControl + { + get + { object obj = ResourceManager.GetObject("DashboardControl", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] DashboardLayout { - get { + public static Byte[] DashboardLayout + { + get + { object obj = ResourceManager.GetObject("DashboardLayout", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] DataAccessCredentials { - get { + public static Byte[] DataAccessCredentials + { + get + { object obj = ResourceManager.GetObject("DataAccessCredentials", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] Database { - get { + public static Byte[] Database + { + get + { object obj = ResourceManager.GetObject("Database", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] DatabaseRefresh { - get { + public static Byte[] DatabaseRefresh + { + get + { object obj = ResourceManager.GetObject("DatabaseRefresh", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] DecryptionPrivateKeyNode { - get { + public static Byte[] DecryptionPrivateKeyNode + { + get + { object obj = ResourceManager.GetObject("DecryptionPrivateKeyNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] Diff { - get { + public static Byte[] Diff + { + get + { object obj = ResourceManager.GetObject("Diff", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] DocumentationNode { - get { + public static Byte[] DocumentationNode + { + get + { object obj = ResourceManager.GetObject("DocumentationNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] DQE { - get { + public static Byte[] DQE + { + get + { object obj = ResourceManager.GetObject("DQE", resourceCulture); return (Byte[])obj; } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] DQEGraphAnnotation { - get { + public static Byte[] DQEGraphAnnotation + { + get + { object obj = ResourceManager.GetObject("DQEGraphAnnotation", resourceCulture); return (Byte[])obj; } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] DropHere { - get { + public static Byte[] DropHere + { + get + { object obj = ResourceManager.GetObject("DropHere", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] EmptyProject { - get { + public static Byte[] EmptyProject + { + get + { object obj = ResourceManager.GetObject("EmptyProject", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] Evaluation { - get { + public static Byte[] Evaluation + { + get + { object obj = ResourceManager.GetObject("Evaluation", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] EXCEPT { - get { + public static Byte[] EXCEPT + { + get + { object obj = ResourceManager.GetObject("EXCEPT", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] EXCEPTCohortAggregate { - get { + public static Byte[] EXCEPTCohortAggregate + { + get + { object obj = ResourceManager.GetObject("EXCEPTCohortAggregate", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] Exe { - get { + public static Byte[] Exe + { + get + { object obj = ResourceManager.GetObject("Exe", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ExecuteArrow { - get { + public static Byte[] ExecuteArrow + { + get + { object obj = ResourceManager.GetObject("ExecuteArrow", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ExecuteArrowWithTickOverlay { - get { + public static Byte[] ExecuteArrowWithTickOverlay + { + get + { object obj = ResourceManager.GetObject("ExecuteArrowWithTickOverlay", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ExpandAllNodes { - get { + public static Byte[] ExpandAllNodes + { + get + { object obj = ResourceManager.GetObject("ExpandAllNodes", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type System.Byte[]. /// - public static Byte[] ExtendedProperty { - get { + public static Byte[] ExtendedProperty + { + get + { object obj = ResourceManager.GetObject("ExtendedProperty", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ExternalCohortTable { - get { + public static Byte[] ExternalCohortTable + { + get + { object obj = ResourceManager.GetObject("ExternalCohortTable", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ExternalDatabaseServer { - get { + public static Byte[] ExternalDatabaseServer + { + get + { object obj = ResourceManager.GetObject("ExternalDatabaseServer", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ExternalDatabaseServer_ANO { - get { + public static Byte[] ExternalDatabaseServer_ANO + { + get + { object obj = ResourceManager.GetObject("ExternalDatabaseServer_ANO", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ExternalDatabaseServer_Cache { - get { + public static Byte[] ExternalDatabaseServer_Cache + { + get + { object obj = ResourceManager.GetObject("ExternalDatabaseServer_Cache", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ExternalDatabaseServer_DQE { - get { + public static Byte[] ExternalDatabaseServer_DQE + { + get + { object obj = ResourceManager.GetObject("ExternalDatabaseServer_DQE", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ExternalDatabaseServer_IdentifierDump { - get { + public static Byte[] ExternalDatabaseServer_IdentifierDump + { + get + { object obj = ResourceManager.GetObject("ExternalDatabaseServer_IdentifierDump", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ExternalDatabaseServer_Logging { - get { + public static Byte[] ExternalDatabaseServer_Logging + { + get + { object obj = ResourceManager.GetObject("ExternalDatabaseServer_Logging", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ExtractableCohort { - get { + public static Byte[] ExtractableCohort + { + get + { object obj = ResourceManager.GetObject("ExtractableCohort", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ExtractableColumn { - get { + public static Byte[] ExtractableColumn + { + get + { object obj = ResourceManager.GetObject("ExtractableColumn", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ExtractableDataSet { - get { + public static Byte[] ExtractableDataSet + { + get + { object obj = ResourceManager.GetObject("ExtractableDataSet", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ExtractableDataSetDisabled { - get { + public static Byte[] ExtractableDataSetDisabled + { + get + { object obj = ResourceManager.GetObject("ExtractableDataSetDisabled", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ExtractableDataSetPackage { - get { + public static Byte[] ExtractableDataSetPackage + { + get + { object obj = ResourceManager.GetObject("ExtractableDataSetPackage", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ExtractableDataSetsNode { - get { + public static Byte[] ExtractableDataSetsNode + { + get + { object obj = ResourceManager.GetObject("ExtractableDataSetsNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ExtractionConfiguration { - get { + public static Byte[] ExtractionConfiguration + { + get + { object obj = ResourceManager.GetObject("ExtractionConfiguration", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ExtractionConfigurationsNode { - get { + public static Byte[] ExtractionConfigurationsNode + { + get + { object obj = ResourceManager.GetObject("ExtractionConfigurationsNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ExtractionDirectoryNode { - get { + public static Byte[] ExtractionDirectoryNode + { + get + { object obj = ResourceManager.GetObject("ExtractionDirectoryNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ExtractionFilterParameterSet { - get { + public static Byte[] ExtractionFilterParameterSet + { + get + { object obj = ResourceManager.GetObject("ExtractionFilterParameterSet", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ExtractionInformation { - get { + public static Byte[] ExtractionInformation + { + get + { object obj = ResourceManager.GetObject("ExtractionInformation", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ExtractionInformation_ProjectSpecific { - get { + public static Byte[] ExtractionInformation_ProjectSpecific + { + get + { object obj = ResourceManager.GetObject("ExtractionInformation_ProjectSpecific", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ExtractionInformation_SpecialApproval { - get { + public static Byte[] ExtractionInformation_SpecialApproval + { + get + { object obj = ResourceManager.GetObject("ExtractionInformation_SpecialApproval", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ExtractionInformation_Supplemental { - get { + public static Byte[] ExtractionInformation_Supplemental + { + get + { object obj = ResourceManager.GetObject("ExtractionInformation_Supplemental", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] Failed { - get { + public static Byte[] Failed + { + get + { object obj = ResourceManager.GetObject("Failed", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] Favourite { - get { + public static Byte[] Favourite + { + get + { object obj = ResourceManager.GetObject("Favourite", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] File { - get { + public static Byte[] File + { + get + { object obj = ResourceManager.GetObject("File", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] FileMissing { - get { + public static Byte[] FileMissing + { + get + { object obj = ResourceManager.GetObject("FileMissing", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] Filter { - get { + public static Byte[] Filter + { + get + { object obj = ResourceManager.GetObject("Filter", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] FilterContainer { - get { + public static Byte[] FilterContainer + { + get + { object obj = ResourceManager.GetObject("FilterContainer", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] FrozenCohortIdentificationConfiguration { - get { + public static Byte[] FrozenCohortIdentificationConfiguration + { + get + { object obj = ResourceManager.GetObject("FrozenCohortIdentificationConfiguration", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] FrozenExtractionConfiguration { - get { + public static Byte[] FrozenExtractionConfiguration + { + get + { object obj = ResourceManager.GetObject("FrozenExtractionConfiguration", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] FrozenExtractionConfigurationsNode { - get { + public static Byte[] FrozenExtractionConfigurationsNode + { + get + { object obj = ResourceManager.GetObject("FrozenExtractionConfigurationsNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] GetFilesStage { - get { + public static Byte[] GetFilesStage + { + get + { object obj = ResourceManager.GetObject("GetFilesStage", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] GovernanceDocument { - get { + public static Byte[] GovernanceDocument + { + get + { object obj = ResourceManager.GetObject("GovernanceDocument", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] GovernancePeriod { - get { + public static Byte[] GovernancePeriod + { + get + { object obj = ResourceManager.GetObject("GovernancePeriod", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] Graph { - get { + public static Byte[] Graph + { + get + { object obj = ResourceManager.GetObject("Graph", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] GreenIssue { - get { + public static Byte[] GreenIssue + { + get + { object obj = ResourceManager.GetObject("GreenIssue", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] Help { - get { + public static Byte[] Help + { + get + { object obj = ResourceManager.GetObject("Help", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ImportFile { - get { + public static Byte[] ImportFile + { + get + { object obj = ResourceManager.GetObject("ImportFile", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] INTERSECT { - get { + public static Byte[] INTERSECT + { + get + { object obj = ResourceManager.GetObject("INTERSECT", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] INTERSECTCohortAggregate { - get { + public static Byte[] INTERSECTCohortAggregate + { + get + { object obj = ResourceManager.GetObject("INTERSECTCohortAggregate", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] JoinableCollectionNode { - get { + public static Byte[] JoinableCollectionNode + { + get + { object obj = ResourceManager.GetObject("JoinableCollectionNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] JoinInfo { - get { + public static Byte[] JoinInfo + { + get + { object obj = ResourceManager.GetObject("JoinInfo", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] Kill { - get { + public static Byte[] Kill + { + get + { object obj = ResourceManager.GetObject("Kill", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] LoadBubble { - get { + public static Byte[] LoadBubble + { + get + { object obj = ResourceManager.GetObject("LoadBubble", resourceCulture); return ((Byte[])(obj)); } @@ -1296,838 +1558,1006 @@ public static Byte[] LoadMetadataCatalogueLinkage /// /// Looks up a localized resource of type Image. /// - public static Byte[] LoadBubbleMounting { - get { + public static Byte[] LoadBubbleMounting + { + get + { object obj = ResourceManager.GetObject("LoadBubbleMounting", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] LoadDirectoryNode { - get { + public static Byte[] LoadDirectoryNode + { + get + { object obj = ResourceManager.GetObject("LoadDirectoryNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] LoadFinalDatabase { - get { + public static Byte[] LoadFinalDatabase + { + get + { object obj = ResourceManager.GetObject("LoadFinalDatabase", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] LoadMetadata { - get { + public static Byte[] LoadMetadata + { + get + { object obj = ResourceManager.GetObject("LoadMetadata", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] LoadMetadataScheduleNode { - get { + public static Byte[] LoadMetadataScheduleNode + { + get + { object obj = ResourceManager.GetObject("LoadMetadataScheduleNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] LoadModuleAssembly { - get { + public static Byte[] LoadModuleAssembly + { + get + { object obj = ResourceManager.GetObject("LoadModuleAssembly", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] LoadPeriodically { - get { + public static Byte[] LoadPeriodically + { + get + { object obj = ResourceManager.GetObject("LoadPeriodically", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] LoadProgress { - get { + public static Byte[] LoadProgress + { + get + { object obj = ResourceManager.GetObject("LoadProgress", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] Logging { - get { + public static Byte[] Logging + { + get + { object obj = ResourceManager.GetObject("Logging", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] Lookup { - get { + public static Byte[] Lookup + { + get + { object obj = ResourceManager.GetObject("Lookup", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] LookupCompositeJoinInfo { - get { + public static Byte[] LookupCompositeJoinInfo + { + get + { object obj = ResourceManager.GetObject("LookupCompositeJoinInfo", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] Main { - get { + public static Byte[] Main + { + get + { object obj = ResourceManager.GetObject("Main", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] MakeProjectSpecificCatalogueNormalAgain { - get { + public static Byte[] MakeProjectSpecificCatalogueNormalAgain + { + get + { object obj = ResourceManager.GetObject("MakeProjectSpecificCatalogueNormalAgain", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] Menu { - get { + public static Byte[] Menu + { + get + { object obj = ResourceManager.GetObject("Menu", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] NoIconAvailable { - get { + public static Byte[] NoIconAvailable + { + get + { object obj = ResourceManager.GetObject("NoIconAvailable", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ObjectExport { - get { + public static Byte[] ObjectExport + { + get + { object obj = ResourceManager.GetObject("ObjectExport", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ObjectImport { - get { + public static Byte[] ObjectImport + { + get + { object obj = ResourceManager.GetObject("ObjectImport", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] OrangeIssue { - get { + public static Byte[] OrangeIssue + { + get + { object obj = ResourceManager.GetObject("OrangeIssue", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] OtherPipelinesNode { - get { + public static Byte[] OtherPipelinesNode + { + get + { object obj = ResourceManager.GetObject("OtherPipelinesNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ParametersNode { - get { + public static Byte[] ParametersNode + { + get + { object obj = ResourceManager.GetObject("ParametersNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] PatientIndexTable { - get { + public static Byte[] PatientIndexTable + { + get + { object obj = ResourceManager.GetObject("PatientIndexTable", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] PermissionWindow { - get { + public static Byte[] PermissionWindow + { + get + { object obj = ResourceManager.GetObject("PermissionWindow", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] Pipeline { - get { + public static Byte[] Pipeline + { + get + { object obj = ResourceManager.GetObject("Pipeline", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] PipelineComponent { - get { + public static Byte[] PipelineComponent + { + get + { object obj = ResourceManager.GetObject("PipelineComponent", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] PipelineComponentArgument { - get { + public static Byte[] PipelineComponentArgument + { + get + { object obj = ResourceManager.GetObject("PipelineComponentArgument", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] PipelineComponentDestination { - get { + public static Byte[] PipelineComponentDestination + { + get + { object obj = ResourceManager.GetObject("PipelineComponentDestination", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] PipelineComponentSource { - get { + public static Byte[] PipelineComponentSource + { + get + { object obj = ResourceManager.GetObject("PipelineComponentSource", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] PivotField { - get { + public static Byte[] PivotField + { + get + { object obj = ResourceManager.GetObject("PivotField", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] Plugin { - get { + public static Byte[] Plugin + { + get + { object obj = ResourceManager.GetObject("Plugin", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] PreLoadDiscardedColumn { - get { + public static Byte[] PreLoadDiscardedColumn + { + get + { object obj = ResourceManager.GetObject("PreLoadDiscardedColumn", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] PreLoadDiscardedColumnsNode { - get { + public static Byte[] PreLoadDiscardedColumnsNode + { + get + { object obj = ResourceManager.GetObject("PreLoadDiscardedColumnsNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ProcessTask { - get { + public static Byte[] ProcessTask + { + get + { object obj = ResourceManager.GetObject("ProcessTask", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ProcessTaskArgument { - get { + public static Byte[] ProcessTaskArgument + { + get + { object obj = ResourceManager.GetObject("ProcessTaskArgument", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] Project { - get { + public static Byte[] Project + { + get + { object obj = ResourceManager.GetObject("Project", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ProjectCatalogue { - get { + public static Byte[] ProjectCatalogue + { + get + { object obj = ResourceManager.GetObject("ProjectCatalogue", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ProjectCataloguesNode { - get { + public static Byte[] ProjectCataloguesNode + { + get + { object obj = ResourceManager.GetObject("ProjectCataloguesNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ProjectCohortIdentificationConfigurationAssociation { - get { + public static Byte[] ProjectCohortIdentificationConfigurationAssociation + { + get + { object obj = ResourceManager.GetObject("ProjectCohortIdentificationConfigurationAssociation", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ProjectCohortIdentificationConfigurationAssociationsNode { - get { + public static Byte[] ProjectCohortIdentificationConfigurationAssociationsNode + { + get + { object obj = ResourceManager.GetObject("ProjectCohortIdentificationConfigurationAssociationsNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ProjectCohortsNode { - get { + public static Byte[] ProjectCohortsNode + { + get + { object obj = ResourceManager.GetObject("ProjectCohortsNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ProjectSavedCohortsNode { - get { + public static Byte[] ProjectSavedCohortsNode + { + get + { object obj = ResourceManager.GetObject("ProjectSavedCohortsNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ProjectsNode { - get { + public static Byte[] ProjectsNode + { + get + { object obj = ResourceManager.GetObject("ProjectsNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] RedIssue { - get { + public static Byte[] RedIssue + { + get + { object obj = ResourceManager.GetObject("RedIssue", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] Release { - get { + public static Byte[] Release + { + get + { object obj = ResourceManager.GetObject("Release", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ReleaseLog { - get { + public static Byte[] ReleaseLog + { + get + { object obj = ResourceManager.GetObject("ReleaseLog", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] RemoteRDMP { - get { + public static Byte[] RemoteRDMP + { + get + { object obj = ResourceManager.GetObject("RemoteRDMP", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] ReOrder { - get { + public static Byte[] ReOrder + { + get + { object obj = ResourceManager.GetObject("ReOrder", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] RowCounts_Ignore { - get { + public static Byte[] RowCounts_Ignore + { + get + { object obj = ResourceManager.GetObject("RowCounts_Ignore", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] RowCounts_Respect { - get { + public static Byte[] RowCounts_Respect + { + get + { object obj = ResourceManager.GetObject("RowCounts_Respect", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] SelectedDataSetsForcedJoin { - get { + public static Byte[] SelectedDataSetsForcedJoin + { + get + { object obj = ResourceManager.GetObject("SelectedDataSetsForcedJoin", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] Spanner { - get { + public static Byte[] Spanner + { + get + { object obj = ResourceManager.GetObject("Spanner", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] SpontaneouslyInventedColumn { - get { + public static Byte[] SpontaneouslyInventedColumn + { + get + { object obj = ResourceManager.GetObject("SpontaneouslyInventedColumn", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] SQL { - get { + public static Byte[] SQL + { + get + { object obj = ResourceManager.GetObject("SQL", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] SqlThenVSNow { - get { + public static Byte[] SqlThenVSNow + { + get + { object obj = ResourceManager.GetObject("SqlThenVSNow", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] StandardPipelineUseCaseNode { - get { + public static Byte[] StandardPipelineUseCaseNode + { + get + { object obj = ResourceManager.GetObject("StandardPipelineUseCaseNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] StandardRegex { - get { + public static Byte[] StandardRegex + { + get + { object obj = ResourceManager.GetObject("StandardRegex", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] StarHollow { - get { + public static Byte[] StarHollow + { + get + { object obj = ResourceManager.GetObject("StarHollow", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] SupplementalExtractionResults { - get { + public static Byte[] SupplementalExtractionResults + { + get + { object obj = ResourceManager.GetObject("SupplementalExtractionResults", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] SupportingDocument { - get { + public static Byte[] SupportingDocument + { + get + { object obj = ResourceManager.GetObject("SupportingDocument", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] SupportingDocumentExtractable { - get { + public static Byte[] SupportingDocumentExtractable + { + get + { object obj = ResourceManager.GetObject("SupportingDocumentExtractable", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] SupportingDocumentExtractableGlobal { - get { + public static Byte[] SupportingDocumentExtractableGlobal + { + get + { object obj = ResourceManager.GetObject("SupportingDocumentExtractableGlobal", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] SupportingDocumentGlobal { - get { + public static Byte[] SupportingDocumentGlobal + { + get + { object obj = ResourceManager.GetObject("SupportingDocumentGlobal", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] SupportingSqlExtractable { - get { + public static Byte[] SupportingSqlExtractable + { + get + { object obj = ResourceManager.GetObject("SupportingSqlExtractable", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] SupportingSqlExtractableGlobal { - get { + public static Byte[] SupportingSqlExtractableGlobal + { + get + { object obj = ResourceManager.GetObject("SupportingSqlExtractableGlobal", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] SupportingSqlGlobal { - get { + public static Byte[] SupportingSqlGlobal + { + get + { object obj = ResourceManager.GetObject("SupportingSqlGlobal", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] SupportingSQLTable { - get { + public static Byte[] SupportingSQLTable + { + get + { object obj = ResourceManager.GetObject("SupportingSQLTable", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] Sync { - get { + public static Byte[] Sync + { + get + { object obj = ResourceManager.GetObject("Sync", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] TableInfo { - get { + public static Byte[] TableInfo + { + get + { object obj = ResourceManager.GetObject("TableInfo", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] TableInfoServerNode { - get { + public static Byte[] TableInfoServerNode + { + get + { object obj = ResourceManager.GetObject("TableInfoServerNode", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] TableInfoTableValuedFunction { - get { + public static Byte[] TableInfoTableValuedFunction + { + get + { object obj = ResourceManager.GetObject("TableInfoTableValuedFunction", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] Tick { - get { + public static Byte[] Tick + { + get + { object obj = ResourceManager.GetObject("Tick", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] TicketingSystemConfiguration { - get { + public static Byte[] TicketingSystemConfiguration + { + get + { object obj = ResourceManager.GetObject("TicketingSystemConfiguration", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] TimeCoverageField { - get { + public static Byte[] TimeCoverageField + { + get + { object obj = ResourceManager.GetObject("TimeCoverageField", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] TinyGreen { - get { + public static Byte[] TinyGreen + { + get + { object obj = ResourceManager.GetObject("TinyGreen", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] TinyRed { - get { + public static Byte[] TinyRed + { + get + { object obj = ResourceManager.GetObject("TinyRed", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] TinyYellow { - get { + public static Byte[] TinyYellow + { + get + { object obj = ResourceManager.GetObject("TinyYellow", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] UnfreezeExtractionConfiguration { - get { + public static Byte[] UnfreezeExtractionConfiguration + { + get + { object obj = ResourceManager.GetObject("UnfreezeExtractionConfiguration", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] UNION { - get { + public static Byte[] UNION + { + get + { object obj = ResourceManager.GetObject("UNION", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] UNIONCohortAggregate { - get { + public static Byte[] UNIONCohortAggregate + { + get + { object obj = ResourceManager.GetObject("UNIONCohortAggregate", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] Waiting { - get { + public static Byte[] Waiting + { + get + { object obj = ResourceManager.GetObject("Waiting", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] WaitingForDatabase { - get { + public static Byte[] WaitingForDatabase + { + get + { object obj = ResourceManager.GetObject("WaitingForDatabase", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] Warning { - get { + public static Byte[] Warning + { + get + { object obj = ResourceManager.GetObject("Warning", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] WhatIsACohort { - get { + public static Byte[] WhatIsACohort + { + get + { object obj = ResourceManager.GetObject("WhatIsACohort", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] WindowLayout { - get { + public static Byte[] WindowLayout + { + get + { object obj = ResourceManager.GetObject("WindowLayout", resourceCulture); return ((Byte[])(obj)); } } - + /// /// Looks up a localized resource of type Image. /// - public static Byte[] Writing { - get { + public static Byte[] Writing + { + get + { object obj = ResourceManager.GetObject("Writing", resourceCulture); return ((Byte[])(obj)); } diff --git a/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx b/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx index fa09fce753..11c11ffe0f 100644 --- a/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx +++ b/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx @@ -622,6 +622,12 @@ ..\AllPipelinesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + ..\AllPipelinesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllPipelinesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + ..\StandardPipelineUseCaseNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 diff --git a/Rdmp.Core/Icons/IconProvision/RDMPConcept.cs b/Rdmp.Core/Icons/IconProvision/RDMPConcept.cs index 8600117d46..59c18d4d79 100644 --- a/Rdmp.Core/Icons/IconProvision/RDMPConcept.cs +++ b/Rdmp.Core/Icons/IconProvision/RDMPConcept.cs @@ -204,6 +204,7 @@ public enum RDMPConcept Memento, TableInfoDatabaseNode, Dataset, + AllDatasetsNode, LoadMetadataCatalogueLinkage, Setting, TicketingSystemReleaseStatus, diff --git a/Rdmp.Core/Providers/CatalogueChildProvider.cs b/Rdmp.Core/Providers/CatalogueChildProvider.cs index 296d18fa1a..15cf672ca0 100644 --- a/Rdmp.Core/Providers/CatalogueChildProvider.cs +++ b/Rdmp.Core/Providers/CatalogueChildProvider.cs @@ -7,6 +7,7 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Data; using System.Diagnostics; using System.Linq; using System.Threading.Tasks; @@ -21,6 +22,7 @@ using Rdmp.Core.Curation.Data.ImportExport; using Rdmp.Core.Curation.Data.Pipelines; using Rdmp.Core.Curation.Data.Remoting; +using Rdmp.Core.Curation.DataHelper.RegexRedaction; using Rdmp.Core.MapsDirectlyToDatabaseTable; using Rdmp.Core.Providers.Nodes; using Rdmp.Core.Providers.Nodes.CohortNodes; @@ -66,6 +68,8 @@ public class CatalogueChildProvider : ICoreChildProvider //Catalogue side of things public Catalogue[] AllCatalogues { get; set; } public Curation.Data.Dataset[] AllDatasets { get; set; } + + public RegexRedactionConfiguration[] AllRegexRedactionConfigurations { get; set; } public Dictionary AllCataloguesDictionary { get; private set; } public SupportingDocument[] AllSupportingDocuments { get; set; } @@ -91,6 +95,10 @@ public class CatalogueChildProvider : ICoreChildProvider public AggregateContinuousDateAxis[] AllAggregateContinuousDateAxis { get; private set; } public AllRDMPRemotesNode AllRDMPRemotesNode { get; private set; } + + public AllDatasetsNode AllDatasetsNode { get; private set; } + public AllRegexRedactionConfigurationsNode AllRegexRedactionConfigurationsNode { get; private set; } + public RemoteRDMP[] AllRemoteRDMPs { get; set; } public AllDashboardsNode AllDashboardsNode { get; set; } @@ -248,7 +256,7 @@ public CatalogueChildProvider(ICatalogueRepository repository, IChildProvider[] AllCataloguesDictionary = AllCatalogues.ToDictionaryEx(i => i.ID, o => o); AllDatasets = GetAllObjects(repository); - + AllRegexRedactionConfigurations = GetAllObjects(repository); AllLoadMetadatas = GetAllObjects(repository); AllLoadMetadataLinkage = GetAllObjects(repository); AllProcessTasks = GetAllObjects(repository); @@ -377,6 +385,8 @@ public CatalogueChildProvider(ICatalogueRepository repository, IChildProvider[] //All the things for TableInfoCollectionUI BuildServerNodes(); + BuildDatasetNodes(); + BuildAllRegexRedactionConfigurationsNode(); ReportProgress("BuildServerNodes"); //add a new CatalogueItemNodes @@ -747,6 +757,30 @@ private void AddChildren(PipelineComponent pipelineComponent, DescendancyList de AddToDictionaries(children, descendancy); } + + private void BuildAllRegexRedactionConfigurationsNode() + { + AllRegexRedactionConfigurationsNode = new AllRegexRedactionConfigurationsNode(); + var descendancy = new DescendancyList(AllRegexRedactionConfigurationsNode); + //foreach (var regex in AllRegexRedactionConfigurations) + //{ + // AddChildren(regex, descendancy.Add(regex)); + //} + AddChildren(AllRegexRedactionConfigurationsNode); + } + + + private void BuildDatasetNodes() + { + AllDatasetsNode = new AllDatasetsNode(); + var descendancy = new DescendancyList(AllDatasetsNode); + foreach (var dataset in AllDatasets) + { + AddChildren(dataset, descendancy.Add(dataset)); + } + + } + private void BuildServerNodes() { //add a root node for all the servers to be children of @@ -802,6 +836,12 @@ private void AddChildren(AllANOTablesNode anoTablesNode) AddToDictionaries(new HashSet(AllANOTables), new DescendancyList(anoTablesNode)); } + private void AddChildren(AllRegexRedactionConfigurationsNode allRegexRedactionConfigurationsNode) + { + AddToDictionaries(new HashSet(AllRegexRedactionConfigurations), new DescendancyList(allRegexRedactionConfigurationsNode)); + + } + private void AddChildren(FolderNode folder, DescendancyList descendancy) { foreach (var child in folder.ChildFolders) @@ -866,6 +906,12 @@ private void AddChildren(FolderNode folder, D ); } + private void AddChildren(RegexRedactionConfiguration regex, DescendancyList descendancy) + { + var childObjects = new List(); + AddToDictionaries(new HashSet(childObjects), descendancy); + } + private void AddChildren(Curation.Data.Dataset lmd, DescendancyList descendancy) { var childObjects = new List(); diff --git a/Rdmp.Core/Providers/ICoreChildProvider.cs b/Rdmp.Core/Providers/ICoreChildProvider.cs index bdd068dbf0..86d4b9da32 100644 --- a/Rdmp.Core/Providers/ICoreChildProvider.cs +++ b/Rdmp.Core/Providers/ICoreChildProvider.cs @@ -80,6 +80,10 @@ public interface ICoreChildProvider : IChildProvider AllDashboardsNode AllDashboardsNode { get; } DashboardLayout[] AllDashboards { get; } + AllDatasetsNode AllDatasetsNode { get; } + + AllRegexRedactionConfigurationsNode AllRegexRedactionConfigurationsNode { get; } + AllObjectSharingNode AllObjectSharingNode { get; } ObjectImport[] AllImports { get; } ObjectExport[] AllExports { get; } diff --git a/Rdmp.Core/Providers/Nodes/AllDatasetsNode.cs b/Rdmp.Core/Providers/Nodes/AllDatasetsNode.cs new file mode 100644 index 0000000000..1cdeff8cbe --- /dev/null +++ b/Rdmp.Core/Providers/Nodes/AllDatasetsNode.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Providers.Nodes +{ + public class AllDatasetsNode : SingletonNode + { + public AllDatasetsNode() : base("Datasets") + { + } + } +} diff --git a/Rdmp.Core/Providers/Nodes/AllRegexRedactionConfigurationsNode.cs b/Rdmp.Core/Providers/Nodes/AllRegexRedactionConfigurationsNode.cs new file mode 100644 index 0000000000..3de546e459 --- /dev/null +++ b/Rdmp.Core/Providers/Nodes/AllRegexRedactionConfigurationsNode.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Providers.Nodes +{ + public class AllRegexRedactionConfigurationsNode: SingletonNode + { + public AllRegexRedactionConfigurationsNode():base("Regex Redaction Configurations") + { } + } +} diff --git a/Rdmp.Core/RDMPCollection.cs b/Rdmp.Core/RDMPCollection.cs index 0d61d44ee9..0595e4c53d 100644 --- a/Rdmp.Core/RDMPCollection.cs +++ b/Rdmp.Core/RDMPCollection.cs @@ -17,8 +17,9 @@ public enum RDMPCollection DataExport, SavedCohorts, Favourites, + Datasets, Cohort, DataLoad, - Datasets + Configurations } \ No newline at end of file diff --git a/Rdmp.UI/Collections/DatasetsCollectionUI.Designer.cs b/Rdmp.UI/Collections/ConfigurationsCollectionUI.Designer.cs similarity index 57% rename from Rdmp.UI/Collections/DatasetsCollectionUI.Designer.cs rename to Rdmp.UI/Collections/ConfigurationsCollectionUI.Designer.cs index b2ec676936..37ad0957f2 100644 --- a/Rdmp.UI/Collections/DatasetsCollectionUI.Designer.cs +++ b/Rdmp.UI/Collections/ConfigurationsCollectionUI.Designer.cs @@ -1,6 +1,6 @@ namespace Rdmp.UI.Collections { - partial class DatasetsCollectionUI + partial class ConfigurationsCollectionUI { /// /// Required designer variable. @@ -30,52 +30,50 @@ protected override void Dispose(bool disposing) private void InitializeComponent() { this.components = new System.ComponentModel.Container(); - this.tlvDatasets = new BrightIdeasSoftware.TreeListView(); + this.tlvConfigurations = new BrightIdeasSoftware.TreeListView(); this.olvName = ((BrightIdeasSoftware.OLVColumn)(new BrightIdeasSoftware.OLVColumn())); - ((System.ComponentModel.ISupportInitialize)(this.tlvDatasets)).BeginInit(); + ((System.ComponentModel.ISupportInitialize)(this.tlvConfigurations)).BeginInit(); this.SuspendLayout(); // // tlvFavourites // - this.tlvDatasets.AllColumns.Add(this.olvName); - this.tlvDatasets.CellEditUseWholeCell = false; - this.tlvDatasets.Columns.AddRange(new System.Windows.Forms.ColumnHeader[] { + this.tlvConfigurations.AllColumns.Add(this.olvName); + this.tlvConfigurations.CellEditUseWholeCell = false; + this.tlvConfigurations.Columns.AddRange(new System.Windows.Forms.ColumnHeader[] { this.olvName}); - this.tlvDatasets.Cursor = System.Windows.Forms.Cursors.Default; - this.tlvDatasets.Dock = System.Windows.Forms.DockStyle.Fill; - this.tlvDatasets.FullRowSelect = true; - this.tlvDatasets.HideSelection = false; - this.tlvDatasets.Location = new System.Drawing.Point(0, 0); - this.tlvDatasets.Name = "tlvFavourites"; - this.tlvDatasets.ShowGroups = false; - this.tlvDatasets.Size = new System.Drawing.Size(322, 557); - this.tlvDatasets.TabIndex = 2; - this.tlvDatasets.UseCompatibleStateImageBehavior = false; - this.tlvDatasets.View = System.Windows.Forms.View.Details; - this.tlvDatasets.VirtualMode = true; + this.tlvConfigurations.Cursor = System.Windows.Forms.Cursors.Default; + this.tlvConfigurations.Dock = System.Windows.Forms.DockStyle.Fill; + this.tlvConfigurations.FullRowSelect = true; + this.tlvConfigurations.HideSelection = false; + this.tlvConfigurations.Location = new System.Drawing.Point(0, 0); + this.tlvConfigurations.ShowGroups = false; + this.tlvConfigurations.Size = new System.Drawing.Size(322, 557); + this.tlvConfigurations.TabIndex = 2; + this.tlvConfigurations.UseCompatibleStateImageBehavior = false; + this.tlvConfigurations.View = System.Windows.Forms.View.Details; + this.tlvConfigurations.VirtualMode = true; // // olvName // this.olvName.AspectName = "ToString"; - this.olvName.CellEditUseWholeCell = true; - this.olvName.Text = "Datasets"; + this.olvName.Text = "Name"; this.olvName.MinimumWidth = 100; // // FavouritesCollectionUI // this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F); this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; - this.Controls.Add(this.tlvDatasets); - this.Name = "DatasetsCollectionUI"; + this.Controls.Add(this.tlvConfigurations); + this.Name = "ConfigurationsCollectionUI"; this.Size = new System.Drawing.Size(322, 557); - ((System.ComponentModel.ISupportInitialize)(this.tlvDatasets)).EndInit(); + ((System.ComponentModel.ISupportInitialize)(this.tlvConfigurations)).EndInit(); this.ResumeLayout(false); - this.Text = "DatasetsCollectionUI"; + this.Text = "ConfigurationsCollection"; } #endregion - private BrightIdeasSoftware.TreeListView tlvDatasets; + private BrightIdeasSoftware.TreeListView tlvConfigurations; private BrightIdeasSoftware.OLVColumn olvName; } } \ No newline at end of file diff --git a/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs b/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs new file mode 100644 index 0000000000..a616a8f1e3 --- /dev/null +++ b/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs @@ -0,0 +1,55 @@ +using Rdmp.Core.CommandExecution.AtomicCommands; +using Rdmp.Core; +using Rdmp.Core.MapsDirectlyToDatabaseTable; +using Rdmp.UI.CommandExecution.AtomicCommands; +using Rdmp.UI.ItemActivation; +using Rdmp.UI.Refreshing; +using System; +using System.Collections.Generic; +using System.Linq; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Icons.IconProvision; +using System.Windows.Forms; +using Rdmp.Core.Curation.DataHelper.RegexRedaction; +using Rdmp.Core.Providers.Nodes.PipelineNodes; +using Rdmp.Core.Providers.Nodes.SharingNodes; +using Rdmp.Core.Providers.Nodes; + +namespace Rdmp.UI.Collections; + +public partial class ConfigurationsCollectionUI : RDMPCollectionUI, ILifetimeSubscriber +{ + + //private Dataset[] _datasets; + //private bool _firstTime = true; + private IActivateItems _activator; + + public ConfigurationsCollectionUI() + { + InitializeComponent(); + } + + public override void SetItemActivator(IActivateItems activator) + { + base.SetItemActivator(activator); + _activator = activator; + CommonTreeFunctionality.SetUp(RDMPCollection.Configurations, tlvConfigurations, activator, olvName, olvName, + new RDMPCollectionCommonFunctionalitySettings()); + Activator.RefreshBus.EstablishLifetimeSubscription(this); + tlvConfigurations.AddObject(Activator.CoreChildProvider.AllDatasetsNode); + tlvConfigurations.AddObject(Activator.CoreChildProvider.AllRegexRedactionConfigurationsNode); + tlvConfigurations.Refresh(); + } + + public void RefreshBus_RefreshObject(object sender, RefreshObjectEventArgs e) + { + switch (e.Object) + { + case Dataset: + tlvConfigurations.RefreshObject(tlvConfigurations.Objects.OfType()); + break; + } + } + + public static bool IsRootObject(object root) => root is AllDatasetsNode; +} \ No newline at end of file diff --git a/Rdmp.UI/Collections/DatasetsCollectionUI.resx b/Rdmp.UI/Collections/ConfigurationsCollectionUI.resx similarity index 100% rename from Rdmp.UI/Collections/DatasetsCollectionUI.resx rename to Rdmp.UI/Collections/ConfigurationsCollectionUI.resx diff --git a/Rdmp.UI/Collections/DatasetsCollectionUI.cs b/Rdmp.UI/Collections/DatasetsCollectionUI.cs deleted file mode 100644 index 6a72d8d3d7..0000000000 --- a/Rdmp.UI/Collections/DatasetsCollectionUI.cs +++ /dev/null @@ -1,127 +0,0 @@ -using Rdmp.Core.CommandExecution.AtomicCommands; -using Rdmp.Core; -using Rdmp.Core.MapsDirectlyToDatabaseTable; -using Rdmp.UI.CommandExecution.AtomicCommands; -using Rdmp.UI.ItemActivation; -using Rdmp.UI.Refreshing; -using System; -using System.Collections.Generic; -using System.Linq; -using Rdmp.Core.Curation.Data; -using Rdmp.Core.Icons.IconProvision; -using System.Windows.Forms; - -namespace Rdmp.UI.Collections; - -public partial class DatasetsCollectionUI : RDMPCollectionUI, ILifetimeSubscriber -{ - - private Dataset[] _datasets; - private bool _firstTime = true; - - public DatasetsCollectionUI() - { - InitializeComponent(); - } - - public override void SetItemActivator(IActivateItems activator) - { - base.SetItemActivator(activator); - - CommonTreeFunctionality.SetUp(RDMPCollection.Datasets, tlvDatasets, Activator, olvName, olvName, - new RDMPCollectionCommonFunctionalitySettings()); - CommonTreeFunctionality.WhitespaceRightClickMenuCommandsGetter = - a => new IAtomicCommand[] - { - new ExecuteCommandCreateNewDatasetUI(Activator) - { OverrideCommandName = "Add New Dataset", Weight = -50.9f } - }; - Activator.RefreshBus.EstablishLifetimeSubscription(this); - tlvDatasets.AddObject(activator.CoreChildProvider.DatasetRootFolder); - - RefreshDatasets(Activator.CoreChildProvider.DatasetRootFolder); - - if (_firstTime) - { - CommonTreeFunctionality.SetupColumnTracking(olvName, new Guid("f8b0481e-378c-4996-9400-cb039c2efc5c")); - _firstTime = false; - var _refresh = new ToolStripMenuItem - { - Visible = true, - Image = FamFamFamIcons.arrow_refresh.ImageToBitmap(), - Alignment = ToolStripItemAlignment.Right, - ToolTipText = "Refresh Object" - }; - _refresh.Click += delegate (object sender, EventArgs e) { - var dataset = Activator.CoreChildProvider.AllDatasets.First(); - if (dataset is not null) - { - var cmd = new ExecuteCommandRefreshObject(Activator, dataset); - cmd.Execute(); - } - }; - CommonFunctionality.Add(_refresh); - } - } - - public void RefreshBus_RefreshObject(object sender, RefreshObjectEventArgs e) - { - RefreshDatasets(Activator.CoreChildProvider.DatasetRootFolder); - } - - private void RefreshDatasets(object oRefreshFrom) - { - var rootFolder = Activator.CoreChildProvider.DatasetRootFolder; - if (_datasets != null) - { - var newCatalogues = CommonTreeFunctionality.CoreChildProvider.AllDatasets.Except(_datasets); - if (newCatalogues.Any()) - { - oRefreshFrom = rootFolder; //refresh from the root instead - tlvDatasets.RefreshObject(oRefreshFrom); - } - } - _datasets = CommonTreeFunctionality.CoreChildProvider.AllDatasets; - if (_firstTime || Equals(oRefreshFrom, rootFolder)) - { - tlvDatasets.RefreshObject(rootFolder); - tlvDatasets.Expand(rootFolder); - _firstTime = false; - } - - } - - - /// - /// Returns all root objects in RDMP that match the . Handles unpicking tree collisions e.g. where matches 2 objects with one being the child of the other - /// - /// - /// - /// - public static List FindRootObjects(IActivateItems activator, - Func condition) - { - var datasets = - activator.RepositoryLocator.CatalogueRepository.GetAllObjects(); - - var actualRootFavourites = new List(); - - foreach (var currentFavourite in datasets) - actualRootFavourites.Add(currentFavourite); - - return actualRootFavourites; - } - - - /// - /// Return true if the object should be displayed in this pane - /// - /// - /// - protected virtual bool IncludeObject(IMapsDirectlyToDatabaseTable key) => - Activator.RepositoryLocator.CatalogueRepository.GetAllObjects().Contains(key); - - public static bool IsRootObject(IActivateItems activator, object root) => - //never favourite - false; -} \ No newline at end of file diff --git a/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.Designer.cs b/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.Designer.cs index ab5cf1dfdd..026339607f 100644 --- a/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.Designer.cs +++ b/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.Designer.cs @@ -32,6 +32,10 @@ protected override void Dispose(bool disposing) private void InitializeComponent() { panel1 = new System.Windows.Forms.Panel(); + folv = new FastObjectListView(); + folvFoundValue = new OLVColumn(); + folvReplacmentValue = new OLVColumn(); + folvRedactButton = new OLVColumn(); label4 = new System.Windows.Forms.Label(); tbMaxCount = new System.Windows.Forms.TextBox(); btnRedact = new System.Windows.Forms.Button(); @@ -42,16 +46,14 @@ private void InitializeComponent() label2 = new System.Windows.Forms.Label(); tbCatalogueName = new System.Windows.Forms.TextBox(); label1 = new System.Windows.Forms.Label(); - folv = new BrightIdeasSoftware.FastObjectListView(); - folvFoundValue = new OLVColumn(); - folvReplacmentValue = new OLVColumn(); - folvRedactButton = new OLVColumn(); + btnNewRegex = new System.Windows.Forms.Button(); panel1.SuspendLayout(); ((System.ComponentModel.ISupportInitialize)folv).BeginInit(); SuspendLayout(); // // panel1 // + panel1.Controls.Add(btnNewRegex); panel1.Controls.Add(folv); panel1.Controls.Add(label4); panel1.Controls.Add(tbMaxCount); @@ -70,6 +72,43 @@ private void InitializeComponent() panel1.Size = new System.Drawing.Size(628, 220); panel1.TabIndex = 18; // + // folv + // + folv.AllColumns.Add(folvFoundValue); + folv.AllColumns.Add(folvReplacmentValue); + folv.AllColumns.Add(folvRedactButton); + folv.Location = new System.Drawing.Point(13, 201); + folv.Name = "folv"; + folv.ShowGroups = false; + folv.Size = new System.Drawing.Size(612, 287); + folv.TabIndex = 10; + folv.View = System.Windows.Forms.View.Details; + folv.VirtualMode = true; + // + // folvFoundValue + // + folvFoundValue.AspectName = "FoundValue"; + folvFoundValue.FillsFreeSpace = true; + folvFoundValue.MinimumWidth = 100; + folvFoundValue.Text = "FoundValue"; + folvFoundValue.Width = 100; + // + // folvReplacmentValue + // + folvReplacmentValue.AspectName = "RedactionValue"; + folvReplacmentValue.FillsFreeSpace = true; + folvReplacmentValue.MinimumWidth = 100; + folvReplacmentValue.Text = "Redaction Value"; + folvReplacmentValue.Width = 100; + // + // folvRedactButton + // + folvRedactButton.AspectName = ""; + folvRedactButton.FillsFreeSpace = true; + folvRedactButton.MinimumWidth = 100; + folvRedactButton.Text = "Redact"; + folvRedactButton.Width = 100; + // // label4 // label4.AutoSize = true; @@ -157,42 +196,15 @@ private void InitializeComponent() label1.TabIndex = 0; label1.Text = "Catalogue:"; // - // folv - // - folv.Location = new System.Drawing.Point(13, 201); - folv.Name = "folv"; - folv.ShowGroups = false; - folv.Size = new System.Drawing.Size(612, 287); - folv.TabIndex = 10; - folv.View = System.Windows.Forms.View.Details; - folv.VirtualMode = true; - folv.AllColumns.Add(folvFoundValue); - folv.AllColumns.Add(folvReplacmentValue); - folv.AllColumns.Add(folvRedactButton); - // - // folvFoundValue + // btnNewRegex // - folvFoundValue.AspectName = "FoundValue"; - folvFoundValue.FillsFreeSpace = true; - folvFoundValue.MinimumWidth = 100; - folvFoundValue.Text = "FoundValue"; - folvFoundValue.Width = 100; - // - // folvReplacmentValue - // - folvReplacmentValue.AspectName = "RedactionValue"; - folvReplacmentValue.FillsFreeSpace = true; - folvReplacmentValue.MinimumWidth = 100; - folvReplacmentValue.Text = "Redaction Value"; - folvReplacmentValue.Width = 100; - // - // folvRedactButton - // - folvRedactButton.AspectName = ""; - folvRedactButton.FillsFreeSpace = true; - folvRedactButton.MinimumWidth = 100; - folvRedactButton.Text = "Redact"; - folvRedactButton.Width = 100; + btnNewRegex.Location = new System.Drawing.Point(388, 43); + btnNewRegex.Name = "btnNewRegex"; + btnNewRegex.Size = new System.Drawing.Size(154, 23); + btnNewRegex.TabIndex = 11; + btnNewRegex.Text = "New Regex Configuration"; + btnNewRegex.UseVisualStyleBackColor = true; + btnNewRegex.Click += btnNewRegex_Click; // // RedactCatalogueUI // @@ -224,4 +236,5 @@ private void InitializeComponent() private OLVColumn folvFoundValue; private OLVColumn folvReplacmentValue; private OLVColumn folvRedactButton; + private System.Windows.Forms.Button btnNewRegex; } diff --git a/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.cs b/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.cs index 21337f9ef7..ee490deccb 100644 --- a/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.cs +++ b/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.cs @@ -54,7 +54,11 @@ public override void SetDatabaseObject(IActivateItems activator, Catalogue datab checkedListBox1.Items.AddRange(nonPKColumns); } - + private void btnNewRegex_Click(object sender, EventArgs e) + { + var form = new RegexRedactionConfigurationCreationUI(_activator); + form.ShowDialog(); + } private void btnIdentify_Click(object sender, EventArgs e) { int? max = string.IsNullOrWhiteSpace(tbMaxCount.Text) ? null : int.Parse(tbMaxCount.Text); @@ -66,7 +70,11 @@ private void btnIdentify_Click(object sender, EventArgs e) var config = comboBox1.SelectedItem as RegexRedactionConfiguration; var cmd = new ExecuteCommandIdentifyRegexRedactionsInCatalogue(_activator, _catalogue, config, columns, max); cmd.Execute(); - var x = cmd.results; + if(cmd.results is null) + { + //Do Something! + return; + } foreach(DataRow row in cmd.results.Rows) { object o = new diff --git a/Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationCreationUI.Designer.cs b/Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationCreationUI.Designer.cs new file mode 100644 index 0000000000..73da22aa53 --- /dev/null +++ b/Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationCreationUI.Designer.cs @@ -0,0 +1,59 @@ +namespace Rdmp.UI.SimpleDialogs +{ + partial class RegexRedactionConfigurationCreationUI + { + /// + /// Required designer variable. + /// + private System.ComponentModel.IContainer components = null; + + /// + /// Clean up any resources being used. + /// + /// true if managed resources should be disposed; otherwise, false. + protected override void Dispose(bool disposing) + { + if (disposing && (components != null)) + { + components.Dispose(); + } + base.Dispose(disposing); + } + + #region Windows Form Designer generated code + + /// + /// Required method for Designer support - do not modify + /// the contents of this method with the code editor. + /// + private void InitializeComponent() + { + label1 = new System.Windows.Forms.Label(); + SuspendLayout(); + // + // label1 + // + label1.AutoSize = true; + label1.Location = new System.Drawing.Point(305, 184); + label1.Name = "label1"; + label1.Size = new System.Drawing.Size(38, 15); + label1.TabIndex = 0; + label1.Text = "label1"; + // + // RegexRedactionConfigurationCreationUI + // + AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); + AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; + ClientSize = new System.Drawing.Size(800, 450); + Controls.Add(label1); + Name = "RegexRedactionConfigurationCreationUI"; + Text = "Create New "; + ResumeLayout(false); + PerformLayout(); + } + + #endregion + + private System.Windows.Forms.Label label1; + } +} \ No newline at end of file diff --git a/Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationCreationUI.cs b/Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationCreationUI.cs new file mode 100644 index 0000000000..e3052d50ea --- /dev/null +++ b/Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationCreationUI.cs @@ -0,0 +1,22 @@ +using Rdmp.Core.CommandExecution; +using Rdmp.UI.TestsAndSetup.ServicePropogation; +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Data; +using System.Drawing; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Windows.Forms; + +namespace Rdmp.UI.SimpleDialogs +{ + public partial class RegexRedactionConfigurationCreationUI : Form + { + public RegexRedactionConfigurationCreationUI(IBasicActivateItems activator) + { + InitializeComponent(); + } + } +} diff --git a/Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationCreationUI.resx b/Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationCreationUI.resx new file mode 100644 index 0000000000..af32865ec1 --- /dev/null +++ b/Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationCreationUI.resx @@ -0,0 +1,120 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + \ No newline at end of file diff --git a/Rdmp.UI/SimpleDialogs/SelectDialog`1.resx b/Rdmp.UI/SimpleDialogs/SelectDialog`1.resx deleted file mode 100644 index 6252b1ef07..0000000000 --- a/Rdmp.UI/SimpleDialogs/SelectDialog`1.resx +++ /dev/null @@ -1,63 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - text/microsoft-resx - - - 2.0 - - - System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - True - - \ No newline at end of file diff --git a/Rdmp.UI/Theme/BackColorProvider.cs b/Rdmp.UI/Theme/BackColorProvider.cs index 8e61ecd7d5..e077c53c09 100644 --- a/Rdmp.UI/Theme/BackColorProvider.cs +++ b/Rdmp.UI/Theme/BackColorProvider.cs @@ -30,6 +30,7 @@ public static Color GetColor(RDMPCollection collection) RDMPCollection.Cohort => Color.FromArgb(210, 240, 255), RDMPCollection.DataLoad => Color.DarkGray, RDMPCollection.Datasets => Color.PaleVioletRed, + RDMPCollection.Configurations => Color.DarkSeaGreen, _ => throw new ArgumentOutOfRangeException(nameof(collection)) }; } From 5904882adf16f18ad208e7e36a767dce4f0f8648 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 9 Sep 2024 09:30:49 +0100 Subject: [PATCH 037/106] start of ui --- ...ommandAddNewRegexRedactionConfiguration.cs | 4 +-- Rdmp.Core/Providers/CatalogueChildProvider.cs | 17 +++++++++--- .../Collections/ConfigurationsCollectionUI.cs | 15 +++++++++++ Rdmp.UI/Collections/TableInfoCollectionUI.cs | 6 ++++- ...mandAddNewRegexRedactionConfigurationUI.cs | 27 +++++++++++++++++++ .../ExecuteCommandCreateNewDatasetUI.cs | 2 +- ...ommandCreateRegexRedactionConfiguration.cs | 18 +++++++++++++ 7 files changed, 82 insertions(+), 7 deletions(-) create mode 100644 Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandAddNewRegexRedactionConfigurationUI.cs create mode 100644 Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandCreateRegexRedactionConfiguration.cs diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewRegexRedactionConfiguration.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewRegexRedactionConfiguration.cs index 0e285edc90..59eac6c9be 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewRegexRedactionConfiguration.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewRegexRedactionConfiguration.cs @@ -23,7 +23,7 @@ public class ExecuteCommandAddNewRegexRedactionConfiguration : BasicCommandExecu private IBasicActivateItems _activator; - public ExecuteCommandAddNewRegexRedactionConfiguration(IBasicActivateItems activator, [DemandsInitialization("Name")]string name, [DemandsInitialization("pattern")] string redactionPattern, [DemandsInitialization("redaction")] string redactionString, string description=null) : base(activator) + public ExecuteCommandAddNewRegexRedactionConfiguration(IBasicActivateItems activator, [DemandsInitialization("Name")] string name, [DemandsInitialization("pattern")] string redactionPattern, [DemandsInitialization("redaction")] string redactionString, string description = null) : base(activator) { _activator = activator; _name = name; @@ -35,7 +35,7 @@ public ExecuteCommandAddNewRegexRedactionConfiguration(IBasicActivateItems activ public override void Execute() { base.Execute(); - var config = new RegexRedactionConfiguration(_activator.RepositoryLocator.CatalogueRepository, _name, new Regex(_redactionPattern), _redactionString,_description); + var config = new RegexRedactionConfiguration(_activator.RepositoryLocator.CatalogueRepository, _name, new Regex(_redactionPattern), _redactionString, _description); config.SaveToDatabase(); } diff --git a/Rdmp.Core/Providers/CatalogueChildProvider.cs b/Rdmp.Core/Providers/CatalogueChildProvider.cs index 15cf672ca0..ce47d6e399 100644 --- a/Rdmp.Core/Providers/CatalogueChildProvider.cs +++ b/Rdmp.Core/Providers/CatalogueChildProvider.cs @@ -778,6 +778,7 @@ private void BuildDatasetNodes() { AddChildren(dataset, descendancy.Add(dataset)); } + AddChildren(AllDatasetsNode); } @@ -842,6 +843,12 @@ private void AddChildren(AllRegexRedactionConfigurationsNode allRegexRedactionCo } + private void AddChildren(AllDatasetsNode allDatasetsNode) + { + AddToDictionaries(new HashSet(AllDatasets), new DescendancyList(allDatasetsNode)); + + } + private void AddChildren(FolderNode folder, DescendancyList descendancy) { foreach (var child in folder.ChildFolders) @@ -908,13 +915,17 @@ private void AddChildren(FolderNode folder, D private void AddChildren(RegexRedactionConfiguration regex, DescendancyList descendancy) { - var childObjects = new List(); + var childObjects = new List() { regex}; AddToDictionaries(new HashSet(childObjects), descendancy); } - private void AddChildren(Curation.Data.Dataset lmd, DescendancyList descendancy) + private void AddChildren(Curation.Data.Dataset dataset, DescendancyList descendancy) { - var childObjects = new List(); + var childObjects = new List + { + dataset + }; + AddToDictionaries(new HashSet(childObjects), descendancy); } diff --git a/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs b/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs index a616a8f1e3..12aad41535 100644 --- a/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs +++ b/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs @@ -29,12 +29,27 @@ public ConfigurationsCollectionUI() InitializeComponent(); } + private IAtomicCommand[] GetWhitespaceRightClickMenu() + { + return new IAtomicCommand[] + { + new ExecuteCommandCreateNewDatasetUI(_activator){ + OverrideCommandName="Add New Dataset" + }, + new ExecuteCommandAddNewRegexRedactionConfigurationUI(_activator) + //{ + // //OverrideCommandName="Add New Regex Redaction Configuration" + //} + }; + } + public override void SetItemActivator(IActivateItems activator) { base.SetItemActivator(activator); _activator = activator; CommonTreeFunctionality.SetUp(RDMPCollection.Configurations, tlvConfigurations, activator, olvName, olvName, new RDMPCollectionCommonFunctionalitySettings()); + CommonTreeFunctionality.WhitespaceRightClickMenuCommandsGetter = e => GetWhitespaceRightClickMenu(); Activator.RefreshBus.EstablishLifetimeSubscription(this); tlvConfigurations.AddObject(Activator.CoreChildProvider.AllDatasetsNode); tlvConfigurations.AddObject(Activator.CoreChildProvider.AllRegexRedactionConfigurationsNode); diff --git a/Rdmp.UI/Collections/TableInfoCollectionUI.cs b/Rdmp.UI/Collections/TableInfoCollectionUI.cs index ec0683405d..f03f2928c9 100644 --- a/Rdmp.UI/Collections/TableInfoCollectionUI.cs +++ b/Rdmp.UI/Collections/TableInfoCollectionUI.cs @@ -5,9 +5,11 @@ // You should have received a copy of the GNU General Public License along with RDMP. If not, see . using System; +using System.DirectoryServices.Protocols; using System.Linq; using System.Windows.Forms; using Rdmp.Core; +using Rdmp.Core.CommandExecution; using Rdmp.Core.CommandExecution.AtomicCommands; using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.Data.DataLoad; @@ -16,6 +18,7 @@ using Rdmp.Core.Providers.Nodes; using Rdmp.Core.Providers.Nodes.PipelineNodes; using Rdmp.Core.Providers.Nodes.SharingNodes; +using Rdmp.UI.CommandExecution.AtomicCommands; using Rdmp.UI.ItemActivation; using Rdmp.UI.LocationsMenu; using Rdmp.UI.Refreshing; @@ -36,6 +39,7 @@ namespace Rdmp.UI.Collections; public partial class TableInfoCollectionUI : RDMPCollectionUI, ILifetimeSubscriber { private bool _isFirstTime = true; + private IActivateItems _activator; public TableInfoCollectionUI() { @@ -88,7 +92,7 @@ private void olvTableInfos_KeyUp(object sender, KeyEventArgs e) public override void SetItemActivator(IActivateItems activator) { base.SetItemActivator(activator); - + _activator = activator; CommonTreeFunctionality.SetUp( RDMPCollection.Tables, tlvTableInfos, diff --git a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandAddNewRegexRedactionConfigurationUI.cs b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandAddNewRegexRedactionConfigurationUI.cs new file mode 100644 index 0000000000..f6bfadbbeb --- /dev/null +++ b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandAddNewRegexRedactionConfigurationUI.cs @@ -0,0 +1,27 @@ +using Rdmp.Core.CommandExecution.AtomicCommands; +using Rdmp.UI.ItemActivation; +using Rdmp.UI.TestsAndSetup.ServicePropogation; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.UI.CommandExecution.AtomicCommands +{ + public class ExecuteCommandAddNewRegexRedactionConfigurationUI : ExecuteCommandCreateRegexRedactionConfiguration + { + private readonly IActivateItems _activator; + + public ExecuteCommandAddNewRegexRedactionConfigurationUI(IActivateItems activator) : base(activator) + { + _activator = activator; + } + + //public override void Execute() + //{ + // //var ui = new + //} + } + +} diff --git a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandCreateNewDatasetUI.cs b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandCreateNewDatasetUI.cs index 0ef2fc15df..1ae48586ec 100644 --- a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandCreateNewDatasetUI.cs +++ b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandCreateNewDatasetUI.cs @@ -23,6 +23,6 @@ public ExecuteCommandCreateNewDatasetUI(IActivateItems activator) : base( public override void Execute() { var ui = new CreateNewDatasetUI(_activator, this); - ui.ShowDialog(); + ui.Show(); } } \ No newline at end of file diff --git a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandCreateRegexRedactionConfiguration.cs b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandCreateRegexRedactionConfiguration.cs new file mode 100644 index 0000000000..c37da09747 --- /dev/null +++ b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandCreateRegexRedactionConfiguration.cs @@ -0,0 +1,18 @@ +using Rdmp.Core.CommandExecution; +using Rdmp.UI.ItemActivation; +using Rdmp.UI.TestsAndSetup.ServicePropogation; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.UI.CommandExecution.AtomicCommands +{ + public class ExecuteCommandCreateRegexRedactionConfiguration: BasicCommandExecution + { + public ExecuteCommandCreateRegexRedactionConfiguration(IActivateItems activator) : base(activator) + { + } + } +} From 5c47cf7050958fc9df0364401cf9e87874bbf7d4 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 9 Sep 2024 14:00:00 +0100 Subject: [PATCH 038/106] basic configuraion --- .../RegexRedactionConfiguration.cs | 8 +- .../up/086_AddRegexRedactionConfiguration.sql | 1 + Rdmp.Core/Icons/AllDatasetsNode.png | Bin 0 -> 726 bytes .../AllRegexRedactionConfigurationsNode.png | Bin 0 -> 267 bytes .../Icons/IconProvision/CatalogueIcons.resx | 12 +- Rdmp.Core/Icons/IconProvision/RDMPConcept.cs | 3 +- Rdmp.Core/Providers/CatalogueChildProvider.cs | 52 ++++- .../Collections/ConfigurationsCollectionUI.cs | 6 +- ...mandAddNewRegexRedactionConfigurationUI.cs | 16 +- ...ommandCreateRegexRedactionConfiguration.cs | 18 -- .../ProposeExecutionWhenTargetIsDataset.cs | 1 + ...WhenTargetIsRegexRedactionConfiguration.cs | 34 +++ Rdmp.UI/Menus/DatasetMenu.cs | 6 +- Rdmp.UI/Rdmp.UI.csproj | 1 + ...wRegexRedactionConfigurationUI.Designer.cs | 205 ++++++++++++++++++ .../CreateNewRegexRedactionConfigurationUI.cs | 94 ++++++++ ...reateNewRegexRedactionConfigurationUI.resx | 120 ++++++++++ .../RegexRedactionConfigurationUI.Designer.cs | 172 +++++++++++++++ .../RegexRedactionConfigurationUI.cs | 63 ++++++ .../RegexRedactionConfigurationUI.resx | 123 +++++++++++ 20 files changed, 886 insertions(+), 49 deletions(-) create mode 100644 Rdmp.Core/Icons/AllDatasetsNode.png create mode 100644 Rdmp.Core/Icons/AllRegexRedactionConfigurationsNode.png delete mode 100644 Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandCreateRegexRedactionConfiguration.cs create mode 100644 Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsRegexRedactionConfiguration.cs create mode 100644 Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationForm/CreateNewRegexRedactionConfigurationUI.Designer.cs create mode 100644 Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationForm/CreateNewRegexRedactionConfigurationUI.cs create mode 100644 Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationForm/CreateNewRegexRedactionConfigurationUI.resx create mode 100644 Rdmp.UI/SubComponents/RegexRedactionConfigurationUI.Designer.cs create mode 100644 Rdmp.UI/SubComponents/RegexRedactionConfigurationUI.cs create mode 100644 Rdmp.UI/SubComponents/RegexRedactionConfigurationUI.resx diff --git a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionConfiguration.cs b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionConfiguration.cs index 9b2e90f8c7..6f26724cd2 100644 --- a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionConfiguration.cs +++ b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionConfiguration.cs @@ -32,7 +32,7 @@ public class RegexRedactionConfiguration : DatabaseEntity, IRegexRedactionConfig public override string ToString() { - return $"{_name}({_regexPattern})"; + return $"{_name}"; } [Unique] @@ -67,14 +67,15 @@ public string RedactionString public RegexRedactionConfiguration() { } - public RegexRedactionConfiguration(ICatalogueRepository repository, string name, Regex regexPattern, string redactionString, string description = null) + public RegexRedactionConfiguration(ICatalogueRepository repository, string name, Regex regexPattern, string redactionString, string description = null,string folder="\\") { repository.InsertAndHydrate(this, new Dictionary { { "Name", name }, { "RegexPattern", regexPattern.ToString() }, {"RedactionString", redactionString }, - {"Description",description } + {"Description",description }, + {"Folder",folder } }); } @@ -84,6 +85,7 @@ internal RegexRedactionConfiguration(ICatalogueRepository repository, DbDataRead Description = r["Description"].ToString(); RedactionString = r["RedactionString"].ToString(); RegexPattern = r["RegexPattern"].ToString(); + Folder = r["Folder"].ToString(); } diff --git a/Rdmp.Core/Databases/CatalogueDatabase/up/086_AddRegexRedactionConfiguration.sql b/Rdmp.Core/Databases/CatalogueDatabase/up/086_AddRegexRedactionConfiguration.sql index 21b8cf216a..65140dc004 100644 --- a/Rdmp.Core/Databases/CatalogueDatabase/up/086_AddRegexRedactionConfiguration.sql +++ b/Rdmp.Core/Databases/CatalogueDatabase/up/086_AddRegexRedactionConfiguration.sql @@ -10,6 +10,7 @@ CREATE TABLE [dbo].RegexRedactionConfiguration( Description [nvarchar](250), RegexPattern [nvarchar](250) NOT NULL, RedactionString [nvarchar](250) NOT NULL, + Folder [nvarchar](max) NOT NULL DEFAULT '\' CONSTRAINT [PK_RegexRedactionConfiguration] PRIMARY KEY CLUSTERED ( [ID] ASC diff --git a/Rdmp.Core/Icons/AllDatasetsNode.png b/Rdmp.Core/Icons/AllDatasetsNode.png new file mode 100644 index 0000000000000000000000000000000000000000..693709cbc1b156839a754e53cbaf409edec69567 GIT binary patch literal 726 zcmV;{0xA88P)T&uF!>{(iS?1OX-eX zKw9bunxR5FrF6QaYs~9>A4#zW^dwIvCpq(+cfR?U`T6-{9LHUqo16RKcDwUVr?cX4 zIN~hJDs48~aRAJ}U_2g=KAB9SP$;0;Y@*$6Ly{z<(`i^NmbL#1W@l#$wOS3;YPBOE zJ;7`?L*?Ga6XzC292wl75}>gDz`(>h?is$JPxm#0jGnotoK|nAVM5$DQ z!C*kO-aeF@+Ejy?nVHEp8V&F~k7BWicsx!aH9kHLRpcQ?L&JFBAB4i&kAaVUxVvzh z3a-EY0%m%8nhI7|SE(QpiBL#sG#VUMM9}*(0mg2(Q$Zq;z|PJNd_Euiem@;jtJN@i za|c2MmsL?PR;yKNwOUA}QuO=7;V@#c7!{~gs?J7hAlsE7U#g?$aRkhSTqLq6iuCu9 z10_j_=;?Dc?4cZ386qH0HkgHTDT|HmGR`W4V2noNQJqfLJEot)q{V_UtsW+m31cP~ zDwWEi3HYBSoF4M;T?VaIdqinn1HZ9}32qs-PdwPbCf+WI6n9jl0-8cjV3%1FB%B&r z+`mzSliyLSH0dxYE}rk&=!uCa*V>()2znj`_XYjtbt>@4FLHnJE|G`xv)Ba@oLBny z1%3K7c4fiB^4{k6E8Pif0kNy62}b@9+NS%G}c0*}aI z1_r*pAk66gd%{|vpk#?_L`iUdT1k0gQ7S`0VrE{6US4X6f{C7io}rmb=K8NdMID|l zjv*HQdwUM@HW=_QTeJFqi|?^yvdM`I3%b=*AyXtZO2y^bDsStwyZa6V03n^J};TM(`}3JQt|f_)$IHp1Kq^n>FVdQ I&MBb@0L{i-;{X5v literal 0 HcmV?d00001 diff --git a/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx b/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx index 11c11ffe0f..a821d3ddd4 100644 --- a/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx +++ b/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx @@ -622,12 +622,12 @@ ..\AllPipelinesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - ..\AllPipelinesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllPipelinesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - + + ..\AllDatasetsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllRegexRedactionConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + ..\StandardPipelineUseCaseNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 diff --git a/Rdmp.Core/Icons/IconProvision/RDMPConcept.cs b/Rdmp.Core/Icons/IconProvision/RDMPConcept.cs index 59c18d4d79..54bfb31e8b 100644 --- a/Rdmp.Core/Icons/IconProvision/RDMPConcept.cs +++ b/Rdmp.Core/Icons/IconProvision/RDMPConcept.cs @@ -210,5 +210,6 @@ public enum RDMPConcept TicketingSystemReleaseStatus, RegexRedaction, RegexRedactionConfiguration, - RegexRedactionKey + RegexRedactionKey, + AllRegexRedactionConfigurationsNode } \ No newline at end of file diff --git a/Rdmp.Core/Providers/CatalogueChildProvider.cs b/Rdmp.Core/Providers/CatalogueChildProvider.cs index ce47d6e399..3b010b9b90 100644 --- a/Rdmp.Core/Providers/CatalogueChildProvider.cs +++ b/Rdmp.Core/Providers/CatalogueChildProvider.cs @@ -151,6 +151,7 @@ public class CatalogueChildProvider : ICoreChildProvider public FolderNode LoadMetadataRootFolder { get; set; } public FolderNode DatasetRootFolder { get; set; } + public FolderNode RegexRedactionConfigurationRootFolder { get; set; } public FolderNode CohortIdentificationConfigurationRootFolder { get; set; } public FolderNode CohortIdentificationConfigurationRootFolderWithoutVersionedConfigurations { get; set; } @@ -385,8 +386,6 @@ public CatalogueChildProvider(ICatalogueRepository repository, IChildProvider[] //All the things for TableInfoCollectionUI BuildServerNodes(); - BuildDatasetNodes(); - BuildAllRegexRedactionConfigurationsNode(); ReportProgress("BuildServerNodes"); //add a new CatalogueItemNodes @@ -399,6 +398,9 @@ public CatalogueChildProvider(ICatalogueRepository repository, IChildProvider[] DatasetRootFolder = FolderHelper.BuildFolderTree(AllDatasets); AddChildren(DatasetRootFolder, new DescendancyList(DatasetRootFolder)); + RegexRedactionConfigurationRootFolder = FolderHelper.BuildFolderTree(AllRegexRedactionConfigurations); + AddChildren(RegexRedactionConfigurationRootFolder, new DescendancyList(RegexRedactionConfigurationRootFolder)); + ReportProgress("Build Catalogue Folder Root"); LoadMetadataRootFolder = FolderHelper.BuildFolderTree(AllLoadMetadatas); @@ -421,6 +423,10 @@ public CatalogueChildProvider(ICatalogueRepository repository, IChildProvider[] TemplateAggregateConfigurations = AllAggregateConfigurations .Where(ac => templateAggregateConfigurationIds.Contains(ac.ID)).ToArray(); + + BuildDatasetNodes(); + BuildAllRegexRedactionConfigurationsNode(); + //add the orphans under the orphan folder AddToDictionaries(new HashSet(OrphanAggregateConfigurations), new DescendancyList(OrphanAggregateConfigurationsNode)); @@ -766,7 +772,8 @@ private void BuildAllRegexRedactionConfigurationsNode() //{ // AddChildren(regex, descendancy.Add(regex)); //} - AddChildren(AllRegexRedactionConfigurationsNode); + //AddChildren(AllRegexRedactionConfigurationsNode); + AddChildren(RegexRedactionConfigurationRootFolder, descendancy); } @@ -774,11 +781,7 @@ private void BuildDatasetNodes() { AllDatasetsNode = new AllDatasetsNode(); var descendancy = new DescendancyList(AllDatasetsNode); - foreach (var dataset in AllDatasets) - { - AddChildren(dataset, descendancy.Add(dataset)); - } - AddChildren(AllDatasetsNode); + AddChildren(DatasetRootFolder, descendancy); } @@ -849,6 +852,19 @@ private void AddChildren(AllDatasetsNode allDatasetsNode) } + private void Addchildren(FolderNode folder, DescendancyList descendancy) + { + foreach (var child in folder.ChildFolders) + //add subfolder children + AddChildren(child, descendancy.Add(child)); + foreach (var c in folder.ChildObjects) AddChildren(c, descendancy.Add(c)); + AddToDictionaries(new HashSet( + folder.ChildFolders.Cast() + .Union(folder.ChildObjects)), descendancy + ); + + } + private void AddChildren(FolderNode folder, DescendancyList descendancy) { foreach (var child in folder.ChildFolders) @@ -880,8 +896,26 @@ private void AddChildren(FolderNode folder, DescendancyList descen ); } + private void AddChildren(FolderNode folder, DescendancyList descendancy) + { + if (folder is null) return; + foreach (var child in folder.ChildFolders) + //add subfolder children + AddChildren(child, descendancy.Add(child)); + + //add loads in folder + foreach (var ds in folder.ChildObjects) AddChildren(ds, descendancy.Add(ds)); + + //// Children are the folders + objects + AddToDictionaries(new HashSet( + folder.ChildFolders.Cast() + .Union(folder.ChildObjects)), descendancy + ); + } + private void AddChildren(FolderNode folder, DescendancyList descendancy) { + if (folder is null) return; foreach (var child in folder.ChildFolders) //add subfolder children AddChildren(child, descendancy.Add(child)); @@ -915,7 +949,7 @@ private void AddChildren(FolderNode folder, D private void AddChildren(RegexRedactionConfiguration regex, DescendancyList descendancy) { - var childObjects = new List() { regex}; + var childObjects = new List() { regex }; AddToDictionaries(new HashSet(childObjects), descendancy); } diff --git a/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs b/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs index 12aad41535..fe1bc9418b 100644 --- a/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs +++ b/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs @@ -37,9 +37,9 @@ private IAtomicCommand[] GetWhitespaceRightClickMenu() OverrideCommandName="Add New Dataset" }, new ExecuteCommandAddNewRegexRedactionConfigurationUI(_activator) - //{ - // //OverrideCommandName="Add New Regex Redaction Configuration" - //} + { + OverrideCommandName="Add New Regex Redaction Configuration" + } }; } diff --git a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandAddNewRegexRedactionConfigurationUI.cs b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandAddNewRegexRedactionConfigurationUI.cs index f6bfadbbeb..4ed68a9510 100644 --- a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandAddNewRegexRedactionConfigurationUI.cs +++ b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandAddNewRegexRedactionConfigurationUI.cs @@ -1,5 +1,8 @@ -using Rdmp.Core.CommandExecution.AtomicCommands; +using Rdmp.Core.CommandExecution; +using Rdmp.Core.CommandExecution.AtomicCommands; using Rdmp.UI.ItemActivation; +using Rdmp.UI.SimpleDialogs.Datasets; +using Rdmp.UI.SimpleDialogs.RegexRedactionConfigurationForm; using Rdmp.UI.TestsAndSetup.ServicePropogation; using System; using System.Collections.Generic; @@ -9,7 +12,7 @@ namespace Rdmp.UI.CommandExecution.AtomicCommands { - public class ExecuteCommandAddNewRegexRedactionConfigurationUI : ExecuteCommandCreateRegexRedactionConfiguration + public class ExecuteCommandAddNewRegexRedactionConfigurationUI : BasicCommandExecution { private readonly IActivateItems _activator; @@ -18,10 +21,11 @@ public ExecuteCommandAddNewRegexRedactionConfigurationUI(IActivateItems activato _activator = activator; } - //public override void Execute() - //{ - // //var ui = new - //} + public override void Execute() + { + var ui = new CreateNewRegexRedactionConfigurationUI(_activator); + ui.Show(); + } } } diff --git a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandCreateRegexRedactionConfiguration.cs b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandCreateRegexRedactionConfiguration.cs deleted file mode 100644 index c37da09747..0000000000 --- a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandCreateRegexRedactionConfiguration.cs +++ /dev/null @@ -1,18 +0,0 @@ -using Rdmp.Core.CommandExecution; -using Rdmp.UI.ItemActivation; -using Rdmp.UI.TestsAndSetup.ServicePropogation; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Rdmp.UI.CommandExecution.AtomicCommands -{ - public class ExecuteCommandCreateRegexRedactionConfiguration: BasicCommandExecution - { - public ExecuteCommandCreateRegexRedactionConfiguration(IActivateItems activator) : base(activator) - { - } - } -} diff --git a/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsDataset.cs b/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsDataset.cs index 7bc681d63e..5ff10a9b23 100644 --- a/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsDataset.cs +++ b/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsDataset.cs @@ -6,6 +6,7 @@ using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.DataHelper.RegexRedaction; using Rdmp.Core.ReusableLibraryCode.Annotations; using Rdmp.UI.ItemActivation; using Rdmp.UI.SubComponents; diff --git a/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsRegexRedactionConfiguration.cs b/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsRegexRedactionConfiguration.cs new file mode 100644 index 0000000000..9e248f39fb --- /dev/null +++ b/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsRegexRedactionConfiguration.cs @@ -0,0 +1,34 @@ +using Rdmp.Core.CommandExecution; +using Rdmp.Core.Curation.DataHelper.RegexRedaction; +using Rdmp.Core.ReusableLibraryCode.Annotations; +using Rdmp.UI.ItemActivation; +using Rdmp.UI.SubComponents; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.UI.CommandExecution.Proposals +{ + internal class ProposeExecutionWhenTargetIsRegexRedactionConfiguration: RDMPCommandExecutionProposal + { + public ProposeExecutionWhenTargetIsRegexRedactionConfiguration(IActivateItems itemActivator) : base( + itemActivator) + { + } + + public override bool CanActivate(RegexRedactionConfiguration target) => true; + + public override void Activate(RegexRedactionConfiguration target) + { + ItemActivator.Activate(target); + } + + [CanBeNull] + public override ICommandExecution ProposeExecution(ICombineToMakeCommand cmd, + RegexRedactionConfiguration target, + InsertOption insertOption = InsertOption.Default) => + null; + } +} diff --git a/Rdmp.UI/Menus/DatasetMenu.cs b/Rdmp.UI/Menus/DatasetMenu.cs index 8326c6342e..5b833b525f 100644 --- a/Rdmp.UI/Menus/DatasetMenu.cs +++ b/Rdmp.UI/Menus/DatasetMenu.cs @@ -10,11 +10,11 @@ namespace Rdmp.UI.Menus; /// /// Menu item for datasets /// -public class DatasetMenu: RDMPContextMenuStrip +public class DatasetMenu : RDMPContextMenuStrip { - public DatasetMenu(RDMPContextMenuStripArgs args, Dataset dataset): base(args, dataset) + public DatasetMenu(RDMPContextMenuStripArgs args, Dataset dataset) : base(args, dataset) { - Add(new ExecuteCommandDeleteDatasetUI(_activator,dataset)); + Add(new ExecuteCommandDeleteDatasetUI(_activator, dataset) { OverrideCommandName = "Delete Dataset" }); } } diff --git a/Rdmp.UI/Rdmp.UI.csproj b/Rdmp.UI/Rdmp.UI.csproj index f09049bec8..8ad6149ca3 100644 --- a/Rdmp.UI/Rdmp.UI.csproj +++ b/Rdmp.UI/Rdmp.UI.csproj @@ -242,6 +242,7 @@ WideMessageBox.cs + Component diff --git a/Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationForm/CreateNewRegexRedactionConfigurationUI.Designer.cs b/Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationForm/CreateNewRegexRedactionConfigurationUI.Designer.cs new file mode 100644 index 0000000000..48a044f19d --- /dev/null +++ b/Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationForm/CreateNewRegexRedactionConfigurationUI.Designer.cs @@ -0,0 +1,205 @@ +namespace Rdmp.UI.SimpleDialogs.RegexRedactionConfigurationForm +{ + partial class CreateNewRegexRedactionConfigurationUI + { + /// + /// Required designer variable. + /// + private System.ComponentModel.IContainer components = null; + + /// + /// Clean up any resources being used. + /// + /// true if managed resources should be disposed; otherwise, false. + protected override void Dispose(bool disposing) + { + if (disposing && (components != null)) + { + components.Dispose(); + } + base.Dispose(disposing); + } + + #region Windows Form Designer generated code + + /// + /// Required method for Designer support - do not modify + /// the contents of this method with the code editor. + /// + private void InitializeComponent() + { + label1 = new System.Windows.Forms.Label(); + tbName = new System.Windows.Forms.TextBox(); + tbRegexPattern = new System.Windows.Forms.TextBox(); + label2 = new System.Windows.Forms.Label(); + tbRedactionString = new System.Windows.Forms.TextBox(); + label3 = new System.Windows.Forms.Label(); + tbDescription = new System.Windows.Forms.TextBox(); + label4 = new System.Windows.Forms.Label(); + btnCreate = new System.Windows.Forms.Button(); + btnCancel = new System.Windows.Forms.Button(); + lblError = new System.Windows.Forms.Label(); + label5 = new System.Windows.Forms.Label(); + tbFolder = new System.Windows.Forms.TextBox(); + label6 = new System.Windows.Forms.Label(); + SuspendLayout(); + // + // label1 + // + label1.AutoSize = true; + label1.Location = new System.Drawing.Point(26, 30); + label1.Name = "label1"; + label1.Size = new System.Drawing.Size(47, 15); + label1.TabIndex = 0; + label1.Text = "Name *"; + // + // tbName + // + tbName.Location = new System.Drawing.Point(32, 47); + tbName.Name = "tbName"; + tbName.Size = new System.Drawing.Size(516, 23); + tbName.TabIndex = 1; + // + // tbRegexPattern + // + tbRegexPattern.Location = new System.Drawing.Point(32, 114); + tbRegexPattern.Name = "tbRegexPattern"; + tbRegexPattern.Size = new System.Drawing.Size(516, 23); + tbRegexPattern.TabIndex = 3; + // + // label2 + // + label2.AutoSize = true; + label2.Location = new System.Drawing.Point(26, 97); + label2.Name = "label2"; + label2.Size = new System.Drawing.Size(88, 15); + label2.TabIndex = 2; + label2.Text = "Regex Pattern *"; + // + // tbRedactionString + // + tbRedactionString.Location = new System.Drawing.Point(32, 176); + tbRedactionString.Name = "tbRedactionString"; + tbRedactionString.Size = new System.Drawing.Size(516, 23); + tbRedactionString.TabIndex = 5; + // + // label3 + // + label3.AutoSize = true; + label3.Location = new System.Drawing.Point(26, 159); + label3.Name = "label3"; + label3.Size = new System.Drawing.Size(118, 15); + label3.TabIndex = 4; + label3.Text = " Replacement String*"; + // + // tbDescription + // + tbDescription.Location = new System.Drawing.Point(32, 237); + tbDescription.Name = "tbDescription"; + tbDescription.Size = new System.Drawing.Size(516, 23); + tbDescription.TabIndex = 7; + // + // label4 + // + label4.AutoSize = true; + label4.Location = new System.Drawing.Point(26, 220); + label4.Name = "label4"; + label4.Size = new System.Drawing.Size(67, 15); + label4.TabIndex = 6; + label4.Text = "Description"; + // + // btnCreate + // + btnCreate.Location = new System.Drawing.Point(471, 351); + btnCreate.Name = "btnCreate"; + btnCreate.Size = new System.Drawing.Size(75, 23); + btnCreate.TabIndex = 8; + btnCreate.Text = "Create"; + btnCreate.UseVisualStyleBackColor = true; + // + // btnCancel + // + btnCancel.Location = new System.Drawing.Point(390, 351); + btnCancel.Name = "btnCancel"; + btnCancel.Size = new System.Drawing.Size(75, 23); + btnCancel.TabIndex = 9; + btnCancel.Text = "Cancel"; + btnCancel.UseVisualStyleBackColor = true; + // + // lblError + // + lblError.AutoSize = true; + lblError.Location = new System.Drawing.Point(30, 351); + lblError.Name = "lblError"; + lblError.Size = new System.Drawing.Size(39, 15); + lblError.TabIndex = 10; + lblError.Text = "Name"; + // + // label5 + // + label5.AutoSize = true; + label5.Location = new System.Drawing.Point(487, 329); + label5.Name = "label5"; + label5.Size = new System.Drawing.Size(62, 15); + label5.TabIndex = 11; + label5.Text = "* Required"; + // + // tbFolder + // + tbFolder.Location = new System.Drawing.Point(33, 291); + tbFolder.Name = "tbFolder"; + tbFolder.Size = new System.Drawing.Size(516, 23); + tbFolder.TabIndex = 13; + // + // label6 + // + label6.AutoSize = true; + label6.Location = new System.Drawing.Point(27, 274); + label6.Name = "label6"; + label6.Size = new System.Drawing.Size(48, 15); + label6.TabIndex = 12; + label6.Text = "Folder *"; + // + // CreateNewRegexRedactionConfigurationUI + // + AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); + AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; + ClientSize = new System.Drawing.Size(800, 450); + Controls.Add(tbFolder); + Controls.Add(label6); + Controls.Add(label5); + Controls.Add(lblError); + Controls.Add(btnCancel); + Controls.Add(btnCreate); + Controls.Add(tbDescription); + Controls.Add(label4); + Controls.Add(tbRedactionString); + Controls.Add(label3); + Controls.Add(tbRegexPattern); + Controls.Add(label2); + Controls.Add(tbName); + Controls.Add(label1); + Name = "CreateNewRegexRedactionConfigurationUI"; + Text = "Create New Regex Redaction Configuration"; + ResumeLayout(false); + PerformLayout(); + } + + #endregion + + private System.Windows.Forms.Label label1; + private System.Windows.Forms.TextBox tbName; + private System.Windows.Forms.TextBox tbRegexPattern; + private System.Windows.Forms.Label label2; + private System.Windows.Forms.TextBox tbRedactionString; + private System.Windows.Forms.Label label3; + private System.Windows.Forms.TextBox tbDescription; + private System.Windows.Forms.Label label4; + private System.Windows.Forms.Button btnCreate; + private System.Windows.Forms.Button btnCancel; + private System.Windows.Forms.Label lblError; + private System.Windows.Forms.Label label5; + private System.Windows.Forms.TextBox tbFolder; + private System.Windows.Forms.Label label6; + } +} \ No newline at end of file diff --git a/Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationForm/CreateNewRegexRedactionConfigurationUI.cs b/Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationForm/CreateNewRegexRedactionConfigurationUI.cs new file mode 100644 index 0000000000..f33cb98048 --- /dev/null +++ b/Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationForm/CreateNewRegexRedactionConfigurationUI.cs @@ -0,0 +1,94 @@ +using Rdmp.Core.CommandExecution.AtomicCommands; +using Rdmp.Core.Curation.DataHelper.RegexRedaction; +using Rdmp.UI.CommandExecution.AtomicCommands; +using Rdmp.UI.ItemActivation; +using Rdmp.UI.TestsAndSetup.ServicePropogation; +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Data; +using System.Drawing; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using System.Windows.Forms; + +namespace Rdmp.UI.SimpleDialogs.RegexRedactionConfigurationForm +{ + public partial class CreateNewRegexRedactionConfigurationUI : RDMPForm + { + private readonly IActivateItems _activator; + + public CreateNewRegexRedactionConfigurationUI(IActivateItems activator) : base(activator) + { + _activator = activator; + InitializeComponent(); + this.btnCreate.Enabled = false; + this.tbName.TextChanged += OnChange; + this.tbRegexPattern.TextChanged += OnChange; + this.tbRedactionString.TextChanged += OnChange; + this.btnCancel.Click += Cancel; + this.btnCreate.Click += Create; + this.lblError.Text = ""; + this.lblError.Visible = false; + this.tbFolder.Text = "\\"; + } + + private void Cancel(object sender, EventArgs e) + { + Close(); + } + private void Create(object sender, EventArgs e) + { + var config = new RegexRedactionConfiguration(_activator.RepositoryLocator.CatalogueRepository, tbName.Text, new Regex(tbRegexPattern.Text), tbRedactionString.Text, tbDescription.Text); + config.Folder = string.IsNullOrWhiteSpace(tbFolder.Text) ? "\\" : tbFolder.Text; + config.SaveToDatabase(); + _activator.Publish(config); + Close(); + } + private bool ValidRegex() + { + var regexString = this.tbRegexPattern.Text; + if (string.IsNullOrWhiteSpace(regexString)) return false; + try + { + Regex.Match("", regexString); + } + catch (ArgumentException) + { + return false; + } + + return true; + } + + private void OnChange(object sender, EventArgs e) + { + var validRegex = ValidRegex(); + if (validRegex) + { + lblError.Text = ""; + lblError.Visible = false; + } + else + { + if (!string.IsNullOrWhiteSpace(tbRegexPattern.Text)) + { + lblError.Text = "Regex Pattern is invalid"; + lblError.Visible = true; + lblError.ForeColor = Color.Red; + } + } + if (!string.IsNullOrWhiteSpace(this.tbName.Text) + && validRegex && !string.IsNullOrWhiteSpace(this.tbRedactionString.Text)) + { + this.btnCreate.Enabled = true; + } + else + { + this.btnCreate.Enabled = false; + } + } + } +} diff --git a/Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationForm/CreateNewRegexRedactionConfigurationUI.resx b/Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationForm/CreateNewRegexRedactionConfigurationUI.resx new file mode 100644 index 0000000000..af32865ec1 --- /dev/null +++ b/Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationForm/CreateNewRegexRedactionConfigurationUI.resx @@ -0,0 +1,120 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + \ No newline at end of file diff --git a/Rdmp.UI/SubComponents/RegexRedactionConfigurationUI.Designer.cs b/Rdmp.UI/SubComponents/RegexRedactionConfigurationUI.Designer.cs new file mode 100644 index 0000000000..666d3bc5df --- /dev/null +++ b/Rdmp.UI/SubComponents/RegexRedactionConfigurationUI.Designer.cs @@ -0,0 +1,172 @@ +namespace Rdmp.UI.SubComponents; + +partial class RegexRedactionConfigurationUI +{ + /// + /// Required designer variable. + /// + private System.ComponentModel.IContainer components = null; + + /// + /// Clean up any resources being used. + /// + /// true if managed resources should be disposed; otherwise, false. + protected override void Dispose(bool disposing) + { + if (disposing && (components != null)) + { + components.Dispose(); + } + base.Dispose(disposing); + } + + #region Windows Form Designer generated code + + /// + /// Required method for Designer support - do not modify + /// the contents of this method with the code editor. + /// + private void InitializeComponent() + { + label1 = new System.Windows.Forms.Label(); + tbName = new System.Windows.Forms.TextBox(); + columnButtonRenderer1 = new BrightIdeasSoftware.ColumnButtonRenderer(); + tbRegexPattern = new System.Windows.Forms.TextBox(); + label2 = new System.Windows.Forms.Label(); + tbRedactionString = new System.Windows.Forms.TextBox(); + label3 = new System.Windows.Forms.Label(); + tbDescription = new System.Windows.Forms.TextBox(); + label4 = new System.Windows.Forms.Label(); + label5 = new System.Windows.Forms.Label(); + tbFolder = new System.Windows.Forms.TextBox(); + label6 = new System.Windows.Forms.Label(); + SuspendLayout(); + // + // label1 + // + label1.AutoSize = true; + label1.Location = new System.Drawing.Point(80, 63); + label1.Name = "label1"; + label1.Size = new System.Drawing.Size(47, 15); + label1.TabIndex = 0; + label1.Text = "Name *"; + // + // tbName + // + tbName.Location = new System.Drawing.Point(80, 81); + tbName.Name = "tbName"; + tbName.Size = new System.Drawing.Size(320, 23); + tbName.TabIndex = 1; + // + // columnButtonRenderer1 + // + columnButtonRenderer1.ButtonPadding = new System.Drawing.Size(10, 10); + // + // tbRegexPattern + // + tbRegexPattern.Location = new System.Drawing.Point(80, 134); + tbRegexPattern.Name = "tbRegexPattern"; + tbRegexPattern.Size = new System.Drawing.Size(320, 23); + tbRegexPattern.TabIndex = 4; + // + // label2 + // + label2.AutoSize = true; + label2.Location = new System.Drawing.Point(80, 116); + label2.Name = "label2"; + label2.Size = new System.Drawing.Size(88, 15); + label2.TabIndex = 3; + label2.Text = "Regex Pattern *"; + // + // tbRedactionString + // + tbRedactionString.Location = new System.Drawing.Point(80, 193); + tbRedactionString.Name = "tbRedactionString"; + tbRedactionString.Size = new System.Drawing.Size(320, 23); + tbRedactionString.TabIndex = 6; + // + // label3 + // + label3.AutoSize = true; + label3.Location = new System.Drawing.Point(80, 175); + label3.Name = "label3"; + label3.Size = new System.Drawing.Size(118, 15); + label3.TabIndex = 5; + label3.Text = "Replacement String *"; + // + // tbDescription + // + tbDescription.Location = new System.Drawing.Point(80, 244); + tbDescription.Name = "tbDescription"; + tbDescription.Size = new System.Drawing.Size(320, 23); + tbDescription.TabIndex = 8; + // + // label4 + // + label4.AutoSize = true; + label4.Location = new System.Drawing.Point(80, 226); + label4.Name = "label4"; + label4.Size = new System.Drawing.Size(67, 15); + label4.TabIndex = 7; + label4.Text = "Description"; + // + // label5 + // + label5.AutoSize = true; + label5.Location = new System.Drawing.Point(338, 333); + label5.Name = "label5"; + label5.Size = new System.Drawing.Size(62, 15); + label5.TabIndex = 9; + label5.Text = "* Required"; + label5.Click += label5_Click; + // + // tbFolder + // + tbFolder.Location = new System.Drawing.Point(80, 298); + tbFolder.Name = "tbFolder"; + tbFolder.Size = new System.Drawing.Size(320, 23); + tbFolder.TabIndex = 11; + // + // label6 + // + label6.AutoSize = true; + label6.Location = new System.Drawing.Point(80, 280); + label6.Name = "label6"; + label6.Size = new System.Drawing.Size(51, 15); + label6.TabIndex = 10; + label6.Text = "Folder * "; + // + // RegexRedactionConfigurationUI + // + Controls.Add(tbFolder); + Controls.Add(label6); + Controls.Add(label5); + Controls.Add(tbDescription); + Controls.Add(label4); + Controls.Add(tbRedactionString); + Controls.Add(label3); + Controls.Add(tbRegexPattern); + Controls.Add(label2); + Controls.Add(tbName); + Controls.Add(label1); + Name = "RegexRedactionConfigurationUI"; + Size = new System.Drawing.Size(800, 450); + ResumeLayout(false); + PerformLayout(); + } + + #endregion + + private System.Windows.Forms.Label label1; + private System.Windows.Forms.TextBox tbName; + private BrightIdeasSoftware.ColumnButtonRenderer columnButtonRenderer1; + private System.Windows.Forms.TextBox tbRegexPattern; + private System.Windows.Forms.Label label2; + private System.Windows.Forms.TextBox tbRedactionString; + private System.Windows.Forms.Label label3; + private System.Windows.Forms.TextBox tbDescription; + private System.Windows.Forms.Label label4; + private System.Windows.Forms.Label label5; + private System.Windows.Forms.TextBox tbFolder; + private System.Windows.Forms.Label label6; +} diff --git a/Rdmp.UI/SubComponents/RegexRedactionConfigurationUI.cs b/Rdmp.UI/SubComponents/RegexRedactionConfigurationUI.cs new file mode 100644 index 0000000000..a18c217c90 --- /dev/null +++ b/Rdmp.UI/SubComponents/RegexRedactionConfigurationUI.cs @@ -0,0 +1,63 @@ +using Rdmp.UI.TestsAndSetup.ServicePropogation; +using System; +using System.ComponentModel; +using System.Linq; +using System.Windows.Forms; +using Rdmp.UI.Refreshing; +using Rdmp.Core.Dataset; +using Rdmp.Core.Curation.Data; +using Rdmp.UI.ItemActivation; +using Rdmp.Core.Curation.DataHelper.RegexRedaction; +using Microsoft.Data.SqlClient; +using System.Text.RegularExpressions; +using System.Drawing; +using Rdmp.Core.MapsDirectlyToDatabaseTable; + +namespace Rdmp.UI.SubComponents; +public partial class RegexRedactionConfigurationUI : RegexRedactionConfigurationUI_Design, IRefreshBusSubscriber +{ + public RegexRedactionConfigurationUI() + { + InitializeComponent(); + } + + public override void SetDatabaseObject(IActivateItems activator, RegexRedactionConfiguration databaseObject) + { + base.SetDatabaseObject(activator, databaseObject); + + Bind(tbName, "Text", "Name", static c => c.Name); + Bind(tbRegexPattern, "Text", "RegexPattern", static c => c.RegexPattern); + Bind(tbRedactionString, "Text", "RedactionString", static c => c.RedactionString); + Bind(tbDescription, "Text", "Description", static c => c.Description); + Bind(tbFolder, "Text", "Folder", static c => c.Folder); + var s = GetObjectSaverButton(); + s.SetupFor(this, databaseObject, activator); + GetObjectSaverButton()?.Enable(false); + } + + + public void RefreshBus_RefreshObject(object sender, RefreshObjectEventArgs e) + { + } + + private void label1_Click(object sender, EventArgs e) + { + + } + + private void label4_Click(object sender, EventArgs e) + { + + } + + private void label5_Click(object sender, EventArgs e) + { + + } +} +[TypeDescriptionProvider( + typeof(AbstractControlDescriptionProvider))] +public abstract class + RegexRedactionConfigurationUI_Design : RDMPSingleDatabaseObjectControl +{ +} \ No newline at end of file diff --git a/Rdmp.UI/SubComponents/RegexRedactionConfigurationUI.resx b/Rdmp.UI/SubComponents/RegexRedactionConfigurationUI.resx new file mode 100644 index 0000000000..5fab7db9b7 --- /dev/null +++ b/Rdmp.UI/SubComponents/RegexRedactionConfigurationUI.resx @@ -0,0 +1,123 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + 17, 17 + + \ No newline at end of file From f0ee020431fc8d6957255b066abd65bfdb3a4201 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 9 Sep 2024 16:03:50 +0100 Subject: [PATCH 039/106] working add --- ...mmandIdentifyRegexRedactionsInCatalogue.cs | 8 +++-- .../RegexRedaction/RegexRedactionHelper.cs | 2 +- Rdmp.UI/Menus/CatalogueMenu.cs | 2 +- .../RedactCatalogueUI.Designer.cs | 35 +++++++++++-------- Rdmp.UI/SimpleDialogs/RedactCatalogueUI.cs | 34 ++++++++++++++++-- 5 files changed, 58 insertions(+), 23 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandIdentifyRegexRedactionsInCatalogue.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandIdentifyRegexRedactionsInCatalogue.cs index d5b714dd93..ef89b5714e 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandIdentifyRegexRedactionsInCatalogue.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandIdentifyRegexRedactionsInCatalogue.cs @@ -69,10 +69,12 @@ public override void Execute() using var da = server.GetDataAdapter(cmd); da.Fill(dt); } - dt.Columns.Add("Proposed Redaction"); - foreach(DataRow row in dt.Rows) + dt.Columns.Add("RedactionValue"); + dt.Columns.Add("WhereSQL"); + foreach (DataRow row in dt.Rows) { - row["Proposed Redaction"] = "TODO"; + row["RedactionValue"] = "TODO"; + //row["WhereSQL"] = string.Join(" AND ", row.ItemArray.Skip(1).SkipLast(1).ToArray().Select((ri, index) => $"{_cataloguePKs[index].ColumnInfo.GetFullyQualifiedName()}={ri}")); } results = dt; } diff --git a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs index a9c029c90a..ebd4122e30 100644 --- a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs +++ b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs @@ -86,7 +86,7 @@ public static string GetRedactionValue(string value, ColumnInfo column, DataRow string replacementValue = _redactionConfiguration.RedactionString; var lengthDiff = (float)foundMatch.Length - replacementValue.Length; - if (lengthDiff < 1) + if (lengthDiff < 0) { throw new Exception($"Redaction string '{_redactionConfiguration.RedactionString}' is longer than found match '{foundMatch}'."); } diff --git a/Rdmp.UI/Menus/CatalogueMenu.cs b/Rdmp.UI/Menus/CatalogueMenu.cs index e08d71dd8e..800f099f1a 100644 --- a/Rdmp.UI/Menus/CatalogueMenu.cs +++ b/Rdmp.UI/Menus/CatalogueMenu.cs @@ -24,7 +24,7 @@ internal class CatalogueMenu : RDMPContextMenuStrip public CatalogueMenu(RDMPContextMenuStripArgs args, Catalogue catalogue) : base(args, catalogue) { var isApiCall = catalogue.IsApiCall(); - Add(new ExecuteCommandLinkCatalogueToDatasetUI(_activator, catalogue)); + Add(new ExecuteCommandLinkCatalogueToDatasetUI(_activator, catalogue) { OverrideCommandName="Link Catalogue To Dataset"}); Add(new ExecuteCommandGenerateMetadataReport(_activator, catalogue) { Weight = -99.059f diff --git a/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.Designer.cs b/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.Designer.cs index 026339607f..5eb9f37971 100644 --- a/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.Designer.cs +++ b/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.Designer.cs @@ -71,19 +71,7 @@ private void InitializeComponent() panel1.Name = "panel1"; panel1.Size = new System.Drawing.Size(628, 220); panel1.TabIndex = 18; - // - // folv - // - folv.AllColumns.Add(folvFoundValue); - folv.AllColumns.Add(folvReplacmentValue); - folv.AllColumns.Add(folvRedactButton); - folv.Location = new System.Drawing.Point(13, 201); - folv.Name = "folv"; - folv.ShowGroups = false; - folv.Size = new System.Drawing.Size(612, 287); - folv.TabIndex = 10; - folv.View = System.Windows.Forms.View.Details; - folv.VirtualMode = true; + // // folvFoundValue // @@ -103,11 +91,29 @@ private void InitializeComponent() // // folvRedactButton // - folvRedactButton.AspectName = ""; + folvRedactButton.AspectName = "Redact"; folvRedactButton.FillsFreeSpace = true; folvRedactButton.MinimumWidth = 100; folvRedactButton.Text = "Redact"; folvRedactButton.Width = 100; + folvRedactButton.IsButton = true; + folvRedactButton.ButtonSizing = OLVColumn.ButtonSizingMode.TextBounds; + // + // folv + // + folv.Columns.Add(folvFoundValue); + folv.Columns.Add(folvReplacmentValue); + folv.Columns.Add(folvRedactButton); + folv.AllColumns.Add(folvFoundValue); + folv.AllColumns.Add(folvReplacmentValue); + folv.AllColumns.Add(folvRedactButton); + folv.Location = new System.Drawing.Point(13, 201); + folv.Name = "folv"; + folv.ShowGroups = false; + folv.Size = new System.Drawing.Size(612, 287); + folv.TabIndex = 10; + //folv.View = System.Windows.Forms.View.Details; + //folv.VirtualMode = true; // // label4 // @@ -204,7 +210,6 @@ private void InitializeComponent() btnNewRegex.TabIndex = 11; btnNewRegex.Text = "New Regex Configuration"; btnNewRegex.UseVisualStyleBackColor = true; - btnNewRegex.Click += btnNewRegex_Click; // // RedactCatalogueUI // diff --git a/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.cs b/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.cs index ee490deccb..ab8bce4e80 100644 --- a/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.cs +++ b/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.cs @@ -8,8 +8,10 @@ using System.Collections.Generic; using System.ComponentModel; using System.Data; +using System.Diagnostics; using System.Linq; using System.Windows.Forms; +using BrightIdeasSoftware; using NPOI.SS.Formula.Functions; using Rdmp.Core; using Rdmp.Core.CommandExecution; @@ -18,9 +20,13 @@ using Rdmp.Core.Curation.Data.Governance; using Rdmp.Core.Curation.Data.Remoting; using Rdmp.Core.Curation.DataHelper.RegexRedaction; +using Rdmp.Core.QueryBuilding; +using Rdmp.UI.CommandExecution.AtomicCommands; using Rdmp.UI.ItemActivation; +using Rdmp.UI.MainFormUITabs; using Rdmp.UI.Rules; using Rdmp.UI.SimpleControls; +using Rdmp.UI.SimpleDialogs.RegexRedactionConfigurationForm; using Rdmp.UI.TestsAndSetup.ServicePropogation; @@ -36,7 +42,26 @@ public partial class RedactCatalogueUI : RedactCatalogueUI_Design public RedactCatalogueUI() { InitializeComponent(); + this.comboBox1.Items.Clear(); AssociatedCollection = RDMPCollection.Tables; + this.folv.ButtonClick += Redact; + this.btnNewRegex.Click += HandleNewRegex; + } + + public void HandleNewRegex(object sender, EventArgs e) + { + //_activator + var ui = new CreateNewRegexRedactionConfigurationUI(_activator); + var res = ui.ShowDialog(); + var cmd = new ExecuteCommandRefreshObject(_activator, _catalogue); + cmd.Execute(); + + } + + public void Redact(object sender, CellClickEventArgs e) + { + Debug.WriteLine(String.Format("Button clicked: ({0}, {1}, {2})", e.RowIndex, e.SubItem, e.Model)); + //todo want to run a "redact single" with the PKS } public override void SetDatabaseObject(IActivateItems activator, Catalogue databaseObject) @@ -47,6 +72,7 @@ public override void SetDatabaseObject(IActivateItems activator, Catalogue datab _activator = activator; _catalogue = databaseObject; var regexConfigurations = _activator.RepositoryLocator.CatalogueRepository.GetAllObjects().ToArray(); + this.comboBox1.Items.Clear(); comboBox1.Items.AddRange(regexConfigurations); comboBox1.DisplayMember = "Name"; @@ -79,8 +105,10 @@ private void btnIdentify_Click(object sender, EventArgs e) { object o = new { - FoundValue = row.ItemArray[0], - RedactionValue = row.ItemArray[1] + FoundValue = row[0], + RedactionValue = row["RedactionValue"], + Redact = "Redact", + PKs = row.ItemArray.Skip(1).SkipLast(1).ToArray() }; folv.AddObject(o); } @@ -96,7 +124,7 @@ private void btnRedact_Click(object sender, EventArgs e) { columns.Add(((CatalogueItem)item).ColumnInfo); } - var cmd = new ExecuteCommandPerformRegexRedactionOnCatalogue(_activator, _catalogue, (RegexRedactionConfiguration)comboBox1.SelectedValue, columns, max); + var cmd = new ExecuteCommandPerformRegexRedactionOnCatalogue(_activator, _catalogue, (RegexRedactionConfiguration)comboBox1.SelectedItem, columns, max); cmd.Execute(); //TODO some sort up update to let the user know how many we updated? } From 5c4626eee7134f765f42c3a458a58632baeb7df8 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 10 Sep 2024 10:38:20 +0100 Subject: [PATCH 040/106] user interface --- ...mmandIdentifyRegexRedactionsInCatalogue.cs | 88 ------------ ...CommandPerformRegexRedactionOnCatalogue.cs | 2 + .../IRegexRedactionConfiguration.cs | 2 +- .../RegexRedactionConfiguration.cs | 8 +- .../up/086_AddRegexRedactionConfiguration.sql | 1 - Rdmp.Core/Providers/CatalogueChildProvider.cs | 126 +++++------------- Rdmp.UI/Menus/CatalogueMenu.cs | 2 +- .../RedactCatalogueUI.Designer.cs | 91 ++++++------- Rdmp.UI/SimpleDialogs/RedactCatalogueUI.cs | 58 +++----- ...dactionConfigurationCreationUI.Designer.cs | 59 -------- .../RegexRedactionConfigurationCreationUI.cs | 22 --- ...RegexRedactionConfigurationCreationUI.resx | 120 ----------------- ...wRegexRedactionConfigurationUI.Designer.cs | 30 +---- .../CreateNewRegexRedactionConfigurationUI.cs | 2 - .../DatasetConfigurationUI.Designer.cs | 27 +--- .../SubComponents/DatasetConfigurationUI.cs | 1 - .../RegexRedactionConfigurationUI.Designer.cs | 24 +--- .../RegexRedactionConfigurationUI.cs | 1 - 18 files changed, 103 insertions(+), 561 deletions(-) delete mode 100644 Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandIdentifyRegexRedactionsInCatalogue.cs delete mode 100644 Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationCreationUI.Designer.cs delete mode 100644 Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationCreationUI.cs delete mode 100644 Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationCreationUI.resx diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandIdentifyRegexRedactionsInCatalogue.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandIdentifyRegexRedactionsInCatalogue.cs deleted file mode 100644 index ef89b5714e..0000000000 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandIdentifyRegexRedactionsInCatalogue.cs +++ /dev/null @@ -1,88 +0,0 @@ -using FAnsi.Discovery; -using FAnsi.Discovery.QuerySyntax; -using MongoDB.Driver.Core.Servers; -using Rdmp.Core.Curation.Data; -using Rdmp.Core.Curation.DataHelper.RegexRedaction; -using Rdmp.Core.QueryBuilding; -using Rdmp.Core.Repositories; -using Rdmp.Core.ReusableLibraryCode.DataAccess; -using System; -using System.Collections.Generic; -using System.Data; -using System.Linq; -using System.Text; -using System.Text.RegularExpressions; -using System.Threading.Tasks; - -namespace Rdmp.Core.CommandExecution.AtomicCommands; - -public class ExecuteCommandIdentifyRegexRedactionsInCatalogue : BasicCommandExecution, IAtomicCommand -{ - private readonly IBasicActivateItems _activator; - private readonly ICatalogue _catalogue; - private readonly RegexRedactionConfiguration _redactionConfiguration; - private readonly List _columns; - private readonly int? _readLimit; - public DataTable results; - - public ExecuteCommandIdentifyRegexRedactionsInCatalogue(IBasicActivateItems activator, ICatalogue catalogue, RegexRedactionConfiguration redactionConfiguration, List columns = null, int? readLimit = null) - { - _activator = activator; - _catalogue = catalogue; - _redactionConfiguration = redactionConfiguration; - _columns = columns; - _readLimit = readLimit; - } - - public override void Execute() - { - base.Execute(); - var memoryRepo = new MemoryCatalogueRepository(); - foreach (var columnInfo in _columns.Where(c => !c.IsPrimaryKey)) - { - var server = _catalogue.GetDistinctLiveDatabaseServer(DataAccessContext.InternalDataProcessing, false); - var columnName = columnInfo.Name; - var table = columnInfo.TableInfo.Name; - var _discoveredTable = columnInfo.TableInfo.Discover(DataAccessContext.InternalDataProcessing); - DiscoveredColumn[] discoveredColumns = _discoveredTable.DiscoverColumns(); - var _discoveredPKColumns = discoveredColumns.Where(c => c.IsPrimaryKey).ToArray(); - if (_discoveredPKColumns.Length != 0) - { - var _cataloguePKs = _catalogue.CatalogueItems.Where(c => c.ColumnInfo.IsPrimaryKey).ToList(); - var qb = new QueryBuilder(null, null, null); - qb.AddColumn(new ColumnInfoToIColumn(memoryRepo, columnInfo)); - foreach (var pk in _discoveredPKColumns) - { - var cataloguePk = _cataloguePKs.FirstOrDefault(c => c.ColumnInfo.GetRuntimeName() == pk.GetRuntimeName()); - qb.AddColumn(new ColumnInfoToIColumn(memoryRepo, cataloguePk.ColumnInfo)); - } - qb.AddCustomLine($"{columnInfo.GetRuntimeName()} LIKE '%{_redactionConfiguration.RegexPattern}%'", QueryComponent.WHERE); - if (_readLimit is not null) - { - qb.TopX = (int)_readLimit; - } - var sql = qb.SQL; - var dt = new DataTable(); - dt.BeginLoadData(); - using (var cmd = server.GetCommand(sql, server.GetConnection())) - { - using var da = server.GetDataAdapter(cmd); - da.Fill(dt); - } - dt.Columns.Add("RedactionValue"); - dt.Columns.Add("WhereSQL"); - foreach (DataRow row in dt.Rows) - { - row["RedactionValue"] = "TODO"; - //row["WhereSQL"] = string.Join(" AND ", row.ItemArray.Skip(1).SkipLast(1).ToArray().Select((ri, index) => $"{_cataloguePKs[index].ColumnInfo.GetFullyQualifiedName()}={ri}")); - } - results = dt; - } - else - { - //Unable to find any primary keys - throw new Exception($"Unable to identify any primary keys in table '{table}'. Redactions cannot be performed on tables without primary keys"); - } - } - } -} diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs index 1d8f7254f4..5ea12b8584 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs @@ -34,6 +34,7 @@ public class ExecuteCommandPerformRegexRedactionOnCatalogue : BasicCommandExecut private readonly DataTable pksToSave; private readonly int? _readLimit; private DataTable redactionUpates = new(); + public int resultCount = 0; public ExecuteCommandPerformRegexRedactionOnCatalogue(IBasicActivateItems activator, ICatalogue catalogue, RegexRedactionConfiguration redactionConfiguration, List columns, int? readLimit = null) { @@ -128,5 +129,6 @@ public override void Execute() } timer.Stop(); var x = timer.ElapsedMilliseconds; + resultCount = redactionUpates.Rows.Count; } } diff --git a/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedactionConfiguration.cs b/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedactionConfiguration.cs index a4f795b688..dfa257f5d4 100644 --- a/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedactionConfiguration.cs +++ b/Rdmp.Core/Curation/DataHelper/RegexRedaction/IRegexRedactionConfiguration.cs @@ -12,7 +12,7 @@ namespace Rdmp.Core.Curation.DataHelper.RegexRedaction; /// /// Stores the configuration to run regex redactions on data /// -public interface IRegexRedactionConfiguration: IMapsDirectlyToDatabaseTable, IHasFolder +public interface IRegexRedactionConfiguration: IMapsDirectlyToDatabaseTable { ICatalogueRepository CatalogueRepository { get; } diff --git a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionConfiguration.cs b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionConfiguration.cs index 6f26724cd2..ffa9140532 100644 --- a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionConfiguration.cs +++ b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionConfiguration.cs @@ -26,9 +26,6 @@ public class RegexRedactionConfiguration : DatabaseEntity, IRegexRedactionConfig private string _redactionString; private string _name; private string _description; - private string _folder; - - public override string ToString() { @@ -63,7 +60,6 @@ public string RedactionString get => _redactionString; set => SetField(ref _redactionString, value); } - public string Folder { get => _folder; set => SetField(ref _folder, value); } public RegexRedactionConfiguration() { } @@ -74,8 +70,7 @@ public RegexRedactionConfiguration(ICatalogueRepository repository, string name, { "Name", name }, { "RegexPattern", regexPattern.ToString() }, {"RedactionString", redactionString }, - {"Description",description }, - {"Folder",folder } + {"Description",description } }); } @@ -85,7 +80,6 @@ internal RegexRedactionConfiguration(ICatalogueRepository repository, DbDataRead Description = r["Description"].ToString(); RedactionString = r["RedactionString"].ToString(); RegexPattern = r["RegexPattern"].ToString(); - Folder = r["Folder"].ToString(); } diff --git a/Rdmp.Core/Databases/CatalogueDatabase/up/086_AddRegexRedactionConfiguration.sql b/Rdmp.Core/Databases/CatalogueDatabase/up/086_AddRegexRedactionConfiguration.sql index 65140dc004..21b8cf216a 100644 --- a/Rdmp.Core/Databases/CatalogueDatabase/up/086_AddRegexRedactionConfiguration.sql +++ b/Rdmp.Core/Databases/CatalogueDatabase/up/086_AddRegexRedactionConfiguration.sql @@ -10,7 +10,6 @@ CREATE TABLE [dbo].RegexRedactionConfiguration( Description [nvarchar](250), RegexPattern [nvarchar](250) NOT NULL, RedactionString [nvarchar](250) NOT NULL, - Folder [nvarchar](max) NOT NULL DEFAULT '\' CONSTRAINT [PK_RegexRedactionConfiguration] PRIMARY KEY CLUSTERED ( [ID] ASC diff --git a/Rdmp.Core/Providers/CatalogueChildProvider.cs b/Rdmp.Core/Providers/CatalogueChildProvider.cs index 3b010b9b90..c45b086f6f 100644 --- a/Rdmp.Core/Providers/CatalogueChildProvider.cs +++ b/Rdmp.Core/Providers/CatalogueChildProvider.cs @@ -7,7 +7,6 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; -using System.Data; using System.Diagnostics; using System.Linq; using System.Threading.Tasks; @@ -68,8 +67,6 @@ public class CatalogueChildProvider : ICoreChildProvider //Catalogue side of things public Catalogue[] AllCatalogues { get; set; } public Curation.Data.Dataset[] AllDatasets { get; set; } - - public RegexRedactionConfiguration[] AllRegexRedactionConfigurations { get; set; } public Dictionary AllCataloguesDictionary { get; private set; } public SupportingDocument[] AllSupportingDocuments { get; set; } @@ -95,10 +92,6 @@ public class CatalogueChildProvider : ICoreChildProvider public AggregateContinuousDateAxis[] AllAggregateContinuousDateAxis { get; private set; } public AllRDMPRemotesNode AllRDMPRemotesNode { get; private set; } - - public AllDatasetsNode AllDatasetsNode { get; private set; } - public AllRegexRedactionConfigurationsNode AllRegexRedactionConfigurationsNode { get; private set; } - public RemoteRDMP[] AllRemoteRDMPs { get; set; } public AllDashboardsNode AllDashboardsNode { get; set; } @@ -151,7 +144,6 @@ public class CatalogueChildProvider : ICoreChildProvider public FolderNode LoadMetadataRootFolder { get; set; } public FolderNode DatasetRootFolder { get; set; } - public FolderNode RegexRedactionConfigurationRootFolder { get; set; } public FolderNode CohortIdentificationConfigurationRootFolder { get; set; } public FolderNode CohortIdentificationConfigurationRootFolderWithoutVersionedConfigurations { get; set; } @@ -213,6 +205,11 @@ public class CatalogueChildProvider : ICoreChildProvider public AllTemplateAggregateConfigurationsNode TemplateAggregateConfigurationsNode { get; set; } = new(); public FolderNode CatalogueRootFolder { get; private set; } + public AllDatasetsNode AllDatasetsNode { get; private set; } + + public RegexRedactionConfiguration[] AllRegexRedactionConfigurations { get; private set; } + public AllRegexRedactionConfigurationsNode AllRegexRedactionConfigurationsNode { get; private set; } + public HashSet OrphanAggregateConfigurations; public AggregateConfiguration[] TemplateAggregateConfigurations; @@ -257,7 +254,7 @@ public CatalogueChildProvider(ICatalogueRepository repository, IChildProvider[] AllCataloguesDictionary = AllCatalogues.ToDictionaryEx(i => i.ID, o => o); AllDatasets = GetAllObjects(repository); - AllRegexRedactionConfigurations = GetAllObjects(repository); + AllLoadMetadatas = GetAllObjects(repository); AllLoadMetadataLinkage = GetAllObjects(repository); AllProcessTasks = GetAllObjects(repository); @@ -398,9 +395,6 @@ public CatalogueChildProvider(ICatalogueRepository repository, IChildProvider[] DatasetRootFolder = FolderHelper.BuildFolderTree(AllDatasets); AddChildren(DatasetRootFolder, new DescendancyList(DatasetRootFolder)); - RegexRedactionConfigurationRootFolder = FolderHelper.BuildFolderTree(AllRegexRedactionConfigurations); - AddChildren(RegexRedactionConfigurationRootFolder, new DescendancyList(RegexRedactionConfigurationRootFolder)); - ReportProgress("Build Catalogue Folder Root"); LoadMetadataRootFolder = FolderHelper.BuildFolderTree(AllLoadMetadatas); @@ -423,10 +417,6 @@ public CatalogueChildProvider(ICatalogueRepository repository, IChildProvider[] TemplateAggregateConfigurations = AllAggregateConfigurations .Where(ac => templateAggregateConfigurationIds.Contains(ac.ID)).ToArray(); - - BuildDatasetNodes(); - BuildAllRegexRedactionConfigurationsNode(); - //add the orphans under the orphan folder AddToDictionaries(new HashSet(OrphanAggregateConfigurations), new DescendancyList(OrphanAggregateConfigurationsNode)); @@ -460,6 +450,17 @@ public CatalogueChildProvider(ICatalogueRepository repository, IChildProvider[] ReportProgress("After Plugins"); + AllRegexRedactionConfigurations = GetAllObjects(repository); + AllRegexRedactionConfigurationsNode = new AllRegexRedactionConfigurationsNode(); + AddChildren(AllRegexRedactionConfigurationsNode); + + + AllDatasets = GetAllObjects(repository); + AllDatasetsNode = new AllDatasetsNode(); + AddChildren(AllDatasetsNode); + + ReportProgress("After Configurations"); + var searchables = new Dictionary>(); foreach (var o in _descendancyDictionary.Keys.OfType()) @@ -605,6 +606,20 @@ private void AddChildren(AllPluginsNode allPluginsNode) AddToDictionaries(children, descendancy); } + private void AddChildren(AllRegexRedactionConfigurationsNode allRegexRedactionConfigurationsNode) + { + var children = new HashSet(AllRegexRedactionConfigurations); + var descendancy = new DescendancyList(allRegexRedactionConfigurationsNode); + AddToDictionaries(children, descendancy); + } + + private void AddChildren(AllDatasetsNode allDatasetsNode) + { + var children = new HashSet(AllDatasets); + var descendancy = new DescendancyList(allDatasetsNode); + AddToDictionaries(children, descendancy); + } + private void AddChildren(AllGovernanceNode allGovernanceNode) { var children = new HashSet(); @@ -763,28 +778,6 @@ private void AddChildren(PipelineComponent pipelineComponent, DescendancyList de AddToDictionaries(children, descendancy); } - - private void BuildAllRegexRedactionConfigurationsNode() - { - AllRegexRedactionConfigurationsNode = new AllRegexRedactionConfigurationsNode(); - var descendancy = new DescendancyList(AllRegexRedactionConfigurationsNode); - //foreach (var regex in AllRegexRedactionConfigurations) - //{ - // AddChildren(regex, descendancy.Add(regex)); - //} - //AddChildren(AllRegexRedactionConfigurationsNode); - AddChildren(RegexRedactionConfigurationRootFolder, descendancy); - } - - - private void BuildDatasetNodes() - { - AllDatasetsNode = new AllDatasetsNode(); - var descendancy = new DescendancyList(AllDatasetsNode); - AddChildren(DatasetRootFolder, descendancy); - - } - private void BuildServerNodes() { //add a root node for all the servers to be children of @@ -840,31 +833,6 @@ private void AddChildren(AllANOTablesNode anoTablesNode) AddToDictionaries(new HashSet(AllANOTables), new DescendancyList(anoTablesNode)); } - private void AddChildren(AllRegexRedactionConfigurationsNode allRegexRedactionConfigurationsNode) - { - AddToDictionaries(new HashSet(AllRegexRedactionConfigurations), new DescendancyList(allRegexRedactionConfigurationsNode)); - - } - - private void AddChildren(AllDatasetsNode allDatasetsNode) - { - AddToDictionaries(new HashSet(AllDatasets), new DescendancyList(allDatasetsNode)); - - } - - private void Addchildren(FolderNode folder, DescendancyList descendancy) - { - foreach (var child in folder.ChildFolders) - //add subfolder children - AddChildren(child, descendancy.Add(child)); - foreach (var c in folder.ChildObjects) AddChildren(c, descendancy.Add(c)); - AddToDictionaries(new HashSet( - folder.ChildFolders.Cast() - .Union(folder.ChildObjects)), descendancy - ); - - } - private void AddChildren(FolderNode folder, DescendancyList descendancy) { foreach (var child in folder.ChildFolders) @@ -896,26 +864,8 @@ private void AddChildren(FolderNode folder, DescendancyList descen ); } - private void AddChildren(FolderNode folder, DescendancyList descendancy) - { - if (folder is null) return; - foreach (var child in folder.ChildFolders) - //add subfolder children - AddChildren(child, descendancy.Add(child)); - - //add loads in folder - foreach (var ds in folder.ChildObjects) AddChildren(ds, descendancy.Add(ds)); - - //// Children are the folders + objects - AddToDictionaries(new HashSet( - folder.ChildFolders.Cast() - .Union(folder.ChildObjects)), descendancy - ); - } - private void AddChildren(FolderNode folder, DescendancyList descendancy) { - if (folder is null) return; foreach (var child in folder.ChildFolders) //add subfolder children AddChildren(child, descendancy.Add(child)); @@ -947,19 +897,9 @@ private void AddChildren(FolderNode folder, D ); } - private void AddChildren(RegexRedactionConfiguration regex, DescendancyList descendancy) - { - var childObjects = new List() { regex }; - AddToDictionaries(new HashSet(childObjects), descendancy); - } - - private void AddChildren(Curation.Data.Dataset dataset, DescendancyList descendancy) + private void AddChildren(Curation.Data.Dataset lmd, DescendancyList descendancy) { - var childObjects = new List - { - dataset - }; - + var childObjects = new List(); AddToDictionaries(new HashSet(childObjects), descendancy); } diff --git a/Rdmp.UI/Menus/CatalogueMenu.cs b/Rdmp.UI/Menus/CatalogueMenu.cs index 800f099f1a..dcf3aec406 100644 --- a/Rdmp.UI/Menus/CatalogueMenu.cs +++ b/Rdmp.UI/Menus/CatalogueMenu.cs @@ -74,7 +74,7 @@ public CatalogueMenu(RDMPContextMenuStripArgs args, Catalogue catalogue) : base( Add(new ExecuteCommandReOrderColumns(_activator, catalogue) { SuggestedCategory = CatalogueItems, Weight = -99.046f }); Add(new ExecuteCommandRegexRedaction(_activator, catalogue) - { SuggestedCategory = CatalogueItems, Weight = -99.046f, OverrideCommandName="Perform Regex Redaction" }); + { SuggestedCategory = CatalogueItems, Weight = -99.046f, OverrideCommandName="Regex Redactions" }); Add(new ExecuteCommandGuessAssociatedColumns(_activator, catalogue, null) { SuggestedCategory = CatalogueItems, Weight = -99.045f, PromptForPartialMatching = true }); Add(new ExecuteCommandChangeExtractionCategory(_activator, diff --git a/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.Designer.cs b/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.Designer.cs index 5eb9f37971..6b266c7fda 100644 --- a/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.Designer.cs +++ b/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.Designer.cs @@ -32,6 +32,7 @@ protected override void Dispose(bool disposing) private void InitializeComponent() { panel1 = new System.Windows.Forms.Panel(); + btnNewRegex = new System.Windows.Forms.Button(); folv = new FastObjectListView(); folvFoundValue = new OLVColumn(); folvReplacmentValue = new OLVColumn(); @@ -39,14 +40,12 @@ private void InitializeComponent() label4 = new System.Windows.Forms.Label(); tbMaxCount = new System.Windows.Forms.TextBox(); btnRedact = new System.Windows.Forms.Button(); - btnIdentify = new System.Windows.Forms.Button(); label3 = new System.Windows.Forms.Label(); checkedListBox1 = new System.Windows.Forms.CheckedListBox(); comboBox1 = new System.Windows.Forms.ComboBox(); label2 = new System.Windows.Forms.Label(); tbCatalogueName = new System.Windows.Forms.TextBox(); label1 = new System.Windows.Forms.Label(); - btnNewRegex = new System.Windows.Forms.Button(); panel1.SuspendLayout(); ((System.ComponentModel.ISupportInitialize)folv).BeginInit(); SuspendLayout(); @@ -58,7 +57,6 @@ private void InitializeComponent() panel1.Controls.Add(label4); panel1.Controls.Add(tbMaxCount); panel1.Controls.Add(btnRedact); - panel1.Controls.Add(btnIdentify); panel1.Controls.Add(label3); panel1.Controls.Add(checkedListBox1); panel1.Controls.Add(comboBox1); @@ -69,56 +67,63 @@ private void InitializeComponent() panel1.Location = new System.Drawing.Point(0, 0); panel1.Margin = new System.Windows.Forms.Padding(4, 3, 4, 3); panel1.Name = "panel1"; - panel1.Size = new System.Drawing.Size(628, 220); + panel1.Size = new System.Drawing.Size(682, 556); panel1.TabIndex = 18; - + // + // btnNewRegex + // + btnNewRegex.Location = new System.Drawing.Point(388, 43); + btnNewRegex.Name = "btnNewRegex"; + btnNewRegex.Size = new System.Drawing.Size(154, 23); + btnNewRegex.TabIndex = 11; + btnNewRegex.Text = "New Regex Configuration"; + btnNewRegex.UseVisualStyleBackColor = true; + // + // folv + // + folv.AllColumns.Add(folvFoundValue); + folv.AllColumns.Add(folvReplacmentValue); + folv.AllColumns.Add(folvRedactButton); + folv.Columns.AddRange(new System.Windows.Forms.ColumnHeader[] { folvFoundValue, folvReplacmentValue, folvRedactButton }); + folv.Location = new System.Drawing.Point(13, 201); + folv.Name = "folv"; + folv.ShowGroups = false; + folv.Size = new System.Drawing.Size(612, 287); + folv.TabIndex = 10; + folv.View = System.Windows.Forms.View.Details; + folv.VirtualMode = true; // // folvFoundValue // - folvFoundValue.AspectName = "FoundValue"; + folvFoundValue.AspectName = "RedactedValue"; folvFoundValue.FillsFreeSpace = true; folvFoundValue.MinimumWidth = 100; - folvFoundValue.Text = "FoundValue"; + folvFoundValue.Text = "Redacted Value"; folvFoundValue.Width = 100; // // folvReplacmentValue // - folvReplacmentValue.AspectName = "RedactionValue"; + folvReplacmentValue.AspectName = "ReplacementValue"; folvReplacmentValue.FillsFreeSpace = true; folvReplacmentValue.MinimumWidth = 100; - folvReplacmentValue.Text = "Redaction Value"; + folvReplacmentValue.Text = "Replacement Value"; folvReplacmentValue.Width = 100; // // folvRedactButton // - folvRedactButton.AspectName = "Redact"; + folvRedactButton.AspectName = "ID"; + folvRedactButton.ButtonSizing = OLVColumn.ButtonSizingMode.CellBounds; folvRedactButton.FillsFreeSpace = true; + folvRedactButton.IsButton = true; folvRedactButton.MinimumWidth = 100; - folvRedactButton.Text = "Redact"; + folvRedactButton.Text = "Restore"; folvRedactButton.Width = 100; - folvRedactButton.IsButton = true; - folvRedactButton.ButtonSizing = OLVColumn.ButtonSizingMode.TextBounds; - // - // folv - // - folv.Columns.Add(folvFoundValue); - folv.Columns.Add(folvReplacmentValue); - folv.Columns.Add(folvRedactButton); - folv.AllColumns.Add(folvFoundValue); - folv.AllColumns.Add(folvReplacmentValue); - folv.AllColumns.Add(folvRedactButton); - folv.Location = new System.Drawing.Point(13, 201); - folv.Name = "folv"; - folv.ShowGroups = false; - folv.Size = new System.Drawing.Size(612, 287); - folv.TabIndex = 10; - //folv.View = System.Windows.Forms.View.Details; - //folv.VirtualMode = true; + folvRedactButton.AspectGetter = delegate { return "Restore"; }; // // label4 // label4.AutoSize = true; - label4.Location = new System.Drawing.Point(509, 176); + label4.Location = new System.Drawing.Point(325, 176); label4.Name = "label4"; label4.Size = new System.Drawing.Size(33, 15); label4.TabIndex = 9; @@ -126,14 +131,14 @@ private void InitializeComponent() // // tbMaxCount // - tbMaxCount.Location = new System.Drawing.Point(548, 172); + tbMaxCount.Location = new System.Drawing.Point(364, 172); tbMaxCount.Name = "tbMaxCount"; tbMaxCount.Size = new System.Drawing.Size(77, 23); tbMaxCount.TabIndex = 8; // // btnRedact // - btnRedact.Location = new System.Drawing.Point(373, 172); + btnRedact.Location = new System.Drawing.Point(208, 171); btnRedact.Name = "btnRedact"; btnRedact.Size = new System.Drawing.Size(113, 23); btnRedact.TabIndex = 7; @@ -141,16 +146,6 @@ private void InitializeComponent() btnRedact.UseVisualStyleBackColor = true; btnRedact.Click += btnRedact_Click; // - // btnIdentify - // - btnIdentify.Location = new System.Drawing.Point(223, 172); - btnIdentify.Name = "btnIdentify"; - btnIdentify.Size = new System.Drawing.Size(144, 23); - btnIdentify.TabIndex = 6; - btnIdentify.Text = "Identify Redactions"; - btnIdentify.UseVisualStyleBackColor = true; - btnIdentify.Click += btnIdentify_Click; - // // label3 // label3.AutoSize = true; @@ -202,15 +197,6 @@ private void InitializeComponent() label1.TabIndex = 0; label1.Text = "Catalogue:"; // - // btnNewRegex - // - btnNewRegex.Location = new System.Drawing.Point(388, 43); - btnNewRegex.Name = "btnNewRegex"; - btnNewRegex.Size = new System.Drawing.Size(154, 23); - btnNewRegex.TabIndex = 11; - btnNewRegex.Text = "New Regex Configuration"; - btnNewRegex.UseVisualStyleBackColor = true; - // // RedactCatalogueUI // AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); @@ -218,7 +204,7 @@ private void InitializeComponent() Controls.Add(panel1); Margin = new System.Windows.Forms.Padding(4, 3, 4, 3); Name = "RedactCatalogueUI"; - Size = new System.Drawing.Size(628, 220); + Size = new System.Drawing.Size(682, 556); panel1.ResumeLayout(false); panel1.PerformLayout(); ((System.ComponentModel.ISupportInitialize)folv).EndInit(); @@ -236,7 +222,6 @@ private void InitializeComponent() private System.Windows.Forms.Label label4; private System.Windows.Forms.TextBox tbMaxCount; private System.Windows.Forms.Button btnRedact; - private System.Windows.Forms.Button btnIdentify; private BrightIdeasSoftware.FastObjectListView folv; private OLVColumn folvFoundValue; private OLVColumn folvReplacmentValue; diff --git a/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.cs b/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.cs index ab8bce4e80..3f08f84576 100644 --- a/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.cs +++ b/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.cs @@ -46,6 +46,7 @@ public RedactCatalogueUI() AssociatedCollection = RDMPCollection.Tables; this.folv.ButtonClick += Redact; this.btnNewRegex.Click += HandleNewRegex; + } public void HandleNewRegex(object sender, EventArgs e) @@ -55,13 +56,25 @@ public void HandleNewRegex(object sender, EventArgs e) var res = ui.ShowDialog(); var cmd = new ExecuteCommandRefreshObject(_activator, _catalogue); cmd.Execute(); + } + public void RefreshTable() + { + var columnIds = _catalogue.CatalogueItems.Select(c => c.ColumnInfo_ID).ToList(); + var objects = _activator.RepositoryLocator.CatalogueRepository.GetAllObjects().Where(r => columnIds.Contains(r.ColumnInfo_ID)).ToList(); + folv.ClearObjects(); + folv.AddObjects(objects); } public void Redact(object sender, CellClickEventArgs e) { Debug.WriteLine(String.Format("Button clicked: ({0}, {1}, {2})", e.RowIndex, e.SubItem, e.Model)); - //todo want to run a "redact single" with the PKS + var redaction= (e.Model as RegexRedaction); + var cmd = new ExecuteCommandRestoreRegexRedactedValueInCatalogue(_activator,redaction); + Cursor.Current = Cursors.WaitCursor; + cmd.Execute(); + Cursor.Current = Cursors.Default; + RefreshTable(); } public override void SetDatabaseObject(IActivateItems activator, Catalogue databaseObject) @@ -75,48 +88,14 @@ public override void SetDatabaseObject(IActivateItems activator, Catalogue datab this.comboBox1.Items.Clear(); comboBox1.Items.AddRange(regexConfigurations); comboBox1.DisplayMember = "Name"; - + RefreshTable(); var nonPKColumns = databaseObject.CatalogueItems.Where(c => !c.ColumnInfo.IsPrimaryKey).ToArray(); checkedListBox1.Items.AddRange(nonPKColumns); } - private void btnNewRegex_Click(object sender, EventArgs e) - { - var form = new RegexRedactionConfigurationCreationUI(_activator); - form.ShowDialog(); - } - private void btnIdentify_Click(object sender, EventArgs e) - { - int? max = string.IsNullOrWhiteSpace(tbMaxCount.Text) ? null : int.Parse(tbMaxCount.Text); - var columns = new List(); - foreach (object item in checkedListBox1.CheckedItems) - { - columns.Add(((CatalogueItem)item).ColumnInfo); - } - var config = comboBox1.SelectedItem as RegexRedactionConfiguration; - var cmd = new ExecuteCommandIdentifyRegexRedactionsInCatalogue(_activator, _catalogue, config, columns, max); - cmd.Execute(); - if(cmd.results is null) - { - //Do Something! - return; - } - foreach(DataRow row in cmd.results.Rows) - { - object o = new - { - FoundValue = row[0], - RedactionValue = row["RedactionValue"], - Redact = "Redact", - PKs = row.ItemArray.Skip(1).SkipLast(1).ToArray() - }; - folv.AddObject(o); - } - } - private void btnRedact_Click(object sender, EventArgs e) { - if (_activator.YesNo("Are you sure?", "TODO")) + if (_activator.YesNo("Are you sure you want to preform redactions on the selected columns?", "Redact All matches in this catalogue")) { int? max = string.IsNullOrWhiteSpace(tbMaxCount.Text) ? null : int.Parse(tbMaxCount.Text); var columns = new List(); @@ -125,8 +104,11 @@ private void btnRedact_Click(object sender, EventArgs e) columns.Add(((CatalogueItem)item).ColumnInfo); } var cmd = new ExecuteCommandPerformRegexRedactionOnCatalogue(_activator, _catalogue, (RegexRedactionConfiguration)comboBox1.SelectedItem, columns, max); + Cursor.Current = Cursors.WaitCursor; cmd.Execute(); - //TODO some sort up update to let the user know how many we updated? + Cursor.Current = Cursors.Default; + _activator.Show($"Performed {cmd.resultCount} Redactions."); + RefreshTable(); } } diff --git a/Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationCreationUI.Designer.cs b/Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationCreationUI.Designer.cs deleted file mode 100644 index 73da22aa53..0000000000 --- a/Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationCreationUI.Designer.cs +++ /dev/null @@ -1,59 +0,0 @@ -namespace Rdmp.UI.SimpleDialogs -{ - partial class RegexRedactionConfigurationCreationUI - { - /// - /// Required designer variable. - /// - private System.ComponentModel.IContainer components = null; - - /// - /// Clean up any resources being used. - /// - /// true if managed resources should be disposed; otherwise, false. - protected override void Dispose(bool disposing) - { - if (disposing && (components != null)) - { - components.Dispose(); - } - base.Dispose(disposing); - } - - #region Windows Form Designer generated code - - /// - /// Required method for Designer support - do not modify - /// the contents of this method with the code editor. - /// - private void InitializeComponent() - { - label1 = new System.Windows.Forms.Label(); - SuspendLayout(); - // - // label1 - // - label1.AutoSize = true; - label1.Location = new System.Drawing.Point(305, 184); - label1.Name = "label1"; - label1.Size = new System.Drawing.Size(38, 15); - label1.TabIndex = 0; - label1.Text = "label1"; - // - // RegexRedactionConfigurationCreationUI - // - AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); - AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; - ClientSize = new System.Drawing.Size(800, 450); - Controls.Add(label1); - Name = "RegexRedactionConfigurationCreationUI"; - Text = "Create New "; - ResumeLayout(false); - PerformLayout(); - } - - #endregion - - private System.Windows.Forms.Label label1; - } -} \ No newline at end of file diff --git a/Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationCreationUI.cs b/Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationCreationUI.cs deleted file mode 100644 index e3052d50ea..0000000000 --- a/Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationCreationUI.cs +++ /dev/null @@ -1,22 +0,0 @@ -using Rdmp.Core.CommandExecution; -using Rdmp.UI.TestsAndSetup.ServicePropogation; -using System; -using System.Collections.Generic; -using System.ComponentModel; -using System.Data; -using System.Drawing; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using System.Windows.Forms; - -namespace Rdmp.UI.SimpleDialogs -{ - public partial class RegexRedactionConfigurationCreationUI : Form - { - public RegexRedactionConfigurationCreationUI(IBasicActivateItems activator) - { - InitializeComponent(); - } - } -} diff --git a/Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationCreationUI.resx b/Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationCreationUI.resx deleted file mode 100644 index af32865ec1..0000000000 --- a/Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationCreationUI.resx +++ /dev/null @@ -1,120 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - text/microsoft-resx - - - 2.0 - - - System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - \ No newline at end of file diff --git a/Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationForm/CreateNewRegexRedactionConfigurationUI.Designer.cs b/Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationForm/CreateNewRegexRedactionConfigurationUI.Designer.cs index 48a044f19d..7ae56eb119 100644 --- a/Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationForm/CreateNewRegexRedactionConfigurationUI.Designer.cs +++ b/Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationForm/CreateNewRegexRedactionConfigurationUI.Designer.cs @@ -40,8 +40,6 @@ private void InitializeComponent() btnCancel = new System.Windows.Forms.Button(); lblError = new System.Windows.Forms.Label(); label5 = new System.Windows.Forms.Label(); - tbFolder = new System.Windows.Forms.TextBox(); - label6 = new System.Windows.Forms.Label(); SuspendLayout(); // // label1 @@ -110,7 +108,7 @@ private void InitializeComponent() // // btnCreate // - btnCreate.Location = new System.Drawing.Point(471, 351); + btnCreate.Location = new System.Drawing.Point(476, 288); btnCreate.Name = "btnCreate"; btnCreate.Size = new System.Drawing.Size(75, 23); btnCreate.TabIndex = 8; @@ -119,7 +117,7 @@ private void InitializeComponent() // // btnCancel // - btnCancel.Location = new System.Drawing.Point(390, 351); + btnCancel.Location = new System.Drawing.Point(395, 288); btnCancel.Name = "btnCancel"; btnCancel.Size = new System.Drawing.Size(75, 23); btnCancel.TabIndex = 9; @@ -129,7 +127,7 @@ private void InitializeComponent() // lblError // lblError.AutoSize = true; - lblError.Location = new System.Drawing.Point(30, 351); + lblError.Location = new System.Drawing.Point(35, 288); lblError.Name = "lblError"; lblError.Size = new System.Drawing.Size(39, 15); lblError.TabIndex = 10; @@ -138,35 +136,17 @@ private void InitializeComponent() // label5 // label5.AutoSize = true; - label5.Location = new System.Drawing.Point(487, 329); + label5.Location = new System.Drawing.Point(492, 266); label5.Name = "label5"; label5.Size = new System.Drawing.Size(62, 15); label5.TabIndex = 11; label5.Text = "* Required"; // - // tbFolder - // - tbFolder.Location = new System.Drawing.Point(33, 291); - tbFolder.Name = "tbFolder"; - tbFolder.Size = new System.Drawing.Size(516, 23); - tbFolder.TabIndex = 13; - // - // label6 - // - label6.AutoSize = true; - label6.Location = new System.Drawing.Point(27, 274); - label6.Name = "label6"; - label6.Size = new System.Drawing.Size(48, 15); - label6.TabIndex = 12; - label6.Text = "Folder *"; - // // CreateNewRegexRedactionConfigurationUI // AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; ClientSize = new System.Drawing.Size(800, 450); - Controls.Add(tbFolder); - Controls.Add(label6); Controls.Add(label5); Controls.Add(lblError); Controls.Add(btnCancel); @@ -199,7 +179,5 @@ private void InitializeComponent() private System.Windows.Forms.Button btnCancel; private System.Windows.Forms.Label lblError; private System.Windows.Forms.Label label5; - private System.Windows.Forms.TextBox tbFolder; - private System.Windows.Forms.Label label6; } } \ No newline at end of file diff --git a/Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationForm/CreateNewRegexRedactionConfigurationUI.cs b/Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationForm/CreateNewRegexRedactionConfigurationUI.cs index f33cb98048..064e88ad77 100644 --- a/Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationForm/CreateNewRegexRedactionConfigurationUI.cs +++ b/Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationForm/CreateNewRegexRedactionConfigurationUI.cs @@ -32,7 +32,6 @@ public CreateNewRegexRedactionConfigurationUI(IActivateItems activator) : base(a this.btnCreate.Click += Create; this.lblError.Text = ""; this.lblError.Visible = false; - this.tbFolder.Text = "\\"; } private void Cancel(object sender, EventArgs e) @@ -42,7 +41,6 @@ private void Cancel(object sender, EventArgs e) private void Create(object sender, EventArgs e) { var config = new RegexRedactionConfiguration(_activator.RepositoryLocator.CatalogueRepository, tbName.Text, new Regex(tbRegexPattern.Text), tbRedactionString.Text, tbDescription.Text); - config.Folder = string.IsNullOrWhiteSpace(tbFolder.Text) ? "\\" : tbFolder.Text; config.SaveToDatabase(); _activator.Publish(config); Close(); diff --git a/Rdmp.UI/SubComponents/DatasetConfigurationUI.Designer.cs b/Rdmp.UI/SubComponents/DatasetConfigurationUI.Designer.cs index df11a5de7d..b251b593bc 100644 --- a/Rdmp.UI/SubComponents/DatasetConfigurationUI.Designer.cs +++ b/Rdmp.UI/SubComponents/DatasetConfigurationUI.Designer.cs @@ -34,8 +34,6 @@ private void InitializeComponent() tbSource = new System.Windows.Forms.TextBox(); label2 = new System.Windows.Forms.Label(); label3 = new System.Windows.Forms.Label(); - label4 = new System.Windows.Forms.Label(); - tbFolder = new System.Windows.Forms.TextBox(); lblDatasetUsage = new System.Windows.Forms.Label(); SuspendLayout(); // @@ -88,23 +86,6 @@ private void InitializeComponent() label3.TabIndex = 5; label3.Text = "Dataset Source"; // - // label4 - // - label4.AutoSize = true; - label4.Location = new System.Drawing.Point(28, 282); - label4.Name = "label4"; - label4.Size = new System.Drawing.Size(82, 15); - label4.TabIndex = 6; - label4.Text = "Dataset Folder"; - label4.Click += label4_Click; - // - // tbFolder - // - tbFolder.Location = new System.Drawing.Point(24, 314); - tbFolder.Name = "tbFolder"; - tbFolder.Size = new System.Drawing.Size(273, 23); - tbFolder.TabIndex = 7; - // // lblDatasetUsage // lblDatasetUsage.AutoSize = true; @@ -114,20 +95,18 @@ private void InitializeComponent() lblDatasetUsage.TabIndex = 8; lblDatasetUsage.Text = "label5"; // - // DatsetConfigurationUI + // DatasetConfigurationUI // AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; Controls.Add(lblDatasetUsage); - Controls.Add(tbFolder); - Controls.Add(label4); Controls.Add(label3); Controls.Add(label2); Controls.Add(tbSource); Controls.Add(tbDOI); Controls.Add(tbName); Controls.Add(label1); - Name = "DatsetConfigurationUI"; + Name = "DatasetConfigurationUI"; Size = new System.Drawing.Size(800, 450); ResumeLayout(false); PerformLayout(); @@ -141,7 +120,5 @@ private void InitializeComponent() private System.Windows.Forms.TextBox tbSource; private System.Windows.Forms.Label label2; private System.Windows.Forms.Label label3; - private System.Windows.Forms.Label label4; - private System.Windows.Forms.TextBox tbFolder; private System.Windows.Forms.Label lblDatasetUsage; } diff --git a/Rdmp.UI/SubComponents/DatasetConfigurationUI.cs b/Rdmp.UI/SubComponents/DatasetConfigurationUI.cs index 1e90f872f6..dc7315fb89 100644 --- a/Rdmp.UI/SubComponents/DatasetConfigurationUI.cs +++ b/Rdmp.UI/SubComponents/DatasetConfigurationUI.cs @@ -40,7 +40,6 @@ public override void SetDatabaseObject(IActivateItems activator, Dataset databas Bind(tbName, "Text", "Name", static c => c.Name); Bind(tbDOI, "Text", "DigitalObjectIdentifier", static c => c.DigitalObjectIdentifier); Bind(tbSource, "Text", "Source", static c => c.Source); - Bind(tbFolder, "Text", "Folder", static c => c.Folder); var s = GetObjectSaverButton(); s.SetupFor(this, databaseObject, activator); GetObjectSaverButton()?.Enable(false); diff --git a/Rdmp.UI/SubComponents/RegexRedactionConfigurationUI.Designer.cs b/Rdmp.UI/SubComponents/RegexRedactionConfigurationUI.Designer.cs index 666d3bc5df..c836da8a7c 100644 --- a/Rdmp.UI/SubComponents/RegexRedactionConfigurationUI.Designer.cs +++ b/Rdmp.UI/SubComponents/RegexRedactionConfigurationUI.Designer.cs @@ -38,8 +38,6 @@ private void InitializeComponent() tbDescription = new System.Windows.Forms.TextBox(); label4 = new System.Windows.Forms.Label(); label5 = new System.Windows.Forms.Label(); - tbFolder = new System.Windows.Forms.TextBox(); - label6 = new System.Windows.Forms.Label(); SuspendLayout(); // // label1 @@ -113,33 +111,15 @@ private void InitializeComponent() // label5 // label5.AutoSize = true; - label5.Location = new System.Drawing.Point(338, 333); + label5.Location = new System.Drawing.Point(338, 270); label5.Name = "label5"; label5.Size = new System.Drawing.Size(62, 15); label5.TabIndex = 9; label5.Text = "* Required"; label5.Click += label5_Click; // - // tbFolder - // - tbFolder.Location = new System.Drawing.Point(80, 298); - tbFolder.Name = "tbFolder"; - tbFolder.Size = new System.Drawing.Size(320, 23); - tbFolder.TabIndex = 11; - // - // label6 - // - label6.AutoSize = true; - label6.Location = new System.Drawing.Point(80, 280); - label6.Name = "label6"; - label6.Size = new System.Drawing.Size(51, 15); - label6.TabIndex = 10; - label6.Text = "Folder * "; - // // RegexRedactionConfigurationUI // - Controls.Add(tbFolder); - Controls.Add(label6); Controls.Add(label5); Controls.Add(tbDescription); Controls.Add(label4); @@ -167,6 +147,4 @@ private void InitializeComponent() private System.Windows.Forms.TextBox tbDescription; private System.Windows.Forms.Label label4; private System.Windows.Forms.Label label5; - private System.Windows.Forms.TextBox tbFolder; - private System.Windows.Forms.Label label6; } diff --git a/Rdmp.UI/SubComponents/RegexRedactionConfigurationUI.cs b/Rdmp.UI/SubComponents/RegexRedactionConfigurationUI.cs index a18c217c90..67ad0992f1 100644 --- a/Rdmp.UI/SubComponents/RegexRedactionConfigurationUI.cs +++ b/Rdmp.UI/SubComponents/RegexRedactionConfigurationUI.cs @@ -29,7 +29,6 @@ public override void SetDatabaseObject(IActivateItems activator, RegexRedactionC Bind(tbRegexPattern, "Text", "RegexPattern", static c => c.RegexPattern); Bind(tbRedactionString, "Text", "RedactionString", static c => c.RedactionString); Bind(tbDescription, "Text", "Description", static c => c.Description); - Bind(tbFolder, "Text", "Folder", static c => c.Folder); var s = GetObjectSaverButton(); s.SetupFor(this, databaseObject, activator); GetObjectSaverButton()?.Enable(false); From 2f5bd0ab68db0f73490178780ad232cf1fd5be7a Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 10 Sep 2024 16:17:00 +0100 Subject: [PATCH 041/106] update for tests --- Rdmp.Core/Providers/CatalogueChildProvider.cs | 6 +++--- Rdmp.Core/Providers/DataExportChildProvider.cs | 3 +++ Rdmp.UI.Tests/ChildProviderTests.cs | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Rdmp.Core/Providers/CatalogueChildProvider.cs b/Rdmp.Core/Providers/CatalogueChildProvider.cs index c45b086f6f..fd83e6369e 100644 --- a/Rdmp.Core/Providers/CatalogueChildProvider.cs +++ b/Rdmp.Core/Providers/CatalogueChildProvider.cs @@ -205,10 +205,10 @@ public class CatalogueChildProvider : ICoreChildProvider public AllTemplateAggregateConfigurationsNode TemplateAggregateConfigurationsNode { get; set; } = new(); public FolderNode CatalogueRootFolder { get; private set; } - public AllDatasetsNode AllDatasetsNode { get; private set; } + public AllDatasetsNode AllDatasetsNode { get; set; } - public RegexRedactionConfiguration[] AllRegexRedactionConfigurations { get; private set; } - public AllRegexRedactionConfigurationsNode AllRegexRedactionConfigurationsNode { get; private set; } + public RegexRedactionConfiguration[] AllRegexRedactionConfigurations { get; set; } + public AllRegexRedactionConfigurationsNode AllRegexRedactionConfigurationsNode { get; set; } public HashSet OrphanAggregateConfigurations; public AggregateConfiguration[] TemplateAggregateConfigurations; diff --git a/Rdmp.Core/Providers/DataExportChildProvider.cs b/Rdmp.Core/Providers/DataExportChildProvider.cs index 2d36cec369..1d410285e6 100644 --- a/Rdmp.Core/Providers/DataExportChildProvider.cs +++ b/Rdmp.Core/Providers/DataExportChildProvider.cs @@ -800,6 +800,9 @@ public override void UpdateTo(ICoreChildProvider other) _extractionProgressesBySelectedDataSetID = dxOther._extractionProgressesBySelectedDataSetID; ProjectRootFolder = dxOther.ProjectRootFolder; DatasetRootFolder = dxOther.DatasetRootFolder; + AllDatasetsNode = dxOther.AllDatasetsNode; + AllRegexRedactionConfigurations = dxOther.AllRegexRedactionConfigurations; + AllRegexRedactionConfigurationsNode = dxOther.AllRegexRedactionConfigurationsNode; } } diff --git a/Rdmp.UI.Tests/ChildProviderTests.cs b/Rdmp.UI.Tests/ChildProviderTests.cs index 4e7ac61cce..a6bb56095e 100644 --- a/Rdmp.UI.Tests/ChildProviderTests.cs +++ b/Rdmp.UI.Tests/ChildProviderTests.cs @@ -107,7 +107,7 @@ public void TestUpTo() Assert.That(prop.GetValue(cp2), Is.SameAs(prop.GetValue(cp1)), $"Prop {prop} was not the same between child providers - after UpdateTo"); } - catch (Exception) + catch (Exception e) { badProps.Add(prop.Name); } From 49bf660c1f8f64686feb5beb19f72caee55c48f3 Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 12 Sep 2024 09:34:43 +0100 Subject: [PATCH 042/106] update test --- Directory.Packages.props | 8 ++--- .../RegexRedactionMutilatorTests.cs | 32 +++++++++++++++++-- Rdmp.UI.Tests/ChildProviderTests.cs | 2 +- 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 34de41fbc2..17b777c0cf 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -1,9 +1,9 @@ - - - - + + + + diff --git a/Rdmp.Core.Tests/DataLoad/Modules/Mutilators/RegexRedactionMutilatorTests.cs b/Rdmp.Core.Tests/DataLoad/Modules/Mutilators/RegexRedactionMutilatorTests.cs index 33cfc213e7..774a3e9601 100644 --- a/Rdmp.Core.Tests/DataLoad/Modules/Mutilators/RegexRedactionMutilatorTests.cs +++ b/Rdmp.Core.Tests/DataLoad/Modules/Mutilators/RegexRedactionMutilatorTests.cs @@ -51,6 +51,10 @@ public void RedactionMutilator_BasicTest() if (raw.Exists()) raw.Drop(); + var dleStaging = db.Server.ExpectDatabase($"DLE_STAGING"); + if (!dleStaging.Exists()) + db.Server.CreateDatabase("DLE_STAGING"); + using var dt = new DataTable("MyTable"); dt.Columns.Add("Name"); dt.Columns.Add("DateOfBirth"); @@ -84,7 +88,7 @@ public void RedactionMutilator_BasicTest() var regex = new RegexRedactionConfiguration(CatalogueRepository, "No Yelling", new Regex("Yella"), "FFF"); regex.SaveToDatabase(); - var pt = new ProcessTask(CatalogueRepository, lmd, LoadStage.AdjustRaw) + var pt = new ProcessTask(CatalogueRepository, lmd, LoadStage.AdjustStaging) { Path = typeof(RegexRedactionMutilator).FullName, ProcessTaskType = ProcessTaskType.MutilateDataTable, @@ -132,8 +136,6 @@ public void RedactionMutilator_BasicTest() var redactionKeys = CatalogueRepository.GetAllObjects(); Assert.That(redactionKeys.Length, Is.EqualTo(2)); - Assert.That(redactionKeys[0].Value == "MrMurder"); - Assert.That(redactionKeys[1].Value == "01/01/2001 00:00:00"); foreach (var r in redactions) { r.DeleteInDatabase(); @@ -156,6 +158,10 @@ public void RedactionMutilator_OddStringLength() if (raw.Exists()) raw.Drop(); + var dleStaging = db.Server.ExpectDatabase($"DLE_STAGING"); + if (!dleStaging.Exists()) + db.Server.CreateDatabase("DLE_STAGING"); + using var dt = new DataTable("MyTable"); dt.Columns.Add("Name"); dt.Columns.Add("DateOfBirth"); @@ -261,6 +267,10 @@ public void RedactionMutilator_RedactionTooLong() if (raw.Exists()) raw.Drop(); + var dleStaging = db.Server.ExpectDatabase($"DLE_STAGING"); + if (!dleStaging.Exists()) + db.Server.CreateDatabase("DLE_STAGING"); + using var dt = new DataTable("MyTable"); dt.Columns.Add("Name"); dt.Columns.Add("DateOfBirth"); @@ -352,6 +362,10 @@ public void RedactionMutilator_RedactAPK() if (raw.Exists()) raw.Drop(); + var dleStaging = db.Server.ExpectDatabase($"DLE_STAGING"); + if (!dleStaging.Exists()) + db.Server.CreateDatabase("DLE_STAGING"); + using var dt = new DataTable("MyTable"); dt.Columns.Add("Name"); dt.Columns.Add("DateOfBirth"); @@ -443,6 +457,10 @@ public void RedactionMutilator_NoPKS() if (raw.Exists()) raw.Drop(); + var dleStaging = db.Server.ExpectDatabase($"DLE_STAGING"); + if (!dleStaging.Exists()) + db.Server.CreateDatabase("DLE_STAGING"); + using var dt = new DataTable("MyTable"); dt.Columns.Add("Name"); dt.Columns.Add("DateOfBirth"); @@ -534,6 +552,10 @@ public void RedactionMutilator_MultipleInOneCell() if (raw.Exists()) raw.Drop(); + var dleStaging = db.Server.ExpectDatabase($"DLE_STAGING"); + if (!dleStaging.Exists()) + db.Server.CreateDatabase("DLE_STAGING"); + using var dt = new DataTable("MyTable"); dt.Columns.Add("Name"); dt.Columns.Add("DateOfBirth"); @@ -645,6 +667,10 @@ public void RedactionMutilator_SpeedTest() if (raw.Exists()) raw.Drop(); + var dleStaging = db.Server.ExpectDatabase($"DLE_STAGING"); + if (!dleStaging.Exists()) + db.Server.CreateDatabase("DLE_STAGING"); + using var dt = new DataTable("MyTable"); dt.Columns.Add("Name"); dt.Columns.Add("DateOfBirth"); diff --git a/Rdmp.UI.Tests/ChildProviderTests.cs b/Rdmp.UI.Tests/ChildProviderTests.cs index a6bb56095e..4e7ac61cce 100644 --- a/Rdmp.UI.Tests/ChildProviderTests.cs +++ b/Rdmp.UI.Tests/ChildProviderTests.cs @@ -107,7 +107,7 @@ public void TestUpTo() Assert.That(prop.GetValue(cp2), Is.SameAs(prop.GetValue(cp1)), $"Prop {prop} was not the same between child providers - after UpdateTo"); } - catch (Exception e) + catch (Exception) { badProps.Add(prop.Name); } From c110420ac8a607b6c008c3e4bddc7bf2f06bbe8b Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 12 Sep 2024 10:24:26 +0100 Subject: [PATCH 043/106] tests --- .../Modules/Mutilators/RegexRedactionMutilatorTests.cs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Rdmp.Core.Tests/DataLoad/Modules/Mutilators/RegexRedactionMutilatorTests.cs b/Rdmp.Core.Tests/DataLoad/Modules/Mutilators/RegexRedactionMutilatorTests.cs index 774a3e9601..fd4207f1d4 100644 --- a/Rdmp.Core.Tests/DataLoad/Modules/Mutilators/RegexRedactionMutilatorTests.cs +++ b/Rdmp.Core.Tests/DataLoad/Modules/Mutilators/RegexRedactionMutilatorTests.cs @@ -243,8 +243,6 @@ public void RedactionMutilator_OddStringLength() var redactionKeys = CatalogueRepository.GetAllObjects(); Assert.That(redactionKeys.Length, Is.EqualTo(2)); - Assert.That(redactionKeys[0].Value == "MrMurder"); - Assert.That(redactionKeys[1].Value == "01/01/2001 00:00:00"); foreach (var r in redactions) { r.DeleteInDatabase(); @@ -641,10 +639,6 @@ public void RedactionMutilator_MultipleInOneCell() var redactionKeys = CatalogueRepository.GetAllObjects(); Assert.That(redactionKeys.Length, Is.EqualTo(4)); - Assert.That(redactionKeys[0].Value == "MrMurder"); - Assert.That(redactionKeys[1].Value == "01/01/2001 00:00:00"); - Assert.That(redactionKeys[2].Value == "MrMurder"); - Assert.That(redactionKeys[3].Value == "01/01/2001 00:00:00"); foreach (var r in redactions) { r.DeleteInDatabase(); From 26b8008583449edca7ce675c7d74fed162c9f3f2 Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 12 Sep 2024 13:24:13 +0100 Subject: [PATCH 044/106] tidy up --- ...storeRegexRedactedValueInCatalogueTests.cs | 5 -- .../RegexRedactionMutilatorTests.cs | 13 ----- ...ommandAddNewRegexRedactionConfiguration.cs | 5 -- ...CommandPerformRegexRedactionOnCatalogue.cs | 5 -- ...andRestoreRegexRedactedValueInCatalogue.cs | 3 -- .../RegexRedactionConfiguration.cs | 1 - .../RegexRedaction/RegexRedactionHelper.cs | 26 --------- .../CreateCatalogue.sql | 54 +++++++++++++++++++ 8 files changed, 54 insertions(+), 58 deletions(-) diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandRestoreRegexRedactedValueInCatalogueTests.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandRestoreRegexRedactedValueInCatalogueTests.cs index 9791b82393..ca0919130a 100644 --- a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandRestoreRegexRedactedValueInCatalogueTests.cs +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandRestoreRegexRedactedValueInCatalogueTests.cs @@ -6,15 +6,10 @@ using Rdmp.Core.Curation.DataHelper.RegexRedaction; using Rdmp.Core.Curation; using Rdmp.Core.ReusableLibraryCode.Checks; -using System; -using System.Collections.Generic; using System.Data; using System.Linq; -using System.Text; using System.Text.RegularExpressions; -using System.Threading.Tasks; using Tests.Common; -using Rdmp.Core.Validation.Constraints.Secondary.Predictor; namespace Rdmp.Core.Tests.CommandExecution; diff --git a/Rdmp.Core.Tests/DataLoad/Modules/Mutilators/RegexRedactionMutilatorTests.cs b/Rdmp.Core.Tests/DataLoad/Modules/Mutilators/RegexRedactionMutilatorTests.cs index fd4207f1d4..9c5bc19428 100644 --- a/Rdmp.Core.Tests/DataLoad/Modules/Mutilators/RegexRedactionMutilatorTests.cs +++ b/Rdmp.Core.Tests/DataLoad/Modules/Mutilators/RegexRedactionMutilatorTests.cs @@ -1,37 +1,24 @@ using FAnsi; using FAnsi.Discovery; -using NPOI.SS.Formula.Functions; -using NSubstitute; using NUnit.Framework; -using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.Data.DataLoad; using Rdmp.Core.Curation.Data.Defaults; using Rdmp.Core.Curation.DataHelper.RegexRedaction; using Rdmp.Core.DataFlowPipeline; -using Rdmp.Core.DataLoad; -using Rdmp.Core.DataLoad.Engine.Checks.Checkers; using Rdmp.Core.DataLoad.Engine.DatabaseManagement.EntityNaming; using Rdmp.Core.DataLoad.Engine.Job; using Rdmp.Core.DataLoad.Engine.LoadExecution; -using Rdmp.Core.DataLoad.Engine.LoadExecution.Components.Standard; using Rdmp.Core.DataLoad.Engine.LoadProcess; -using Rdmp.Core.DataLoad.Modules.Attachers; using Rdmp.Core.DataLoad.Modules.Mutilators; using Rdmp.Core.Logging; using Rdmp.Core.ReusableLibraryCode.Checks; using Rdmp.Core.ReusableLibraryCode.Progress; using Rdmp.Core.Tests.DataLoad.Engine.Integration; -using Rdmp.Core.Tests.DataLoad.Engine.Integration.CrossDatabaseTypeTests; using System; -using System.Collections.Generic; using System.Data; -using System.Diagnostics; using System.IO; using System.Linq; -using System.Text; using System.Text.RegularExpressions; -using System.Threading.Tasks; -using Tests.Common; using TypeGuesser; namespace Rdmp.Core.Tests.DataLoad.Modules.Mutilators; diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewRegexRedactionConfiguration.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewRegexRedactionConfiguration.cs index 59eac6c9be..16a1c74fe1 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewRegexRedactionConfiguration.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewRegexRedactionConfiguration.cs @@ -2,11 +2,6 @@ using Rdmp.Core.ReusableLibraryCode.Icons.IconProvision; using SixLabors.ImageSharp.PixelFormats; using SixLabors.ImageSharp; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; using Rdmp.Core.Curation.DataHelper.RegexRedaction; using System.Text.RegularExpressions; using Rdmp.Core.Curation.Data; diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs index 5ea12b8584..ebd090e4e9 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs @@ -1,21 +1,16 @@ using FAnsi.Discovery; using FAnsi.Discovery.QuerySyntax; -using Microsoft.Data.SqlClient; using MongoDB.Driver; using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.DataHelper.RegexRedaction; -using Rdmp.Core.MapsDirectlyToDatabaseTable; using Rdmp.Core.QueryBuilding; using Rdmp.Core.Repositories; using Rdmp.Core.ReusableLibraryCode.DataAccess; using System; using System.Collections.Generic; using System.Data; -using System.Data.Common; using System.Diagnostics; using System.Linq; -using System.Text.RegularExpressions; -using Terminal.Gui; using static System.Linq.Enumerable; namespace Rdmp.Core.CommandExecution.AtomicCommands; diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreRegexRedactedValueInCatalogue.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreRegexRedactedValueInCatalogue.cs index 572389e634..96ac89741c 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreRegexRedactedValueInCatalogue.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreRegexRedactedValueInCatalogue.cs @@ -1,6 +1,5 @@ using FAnsi.Discovery; using FAnsi.Discovery.QuerySyntax; -using MongoDB.Driver.Core.Servers; using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.DataHelper.RegexRedaction; using Rdmp.Core.QueryBuilding; @@ -10,8 +9,6 @@ using System.Collections.Generic; using System.Data; using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace Rdmp.Core.CommandExecution.AtomicCommands { diff --git a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionConfiguration.cs b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionConfiguration.cs index ffa9140532..d7a800b74d 100644 --- a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionConfiguration.cs +++ b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionConfiguration.cs @@ -11,7 +11,6 @@ using Rdmp.Core.Repositories; using System.Collections.Generic; using System.Data.Common; -using Rdmp.Core.MapsDirectlyToDatabaseTable; namespace Rdmp.Core.Curation.DataHelper.RegexRedaction; diff --git a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs index ebd4122e30..b200011ca4 100644 --- a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs +++ b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs @@ -1,45 +1,19 @@ using FAnsi.Discovery.QuerySyntax; using FAnsi.Discovery; -using Microsoft.Data.SqlClient; -using MongoDB.Driver.Core.Servers; -using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation.Data; using Rdmp.Core.MapsDirectlyToDatabaseTable; using System; using System.Collections.Generic; using System.Data; using System.Linq; -using System.Text; using System.Text.RegularExpressions; -using System.Threading.Tasks; -using MongoDB.Driver; using Rdmp.Core.Repositories; -using NPOI.SS.Formula.Functions; -using Rdmp.Core.QueryBuilding; namespace Rdmp.Core.Curation.DataHelper.RegexRedaction { public static class RegexRedactionHelper { - //public static QueryBuilder SQLLikeQuery(ColumnInfo columnInfo, CatalogueItem[] pkCatalogueItems, RegexRedactionConfiguration regexConfiguration,int? readLimit) - //{ - // var memoryRepo = new MemoryCatalogueRepository(); - // var qb = new QueryBuilder(null, null, null); - // qb.AddColumn(new ColumnInfoToIColumn(memoryRepo, columnInfo)); - // foreach(var pk in pkCatalogueItems) - // { - // qb.AddColumn(new ColumnInfoToIColumn(memoryRepo, pk.ColumnInfo)); - // } - // qb.AddCustomLine($"{columnInfo.GetRuntimeName()} LIKE '%{regexConfiguration.RegexPattern}%'", QueryComponent.WHERE); - // if (readLimit is not null) - // { - // qb.TopX = (int)readLimit; - // } - - // return qb; - //} - public static DataTable GenerateRedactionsDataTable() { DataTable redactionsToSaveTable = new DataTable(); diff --git a/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql b/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql index 7e4bc622d0..cf847986e0 100644 --- a/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql +++ b/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql @@ -1344,6 +1344,60 @@ CONSTRAINT [UNIQUE_SettingKey] UNIQUE([Key]), ) ON [PRIMARY] GO + +if not exists (select 1 from sys.tables where name = 'RegexRedactionConfiguration') +BEGIN +CREATE TABLE [dbo].RegexRedactionConfiguration( + [ID] [int] IDENTITY(1,1) NOT NULL, + Name [nvarchar](250) NOT NULL, + Description [nvarchar](250), + RegexPattern [nvarchar](250) NOT NULL, + RedactionString [nvarchar](250) NOT NULL, +CONSTRAINT [PK_RegexRedactionConfiguration] PRIMARY KEY CLUSTERED +( + [ID] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +END +GO + +if not exists (select 1 from sys.tables where name = 'RegexRedaction') +BEGIN +CREATE TABLE [dbo].RegexRedaction( + [ID] [int] IDENTITY(1,1) NOT NULL, + RedactionConfiguration_ID [int] NOT NULL, + ColumnInfo_ID [int] NOT NULL, + StartingIndex [int] NOT NULL, + RedactedValue [nvarchar](250), + ReplacementValue [nvarchar](250) NOT NULL, + CONSTRAINT FK_Redaction_RedactionConfiguration_ID FOREIGN KEY (RedactionConfiguration_ID) REFERENCES RegexRedactionConfiguration(ID), + CONSTRAINT FK_Redaction_ColumnInfo_ID FOREIGN KEY (ColumnInfo_ID) REFERENCES ColumnInfo(ID) ON DELETE CASCADE, +CONSTRAINT [PK_RegexRedaction] PRIMARY KEY CLUSTERED +( + [ID] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +END +GO + +if not exists (select 1 from sys.tables where name = 'RegexRedactionKey') +BEGIN +CREATE TABLE [dbo].RegexRedactionKey( + [ID] [int] IDENTITY(1,1) NOT NULL, + RegexRedaction_ID [int] NOT NULL, + ColumnInfo_ID [int] NOT NULL, + Value [nvarchar](max), + CONSTRAINT FK_RedactionKey_Redaction_ID FOREIGN KEY (RegexRedaction_ID) REFERENCES RegexRedaction(ID) ON DELETE CASCADE, + CONSTRAINT FK_RedactionKey_ColumnInfo_ID FOREIGN KEY (ColumnInfo_ID) REFERENCES ColumnInfo(ID), +CONSTRAINT [PK_RegexRedactionKey] PRIMARY KEY CLUSTERED +( + [ID] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +END +GO + + EXEC sys.sp_addextendedproperty @name=N'MS_Description', @value=N'Table ID' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'Catalogue', @level2type=N'COLUMN',@level2name=N'ID' GO EXEC sys.sp_addextendedproperty @name=N'MS_Description', @value=N'‘SMR01’ for example' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'Catalogue', @level2type=N'COLUMN',@level2name=N'Acronym' From d50c5fc46359f5b5b9038b7cfda659182bcc41d8 Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 13 Sep 2024 14:00:36 +0100 Subject: [PATCH 045/106] updates --- ...CommandPerformRegexRedactionOnCatalogue.cs | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs index ebd090e4e9..443e7a26d2 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs @@ -25,8 +25,8 @@ public class ExecuteCommandPerformRegexRedactionOnCatalogue : BasicCommandExecut private DiscoveredColumn[] _discoveredPKColumns; private DiscoveredTable _discoveredTable; private List _cataloguePKs; - private readonly DataTable redactionsToSaveTable; - private readonly DataTable pksToSave; + private DataTable redactionsToSaveTable; + private DataTable pksToSave; private readonly int? _readLimit; private DataTable redactionUpates = new(); public int resultCount = 0; @@ -39,8 +39,6 @@ public ExecuteCommandPerformRegexRedactionOnCatalogue(IBasicActivateItems activa _columns = columns; _server = _catalogue.GetDistinctLiveDatabaseServer(DataAccessContext.InternalDataProcessing, false); _readLimit = readLimit; - redactionsToSaveTable = RegexRedactionHelper.GenerateRedactionsDataTable(); - pksToSave = RegexRedactionHelper.GeneratePKDataTable(); } public override void Execute() @@ -51,7 +49,8 @@ public override void Execute() var timer = Stopwatch.StartNew(); foreach (var columnInfo in _columns.Where(c => !c.IsPrimaryKey)) { - + redactionsToSaveTable = RegexRedactionHelper.GenerateRedactionsDataTable(); + pksToSave = RegexRedactionHelper.GeneratePKDataTable(); var columnName = columnInfo.Name; var table = columnInfo.TableInfo.Name; _discoveredTable = columnInfo.TableInfo.Discover(DataAccessContext.InternalDataProcessing); @@ -100,7 +99,17 @@ public override void Execute() { pksToSave.Rows[i]["ID"] = i + 1; } + var existing = _discoveredTable.Database.ExpectTable("pksToSave_Temp"); + if (existing.Exists()) + { + existing.Drop(); + } var t1 = _discoveredTable.Database.CreateTable("pksToSave_Temp", pksToSave); + existing = _discoveredTable.Database.ExpectTable("redactionsToSaveTable_Temp"); + if (existing.Exists()) + { + existing.Drop(); + } var t2 = _discoveredTable.Database.CreateTable("redactionsToSaveTable_Temp", redactionsToSaveTable); RegexRedactionHelper.SaveRedactions(_activator.RepositoryLocator.CatalogueRepository, t1, t2, _server); t1.Drop(); @@ -121,9 +130,10 @@ public override void Execute() { throw new Exception($"Unable to identify any primary keys in table '{table}'. Redactions cannot be performed on tables without primary keys"); } + resultCount += redactionUpates.Rows.Count; + } timer.Stop(); var x = timer.ElapsedMilliseconds; - resultCount = redactionUpates.Rows.Count; } } From bb23800dca5fa409bd486cde7f405f98b814c8b1 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 16 Sep 2024 09:57:42 +0100 Subject: [PATCH 046/106] fix merge --- .../Icons/IconProvision/CatalogueIcons.resx | 1429 ----------------- 1 file changed, 1429 deletions(-) diff --git a/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx b/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx index 43a287352b..4684396b1f 100644 --- a/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx +++ b/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx @@ -59,1434 +59,6 @@ : using a System.ComponentModel.TypeConverter : and then encoded with base64 encoding. --> -<<<<<<< HEAD - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - text/microsoft-resx - - - 2.0 - - - System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - - ..\..\Icons\columninfo.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\DQE.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\Lookup.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\ExecuteArrow.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\executeArrowWithTickOverlay.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\spanner.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\tableinfo.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\backup.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\sync.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\StarHollow.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\loadMetadata.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\logging.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\importFile.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\Catalogue.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\Filter.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\ExtractionInformation.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\FilterContainer.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\cohortAggregate.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\greenIssue.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\orangeIssue.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\redIssue.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\Clipboard.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\patientIndexTable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\expandAllNodes.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\ExtractionInformation_SpecialApproval.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\ExtractionInformation_Supplemental.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\AggregatesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\catalogueItemsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\BigCohort.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\BigGraph.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\BigPatientIndexTable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\cohortIdentificationConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\frozenCohortIdentificationConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\supportingDocument.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\documentationNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\supportingDocumentExtractable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\EXCEPT.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\INTERSECT.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\supportingDocumentExtractableGlobal.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\supportingDocumentGlobal.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\supportingSqlExtractableGlobal.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\supportingSqlGlobal.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\UNION.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\collisionResolution.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\AllServersNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\tableinfoTableValuedFunction.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AggregateGraph.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CatalogueFolder.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CatalogueItem.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CatalogueItemTransform.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CatalogueItemIssue.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\DataAccessCredentials.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllDataAccessCredentialsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Database.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractionDirectoryNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\SQL.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\supportingSqlExtractable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ANOColumnInfo.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CloneExtractionConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\EmptyProject.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractableDataSetDisabled.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractionConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Failed.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\FrozenExtractionConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Project.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Release.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\SqlThenVSNow.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ANOTable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllANOTablesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractableCohort.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractableDataSet.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\JoinableCollectionNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ParametersNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\SupportingSQLTable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\TableInfoServerNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\TimeCoverageField.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExternalCohortTable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractableDataSetPackage.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractableDataSetsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\PivotField.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ProjectsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AggregateDimension.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\DashboardControl.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\DashboardLayout.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ReOrder.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\JoinInfo.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\StandardRegex.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Favourite.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractionFilterParameterSet.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AutomationServiceSlot.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CacheProgress.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExternalDatabaseServer.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExternalDatabaseServer_ANO.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExternalDatabaseServer_Cache.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExternalDatabaseServer_DQE.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExternalDatabaseServer_IdentifierDump.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExternalDatabaseServer_Logging.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Plugin.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Kill.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\EXCEPTCohortAggregate.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\INTERSECTCohortAggregate.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\UNIONCohortAggregate.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\RowCounts_Ignore.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\RowCounts_Respect.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\NoIconAvailable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\File.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Help.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\UnfreezeExtractionConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllCataloguesUsedByLoadMetadataNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllProcessTasksUsedByLoadMetadataNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\LoadDirectoryNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\LoadBubble.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\LoadFinalDatabase.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Exe.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\LoadMetadataScheduleNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\LoadPeriodically.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\LoadProgress.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\GetFilesStage.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\LoadBubbleMounting.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\PreLoadDiscardedColumn.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\famfamfam\Tick.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Warning.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllExternalServersNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\DecryptionPrivateKeyNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\PreLoadDiscardedColumnsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\PermissionWindow.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Pipeline.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractionConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllCohortsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ProjectCohortIdentificationConfigurationAssociationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ProjectSavedCohortsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\collapseAllNodes.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllAutomationServerSlotsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllRDMPRemotesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\RemoteRDMP.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AutomateablePipeline.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\PipelineComponent.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\PipelineComponentSource.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\PipelineComponentDestination.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\PipelineComponentArgument.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AggregateTopX.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AutomationServiceException.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllObjectExportsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllObjectImportsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ObjectExport.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ObjectImport.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllObjectSharingNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CohortCustomColumn.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractableColumn.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ProjectCohortsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\FrozenExtractionConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\TinyGreen.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\TinyRed.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\TinyYellow.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractionInformation_ProjectSpecific.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ProjectCatalogue.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ProjectCataloguesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\DropHere.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\MakeProjectSpecificCatalogueNormalAgain.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllPermissionWindowsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Waiting.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\WaitingForDatabase.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Writing.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\FileMissing.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllConnectionStringKeywordsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ConnectionStringKeyword.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllStandardRegexesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllPipelinesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllDatasetsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllRegexRedactionConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\StandardPipelineUseCaseNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\OtherPipelinesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllGovernanceNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\GovernancePeriod.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\GovernanceDocument.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\LookupCompositeJoinInfo.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllFreeCohortIdentificationConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllProjectCohortIdentificationConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Menu.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CatalogueLookupsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\DatabaseRefresh.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Diff.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CohortAggregateContainer.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\WhatIsACohort.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllPluginsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Graph.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Main.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ProcessTask.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CacheFetchFailure.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CumulativeExtractionResults.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\DQEGraphAnnotation.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Evaluation.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\LoadModuleAssembly.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ProcessTaskArgument.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ProjectCohortIdentificationConfigurationAssociation.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ReleaseLog.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\SelectedDataSetsForcedJoin.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\SpontaneouslyInventedColumn.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\SupplementalExtractionResults.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\TicketingSystemConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\WindowLayout.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllDashboardsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllOrphanAggregateConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtendedProperty.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractionProgress.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Attacher.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\DataProvider.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\MutilateDataTables.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllTemplateAggregateConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Commit.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Memento.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\TableInfoDatabaseNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Dataset.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a - - - ..\AggregateContinuousDateAxis.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\LoadMetadataCatalogueLinkage.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\famfamfam\cog.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\TicketingSystemReleaseStatus.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\StandardRegex.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\StandardRegex1.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\StandardRegex2.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - -||||||| 242cf549c - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - text/microsoft-resx - - - 2.0 - - - System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - - ..\..\Icons\columninfo.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\DQE.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\Lookup.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\ExecuteArrow.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\executeArrowWithTickOverlay.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\spanner.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\tableinfo.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\backup.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\sync.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\StarHollow.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\loadMetadata.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\logging.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\importFile.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\Catalogue.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\Filter.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\ExtractionInformation.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\FilterContainer.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\cohortAggregate.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\greenIssue.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\orangeIssue.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\redIssue.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\Clipboard.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\patientIndexTable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\expandAllNodes.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\ExtractionInformation_SpecialApproval.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\ExtractionInformation_Supplemental.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\AggregatesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\catalogueItemsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\BigCohort.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\BigGraph.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\BigPatientIndexTable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\cohortIdentificationConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\frozenCohortIdentificationConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\supportingDocument.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\documentationNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\supportingDocumentExtractable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\EXCEPT.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\INTERSECT.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\supportingDocumentExtractableGlobal.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\supportingDocumentGlobal.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\supportingSqlExtractableGlobal.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\supportingSqlGlobal.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\UNION.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\collisionResolution.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\AllServersNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\tableinfoTableValuedFunction.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AggregateGraph.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CatalogueFolder.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CatalogueItem.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CatalogueItemTransform.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CatalogueItemIssue.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\DataAccessCredentials.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllDataAccessCredentialsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Database.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractionDirectoryNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\SQL.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\supportingSqlExtractable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ANOColumnInfo.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CloneExtractionConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\EmptyProject.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractableDataSetDisabled.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractionConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Failed.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\FrozenExtractionConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Project.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Release.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\SqlThenVSNow.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ANOTable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllANOTablesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractableCohort.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractableDataSet.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\JoinableCollectionNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ParametersNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\SupportingSQLTable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\TableInfoServerNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\TimeCoverageField.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExternalCohortTable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractableDataSetPackage.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractableDataSetsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\PivotField.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ProjectsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AggregateDimension.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\DashboardControl.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\DashboardLayout.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ReOrder.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\JoinInfo.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\StandardRegex.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Favourite.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractionFilterParameterSet.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AutomationServiceSlot.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CacheProgress.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExternalDatabaseServer.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExternalDatabaseServer_ANO.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExternalDatabaseServer_Cache.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExternalDatabaseServer_DQE.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExternalDatabaseServer_IdentifierDump.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExternalDatabaseServer_Logging.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Plugin.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Kill.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\EXCEPTCohortAggregate.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\INTERSECTCohortAggregate.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\UNIONCohortAggregate.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\RowCounts_Ignore.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\RowCounts_Respect.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\NoIconAvailable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\File.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Help.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\UnfreezeExtractionConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllCataloguesUsedByLoadMetadataNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllProcessTasksUsedByLoadMetadataNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\LoadDirectoryNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\LoadBubble.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\LoadFinalDatabase.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Exe.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\LoadMetadataScheduleNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\LoadPeriodically.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\LoadProgress.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\GetFilesStage.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\LoadBubbleMounting.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\PreLoadDiscardedColumn.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\famfamfam\Tick.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Warning.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllExternalServersNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\DecryptionPrivateKeyNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\PreLoadDiscardedColumnsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\PermissionWindow.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Pipeline.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractionConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllCohortsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ProjectCohortIdentificationConfigurationAssociationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ProjectSavedCohortsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\collapseAllNodes.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllAutomationServerSlotsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllRDMPRemotesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\RemoteRDMP.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AutomateablePipeline.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\PipelineComponent.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\PipelineComponentSource.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\PipelineComponentDestination.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\PipelineComponentArgument.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AggregateTopX.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AutomationServiceException.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllObjectExportsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllObjectImportsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ObjectExport.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ObjectImport.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllObjectSharingNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CohortCustomColumn.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractableColumn.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ProjectCohortsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\FrozenExtractionConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\TinyGreen.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\TinyRed.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\TinyYellow.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractionInformation_ProjectSpecific.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ProjectCatalogue.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ProjectCataloguesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\DropHere.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\MakeProjectSpecificCatalogueNormalAgain.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllPermissionWindowsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Waiting.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\WaitingForDatabase.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Writing.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\FileMissing.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllConnectionStringKeywordsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ConnectionStringKeyword.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllStandardRegexesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllPipelinesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\StandardPipelineUseCaseNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\OtherPipelinesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllGovernanceNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\GovernancePeriod.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\GovernanceDocument.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\LookupCompositeJoinInfo.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllFreeCohortIdentificationConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllProjectCohortIdentificationConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Menu.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CatalogueLookupsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\DatabaseRefresh.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Diff.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CohortAggregateContainer.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\WhatIsACohort.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllPluginsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Graph.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Main.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ProcessTask.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CacheFetchFailure.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CumulativeExtractionResults.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\DQEGraphAnnotation.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Evaluation.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\LoadModuleAssembly.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ProcessTaskArgument.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ProjectCohortIdentificationConfigurationAssociation.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ReleaseLog.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\SelectedDataSetsForcedJoin.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\SpontaneouslyInventedColumn.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\SupplementalExtractionResults.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\TicketingSystemConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\WindowLayout.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllDashboardsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllOrphanAggregateConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtendedProperty.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractionProgress.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Attacher.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\DataProvider.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\MutilateDataTables.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllTemplateAggregateConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Commit.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Memento.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\TableInfoDatabaseNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Dataset.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a - - - ..\AggregateContinuousDateAxis.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\LoadMetadataCatalogueLinkage.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\famfamfam\cog.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\TicketingSystemReleaseStatus.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - -======= @@ -2195,4 +767,3 @@ ..\TicketingSystemReleaseStatus.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 ->>>>>>> e887d27056c25fa5bf1b9257be3bec4b38f9808a From 467896dc4557dc4e997813b26e55dba88407725c Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 16 Sep 2024 10:42:54 +0100 Subject: [PATCH 047/106] ad dicons --- ...ommandAddNewRegexRedactionConfiguration.cs | 20 +- ...CommandPerformRegexRedactionOnCatalogue.cs | 41 +- ...andRestoreRegexRedactedValueInCatalogue.cs | 9 +- .../RegexRedaction/RegexRedactionHelper.cs | 40 +- .../Mutilators/RegexRedactionMutilator.cs | 12 +- .../Icons/IconProvision/CatalogueIcons.resx | 1433 +++++++++-------- Rdmp.Core/Icons/IconProvision/RDMPConcept.cs | 6 +- Rdmp.Core/Providers/Nodes/AllDatasetsNode.cs | 7 +- .../AllRegexRedactionConfigurationsNode.cs | 7 +- Rdmp.Core/RDMPCollection.cs | 2 +- .../Collections/ConfigurationsCollectionUI.cs | 10 - Rdmp.UI/Collections/TableInfoCollectionUI.cs | 7 +- .../ExecuteCommandRegexRedaction.cs | 8 +- .../ProposeExecutionWhenTargetIsDataset.cs | 1 - ...WhenTargetIsRegexRedactionConfiguration.cs | 5 - Rdmp.UI/Menus/CatalogueMenu.cs | 2 +- Rdmp.UI/SimpleDialogs/RedactCatalogueUI.cs | 8 - .../RegexRedactionConfigurationUI.cs | 7 - Tools/rdmp/Databases.yaml | 4 +- 19 files changed, 812 insertions(+), 817 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewRegexRedactionConfiguration.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewRegexRedactionConfiguration.cs index 16a1c74fe1..f3ad57e2cc 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewRegexRedactionConfiguration.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewRegexRedactionConfiguration.cs @@ -1,4 +1,10 @@ -using Rdmp.Core.Icons.IconProvision; +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using Rdmp.Core.Icons.IconProvision; using Rdmp.Core.ReusableLibraryCode.Icons.IconProvision; using SixLabors.ImageSharp.PixelFormats; using SixLabors.ImageSharp; @@ -11,11 +17,11 @@ namespace Rdmp.Core.CommandExecution.AtomicCommands; public class ExecuteCommandAddNewRegexRedactionConfiguration : BasicCommandExecution, IAtomicCommand { - private string _name; - private string _redactionPattern; - private string _redactionString; - private string _description; - private IBasicActivateItems _activator; + private readonly string _name; + private readonly string _redactionPattern; + private readonly string _redactionString; + private readonly string _description; + private readonly IBasicActivateItems _activator; public ExecuteCommandAddNewRegexRedactionConfiguration(IBasicActivateItems activator, [DemandsInitialization("Name")] string name, [DemandsInitialization("pattern")] string redactionPattern, [DemandsInitialization("redaction")] string redactionString, string description = null) : base(activator) @@ -35,5 +41,5 @@ public override void Execute() } public override Image GetImage(IIconProvider iconProvider) => - iconProvider.GetImage(RDMPConcept.StandardRegex, OverlayKind.Add); + iconProvider.GetImage(RDMPConcept.RegexRedaction, OverlayKind.Add); } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs index 443e7a26d2..ca10d5c672 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs @@ -1,4 +1,10 @@ -using FAnsi.Discovery; +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using FAnsi.Discovery; using FAnsi.Discovery.QuerySyntax; using MongoDB.Driver; using Rdmp.Core.Curation.Data; @@ -9,7 +15,6 @@ using System; using System.Collections.Generic; using System.Data; -using System.Diagnostics; using System.Linq; using static System.Linq.Enumerable; @@ -39,6 +44,11 @@ public ExecuteCommandPerformRegexRedactionOnCatalogue(IBasicActivateItems activa _columns = columns; _server = _catalogue.GetDistinctLiveDatabaseServer(DataAccessContext.InternalDataProcessing, false); _readLimit = readLimit; + if (_server.DatabaseType != FAnsi.DatabaseType.MicrosoftSQLServer) + { + //This should be implimented for all db types after UAT for the initial purpose + SetImpossible("Regex Redaction are currently only supported on MSSQL databases"); + } } public override void Execute() @@ -46,7 +56,6 @@ public override void Execute() base.Execute(); var memoryRepo = new MemoryCatalogueRepository(); - var timer = Stopwatch.StartNew(); foreach (var columnInfo in _columns.Where(c => !c.IsPrimaryKey)) { redactionsToSaveTable = RegexRedactionHelper.GenerateRedactionsDataTable(); @@ -82,9 +91,6 @@ public override void Execute() da.Fill(dt); } conn.Close(); - timer.Stop(); - Console.WriteLine(timer.ElapsedMilliseconds); - timer.Start(); redactionUpates = dt.Clone(); redactionUpates.BeginLoadData(); foreach (DataRow row in dt.Rows) @@ -92,39 +98,30 @@ public override void Execute() RegexRedactionHelper.Redact(columnInfo, row, _cataloguePKs, _redactionConfiguration, redactionsToSaveTable, pksToSave, redactionUpates); } redactionUpates.EndLoadData(); - timer.Stop(); - Console.WriteLine(timer.ElapsedMilliseconds); - timer.Start(); for (int i = 0; i < pksToSave.Rows.Count; i++) { - pksToSave.Rows[i]["ID"] = i + 1; + pksToSave.Rows[i][nameof(RegexRedactionHelper.Constants.ID)] = i + 1; } - var existing = _discoveredTable.Database.ExpectTable("pksToSave_Temp"); + var existing = _discoveredTable.Database.ExpectTable(nameof(RegexRedactionHelper.Constants.pksToSave_Temp)); if (existing.Exists()) { existing.Drop(); } - var t1 = _discoveredTable.Database.CreateTable("pksToSave_Temp", pksToSave); - existing = _discoveredTable.Database.ExpectTable("redactionsToSaveTable_Temp"); + var t1 = _discoveredTable.Database.CreateTable(nameof(RegexRedactionHelper.Constants.pksToSave_Temp), pksToSave); + existing = _discoveredTable.Database.ExpectTable(nameof(RegexRedactionHelper.Constants.redactionsToSaveTable_Temp)); if (existing.Exists()) { existing.Drop(); } - var t2 = _discoveredTable.Database.CreateTable("redactionsToSaveTable_Temp", redactionsToSaveTable); + var t2 = _discoveredTable.Database.CreateTable(nameof(RegexRedactionHelper.Constants.redactionsToSaveTable_Temp), redactionsToSaveTable); + RegexRedactionHelper.SaveRedactions(_activator.RepositoryLocator.CatalogueRepository, t1, t2, _server); t1.Drop(); t2.Drop(); - timer.Stop(); - Console.WriteLine(timer.ElapsedMilliseconds); - timer.Start(); if (dt.Rows.Count > 0) { RegexRedactionHelper.DoJoinUpdate(columnInfo, _discoveredTable, _server, redactionUpates, _discoveredPKColumns); } - timer.Stop(); - Console.WriteLine(timer.ElapsedMilliseconds); - timer.Start(); - } else { @@ -133,7 +130,5 @@ public override void Execute() resultCount += redactionUpates.Rows.Count; } - timer.Stop(); - var x = timer.ElapsedMilliseconds; } } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreRegexRedactedValueInCatalogue.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreRegexRedactedValueInCatalogue.cs index 96ac89741c..4caffc644f 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreRegexRedactedValueInCatalogue.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreRegexRedactedValueInCatalogue.cs @@ -1,4 +1,10 @@ -using FAnsi.Discovery; +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using FAnsi.Discovery; using FAnsi.Discovery.QuerySyntax; using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.DataHelper.RegexRedaction; @@ -75,7 +81,6 @@ public override void Execute() } var updateSql = updateHelper.BuildUpdate(discoveredTable, discoveredTable, sqlLines); var conn = server.GetConnection(); - //TODO this is borked using (var cmd = server.GetCommand(updateSql, conn)) { conn.Open(); diff --git a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs index b200011ca4..e4944d6321 100644 --- a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs +++ b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs @@ -1,4 +1,10 @@ -using FAnsi.Discovery.QuerySyntax; +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using FAnsi.Discovery.QuerySyntax; using FAnsi.Discovery; using Rdmp.Core.Curation.Data; using Rdmp.Core.MapsDirectlyToDatabaseTable; @@ -13,25 +19,39 @@ namespace Rdmp.Core.Curation.DataHelper.RegexRedaction { public static class RegexRedactionHelper { + public enum Constants { + pksToSave_Temp, + redactionsToSaveTable_Temp, + ID, + RedactionConfiguration_ID, + ColumnInfo_ID, + startingIndex, + ReplacementValue, + RedactedValue, + RegexRedaction_ID, + Value + + }; + public static DataTable GenerateRedactionsDataTable() { DataTable redactionsToSaveTable = new DataTable(); - redactionsToSaveTable.Columns.Add("RedactionConfiguration_ID"); - redactionsToSaveTable.Columns.Add("ColumnInfo_ID"); - redactionsToSaveTable.Columns.Add("startingIndex"); - redactionsToSaveTable.Columns.Add("ReplacementValue"); - redactionsToSaveTable.Columns.Add("RedactedValue"); + redactionsToSaveTable.Columns.Add(nameof(Constants.RedactionConfiguration_ID)); + redactionsToSaveTable.Columns.Add(nameof(Constants.ColumnInfo_ID)); + redactionsToSaveTable.Columns.Add(nameof(Constants.startingIndex)); + redactionsToSaveTable.Columns.Add(nameof(Constants.ReplacementValue)); + redactionsToSaveTable.Columns.Add(nameof(Constants.RedactedValue)); return redactionsToSaveTable; } public static DataTable GeneratePKDataTable() { DataTable pkDataTable = new DataTable(); - pkDataTable.Columns.Add("RegexRedaction_ID"); - pkDataTable.Columns.Add("ColumnInfo_ID"); - pkDataTable.Columns.Add("Value"); - pkDataTable.Columns.Add("ID", typeof(int)); + pkDataTable.Columns.Add(nameof(Constants.RegexRedaction_ID)); + pkDataTable.Columns.Add(nameof(Constants.ColumnInfo_ID)); + pkDataTable.Columns.Add(nameof(Constants.Value)); + pkDataTable.Columns.Add(nameof(Constants.ID), typeof(int)); return pkDataTable; } diff --git a/Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs b/Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs index d1b67f7989..242bebcd17 100644 --- a/Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs +++ b/Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs @@ -1,4 +1,10 @@ -using FAnsi.Discovery; +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using FAnsi.Discovery; using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.Data.DataLoad; using Rdmp.Core.Curation.DataHelper.RegexRedaction; @@ -110,8 +116,8 @@ protected override void MutilateTable(IDataLoadJob job, ITableInfo tableInfo, Di pksToSave.Rows[i]["ID"] = i + 1; } job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"Creating Temporary tables")); - var t1 = table.Database.CreateTable("pksToSave_Temp", pksToSave); - var t2 = table.Database.CreateTable("redactionsToSaveTable_Temp", redactionsToSaveTable); + var t1 = table.Database.CreateTable(nameof(RegexRedactionHelper.Constants.pksToSave_Temp), pksToSave); + var t2 = table.Database.CreateTable(nameof(RegexRedactionHelper.Constants.redactionsToSaveTable_Temp), redactionsToSaveTable); job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"Saving Redactions")); var _server = relatedCatalogues.First().GetDistinctLiveDatabaseServer(DataAccessContext.InternalDataProcessing, false); RegexRedactionHelper.SaveRedactions(job.RepositoryLocator.CatalogueRepository, t1, t2, _server, Timeout * 1000); diff --git a/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx b/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx index 4684396b1f..6abf0c1c75 100644 --- a/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx +++ b/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx @@ -1,6 +1,6 @@  - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - text/microsoft-resx - - - 2.0 - - - System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - - ..\..\Icons\columninfo.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\DQE.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\Lookup.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\ExecuteArrow.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\executeArrowWithTickOverlay.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\spanner.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\tableinfo.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\backup.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\sync.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\StarHollow.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\loadMetadata.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\logging.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\importFile.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\Catalogue.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\Filter.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\ExtractionInformation.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\FilterContainer.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\cohortAggregate.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\greenIssue.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\orangeIssue.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\redIssue.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\Clipboard.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\patientIndexTable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\expandAllNodes.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\ExtractionInformation_SpecialApproval.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\ExtractionInformation_Supplemental.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\AggregatesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\catalogueItemsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\BigCohort.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\BigGraph.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\BigPatientIndexTable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\cohortIdentificationConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\frozenCohortIdentificationConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\supportingDocument.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\documentationNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\supportingDocumentExtractable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\EXCEPT.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\INTERSECT.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\supportingDocumentExtractableGlobal.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\supportingDocumentGlobal.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\supportingSqlExtractableGlobal.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\supportingSqlGlobal.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\UNION.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\collisionResolution.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\AllServersNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\tableinfoTableValuedFunction.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AggregateGraph.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CatalogueFolder.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CatalogueItem.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CatalogueItemTransform.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CatalogueItemIssue.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\DataAccessCredentials.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllDataAccessCredentialsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Database.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractionDirectoryNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\SQL.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\supportingSqlExtractable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ANOColumnInfo.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CloneExtractionConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\EmptyProject.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractableDataSetDisabled.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractionConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Failed.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\FrozenExtractionConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Project.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Release.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\SqlThenVSNow.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ANOTable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllANOTablesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractableCohort.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractableDataSet.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\JoinableCollectionNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ParametersNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\SupportingSQLTable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\TableInfoServerNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\TimeCoverageField.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExternalCohortTable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractableDataSetPackage.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractableDataSetsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\PivotField.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ProjectsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AggregateDimension.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\DashboardControl.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\DashboardLayout.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ReOrder.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\JoinInfo.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\StandardRegex.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Favourite.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractionFilterParameterSet.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AutomationServiceSlot.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CacheProgress.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExternalDatabaseServer.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExternalDatabaseServer_ANO.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExternalDatabaseServer_Cache.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExternalDatabaseServer_DQE.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExternalDatabaseServer_IdentifierDump.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExternalDatabaseServer_Logging.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Plugin.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Kill.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\EXCEPTCohortAggregate.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\INTERSECTCohortAggregate.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\UNIONCohortAggregate.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\RowCounts_Ignore.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\RowCounts_Respect.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\NoIconAvailable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\File.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Help.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\UnfreezeExtractionConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllCataloguesUsedByLoadMetadataNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllProcessTasksUsedByLoadMetadataNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\LoadDirectoryNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\LoadBubble.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\LoadFinalDatabase.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Exe.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\LoadMetadataScheduleNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\LoadPeriodically.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\LoadProgress.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\GetFilesStage.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\LoadBubbleMounting.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\PreLoadDiscardedColumn.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\famfamfam\Tick.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Warning.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllExternalServersNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\DecryptionPrivateKeyNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\PreLoadDiscardedColumnsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\PermissionWindow.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Pipeline.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractionConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllCohortsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ProjectCohortIdentificationConfigurationAssociationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ProjectSavedCohortsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CommittedCohortIdentificationNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\collapseAllNodes.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllAutomationServerSlotsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllRDMPRemotesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\RemoteRDMP.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AutomateablePipeline.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\PipelineComponent.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\PipelineComponentSource.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\PipelineComponentDestination.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\PipelineComponentArgument.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AggregateTopX.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AutomationServiceException.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllObjectExportsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllObjectImportsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ObjectExport.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ObjectImport.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllObjectSharingNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CohortCustomColumn.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractableColumn.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ProjectCohortsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\FrozenExtractionConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\TinyGreen.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\TinyRed.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\TinyYellow.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractionInformation_ProjectSpecific.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ProjectCatalogue.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ProjectCataloguesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\DropHere.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\MakeProjectSpecificCatalogueNormalAgain.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllPermissionWindowsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Waiting.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\WaitingForDatabase.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Writing.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\FileMissing.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllConnectionStringKeywordsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ConnectionStringKeyword.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllStandardRegexesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllPipelinesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\StandardPipelineUseCaseNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\OtherPipelinesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllGovernanceNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\GovernancePeriod.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\GovernanceDocument.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\LookupCompositeJoinInfo.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllFreeCohortIdentificationConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllProjectCohortIdentificationConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Menu.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CatalogueLookupsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\DatabaseRefresh.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Diff.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CohortAggregateContainer.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\WhatIsACohort.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllPluginsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Graph.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Main.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ProcessTask.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CacheFetchFailure.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CumulativeExtractionResults.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\DQEGraphAnnotation.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Evaluation.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\LoadModuleAssembly.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ProcessTaskArgument.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ProjectCohortIdentificationConfigurationAssociation.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ReleaseLog.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\SelectedDataSetsForcedJoin.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\SpontaneouslyInventedColumn.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\SupplementalExtractionResults.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\TicketingSystemConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\WindowLayout.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllDashboardsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllOrphanAggregateConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtendedProperty.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractionProgress.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Attacher.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\DataProvider.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\MutilateDataTables.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllTemplateAggregateConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Commit.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Memento.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\TableInfoDatabaseNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Dataset.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a - - - ..\AggregateContinuousDateAxis.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\LoadMetadataCatalogueLinkage.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\famfamfam\cog.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\TicketingSystemReleaseStatus.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + + ..\..\Icons\columninfo.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\DQE.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\Lookup.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\ExecuteArrow.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\executeArrowWithTickOverlay.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\spanner.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\tableinfo.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\backup.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\sync.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\StarHollow.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\loadMetadata.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\logging.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\importFile.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\Catalogue.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\Filter.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\ExtractionInformation.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\FilterContainer.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\cohortAggregate.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\greenIssue.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\orangeIssue.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\redIssue.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\Clipboard.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\patientIndexTable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\expandAllNodes.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\ExtractionInformation_SpecialApproval.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\ExtractionInformation_Supplemental.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\AggregatesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\catalogueItemsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\BigCohort.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\BigGraph.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\BigPatientIndexTable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\cohortIdentificationConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\frozenCohortIdentificationConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\supportingDocument.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\documentationNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\supportingDocumentExtractable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\EXCEPT.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\INTERSECT.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\supportingDocumentExtractableGlobal.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\supportingDocumentGlobal.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\supportingSqlExtractableGlobal.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\supportingSqlGlobal.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\UNION.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\collisionResolution.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\AllServersNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\tableinfoTableValuedFunction.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AggregateGraph.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\CatalogueFolder.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\CatalogueItem.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\CatalogueItemTransform.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\CatalogueItemIssue.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\DataAccessCredentials.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllDataAccessCredentialsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Database.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ExtractionDirectoryNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\SQL.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\supportingSqlExtractable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ANOColumnInfo.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\CloneExtractionConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\EmptyProject.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ExtractableDataSetDisabled.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ExtractionConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Failed.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\FrozenExtractionConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Project.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Release.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\SqlThenVSNow.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ANOTable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllANOTablesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ExtractableCohort.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ExtractableDataSet.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\JoinableCollectionNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ParametersNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\SupportingSQLTable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\TableInfoServerNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\TimeCoverageField.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ExternalCohortTable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ExtractableDataSetPackage.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ExtractableDataSetsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\PivotField.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ProjectsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AggregateDimension.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\DashboardControl.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\DashboardLayout.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ReOrder.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\JoinInfo.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\StandardRegex.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Favourite.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ExtractionFilterParameterSet.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AutomationServiceSlot.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\CacheProgress.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ExternalDatabaseServer.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ExternalDatabaseServer_ANO.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ExternalDatabaseServer_Cache.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ExternalDatabaseServer_DQE.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ExternalDatabaseServer_IdentifierDump.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ExternalDatabaseServer_Logging.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Plugin.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Kill.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\EXCEPTCohortAggregate.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\INTERSECTCohortAggregate.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\UNIONCohortAggregate.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\RowCounts_Ignore.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\RowCounts_Respect.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\NoIconAvailable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\File.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Help.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\UnfreezeExtractionConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllCataloguesUsedByLoadMetadataNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllProcessTasksUsedByLoadMetadataNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\LoadDirectoryNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\LoadBubble.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\LoadFinalDatabase.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Exe.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\LoadMetadataScheduleNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\LoadPeriodically.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\LoadProgress.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\GetFilesStage.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\LoadBubbleMounting.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\PreLoadDiscardedColumn.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\famfamfam\Tick.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Warning.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllExternalServersNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\DecryptionPrivateKeyNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\PreLoadDiscardedColumnsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\PermissionWindow.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Pipeline.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ExtractionConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllCohortsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ProjectCohortIdentificationConfigurationAssociationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ProjectSavedCohortsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\CommittedCohortIdentificationNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\collapseAllNodes.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllAutomationServerSlotsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllRDMPRemotesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\RemoteRDMP.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AutomateablePipeline.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\PipelineComponent.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\PipelineComponentSource.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\PipelineComponentDestination.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\PipelineComponentArgument.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AggregateTopX.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AutomationServiceException.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllObjectExportsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllObjectImportsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ObjectExport.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ObjectImport.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllObjectSharingNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\CohortCustomColumn.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ExtractableColumn.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ProjectCohortsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\FrozenExtractionConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\TinyGreen.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\TinyRed.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\TinyYellow.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ExtractionInformation_ProjectSpecific.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ProjectCatalogue.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ProjectCataloguesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\DropHere.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\MakeProjectSpecificCatalogueNormalAgain.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllPermissionWindowsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Waiting.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\WaitingForDatabase.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Writing.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\FileMissing.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllConnectionStringKeywordsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ConnectionStringKeyword.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllStandardRegexesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllPipelinesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\StandardPipelineUseCaseNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\OtherPipelinesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllGovernanceNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\GovernancePeriod.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\GovernanceDocument.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\LookupCompositeJoinInfo.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllFreeCohortIdentificationConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllProjectCohortIdentificationConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Menu.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\CatalogueLookupsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\DatabaseRefresh.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Diff.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\CohortAggregateContainer.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\WhatIsACohort.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllPluginsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Graph.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Main.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ProcessTask.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\CacheFetchFailure.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\CumulativeExtractionResults.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\DQEGraphAnnotation.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Evaluation.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\LoadModuleAssembly.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ProcessTaskArgument.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ProjectCohortIdentificationConfigurationAssociation.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ReleaseLog.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\SelectedDataSetsForcedJoin.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\SpontaneouslyInventedColumn.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\SupplementalExtractionResults.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\TicketingSystemConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\WindowLayout.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllDashboardsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllOrphanAggregateConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ExtendedProperty.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ExtractionProgress.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Attacher.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\DataProvider.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\MutilateDataTables.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllTemplateAggregateConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Commit.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Memento.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\TableInfoDatabaseNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Dataset.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a + + + ..\AggregateContinuousDateAxis.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\LoadMetadataCatalogueLinkage.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\famfamfam\cog.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\TicketingSystemReleaseStatus.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Dataset1.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllStandardRegexesNode1.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\StandardRegex3.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\StandardRegex311.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\StandardRegex31.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + \ No newline at end of file diff --git a/Rdmp.Core/Icons/IconProvision/RDMPConcept.cs b/Rdmp.Core/Icons/IconProvision/RDMPConcept.cs index 702f05f252..3fc3fbb0d4 100644 --- a/Rdmp.Core/Icons/IconProvision/RDMPConcept.cs +++ b/Rdmp.Core/Icons/IconProvision/RDMPConcept.cs @@ -204,13 +204,13 @@ public enum RDMPConcept Memento, TableInfoDatabaseNode, Dataset, - AllDatasetsNode, LoadMetadataCatalogueLinkage, Setting, TicketingSystemReleaseStatus, + CommittedCohortIdentificationNode, + AllDatasetsNode, RegexRedaction, RegexRedactionConfiguration, RegexRedactionKey, - AllRegexRedactionConfigurationsNode, - CommittedCohortIdentificationNode + AllRegexRedactionConfigurationsNode } \ No newline at end of file diff --git a/Rdmp.Core/Providers/Nodes/AllDatasetsNode.cs b/Rdmp.Core/Providers/Nodes/AllDatasetsNode.cs index 1cdeff8cbe..c77b7e3169 100644 --- a/Rdmp.Core/Providers/Nodes/AllDatasetsNode.cs +++ b/Rdmp.Core/Providers/Nodes/AllDatasetsNode.cs @@ -1,9 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - + namespace Rdmp.Core.Providers.Nodes { public class AllDatasetsNode : SingletonNode diff --git a/Rdmp.Core/Providers/Nodes/AllRegexRedactionConfigurationsNode.cs b/Rdmp.Core/Providers/Nodes/AllRegexRedactionConfigurationsNode.cs index 3de546e459..2a6570df2d 100644 --- a/Rdmp.Core/Providers/Nodes/AllRegexRedactionConfigurationsNode.cs +++ b/Rdmp.Core/Providers/Nodes/AllRegexRedactionConfigurationsNode.cs @@ -1,9 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - + namespace Rdmp.Core.Providers.Nodes { public class AllRegexRedactionConfigurationsNode: SingletonNode diff --git a/Rdmp.Core/RDMPCollection.cs b/Rdmp.Core/RDMPCollection.cs index 0595e4c53d..35c3336115 100644 --- a/Rdmp.Core/RDMPCollection.cs +++ b/Rdmp.Core/RDMPCollection.cs @@ -17,9 +17,9 @@ public enum RDMPCollection DataExport, SavedCohorts, Favourites, - Datasets, Cohort, DataLoad, + Datasets, Configurations } \ No newline at end of file diff --git a/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs b/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs index fe1bc9418b..41a7a32adf 100644 --- a/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs +++ b/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs @@ -1,18 +1,10 @@ using Rdmp.Core.CommandExecution.AtomicCommands; using Rdmp.Core; -using Rdmp.Core.MapsDirectlyToDatabaseTable; using Rdmp.UI.CommandExecution.AtomicCommands; using Rdmp.UI.ItemActivation; using Rdmp.UI.Refreshing; -using System; -using System.Collections.Generic; using System.Linq; using Rdmp.Core.Curation.Data; -using Rdmp.Core.Icons.IconProvision; -using System.Windows.Forms; -using Rdmp.Core.Curation.DataHelper.RegexRedaction; -using Rdmp.Core.Providers.Nodes.PipelineNodes; -using Rdmp.Core.Providers.Nodes.SharingNodes; using Rdmp.Core.Providers.Nodes; namespace Rdmp.UI.Collections; @@ -20,8 +12,6 @@ namespace Rdmp.UI.Collections; public partial class ConfigurationsCollectionUI : RDMPCollectionUI, ILifetimeSubscriber { - //private Dataset[] _datasets; - //private bool _firstTime = true; private IActivateItems _activator; public ConfigurationsCollectionUI() diff --git a/Rdmp.UI/Collections/TableInfoCollectionUI.cs b/Rdmp.UI/Collections/TableInfoCollectionUI.cs index f03f2928c9..7f4ff38c47 100644 --- a/Rdmp.UI/Collections/TableInfoCollectionUI.cs +++ b/Rdmp.UI/Collections/TableInfoCollectionUI.cs @@ -1,15 +1,13 @@ -// Copyright (c) The University of Dundee 2018-2019 +// Copyright (c) The University of Dundee 2018-2024 // This file is part of the Research Data Management Platform (RDMP). // RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . using System; -using System.DirectoryServices.Protocols; using System.Linq; using System.Windows.Forms; using Rdmp.Core; -using Rdmp.Core.CommandExecution; using Rdmp.Core.CommandExecution.AtomicCommands; using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.Data.DataLoad; @@ -18,7 +16,6 @@ using Rdmp.Core.Providers.Nodes; using Rdmp.Core.Providers.Nodes.PipelineNodes; using Rdmp.Core.Providers.Nodes.SharingNodes; -using Rdmp.UI.CommandExecution.AtomicCommands; using Rdmp.UI.ItemActivation; using Rdmp.UI.LocationsMenu; using Rdmp.UI.Refreshing; @@ -39,7 +36,6 @@ namespace Rdmp.UI.Collections; public partial class TableInfoCollectionUI : RDMPCollectionUI, ILifetimeSubscriber { private bool _isFirstTime = true; - private IActivateItems _activator; public TableInfoCollectionUI() { @@ -92,7 +88,6 @@ private void olvTableInfos_KeyUp(object sender, KeyEventArgs e) public override void SetItemActivator(IActivateItems activator) { base.SetItemActivator(activator); - _activator = activator; CommonTreeFunctionality.SetUp( RDMPCollection.Tables, tlvTableInfos, diff --git a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandRegexRedaction.cs b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandRegexRedaction.cs index 9bf1697773..d780cd57a8 100644 --- a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandRegexRedaction.cs +++ b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandRegexRedaction.cs @@ -2,15 +2,9 @@ using Rdmp.Core.Curation.Data; using Rdmp.Core.Icons.IconProvision; using Rdmp.Core.ReusableLibraryCode.Icons.IconProvision; -using Rdmp.UI.ExtractionUIs; using Rdmp.UI.ItemActivation; using SixLabors.ImageSharp.PixelFormats; using SixLabors.ImageSharp; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; using Rdmp.UI.SimpleDialogs; namespace Rdmp.UI.CommandExecution.AtomicCommands; @@ -32,6 +26,6 @@ public override void Execute() { base.Execute(); - Activator.Activate(_catalogue); + _activator.Activate(_catalogue); } } diff --git a/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsDataset.cs b/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsDataset.cs index 5ff10a9b23..7bc681d63e 100644 --- a/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsDataset.cs +++ b/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsDataset.cs @@ -6,7 +6,6 @@ using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation.Data; -using Rdmp.Core.Curation.DataHelper.RegexRedaction; using Rdmp.Core.ReusableLibraryCode.Annotations; using Rdmp.UI.ItemActivation; using Rdmp.UI.SubComponents; diff --git a/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsRegexRedactionConfiguration.cs b/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsRegexRedactionConfiguration.cs index 9e248f39fb..fbab8356be 100644 --- a/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsRegexRedactionConfiguration.cs +++ b/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsRegexRedactionConfiguration.cs @@ -3,11 +3,6 @@ using Rdmp.Core.ReusableLibraryCode.Annotations; using Rdmp.UI.ItemActivation; using Rdmp.UI.SubComponents; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace Rdmp.UI.CommandExecution.Proposals { diff --git a/Rdmp.UI/Menus/CatalogueMenu.cs b/Rdmp.UI/Menus/CatalogueMenu.cs index dcf3aec406..07f360bb97 100644 --- a/Rdmp.UI/Menus/CatalogueMenu.cs +++ b/Rdmp.UI/Menus/CatalogueMenu.cs @@ -1,4 +1,4 @@ -// Copyright (c) The University of Dundee 2018-2019 +// Copyright (c) The University of Dundee 2018-2024 // This file is part of the Research Data Management Platform (RDMP). // RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. diff --git a/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.cs b/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.cs index 3f08f84576..80cfe67e3a 100644 --- a/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.cs +++ b/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.cs @@ -12,20 +12,12 @@ using System.Linq; using System.Windows.Forms; using BrightIdeasSoftware; -using NPOI.SS.Formula.Functions; using Rdmp.Core; -using Rdmp.Core.CommandExecution; using Rdmp.Core.CommandExecution.AtomicCommands; using Rdmp.Core.Curation.Data; -using Rdmp.Core.Curation.Data.Governance; -using Rdmp.Core.Curation.Data.Remoting; using Rdmp.Core.Curation.DataHelper.RegexRedaction; -using Rdmp.Core.QueryBuilding; using Rdmp.UI.CommandExecution.AtomicCommands; using Rdmp.UI.ItemActivation; -using Rdmp.UI.MainFormUITabs; -using Rdmp.UI.Rules; -using Rdmp.UI.SimpleControls; using Rdmp.UI.SimpleDialogs.RegexRedactionConfigurationForm; using Rdmp.UI.TestsAndSetup.ServicePropogation; diff --git a/Rdmp.UI/SubComponents/RegexRedactionConfigurationUI.cs b/Rdmp.UI/SubComponents/RegexRedactionConfigurationUI.cs index 67ad0992f1..9b47649ed4 100644 --- a/Rdmp.UI/SubComponents/RegexRedactionConfigurationUI.cs +++ b/Rdmp.UI/SubComponents/RegexRedactionConfigurationUI.cs @@ -1,17 +1,10 @@ using Rdmp.UI.TestsAndSetup.ServicePropogation; using System; using System.ComponentModel; -using System.Linq; using System.Windows.Forms; using Rdmp.UI.Refreshing; -using Rdmp.Core.Dataset; -using Rdmp.Core.Curation.Data; using Rdmp.UI.ItemActivation; using Rdmp.Core.Curation.DataHelper.RegexRedaction; -using Microsoft.Data.SqlClient; -using System.Text.RegularExpressions; -using System.Drawing; -using Rdmp.Core.MapsDirectlyToDatabaseTable; namespace Rdmp.UI.SubComponents; public partial class RegexRedactionConfigurationUI : RegexRedactionConfigurationUI_Design, IRefreshBusSubscriber diff --git a/Tools/rdmp/Databases.yaml b/Tools/rdmp/Databases.yaml index c688f62761..434fc12411 100644 --- a/Tools/rdmp/Databases.yaml +++ b/Tools/rdmp/Databases.yaml @@ -1,2 +1,2 @@ -CatalogueConnectionString: Server=(localdb)\MSSQLLocalDB;Database=RDMP_Catalogue;Trusted_Connection=True;TrustServerCertificate=true; -DataExportConnectionString: Server=(localdb)\MSSQLLocalDB;Database=RDMP_DataExport;Trusted_Connection=True;TrustServerCertificate=true; +CatalogueConnectionString: Server=(localdb)\MSSQLLocalDB;Database=TEST_Catalogue;Trusted_Connection=True;TrustServerCertificate=true; +DataExportConnectionString: Server=(localdb)\MSSQLLocalDB;Database=TEST_DataExport;Trusted_Connection=True;TrustServerCertificate=true; From 8409eaf839f032b0eb3231f8ab2f2e1231d838dc Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 16 Sep 2024 11:25:20 +0100 Subject: [PATCH 048/106] add missing icons --- Rdmp.Core/Icons/AllStandardRegexesNode1.png | Bin 0 -> 1142 bytes Rdmp.Core/Icons/Dataset1.png | Bin 0 -> 726 bytes Rdmp.Core/Icons/StandardRegex3.png | Bin 0 -> 267 bytes Rdmp.Core/Icons/StandardRegex31.png | Bin 0 -> 267 bytes Rdmp.Core/Icons/StandardRegex311.png | Bin 0 -> 267 bytes 5 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 Rdmp.Core/Icons/AllStandardRegexesNode1.png create mode 100644 Rdmp.Core/Icons/Dataset1.png create mode 100644 Rdmp.Core/Icons/StandardRegex3.png create mode 100644 Rdmp.Core/Icons/StandardRegex31.png create mode 100644 Rdmp.Core/Icons/StandardRegex311.png diff --git a/Rdmp.Core/Icons/AllStandardRegexesNode1.png b/Rdmp.Core/Icons/AllStandardRegexesNode1.png new file mode 100644 index 0000000000000000000000000000000000000000..9f1f246ee6bce8f39298275426ee8471c197522a GIT binary patch literal 1142 zcmV-+1d02JP)N2bZe?^J zG%heMGBNQWX_Wu~1M^8lK~y+TjgtFMTV)u>>p!q$KX^;roWaBmrUN4oLD4vqg=En% zlSy25adW{;gIh$Mj=3R|t888d$}}*da}Z~P7ZQcA4kio-0S1(w({gD?OX-dF^xWFh zp7ZrNrLe}u#3%XWZJ%@A&-;A)*6{DTZzDiV6U~H76ag^&AMUE8tU38PK#b@w(ppY* z=F4HH{MJln5C?ynPeR6$==p};00YcxCl~mlRgbM>Bq{XKibTyN1Mj=tB@bo5P+ zWpI}r*NW5be^?DZuqi%9y-kEJ?N0J4kwfT_S9=fYvcK$od+ZB?Z+Z#O!eGe?1BaL9 zdJ4_fE4jz)m$#o<=-fHFWO~IJX7*_Uy9?W2oWY~%Jlf7wY9vC%O9zh|Y+ZRg=lmnC zh%P9?Z7940+*f|MD2zLMNpogQM;4F%80M|zQQlG%5%X z?`kgY-niai`!!FH0^@N_wm|WBA?ob`ch;7W`Fa}>7_I;}Rf^!JP4FG+z--e2%oLg6 z>ii|{`0Xe+{$<7&_0QeAu4VUzdki*X9w&K+<4T|#;#?hsnX?E~{+W=u(&pjLa3OfQ z9qyc=L~p9>L}CqJb*SE34U(l}b3JQ(Dsg!R+m$>~a1W^RQa8eu25^(r@Rb`A8~mmQ zNX}No0v8Z8SF9XB_{#o9Oq_>jycSHymL}`Z*=sUISnSti&kbLFZEfp^6-weT4khMRfNZwHmS`3;JB>d2#K9YD5msmu7 zKOy33fPA|W()R3Vgr5uSmQV7}d&B5TQyh^`@3mCSUTophU*Nb(IL&U#4p zk0H$-qe6P(PJsD5MP@z!v>-bAqp{GSCOcawRwGSR2WjpU zMB951*dr9}b&?e$Jn=G3wFBJP3nXjDQtwA#K1=z{tmkrWE|0V_p^3I&V5mA4xbcY^ z7(A&4296W&s{ZSxs=xm&)!$d5`mP;ReWv}Y@9JJPG3GXMYp literal 0 HcmV?d00001 diff --git a/Rdmp.Core/Icons/Dataset1.png b/Rdmp.Core/Icons/Dataset1.png new file mode 100644 index 0000000000000000000000000000000000000000..693709cbc1b156839a754e53cbaf409edec69567 GIT binary patch literal 726 zcmV;{0xA88P)T&uF!>{(iS?1OX-eX zKw9bunxR5FrF6QaYs~9>A4#zW^dwIvCpq(+cfR?U`T6-{9LHUqo16RKcDwUVr?cX4 zIN~hJDs48~aRAJ}U_2g=KAB9SP$;0;Y@*$6Ly{z<(`i^NmbL#1W@l#$wOS3;YPBOE zJ;7`?L*?Ga6XzC292wl75}>gDz`(>h?is$JPxm#0jGnotoK|nAVM5$DQ z!C*kO-aeF@+Ejy?nVHEp8V&F~k7BWicsx!aH9kHLRpcQ?L&JFBAB4i&kAaVUxVvzh z3a-EY0%m%8nhI7|SE(QpiBL#sG#VUMM9}*(0mg2(Q$Zq;z|PJNd_Euiem@;jtJN@i za|c2MmsL?PR;yKNwOUA}QuO=7;V@#c7!{~gs?J7hAlsE7U#g?$aRkhSTqLq6iuCu9 z10_j_=;?Dc?4cZ386qH0HkgHTDT|HmGR`W4V2noNQJqfLJEot)q{V_UtsW+m31cP~ zDwWEi3HYBSoF4M;T?VaIdqinn1HZ9}32qs-PdwPbCf+WI6n9jl0-8cjV3%1FB%B&r z+`mzSliyLSH0dxYE}rk&=!uCa*V>()2znj`_XYjtbt>@4FLHnJE|G`xv)Ba@oLBny z1%3K7c4fiB^4{k6E8Pif0kNy62}b@9+NS%G}c0*}aI z1_r*pAk66gd%{|vpk#?_L`iUdT1k0gQ7S`0VrE{6US4X6f{C7io}rmb=K8NdMID|l zjv*HQdwUM@HW=_QTeJFqi|?^yvdM`I3%b=*AyXtZO2y^bDsStwyZa6V03n^J};TM(`}3JQt|f_)$IHp1Kq^n>FVdQ I&MBb@0L{i-;{X5v literal 0 HcmV?d00001 diff --git a/Rdmp.Core/Icons/StandardRegex31.png b/Rdmp.Core/Icons/StandardRegex31.png new file mode 100644 index 0000000000000000000000000000000000000000..e4ae71daedd18329e7b3da3d9314aac2f7be7ed5 GIT binary patch literal 267 zcmeAS@N?(olHy`uVBq!ia0vp^!XV7S1|*9D%+3HQmUKs7M+SzC{oH>NS%G}c0*}aI z1_r*pAk66gd%{|vpk#?_L`iUdT1k0gQ7S`0VrE{6US4X6f{C7io}rmb=K8NdMID|l zjv*HQdwUM@HW=_QTeJFqi|?^yvdM`I3%b=*AyXtZO2y^bDsStwyZa6V03n^J};TM(`}3JQt|f_)$IHp1Kq^n>FVdQ I&MBb@0L{i-;{X5v literal 0 HcmV?d00001 diff --git a/Rdmp.Core/Icons/StandardRegex311.png b/Rdmp.Core/Icons/StandardRegex311.png new file mode 100644 index 0000000000000000000000000000000000000000..e4ae71daedd18329e7b3da3d9314aac2f7be7ed5 GIT binary patch literal 267 zcmeAS@N?(olHy`uVBq!ia0vp^!XV7S1|*9D%+3HQmUKs7M+SzC{oH>NS%G}c0*}aI z1_r*pAk66gd%{|vpk#?_L`iUdT1k0gQ7S`0VrE{6US4X6f{C7io}rmb=K8NdMID|l zjv*HQdwUM@HW=_QTeJFqi|?^yvdM`I3%b=*AyXtZO2y^bDsStwyZa6V03n^J};TM(`}3JQt|f_)$IHp1Kq^n>FVdQ I&MBb@0L{i-;{X5v literal 0 HcmV?d00001 From fb229235da8113cff60825c8af7f54a52b427999 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 16 Sep 2024 12:02:11 +0100 Subject: [PATCH 049/106] tidyup --- .../ExecuteCommandAddNewRegexRedactionConfiguration.cs | 2 +- .../DataHelper/RegexRedaction/RegexRedactionHelper.cs | 6 +++--- .../Mutilators/MatchingTablesMutilatorWithDataLoadJob.cs | 2 +- .../DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewRegexRedactionConfiguration.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewRegexRedactionConfiguration.cs index f3ad57e2cc..c1f43d0a00 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewRegexRedactionConfiguration.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewRegexRedactionConfiguration.cs @@ -24,7 +24,7 @@ public class ExecuteCommandAddNewRegexRedactionConfiguration : BasicCommandExecu private readonly IBasicActivateItems _activator; - public ExecuteCommandAddNewRegexRedactionConfiguration(IBasicActivateItems activator, [DemandsInitialization("Name")] string name, [DemandsInitialization("pattern")] string redactionPattern, [DemandsInitialization("redaction")] string redactionString, string description = null) : base(activator) + public ExecuteCommandAddNewRegexRedactionConfiguration(IBasicActivateItems activator, [DemandsInitialization("Name")] string name, [DemandsInitialization("Redaction Pattern")] string redactionPattern, [DemandsInitialization("Replacement String")] string redactionString, string description = null) : base(activator) { _activator = activator; _name = name; diff --git a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs index e4944d6321..104797ad5a 100644 --- a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs +++ b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs @@ -29,8 +29,8 @@ public enum Constants { ReplacementValue, RedactedValue, RegexRedaction_ID, - Value - + Value, + TEMP_RedactionUpdates }; @@ -140,7 +140,7 @@ INSERT INTO RegexRedactionKey(RegexRedaction_ID,ColumnInfo_ID,Value) public static void DoJoinUpdate(ColumnInfo column, DiscoveredTable _discoveredTable, DiscoveredServer _server, DataTable redactionUpates, DiscoveredColumn[] _discoveredPKColumns, int timeout = 30000) { - var redactionTable = _discoveredTable.Database.CreateTable("TEMP_RedactionUpdates", redactionUpates); + var redactionTable = _discoveredTable.Database.CreateTable(nameof(Constants.TEMP_RedactionUpdates), redactionUpates); var updateHelper = _server.GetQuerySyntaxHelper().UpdateHelper; var sqlLines = new List diff --git a/Rdmp.Core/DataLoad/Modules/Mutilators/MatchingTablesMutilatorWithDataLoadJob.cs b/Rdmp.Core/DataLoad/Modules/Mutilators/MatchingTablesMutilatorWithDataLoadJob.cs index 1b7ef6bfa9..2a67c72963 100644 --- a/Rdmp.Core/DataLoad/Modules/Mutilators/MatchingTablesMutilatorWithDataLoadJob.cs +++ b/Rdmp.Core/DataLoad/Modules/Mutilators/MatchingTablesMutilatorWithDataLoadJob.cs @@ -61,7 +61,7 @@ public ExitCodeType Mutilate(IDataLoadJob job) TableRegexPattern = new Regex(TableRegexPattern.ToString(), RegexOptions.IgnoreCase); foreach (var tableInfo in job.RegularTablesToLoad) - if (OnlyTables != null && OnlyTables.Any()) + if (OnlyTables != null && OnlyTables.Length != 0) { if (OnlyTables.Contains(tableInfo)) FireMutilate(tableInfo, job); diff --git a/Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs b/Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs index 242bebcd17..630174e6fe 100644 --- a/Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs +++ b/Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs @@ -23,7 +23,7 @@ namespace Rdmp.Core.DataLoad.Modules.Mutilators; public class RegexRedactionMutilator : MatchingTablesMutilatorWithDataLoadJob { [DemandsInitialization("the regex redaction configuration to use")] - public RegexRedactionConfiguration redactionConfiguration { get; set; } + public RegexRedactionConfiguration RedactionConfiguration { get; set; } [DemandsInitialization( "All Columns matching this pattern which have a ColumnInfo defined in the load will be affected by this mutilation", @@ -77,7 +77,7 @@ protected override void MutilateTable(IDataLoadJob job, ITableInfo tableInfo, Di var sql = @$" SELECT {column.GetRuntimeName()} {pkSeparator} {string.Join(", ", pkColumnInfos.Select(c => c.GetRuntimeName()))} FROM {table.GetRuntimeName()} - WHERE {column.GetRuntimeName()} LIKE '%{redactionConfiguration.RegexPattern}%' + WHERE {column.GetRuntimeName()} LIKE '%{RedactionConfiguration.RegexPattern}%' "; var dt = new DataTable(); dt.BeginLoadData(); @@ -101,7 +101,7 @@ protected override void MutilateTable(IDataLoadJob job, ITableInfo tableInfo, Di { try { - RegexRedactionHelper.Redact(columnInfo, row, cataloguePks, redactionConfiguration, redactionsToSaveTable, pksToSave, redactionUpates); + RegexRedactionHelper.Redact(columnInfo, row, cataloguePks, RedactionConfiguration, redactionsToSaveTable, pksToSave, redactionUpates); } catch (Exception e) { From 72f8a294aeb1bf9447a9d346ce11e0c0bc66546e Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 16 Sep 2024 12:56:09 +0100 Subject: [PATCH 050/106] fix tests --- .../Mutilators/RegexRedactionMutilatorTests.cs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Rdmp.Core.Tests/DataLoad/Modules/Mutilators/RegexRedactionMutilatorTests.cs b/Rdmp.Core.Tests/DataLoad/Modules/Mutilators/RegexRedactionMutilatorTests.cs index 9c5bc19428..13a90ed15b 100644 --- a/Rdmp.Core.Tests/DataLoad/Modules/Mutilators/RegexRedactionMutilatorTests.cs +++ b/Rdmp.Core.Tests/DataLoad/Modules/Mutilators/RegexRedactionMutilatorTests.cs @@ -85,7 +85,7 @@ public void RedactionMutilator_BasicTest() pt.CreateArgumentsForClassIfNotExists(); pt.SetArgumentValue("ColumnRegexPattern", ".*"); - pt.SetArgumentValue("redactionConfiguration", regex); + pt.SetArgumentValue("RedactionConfiguration", regex); pt.SetArgumentValue("TableRegexPattern", ".*"); pt.Check(ThrowImmediatelyCheckNotifier.Quiet); @@ -192,7 +192,7 @@ public void RedactionMutilator_OddStringLength() pt.CreateArgumentsForClassIfNotExists(); pt.SetArgumentValue("ColumnRegexPattern", ".*"); - pt.SetArgumentValue("redactionConfiguration", regex); + pt.SetArgumentValue("RedactionConfiguration", regex); pt.SetArgumentValue("TableRegexPattern", ".*"); pt.Check(ThrowImmediatelyCheckNotifier.Quiet); @@ -299,7 +299,7 @@ public void RedactionMutilator_RedactionTooLong() pt.CreateArgumentsForClassIfNotExists(); pt.SetArgumentValue("ColumnRegexPattern", ".*"); - pt.SetArgumentValue("redactionConfiguration", regex); + pt.SetArgumentValue("RedactionConfiguration", regex); pt.SetArgumentValue("TableRegexPattern", ".*"); pt.Check(ThrowImmediatelyCheckNotifier.Quiet); @@ -394,7 +394,7 @@ public void RedactionMutilator_RedactAPK() pt.CreateArgumentsForClassIfNotExists(); pt.SetArgumentValue("ColumnRegexPattern", "DateOfBirth"); - pt.SetArgumentValue("redactionConfiguration", regex); + pt.SetArgumentValue("RedactionConfiguration", regex); pt.SetArgumentValue("TableRegexPattern", ".*"); pt.Check(ThrowImmediatelyCheckNotifier.Quiet); @@ -489,7 +489,7 @@ public void RedactionMutilator_NoPKS() pt.CreateArgumentsForClassIfNotExists(); pt.SetArgumentValue("ColumnRegexPattern", "DateOfBirth"); - pt.SetArgumentValue("redactionConfiguration", regex); + pt.SetArgumentValue("RedactionConfiguration", regex); pt.SetArgumentValue("TableRegexPattern", ".*"); pt.Check(ThrowImmediatelyCheckNotifier.Quiet); @@ -584,7 +584,7 @@ public void RedactionMutilator_MultipleInOneCell() pt.CreateArgumentsForClassIfNotExists(); pt.SetArgumentValue("ColumnRegexPattern", ".*"); - pt.SetArgumentValue("redactionConfiguration", regex); + pt.SetArgumentValue("RedactionConfiguration", regex); pt.SetArgumentValue("TableRegexPattern", ".*"); pt.Check(ThrowImmediatelyCheckNotifier.Quiet); @@ -695,7 +695,7 @@ public void RedactionMutilator_SpeedTest() pt.CreateArgumentsForClassIfNotExists(); pt.SetArgumentValue("ColumnRegexPattern", ".*"); - pt.SetArgumentValue("redactionConfiguration", regex); + pt.SetArgumentValue("RedactionConfiguration", regex); pt.SetArgumentValue("TableRegexPattern", ".*"); pt.Check(ThrowImmediatelyCheckNotifier.Quiet); From 326266aef0faf234bc79e2b1de9210926b2b21cf Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 16 Sep 2024 13:08:45 +0100 Subject: [PATCH 051/106] add missing file --- Rdmp.UI/SimpleDialogs/SelectDialog`1.resx | 63 +++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 Rdmp.UI/SimpleDialogs/SelectDialog`1.resx diff --git a/Rdmp.UI/SimpleDialogs/SelectDialog`1.resx b/Rdmp.UI/SimpleDialogs/SelectDialog`1.resx new file mode 100644 index 0000000000..6252b1ef07 --- /dev/null +++ b/Rdmp.UI/SimpleDialogs/SelectDialog`1.resx @@ -0,0 +1,63 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + True + + \ No newline at end of file From 516cc0b84dfeae754dfafe77f67ea70f600af1fe Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 20 Sep 2024 12:38:43 +0100 Subject: [PATCH 052/106] interim --- .../ExecuteCommandCreateDatasetTests.cs | 2 +- .../ExecuteCommandDeleteDatasetTest.cs | 10 +-- ...ecuteCommandLinkCatalogueToDatasetTests.cs | 8 +- ...teCommandLinkCoulmnInfoWithDatasetTests.cs | 6 +- .../ExecuteCommandCreateDataset.cs | 2 +- .../ExecuteCommandDeleteDataset.cs | 4 +- .../ExecuteCommandLinkCatalogueToDataset.cs | 4 +- .../ExecuteCommandLinkColumnInfoToDataset.cs | 4 +- .../Curation/Data/{ => Datasets}/Dataset.cs | 15 +++- .../Datasets/DatasetProviderConfiguration.cs | 73 +++++++++++++++++++ .../Curation/Data/{ => Datasets}/IDataset.cs | 22 +++++- .../Datasets/IDatasetProviderConfiguration.cs | 21 ++++++ .../up/086_updateDataset.sql | 31 ++++++++ .../Dataset/DatasetConfigurationUICommon.cs | 31 -------- Rdmp.Core/Providers/CatalogueChildProvider.cs | 20 +++-- Rdmp.Core/Providers/ICoreChildProvider.cs | 4 +- .../WordDataReleaseFileGenerator.cs | 4 +- .../Reports/ExtractionTime/WordDataWriter.cs | 4 +- Rdmp.UI/Collections/DatasetsCollectionUI.cs | 2 +- .../ExecuteCommandDeleteDatasetUI.cs | 2 +- .../ExecuteCommandLinkCatalogueToDatasetUI.cs | 1 + ...ExecuteCommandLinkColumnInfoToDataSetUI.cs | 1 + .../ProposeExecutionWhenTargetIsDataset.cs | 2 +- Rdmp.UI/MainFormUITabs/CatalogueItemUI.cs | 1 + Rdmp.UI/MainFormUITabs/CatalogueUI.cs | 1 + Rdmp.UI/Menus/DatasetMenu.cs | 2 +- .../SubComponents/DatasetConfigurationUI.cs | 1 + 27 files changed, 209 insertions(+), 69 deletions(-) rename Rdmp.Core/Curation/Data/{ => Datasets}/Dataset.cs (89%) create mode 100644 Rdmp.Core/Curation/Data/Datasets/DatasetProviderConfiguration.cs rename Rdmp.Core/Curation/Data/{ => Datasets}/IDataset.cs (73%) create mode 100644 Rdmp.Core/Curation/Data/Datasets/IDatasetProviderConfiguration.cs create mode 100644 Rdmp.Core/Databases/CatalogueDatabase/up/086_updateDataset.sql delete mode 100644 Rdmp.Core/Dataset/DatasetConfigurationUICommon.cs diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandCreateDatasetTests.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandCreateDatasetTests.cs index 3a7169231c..f0c786c31a 100644 --- a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandCreateDatasetTests.cs +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandCreateDatasetTests.cs @@ -26,7 +26,7 @@ public void TestDatasetCreationOKExtendedParameters() { var cmd = new ExecuteCommandCreateDataset(GetMockActivator(), "dataset2","somedoi","some source"); Assert.DoesNotThrow(cmd.Execute); - var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(static ds => ds.Name == "dataset2" && ds.DigitalObjectIdentifier == "somedoi" && ds.Source == "some source"); + var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(static ds => ds.Name == "dataset2" && ds.DigitalObjectIdentifier == "somedoi" && ds.Source == "some source"); Assert.That(founddataset,Is.Not.Null); } } diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandDeleteDatasetTest.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandDeleteDatasetTest.cs index 8982ec5ff7..155e2442e0 100644 --- a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandDeleteDatasetTest.cs +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandDeleteDatasetTest.cs @@ -13,20 +13,20 @@ public void TestDeleteExistingDataset() { var cmd = new ExecuteCommandCreateDataset(GetMockActivator(), "dataset"); Assert.DoesNotThrow(() => cmd.Execute()); - Assert.That(GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects(), Has.Length.EqualTo(1)); - var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(static ds => ds.Name == "dataset"); + Assert.That(GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects(), Has.Length.EqualTo(1)); + var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(static ds => ds.Name == "dataset"); var delCmd = new ExecuteCommandDeleteDataset(GetMockActivator(), founddataset); Assert.DoesNotThrow(() => delCmd.Execute()); - Assert.That(GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects(), Is.Empty); + Assert.That(GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects(), Is.Empty); } [Test] public void TestDeleteNonExistantDataset() { - Assert.That(GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects(), Is.Empty); + Assert.That(GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects(), Is.Empty); var founddataset = new Core.Curation.Data.Dataset(); var delCmd = new ExecuteCommandDeleteDataset(GetMockActivator(), founddataset); Assert.Throws(() => delCmd.Execute()); - Assert.That(GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects(), Is.Empty); + Assert.That(GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects(), Is.Empty); } } \ No newline at end of file diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandLinkCatalogueToDatasetTests.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandLinkCatalogueToDatasetTests.cs index 89b7dcb6b1..330e1e3d46 100644 --- a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandLinkCatalogueToDatasetTests.cs +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandLinkCatalogueToDatasetTests.cs @@ -34,7 +34,7 @@ public void TestLinkCatalogueToDataset() var cmd = new ExecuteCommandCreateDataset(GetMockActivator(), "dataset"); Assert.DoesNotThrow(cmd.Execute); - var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); + var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); var foundCatalogue = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(static c => c.Name == "Dataset1"); var linkCmd = new ExecuteCommandLinkCatalogueToDataset(GetMockActivator(), foundCatalogue, founddataset); Assert.DoesNotThrow(linkCmd.Execute); @@ -72,7 +72,7 @@ public void TestLinkCatalogueToDatasetNotAll() var cmd = new ExecuteCommandCreateDataset(GetMockActivator(), "dataset"); Assert.DoesNotThrow(cmd.Execute); - var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); + var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); var foundCatalogue = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(c => c.Name == "Dataset1"); var linkCmd = new ExecuteCommandLinkCatalogueToDataset(GetMockActivator(), foundCatalogue, founddataset, false); Assert.DoesNotThrow(linkCmd.Execute); @@ -94,7 +94,7 @@ public void TestLinkCatalogueToDatasetBadCatalogue() { var cmd = new ExecuteCommandCreateDataset(GetMockActivator(), "dataset"); Assert.DoesNotThrow(cmd.Execute); - var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); + var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); var linkCmd = new ExecuteCommandLinkCatalogueToDataset(GetMockActivator(), null, founddataset, false); Assert.Throws(linkCmd.Execute); } @@ -104,7 +104,7 @@ public void TestLinkCatalogueToDatasetBadDataset() { var cmd = new ExecuteCommandCreateDataset(GetMockActivator(), "dataset"); Assert.DoesNotThrow(cmd.Execute); - var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); + var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); var linkCmd = new ExecuteCommandLinkCatalogueToDataset(GetMockActivator(), new Catalogue(GetMockActivator().RepositoryLocator.CatalogueRepository,"catalogue"), null, false); Assert.Throws(linkCmd.Execute); } diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandLinkCoulmnInfoWithDatasetTests.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandLinkCoulmnInfoWithDatasetTests.cs index 02848abedf..30dcb53432 100644 --- a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandLinkCoulmnInfoWithDatasetTests.cs +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandLinkCoulmnInfoWithDatasetTests.cs @@ -33,7 +33,7 @@ public void TestLinkColumnInfoToDataset() var cmd = new ExecuteCommandCreateDataset(GetMockActivator(), "dataset"); Assert.DoesNotThrow(() => cmd.Execute()); - var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); + var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); var linkCmd = new ExecuteCommandLinkColumnInfoToDataset(GetMockActivator(), _c1, founddataset); Assert.DoesNotThrow(() => linkCmd.Execute()); var columInfo = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects(); @@ -67,7 +67,7 @@ public void TestLinkColumnInfoToDatasetNotAll() var cmd = new ExecuteCommandCreateDataset(GetMockActivator(), "dataset"); Assert.DoesNotThrow(() => cmd.Execute()); - var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); + var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); var linkCmd = new ExecuteCommandLinkColumnInfoToDataset(GetMockActivator(), _c1, founddataset, false); Assert.DoesNotThrow(() => linkCmd.Execute()); var columInfo = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().Where(ci => _cata1.CatalogueItems.Contains(ci)); @@ -87,7 +87,7 @@ public void TestLinkCatalogueToDatasetBadColumnInfo() { var cmd = new ExecuteCommandCreateDataset(GetMockActivator(), "dataset"); Assert.DoesNotThrow(() => cmd.Execute()); - var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); + var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); var linkCmd = new ExecuteCommandLinkColumnInfoToDataset(GetMockActivator(), null, founddataset, false); Assert.Throws(() => linkCmd.Execute()); } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateDataset.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateDataset.cs index c04a7929c8..9309b53b9e 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateDataset.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateDataset.cs @@ -24,7 +24,7 @@ public ExecuteCommandCreateDataset(IBasicActivateItems activator, [DemandsInitia public override void Execute() { base.Execute(); - var dataset = new Curation.Data.Dataset(BasicActivator.RepositoryLocator.CatalogueRepository, _name) { DigitalObjectIdentifier = _doi, Source = _source }; + var dataset = new Curation.Data.Datasets.Dataset(BasicActivator.RepositoryLocator.CatalogueRepository, _name) { DigitalObjectIdentifier = _doi, Source = _source }; dataset.SaveToDatabase(); _activator.Publish(dataset); } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDeleteDataset.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDeleteDataset.cs index 734a81c431..8f65b79385 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDeleteDataset.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDeleteDataset.cs @@ -4,9 +4,9 @@ namespace Rdmp.Core.CommandExecution.AtomicCommands; public sealed class ExecuteCommandDeleteDataset: BasicCommandExecution { - private readonly Curation.Data.Dataset _dataset; + private readonly Curation.Data.Datasets.Dataset _dataset; private readonly IBasicActivateItems _activator; -public ExecuteCommandDeleteDataset(IBasicActivateItems activator, [DemandsInitialization("The Dataset to delete")]Curation.Data.Dataset dataset) +public ExecuteCommandDeleteDataset(IBasicActivateItems activator, [DemandsInitialization("The Dataset to delete")]Curation.Data.Datasets.Dataset dataset) { _dataset = dataset; _activator = activator; diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandLinkCatalogueToDataset.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandLinkCatalogueToDataset.cs index 683d910a14..4e74fae767 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandLinkCatalogueToDataset.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandLinkCatalogueToDataset.cs @@ -13,9 +13,9 @@ namespace Rdmp.Core.CommandExecution.AtomicCommands; public sealed class ExecuteCommandLinkCatalogueToDataset : BasicCommandExecution { private readonly Catalogue _catalogue; - private readonly Curation.Data.Dataset _dataset; + private readonly Curation.Data.Datasets.Dataset _dataset; private readonly bool _linkAll; - public ExecuteCommandLinkCatalogueToDataset(IBasicActivateItems activator, [DemandsInitialization("The catalogue To link")]Catalogue catalogue, [DemandsInitialization("The dataset to link to")]Curation.Data.Dataset dataset, bool linkAllOtherColumns = true) : base(activator) + public ExecuteCommandLinkCatalogueToDataset(IBasicActivateItems activator, [DemandsInitialization("The catalogue To link")]Catalogue catalogue, [DemandsInitialization("The dataset to link to")]Curation.Data.Datasets.Dataset dataset, bool linkAllOtherColumns = true) : base(activator) { _catalogue = catalogue; _dataset = dataset; diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandLinkColumnInfoToDataset.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandLinkColumnInfoToDataset.cs index 7eac784215..57034e3851 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandLinkColumnInfoToDataset.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandLinkColumnInfoToDataset.cs @@ -13,9 +13,9 @@ namespace Rdmp.Core.CommandExecution.AtomicCommands; public sealed class ExecuteCommandLinkColumnInfoToDataset : BasicCommandExecution { private readonly ColumnInfo _columnInfo; - private readonly Curation.Data.Dataset _dataset; + private readonly Curation.Data.Datasets.Dataset _dataset; private readonly bool _linkAll; - public ExecuteCommandLinkColumnInfoToDataset(IBasicActivateItems activator, [DemandsInitialization("The column to link")] ColumnInfo columnInfo, [DemandsInitialization("The dataset to link to")] Curation.Data.Dataset dataset, bool linkAllOtherColumns = true) : base(activator) + public ExecuteCommandLinkColumnInfoToDataset(IBasicActivateItems activator, [DemandsInitialization("The column to link")] ColumnInfo columnInfo, [DemandsInitialization("The dataset to link to")] Curation.Data.Datasets.Dataset dataset, bool linkAllOtherColumns = true) : base(activator) { _columnInfo = columnInfo; _dataset = dataset; diff --git a/Rdmp.Core/Curation/Data/Dataset.cs b/Rdmp.Core/Curation/Data/Datasets/Dataset.cs similarity index 89% rename from Rdmp.Core/Curation/Data/Dataset.cs rename to Rdmp.Core/Curation/Data/Datasets/Dataset.cs index aac0df9eae..475975b747 100644 --- a/Rdmp.Core/Curation/Data/Dataset.cs +++ b/Rdmp.Core/Curation/Data/Datasets/Dataset.cs @@ -12,7 +12,7 @@ using Rdmp.Core.MapsDirectlyToDatabaseTable; using System.Diagnostics.CodeAnalysis; -namespace Rdmp.Core.Curation.Data; +namespace Rdmp.Core.Curation.Data.Datasets; /// @@ -22,6 +22,8 @@ public sealed class Dataset : DatabaseEntity, IDataset, IHasFolder private string _digitalObjectIdentifier; private string _source; private string _folder = FolderHelper.Root; + private string _type = null; + private string _url = null; /// [DoNotImportDescriptions] @@ -53,6 +55,17 @@ public string Source set => SetField(ref _source, value); } + public string Type + { + get => _type; + set => SetField(ref _type, value); + } + + public string Url + { + get => _url; + set => SetField(ref _url, value); + } public override string ToString() => Name; diff --git a/Rdmp.Core/Curation/Data/Datasets/DatasetProviderConfiguration.cs b/Rdmp.Core/Curation/Data/Datasets/DatasetProviderConfiguration.cs new file mode 100644 index 0000000000..2c6ec34740 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/DatasetProviderConfiguration.cs @@ -0,0 +1,73 @@ +using MongoDB.Driver; +using Rdmp.Core.Repositories; +using System; +using System.Collections.Generic; +using System.Data.Common; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets; + +public class DatasetProviderConfiguration : DatabaseEntity, IDatasetProviderConfiguration +{ + + private string _type; + private string _name; + private string _url; + private int _dataAccessCredentials; + private string _organisationId; + + public DatasetProviderConfiguration() { } + + public DatasetProviderConfiguration(ICatalogueRepository repository,string name,string type,string url,int dataAccessCredentialsID, string organisationId) + { + repository.InsertAndHydrate(this, new Dictionary + { + {"Name",name }, + {"Type",type}, + {"URL",url}, + {"DataAccessCredentials_ID",dataAccessCredentialsID }, + {"Oranisation_ID", organisationId } + }); + } + + internal DatasetProviderConfiguration(ICatalogueRepository repository, DbDataReader r) : base(repository, r) + { + Name = r["Name"].ToString(); + Type= r["Type"].ToString(); + Url= r["Url"].ToString(); + DataAccessCredentials_ID = int.Parse(r["DataAccessCredentials_ID"].ToString()); + Organisation_ID = r["Organisation_ID"].ToString(); + } + + public string Type + { + get => _type; + set => SetField(ref _type, value); + } + + public string Name + { + get => _name; + set => SetField(ref _name, value); + } + + public string Url + { + get => _url; + set => SetField(ref _url, value); + } + + public int DataAccessCredentials_ID + { + get => _dataAccessCredentials; + set => SetField(ref _dataAccessCredentials, value); + } + + public string Organisation_ID + { + get => _organisationId; + set => SetField(ref _organisationId, value); + } +} diff --git a/Rdmp.Core/Curation/Data/IDataset.cs b/Rdmp.Core/Curation/Data/Datasets/IDataset.cs similarity index 73% rename from Rdmp.Core/Curation/Data/IDataset.cs rename to Rdmp.Core/Curation/Data/Datasets/IDataset.cs index a7853f2994..67b8c369f9 100644 --- a/Rdmp.Core/Curation/Data/IDataset.cs +++ b/Rdmp.Core/Curation/Data/Datasets/IDataset.cs @@ -7,19 +7,37 @@ using Rdmp.Core.MapsDirectlyToDatabaseTable; using Rdmp.Core.Repositories; -namespace Rdmp.Core.Curation.Data; +namespace Rdmp.Core.Curation.Data.Datasets; /// /// The core of datasets within RDMP. /// Simple objects to link up catalogue data to DOI and datasets /// -public interface IDataset: IMapsDirectlyToDatabaseTable +public interface IDataset : IMapsDirectlyToDatabaseTable { /// /// Returns where the object exists (e.g. database) as or null if the object does not exist in a catalogue repository. /// ICatalogueRepository CatalogueRepository { get; } + /// + /// The Name of the Dataset + /// string Name { get; } + + /// + /// The (optional) DOI of the dataset + /// string DigitalObjectIdentifier { get; } + + /// + /// The Source of the Dataset e.g. Pure, HDR + /// + string Type { get; } + + /// + /// The URL to access the dataset + /// + string Url { get; } + } diff --git a/Rdmp.Core/Curation/Data/Datasets/IDatasetProviderConfiguration.cs b/Rdmp.Core/Curation/Data/Datasets/IDatasetProviderConfiguration.cs new file mode 100644 index 0000000000..117ec3eb81 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/IDatasetProviderConfiguration.cs @@ -0,0 +1,21 @@ +using Org.BouncyCastle.Bcpg.OpenPgp; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets; + +public interface IDatasetProviderConfiguration +{ + public string Type { get; } + public string Name { get; } + public string Url { get; } + + public int DataAccessCredentials_ID { get; } + + public string Organisation_ID { get; } + + +} diff --git a/Rdmp.Core/Databases/CatalogueDatabase/up/086_updateDataset.sql b/Rdmp.Core/Databases/CatalogueDatabase/up/086_updateDataset.sql new file mode 100644 index 0000000000..9f84d462a1 --- /dev/null +++ b/Rdmp.Core/Databases/CatalogueDatabase/up/086_updateDataset.sql @@ -0,0 +1,31 @@ +--Version: 8.3.1 +--Description: Add new fields to the Dataset object + + +if not exists (select 1 from sys.tables where name = 'DatasetProviderConfiguration') +BEGIN +CREATE TABLE [dbo].[DatasetProviderConfiguration]( +[ID] [int] IDENTITY(1,1) NOT NULL, +[Type] [varchar](256) NOT NULL, +[Url] [varchar](500), +[Name] [varchar](256) NOT NULL, +[DataAccessCredentials_ID] [int], +Organisation_ID [varchar](250), +PRIMARY KEY(ID), + CONSTRAINT FK_DataAccessCredentials_ID FOREIGN KEY (DataAccessCredentials_ID) + REFERENCES DataAccessCredentials(ID) +) +END +GO + + +if exists(select 1 from INFORMATION_SCHEMA.COLUMNS where TABLE_NAME='Dataset' and COLUMN_NAME='Url') +BEGIN +ALTER TABLE [dbo].[Dataset] +ADD [Provider] [int] NULL; +ALTER TABLE [dbo].[Dataset] +ADD [Url] [varchar](500) NULL; +ALTER TABLE [dbo].[Dataset] +ADD FOREIGN KEY (Provider) REFERENCES DatasetProviderConfiguration(ID); +END +GO \ No newline at end of file diff --git a/Rdmp.Core/Dataset/DatasetConfigurationUICommon.cs b/Rdmp.Core/Dataset/DatasetConfigurationUICommon.cs deleted file mode 100644 index d51360e1a2..0000000000 --- a/Rdmp.Core/Dataset/DatasetConfigurationUICommon.cs +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) The University of Dundee 2018-2019 -// This file is part of the Research Data Management Platform (RDMP). -// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. -// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. -// You should have received a copy of the GNU General Public License along with RDMP. If not, see . - -using Rdmp.Core.CommandExecution; - -namespace Rdmp.Core.Dataset; - -/// -/// Common methods used by Cohort Builder UI implementations. Eliminates -/// code duplication and makes it possible to add new UI formats later -/// e.g. web/console etc -/// -public class DatasetConfigurationUICommon -{ - /// - /// User interface layer for modal dialogs, showing Exceptions etc - /// - public IBasicActivateItems Activator; - - public Curation.Data.Dataset Dataset; - - - public DatasetConfigurationUICommon() - { - } - - -} \ No newline at end of file diff --git a/Rdmp.Core/Providers/CatalogueChildProvider.cs b/Rdmp.Core/Providers/CatalogueChildProvider.cs index 296d18fa1a..83a62f746f 100644 --- a/Rdmp.Core/Providers/CatalogueChildProvider.cs +++ b/Rdmp.Core/Providers/CatalogueChildProvider.cs @@ -17,10 +17,12 @@ using Rdmp.Core.Curation.Data.Cohort.Joinables; using Rdmp.Core.Curation.Data.Dashboarding; using Rdmp.Core.Curation.Data.DataLoad; +using Rdmp.Core.Curation.Data.Datasets; using Rdmp.Core.Curation.Data.Governance; using Rdmp.Core.Curation.Data.ImportExport; using Rdmp.Core.Curation.Data.Pipelines; using Rdmp.Core.Curation.Data.Remoting; +using Rdmp.Core.Datasets; using Rdmp.Core.MapsDirectlyToDatabaseTable; using Rdmp.Core.Providers.Nodes; using Rdmp.Core.Providers.Nodes.CohortNodes; @@ -65,7 +67,7 @@ public class CatalogueChildProvider : ICoreChildProvider //Catalogue side of things public Catalogue[] AllCatalogues { get; set; } - public Curation.Data.Dataset[] AllDatasets { get; set; } + public Curation.Data.Datasets.Dataset[] AllDatasets { get; set; } public Dictionary AllCataloguesDictionary { get; private set; } public SupportingDocument[] AllSupportingDocuments { get; set; } @@ -142,7 +144,7 @@ public class CatalogueChildProvider : ICoreChildProvider public AllPermissionWindowsNode AllPermissionWindowsNode { get; set; } public FolderNode LoadMetadataRootFolder { get; set; } - public FolderNode DatasetRootFolder { get; set; } + public FolderNode DatasetRootFolder { get; set; } public FolderNode CohortIdentificationConfigurationRootFolder { get; set; } public FolderNode CohortIdentificationConfigurationRootFolderWithoutVersionedConfigurations { get; set; } @@ -247,7 +249,7 @@ public CatalogueChildProvider(ICatalogueRepository repository, IChildProvider[] AllCatalogues = GetAllObjects(repository); AllCataloguesDictionary = AllCatalogues.ToDictionaryEx(i => i.ID, o => o); - AllDatasets = GetAllObjects(repository); + AllDatasets = GetAllObjects(repository); AllLoadMetadatas = GetAllObjects(repository); AllLoadMetadataLinkage = GetAllObjects(repository); @@ -269,6 +271,12 @@ public CatalogueChildProvider(ICatalogueRepository repository, IChildProvider[] AllDataAccessCredentialsNode = new AllDataAccessCredentialsNode(); AddChildren(AllDataAccessCredentialsNode); + + //TESTING + //var x = new DatasetProviderConfiguration(repository, "pure", "TODO", "https://dundee-staging.elsevierpure.com/ws/api",AllDataAccessCredentials.First().ID); + //x.SaveToDatabase(); + var x = new PureDatasetProvider(repository,repository.GetAllObjects().First()); + x.AddExistingDataset("Test", "https://dundee-staging.elsevierpure.com/en/datasets/8ae76754-3fcb-467b-b269-675869470719"); AllConnectionStringKeywordsNode = new AllConnectionStringKeywordsNode(); AllConnectionStringKeywords = GetAllObjects(repository).ToArray(); AddToDictionaries(new HashSet(AllConnectionStringKeywords), @@ -387,6 +395,8 @@ public CatalogueChildProvider(ICatalogueRepository repository, IChildProvider[] DatasetRootFolder = FolderHelper.BuildFolderTree(AllDatasets); + + AddChildren(DatasetRootFolder, new DescendancyList(DatasetRootFolder)); ReportProgress("Build Catalogue Folder Root"); @@ -833,7 +843,7 @@ private void AddChildren(FolderNode folder, DescendancyList descen ); } - private void AddChildren(FolderNode folder, DescendancyList descendancy) + private void AddChildren(FolderNode folder, DescendancyList descendancy) { foreach (var child in folder.ChildFolders) //add subfolder children @@ -866,7 +876,7 @@ private void AddChildren(FolderNode folder, D ); } - private void AddChildren(Curation.Data.Dataset lmd, DescendancyList descendancy) + private void AddChildren(Curation.Data.Datasets.Dataset lmd, DescendancyList descendancy) { var childObjects = new List(); AddToDictionaries(new HashSet(childObjects), descendancy); diff --git a/Rdmp.Core/Providers/ICoreChildProvider.cs b/Rdmp.Core/Providers/ICoreChildProvider.cs index bdd068dbf0..4f0a4fbebb 100644 --- a/Rdmp.Core/Providers/ICoreChildProvider.cs +++ b/Rdmp.Core/Providers/ICoreChildProvider.cs @@ -44,12 +44,12 @@ public interface ICoreChildProvider : IChildProvider JoinableCohortAggregateConfigurationUse[] AllJoinUses { get; set; } FolderNode CatalogueRootFolder { get; } - FolderNode DatasetRootFolder { get; } + FolderNode DatasetRootFolder { get; } FolderNode LoadMetadataRootFolder { get; } FolderNode CohortIdentificationConfigurationRootFolder { get; } FolderNode CohortIdentificationConfigurationRootFolderWithoutVersionedConfigurations { get; } Catalogue[] AllCatalogues { get; } - Curation.Data.Dataset[] AllDatasets { get; } + Curation.Data.Datasets.Dataset[] AllDatasets { get; } Dictionary AllCataloguesDictionary { get; } ExternalDatabaseServer[] AllExternalServers { get; } diff --git a/Rdmp.Core/Reports/ExtractionTime/WordDataReleaseFileGenerator.cs b/Rdmp.Core/Reports/ExtractionTime/WordDataReleaseFileGenerator.cs index 0ccb871060..de05db936a 100644 --- a/Rdmp.Core/Reports/ExtractionTime/WordDataReleaseFileGenerator.cs +++ b/Rdmp.Core/Reports/ExtractionTime/WordDataReleaseFileGenerator.cs @@ -162,7 +162,7 @@ private void CreateCohortDetailsTable(XWPFDocument document) } [NotNull] - private string getDOI([NotNull] Curation.Data.Dataset ds) + private string getDOI([NotNull] Curation.Data.Datasets.Dataset ds) { return !string.IsNullOrWhiteSpace(ds.DigitalObjectIdentifier) ? $" (DOI: {ds.DigitalObjectIdentifier})" : ""; } @@ -188,7 +188,7 @@ private void CreateFileSummary(XWPFDocument document) SetTableCell(table, tableLine, 0, extractableDataset.ToString()); var linkedDatasets = extractableDataset.Catalogue.CatalogueItems.Select(static c => c.ColumnInfo).Where(ci => ci.Dataset_ID != null).Distinct().Select(ci => ci.Dataset_ID); - var datasets = _repository.CatalogueRepository.GetAllObjects().Where(d => linkedDatasets.Contains(d.ID)).ToList(); + var datasets = _repository.CatalogueRepository.GetAllObjects().Where(d => linkedDatasets.Contains(d.ID)).ToList(); var datasetString = string.Join("",datasets.Select(ds=> $"{ds.Name} {getDOI(ds)}, {Environment.NewLine}")); SetTableCell(table, tableLine, 1, result.FiltersUsed); SetTableCell(table, tableLine, 2, filename); diff --git a/Rdmp.Core/Reports/ExtractionTime/WordDataWriter.cs b/Rdmp.Core/Reports/ExtractionTime/WordDataWriter.cs index 45806e2f09..cc2384d162 100644 --- a/Rdmp.Core/Reports/ExtractionTime/WordDataWriter.cs +++ b/Rdmp.Core/Reports/ExtractionTime/WordDataWriter.cs @@ -57,7 +57,7 @@ public WordDataWriter(ExtractionPipelineUseCase executer) [NotNull] - private static string GetDoi([NotNull] Curation.Data.Dataset ds) + private static string GetDoi([NotNull] Curation.Data.Datasets.Dataset ds) { return !string.IsNullOrWhiteSpace(ds.DigitalObjectIdentifier) ? $" (DOI: {ds.DigitalObjectIdentifier})" : ""; } @@ -163,7 +163,7 @@ public void GenerateWordFile() if (foundDatasets.Count > 0) { - var datasets = Executer.Source.Request.Catalogue.Repository.GetAllObjects().ToList(); + var datasets = Executer.Source.Request.Catalogue.Repository.GetAllObjects().ToList(); var datasetString = string.Join(", ", foundDatasets diff --git a/Rdmp.UI/Collections/DatasetsCollectionUI.cs b/Rdmp.UI/Collections/DatasetsCollectionUI.cs index 6a72d8d3d7..6c82090542 100644 --- a/Rdmp.UI/Collections/DatasetsCollectionUI.cs +++ b/Rdmp.UI/Collections/DatasetsCollectionUI.cs @@ -7,9 +7,9 @@ using System; using System.Collections.Generic; using System.Linq; -using Rdmp.Core.Curation.Data; using Rdmp.Core.Icons.IconProvision; using System.Windows.Forms; +using Rdmp.Core.Curation.Data.Datasets; namespace Rdmp.UI.Collections; diff --git a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandDeleteDatasetUI.cs b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandDeleteDatasetUI.cs index 90803cb3a3..e0a1117bef 100644 --- a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandDeleteDatasetUI.cs +++ b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandDeleteDatasetUI.cs @@ -1,4 +1,4 @@ -using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.Datasets; using Rdmp.Core.Icons.IconProvision; using Rdmp.Core.ReusableLibraryCode.Icons.IconProvision; using Rdmp.UI.ItemActivation; diff --git a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandLinkCatalogueToDatasetUI.cs b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandLinkCatalogueToDatasetUI.cs index d0b99eb69d..0fb7fb679a 100644 --- a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandLinkCatalogueToDatasetUI.cs +++ b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandLinkCatalogueToDatasetUI.cs @@ -7,6 +7,7 @@ using Rdmp.Core.CommandExecution; using Rdmp.Core.CommandExecution.AtomicCommands; using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.Datasets; using Rdmp.Core.Icons.IconProvision; using Rdmp.Core.ReusableLibraryCode.Annotations; using Rdmp.Core.ReusableLibraryCode.Icons.IconProvision; diff --git a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandLinkColumnInfoToDataSetUI.cs b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandLinkColumnInfoToDataSetUI.cs index d59809fc9e..2acad8ddd8 100644 --- a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandLinkColumnInfoToDataSetUI.cs +++ b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandLinkColumnInfoToDataSetUI.cs @@ -7,6 +7,7 @@ using Rdmp.Core.CommandExecution; using Rdmp.Core.CommandExecution.AtomicCommands; using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.Datasets; using Rdmp.Core.Icons.IconProvision; using Rdmp.Core.ReusableLibraryCode.Annotations; using Rdmp.Core.ReusableLibraryCode.Icons.IconProvision; diff --git a/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsDataset.cs b/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsDataset.cs index 7bc681d63e..22f816d991 100644 --- a/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsDataset.cs +++ b/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsDataset.cs @@ -5,7 +5,7 @@ // You should have received a copy of the GNU General Public License along with RDMP. If not, see . using Rdmp.Core.CommandExecution; -using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.Datasets; using Rdmp.Core.ReusableLibraryCode.Annotations; using Rdmp.UI.ItemActivation; using Rdmp.UI.SubComponents; diff --git a/Rdmp.UI/MainFormUITabs/CatalogueItemUI.cs b/Rdmp.UI/MainFormUITabs/CatalogueItemUI.cs index 983f0b20c5..949a72e78f 100644 --- a/Rdmp.UI/MainFormUITabs/CatalogueItemUI.cs +++ b/Rdmp.UI/MainFormUITabs/CatalogueItemUI.cs @@ -11,6 +11,7 @@ using Rdmp.Core; using Rdmp.Core.CommandExecution.AtomicCommands; using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.Datasets; using Rdmp.UI.ItemActivation; using Rdmp.UI.Rules; using Rdmp.UI.ScintillaHelper; diff --git a/Rdmp.UI/MainFormUITabs/CatalogueUI.cs b/Rdmp.UI/MainFormUITabs/CatalogueUI.cs index fd74e9144e..a3b951cce7 100644 --- a/Rdmp.UI/MainFormUITabs/CatalogueUI.cs +++ b/Rdmp.UI/MainFormUITabs/CatalogueUI.cs @@ -11,6 +11,7 @@ using System.Windows.Forms; using Rdmp.Core; using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.Datasets; using Rdmp.UI.ItemActivation; using Rdmp.UI.Rules; using Rdmp.UI.ScintillaHelper; diff --git a/Rdmp.UI/Menus/DatasetMenu.cs b/Rdmp.UI/Menus/DatasetMenu.cs index 8326c6342e..5abca0c0fe 100644 --- a/Rdmp.UI/Menus/DatasetMenu.cs +++ b/Rdmp.UI/Menus/DatasetMenu.cs @@ -3,7 +3,7 @@ // RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see .using Rdmp.Core.Curation.Data; -using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.Datasets; using Rdmp.UI.CommandExecution.AtomicCommands; namespace Rdmp.UI.Menus; diff --git a/Rdmp.UI/SubComponents/DatasetConfigurationUI.cs b/Rdmp.UI/SubComponents/DatasetConfigurationUI.cs index 1e90f872f6..47b3ae4126 100644 --- a/Rdmp.UI/SubComponents/DatasetConfigurationUI.cs +++ b/Rdmp.UI/SubComponents/DatasetConfigurationUI.cs @@ -7,6 +7,7 @@ using Rdmp.Core.Dataset; using Rdmp.Core.Curation.Data; using Rdmp.UI.ItemActivation; +using Rdmp.Core.Curation.Data.Datasets; namespace Rdmp.UI.SubComponents; public partial class DatasetConfigurationUI : DatsetConfigurationUI_Design, IRefreshBusSubscriber From d79b9b72ad1b49f7e99309fdcb84bd5efe7e2e3f Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 20 Sep 2024 12:39:45 +0100 Subject: [PATCH 053/106] add missing files --- .../Datasets/DatasetConfigurationUICommon.cs | 31 +++++++ Rdmp.Core/Datasets/IDatasetProvider.cs | 17 ++++ Rdmp.Core/Datasets/InternalDatasetProvider.cs | 27 ++++++ Rdmp.Core/Datasets/PluginDatasetProvider.cs | 32 +++++++ Rdmp.Core/Datasets/PureDatasetProvider.cs | 93 +++++++++++++++++++ 5 files changed, 200 insertions(+) create mode 100644 Rdmp.Core/Datasets/DatasetConfigurationUICommon.cs create mode 100644 Rdmp.Core/Datasets/IDatasetProvider.cs create mode 100644 Rdmp.Core/Datasets/InternalDatasetProvider.cs create mode 100644 Rdmp.Core/Datasets/PluginDatasetProvider.cs create mode 100644 Rdmp.Core/Datasets/PureDatasetProvider.cs diff --git a/Rdmp.Core/Datasets/DatasetConfigurationUICommon.cs b/Rdmp.Core/Datasets/DatasetConfigurationUICommon.cs new file mode 100644 index 0000000000..07ee42c59f --- /dev/null +++ b/Rdmp.Core/Datasets/DatasetConfigurationUICommon.cs @@ -0,0 +1,31 @@ +// Copyright (c) The University of Dundee 2018-2019 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using Rdmp.Core.CommandExecution; + +namespace Rdmp.Core.Dataset; + +/// +/// Common methods used by Cohort Builder UI implementations. Eliminates +/// code duplication and makes it possible to add new UI formats later +/// e.g. web/console etc +/// +public class DatasetConfigurationUICommon +{ + /// + /// User interface layer for modal dialogs, showing Exceptions etc + /// + public IBasicActivateItems Activator; + + public Curation.Data.Datasets.Dataset Dataset; + + + public DatasetConfigurationUICommon() + { + } + + +} \ No newline at end of file diff --git a/Rdmp.Core/Datasets/IDatasetProvider.cs b/Rdmp.Core/Datasets/IDatasetProvider.cs new file mode 100644 index 0000000000..e294c1acb4 --- /dev/null +++ b/Rdmp.Core/Datasets/IDatasetProvider.cs @@ -0,0 +1,17 @@ +using Rdmp.Core.Curation.Data.Datasets; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Datasets; + +public interface IDatasetProvider +{ + + List FetchDatasets(); + + Curation.Data.Datasets.Dataset FetchDatasetByID(int id); + +} diff --git a/Rdmp.Core/Datasets/InternalDatasetProvider.cs b/Rdmp.Core/Datasets/InternalDatasetProvider.cs new file mode 100644 index 0000000000..615985a48c --- /dev/null +++ b/Rdmp.Core/Datasets/InternalDatasetProvider.cs @@ -0,0 +1,27 @@ +using Rdmp.Core.CommandExecution; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Datasets; + +public class InternalDatasetProvider : IDatasetProvider +{ + private readonly IBasicActivateItems _activator; + public InternalDatasetProvider(IBasicActivateItems activator) + { + _activator = activator; + } + + public Curation.Data.Datasets.Dataset FetchDatasetByID(int id) + { + return _activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("ID", id).FirstOrDefault(); + } + + public List FetchDatasets() + { + return _activator.RepositoryLocator.CatalogueRepository.GetAllObjects().ToList(); + } +} diff --git a/Rdmp.Core/Datasets/PluginDatasetProvider.cs b/Rdmp.Core/Datasets/PluginDatasetProvider.cs new file mode 100644 index 0000000000..abbbd00a4b --- /dev/null +++ b/Rdmp.Core/Datasets/PluginDatasetProvider.cs @@ -0,0 +1,32 @@ +using Rdmp.Core.Curation.Data.Datasets; +using Rdmp.Core.Repositories; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Datasets; + +public abstract class PluginDatasetProvider : IDatasetProvider +{ + protected DatasetProviderConfiguration Configuration { get; } + protected ICatalogueRepository Repository { get; } + + protected PluginDatasetProvider(ICatalogueRepository repository, DatasetProviderConfiguration configuration) + { + Configuration = configuration; + Repository = repository; + } + + public abstract Curation.Data.Datasets.Dataset FetchDatasetByID(int id); + + public abstract List FetchDatasets(); + + public abstract void AddExistingDataset(string name, string url); + + public abstract Curation.Data.Datasets.Dataset Create(); + + public abstract Curation.Data.Datasets.Dataset Update(); + +} diff --git a/Rdmp.Core/Datasets/PureDatasetProvider.cs b/Rdmp.Core/Datasets/PureDatasetProvider.cs new file mode 100644 index 0000000000..6a9b392190 --- /dev/null +++ b/Rdmp.Core/Datasets/PureDatasetProvider.cs @@ -0,0 +1,93 @@ +using Amazon.Runtime.Internal.Endpoints.StandardLibrary; +using NPOI.SS.Formula.Functions; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.Datasets; +using Rdmp.Core.Repositories; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Text; +using System.Text.Json.Nodes; +using System.Threading.Tasks; + +namespace Rdmp.Core.Datasets +{ + public class PureDatasetProvider : PluginDatasetProvider + { + private HttpClient _client; + + public PureDatasetProvider(ICatalogueRepository repository,DatasetProviderConfiguration configuration) : base(repository, configuration) + { + _client = new HttpClient(); + var credentials = repository.GetAllObjectsWhere("ID", Configuration.DataAccessCredentials_ID).First(); + var apiKey = credentials.GetDecryptedPassword(); + _client.DefaultRequestHeaders.Add("api-key", apiKey); + //FetchDatasets(); + } + + private string UrltoUUID(string url) => url.Split("/").Last(); + + private bool CheckDatasetExistsAtURL(string url) + { + var uri = $"{Configuration.Url}/data-sets/{UrltoUUID(url)}"; + var response = Task.Run(async () => await _client.GetAsync(uri)).Result; + return response.StatusCode == System.Net.HttpStatusCode.OK; + } + + public override void AddExistingDataset(string name, string url) + { + if (CheckDatasetExistsAtURL(url)) + { + var dataset = new Curation.Data.Datasets.Dataset(Repository, name) + { + Url = url, + Type = this.ToString() + }; + dataset.SaveToDatabase(); + } + else + { + throw new Exception("Cannot access dataset at provided url"); + } + } + + public override Curation.Data.Datasets.Dataset Create() + { + throw new NotImplementedException(); + } + + public override Curation.Data.Datasets.Dataset FetchDatasetByID(int id) + { + return null; + } + + public override List FetchDatasets() + { + ////TODO will have to think about paginations + //var uri = $"{Configuration.Url}/data-sets"; + //var response = Task.Run(async () => await _client.GetAsync(uri)).Result; + //if(response.StatusCode != System.Net.HttpStatusCode.OK) + //{ + // throw new Exception("Error"); + //} + //var detailsString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; + //var jsonObjects = JsonObject.Parse(detailsString); + //foreach(var item in jsonObjects.AsObject()) { + + // if (item.Key == "items") + // { + // Console.WriteLine(item.ToString()); + // //the values here are the datasets + // } + //} + return new List(); + + } + + public override Curation.Data.Datasets.Dataset Update() + { + throw new NotImplementedException(); + } + } +} From 273090487e13319d1aeaa536184b9ab871374e00 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 24 Sep 2024 10:18:37 +0100 Subject: [PATCH 054/106] interim --- Rdmp.Core/Providers/CatalogueChildProvider.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Rdmp.Core/Providers/CatalogueChildProvider.cs b/Rdmp.Core/Providers/CatalogueChildProvider.cs index 83a62f746f..3a39f178ed 100644 --- a/Rdmp.Core/Providers/CatalogueChildProvider.cs +++ b/Rdmp.Core/Providers/CatalogueChildProvider.cs @@ -273,10 +273,10 @@ public CatalogueChildProvider(ICatalogueRepository repository, IChildProvider[] //TESTING - //var x = new DatasetProviderConfiguration(repository, "pure", "TODO", "https://dundee-staging.elsevierpure.com/ws/api",AllDataAccessCredentials.First().ID); - //x.SaveToDatabase(); - var x = new PureDatasetProvider(repository,repository.GetAllObjects().First()); - x.AddExistingDataset("Test", "https://dundee-staging.elsevierpure.com/en/datasets/8ae76754-3fcb-467b-b269-675869470719"); + var x = new DatasetProviderConfiguration(repository, "pure", "TODO", "https://dundee-staging.elsevierpure.com/ws/api",AllDataAccessCredentials.First().ID, "ea4354b7-2253-4ca6-add0-e78220794087"); + x.SaveToDatabase(); + var y = new PureDatasetProvider(repository,repository.GetAllObjects().First()); + y.AddExistingDataset("Test", "https://dundee-staging.elsevierpure.com/en/datasets/8ae76754-3fcb-467b-b269-675869470719"); AllConnectionStringKeywordsNode = new AllConnectionStringKeywordsNode(); AllConnectionStringKeywords = GetAllObjects(repository).ToArray(); AddToDictionaries(new HashSet(AllConnectionStringKeywords), From 45196609ce3ef1a4b8e014d8fd44984fbc2d9d38 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 24 Sep 2024 13:03:46 +0100 Subject: [PATCH 055/106] interim --- .../HomePane/HomeBoxUI.Designer.cs | 162 +++++++++--------- .../WindowManagement/HomePane/HomeBoxUI.cs | 8 + .../WindowManagement/HomePane/HomeBoxUI.resx | 50 +++--- Rdmp.Core/Curation/Data/Datasets/Dataset.cs | 16 +- .../Datasets/DatasetProviderConfiguration.cs | 2 +- .../up/086_updateDataset.sql | 6 +- Rdmp.Core/Datasets/PluginDataset.cs | 14 ++ Rdmp.Core/Datasets/PluginDatasetProvider.cs | 9 +- Rdmp.Core/Datasets/PureDataset.cs | 25 +++ Rdmp.Core/Datasets/PureDatasetProvider.cs | 56 +++++- Rdmp.Core/Providers/CatalogueChildProvider.cs | 6 - 11 files changed, 228 insertions(+), 126 deletions(-) create mode 100644 Rdmp.Core/Datasets/PluginDataset.cs create mode 100644 Rdmp.Core/Datasets/PureDataset.cs diff --git a/Application/ResearchDataManagementPlatform/WindowManagement/HomePane/HomeBoxUI.Designer.cs b/Application/ResearchDataManagementPlatform/WindowManagement/HomePane/HomeBoxUI.Designer.cs index b981915e69..f04524082b 100644 --- a/Application/ResearchDataManagementPlatform/WindowManagement/HomePane/HomeBoxUI.Designer.cs +++ b/Application/ResearchDataManagementPlatform/WindowManagement/HomePane/HomeBoxUI.Designer.cs @@ -28,118 +28,112 @@ protected override void Dispose(bool disposing) /// private void InitializeComponent() { - this.components = new System.ComponentModel.Container(); + components = new System.ComponentModel.Container(); System.ComponentModel.ComponentResourceManager resources = new System.ComponentModel.ComponentResourceManager(typeof(HomeBoxUI)); - this.lblTitle = new System.Windows.Forms.Label(); - this.toolStrip1 = new System.Windows.Forms.ToolStrip(); - this.btnNew = new System.Windows.Forms.ToolStripButton(); - this.btnNewDropdown = new System.Windows.Forms.ToolStripDropDownButton(); - this.toolStripSeparator1 = new System.Windows.Forms.ToolStripSeparator(); - this.btnOpen = new System.Windows.Forms.ToolStripButton(); - this.olvRecent = new BrightIdeasSoftware.TreeListView(); - this.olvName = ((BrightIdeasSoftware.OLVColumn)(new BrightIdeasSoftware.OLVColumn())); - this.toolStrip1.SuspendLayout(); - ((System.ComponentModel.ISupportInitialize)(this.olvRecent)).BeginInit(); - this.SuspendLayout(); + lblTitle = new System.Windows.Forms.Label(); + toolStrip1 = new System.Windows.Forms.ToolStrip(); + btnNew = new System.Windows.Forms.ToolStripButton(); + btnNewDropdown = new System.Windows.Forms.ToolStripDropDownButton(); + toolStripSeparator1 = new System.Windows.Forms.ToolStripSeparator(); + btnOpen = new System.Windows.Forms.ToolStripButton(); + olvRecent = new BrightIdeasSoftware.TreeListView(); + olvName = new BrightIdeasSoftware.OLVColumn(); + toolStrip1.SuspendLayout(); + ((System.ComponentModel.ISupportInitialize)olvRecent).BeginInit(); + SuspendLayout(); // // lblTitle // - this.lblTitle.AutoSize = true; - this.lblTitle.Dock = System.Windows.Forms.DockStyle.Top; - this.lblTitle.Font = new System.Drawing.Font("Microsoft Sans Serif", 21.75F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0))); - this.lblTitle.ForeColor = System.Drawing.SystemColors.Desktop; - this.lblTitle.Location = new System.Drawing.Point(0, 0); - this.lblTitle.Name = "lblTitle"; - this.lblTitle.Size = new System.Drawing.Size(71, 33); - this.lblTitle.TabIndex = 1; - this.lblTitle.Text = "Title"; + lblTitle.AutoSize = true; + lblTitle.Dock = System.Windows.Forms.DockStyle.Top; + lblTitle.Font = new System.Drawing.Font("Microsoft Sans Serif", 21.75F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, 0); + lblTitle.ForeColor = System.Drawing.SystemColors.Desktop; + lblTitle.Location = new System.Drawing.Point(0, 0); + lblTitle.Name = "lblTitle"; + lblTitle.Size = new System.Drawing.Size(71, 33); + lblTitle.TabIndex = 1; + lblTitle.Text = "Title"; // // toolStrip1 // - this.toolStrip1.Items.AddRange(new System.Windows.Forms.ToolStripItem[] { - this.btnNew, - this.btnNewDropdown, - this.toolStripSeparator1, - this.btnOpen}); - this.toolStrip1.Location = new System.Drawing.Point(0, 33); - this.toolStrip1.Name = "toolStrip1"; - this.toolStrip1.Size = new System.Drawing.Size(480, 25); - this.toolStrip1.TabIndex = 2; - this.toolStrip1.Text = "toolStrip1"; + toolStrip1.Items.AddRange(new System.Windows.Forms.ToolStripItem[] { btnNew, btnNewDropdown, toolStripSeparator1, btnOpen }); + toolStrip1.Location = new System.Drawing.Point(0, 33); + toolStrip1.Name = "toolStrip1"; + toolStrip1.Size = new System.Drawing.Size(480, 25); + toolStrip1.TabIndex = 2; + toolStrip1.Text = "toolStrip1"; // // btnNew // - this.btnNew.DisplayStyle = System.Windows.Forms.ToolStripItemDisplayStyle.Image; - this.btnNew.Image = ((System.Drawing.Image)(resources.GetObject("btnNew.Image"))); - this.btnNew.ImageTransparentColor = System.Drawing.Color.Magenta; - this.btnNew.Name = "btnNew"; - this.btnNew.Size = new System.Drawing.Size(23, 22); - this.btnNew.Text = "toolStripButton1"; + btnNew.DisplayStyle = System.Windows.Forms.ToolStripItemDisplayStyle.Image; + btnNew.Image = (System.Drawing.Image)resources.GetObject("btnNew.Image"); + btnNew.ImageTransparentColor = System.Drawing.Color.Magenta; + btnNew.Name = "btnNew"; + btnNew.Size = new System.Drawing.Size(23, 22); + btnNew.Text = "toolStripButton1"; // // btnNewDropdown // - this.btnNewDropdown.DisplayStyle = System.Windows.Forms.ToolStripItemDisplayStyle.Image; - this.btnNewDropdown.Image = ((System.Drawing.Image)(resources.GetObject("btnNewDropdown.Image"))); - this.btnNewDropdown.ImageTransparentColor = System.Drawing.Color.Magenta; - this.btnNewDropdown.Name = "btnNewDropdown"; - this.btnNewDropdown.Size = new System.Drawing.Size(29, 22); - this.btnNewDropdown.Text = "newDropdown"; + btnNewDropdown.DisplayStyle = System.Windows.Forms.ToolStripItemDisplayStyle.Image; + btnNewDropdown.Image = (System.Drawing.Image)resources.GetObject("btnNewDropdown.Image"); + btnNewDropdown.ImageTransparentColor = System.Drawing.Color.Magenta; + btnNewDropdown.Name = "btnNewDropdown"; + btnNewDropdown.Size = new System.Drawing.Size(29, 22); + btnNewDropdown.Text = "newDropdown"; // // toolStripSeparator1 // - this.toolStripSeparator1.Name = "toolStripSeparator1"; - this.toolStripSeparator1.Size = new System.Drawing.Size(6, 25); + toolStripSeparator1.Name = "toolStripSeparator1"; + toolStripSeparator1.Size = new System.Drawing.Size(6, 25); // // btnOpen // - this.btnOpen.DisplayStyle = System.Windows.Forms.ToolStripItemDisplayStyle.Image; - this.btnOpen.Image = ((System.Drawing.Image)(resources.GetObject("btnOpen.Image"))); - this.btnOpen.ImageTransparentColor = System.Drawing.Color.Magenta; - this.btnOpen.Name = "btnOpen"; - this.btnOpen.Size = new System.Drawing.Size(23, 22); - this.btnOpen.Text = "toolStripButton1"; + btnOpen.DisplayStyle = System.Windows.Forms.ToolStripItemDisplayStyle.Image; + btnOpen.Image = (System.Drawing.Image)resources.GetObject("btnOpen.Image"); + btnOpen.ImageTransparentColor = System.Drawing.Color.Magenta; + btnOpen.Name = "btnOpen"; + btnOpen.Size = new System.Drawing.Size(23, 22); + btnOpen.Text = "toolStripButton1"; // // olvRecent // - this.olvRecent.AllColumns.Add(this.olvName); - this.olvRecent.CellEditUseWholeCell = false; - this.olvRecent.Columns.AddRange(new System.Windows.Forms.ColumnHeader[] { - this.olvName}); - this.olvRecent.Cursor = System.Windows.Forms.Cursors.Default; - this.olvRecent.Dock = System.Windows.Forms.DockStyle.Fill; - this.olvRecent.FullRowSelect = true; - this.olvRecent.HideSelection = false; - this.olvRecent.Location = new System.Drawing.Point(0, 58); - this.olvRecent.Name = "olvRecent"; - this.olvRecent.RowHeight = 19; - this.olvRecent.ShowGroups = false; - this.olvRecent.Size = new System.Drawing.Size(480, 441); - this.olvRecent.TabIndex = 3; - this.olvRecent.UseCompatibleStateImageBehavior = false; - this.olvRecent.View = System.Windows.Forms.View.Details; - this.olvRecent.VirtualMode = true; + olvRecent.AllColumns.Add(olvName); + olvRecent.CellEditUseWholeCell = false; + olvRecent.Columns.AddRange(new System.Windows.Forms.ColumnHeader[] { olvName }); + olvRecent.Cursor = System.Windows.Forms.Cursors.Default; + olvRecent.Dock = System.Windows.Forms.DockStyle.Fill; + olvRecent.FullRowSelect = true; + olvRecent.HideSelection = false; + olvRecent.Location = new System.Drawing.Point(0, 58); + olvRecent.Name = "olvRecent"; + olvRecent.RowHeight = 19; + olvRecent.ShowGroups = false; + olvRecent.Size = new System.Drawing.Size(480, 441); + olvRecent.TabIndex = 3; + olvRecent.UseCompatibleStateImageBehavior = false; + olvRecent.View = System.Windows.Forms.View.Details; + olvRecent.VirtualMode = true; // // olvName // - this.olvName.FillsFreeSpace = true; - this.olvName.Groupable = false; - this.olvName.Text = "Recent"; + olvName.FillsFreeSpace = true; + olvName.Groupable = false; + olvName.Text = "Recent"; // // HomeBoxUI // - this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F); - this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; - this.Controls.Add(this.olvRecent); - this.Controls.Add(this.toolStrip1); - this.Controls.Add(this.lblTitle); - this.Name = "HomeBoxUI"; - this.Size = new System.Drawing.Size(480, 499); - this.toolStrip1.ResumeLayout(false); - this.toolStrip1.PerformLayout(); - ((System.ComponentModel.ISupportInitialize)(this.olvRecent)).EndInit(); - this.ResumeLayout(false); - this.PerformLayout(); - + AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F); + AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; + Controls.Add(olvRecent); + Controls.Add(toolStrip1); + Controls.Add(lblTitle); + Name = "HomeBoxUI"; + Size = new System.Drawing.Size(480, 499); + toolStrip1.ResumeLayout(false); + toolStrip1.PerformLayout(); + ((System.ComponentModel.ISupportInitialize)olvRecent).EndInit(); + ResumeLayout(false); + PerformLayout(); } #endregion diff --git a/Application/ResearchDataManagementPlatform/WindowManagement/HomePane/HomeBoxUI.cs b/Application/ResearchDataManagementPlatform/WindowManagement/HomePane/HomeBoxUI.cs index 3577516bc7..1cd196b288 100644 --- a/Application/ResearchDataManagementPlatform/WindowManagement/HomePane/HomeBoxUI.cs +++ b/Application/ResearchDataManagementPlatform/WindowManagement/HomePane/HomeBoxUI.cs @@ -11,6 +11,8 @@ using Rdmp.Core.CommandExecution; using Rdmp.Core.CommandExecution.AtomicCommands; using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.Datasets; +using Rdmp.Core.Datasets; using Rdmp.Core.Icons.IconProvision; using Rdmp.Core.MapsDirectlyToDatabaseTable; using Rdmp.UI; @@ -44,6 +46,12 @@ public void SetUp(IActivateItems activator, string title, Type openType, AtomicC if (!_doneSetup) { _activator = activator; + //TESTING + //var x = new DatasetProviderConfiguration(repository, "pure", "TODO", "https://dundee-staging.elsevierpure.com/ws/api",AllDataAccessCredentials.First().ID, "ea4354b7-2253-4ca6-add0-e78220794087"); + //x.SaveToDatabase(); + var y = new PureDatasetProvider(_activator, _activator.RepositoryLocator.CatalogueRepository.GetAllObjects().First()); + y.FetchDatasetByID(10); + //y.AddExistingDataset("Test", "https://dundee-staging.elsevierpure.com/en/datasets/8ae76754-3fcb-467b-b269-675869470719"); lblTitle.Text = title; btnNew.Image = FamFamFamIcons.add.ImageToBitmap(); diff --git a/Application/ResearchDataManagementPlatform/WindowManagement/HomePane/HomeBoxUI.resx b/Application/ResearchDataManagementPlatform/WindowManagement/HomePane/HomeBoxUI.resx index 0176a37b3c..4f663a28b2 100644 --- a/Application/ResearchDataManagementPlatform/WindowManagement/HomePane/HomeBoxUI.resx +++ b/Application/ResearchDataManagementPlatform/WindowManagement/HomePane/HomeBoxUI.resx @@ -1,17 +1,17 @@  - diff --git a/Rdmp.Core/Curation/Data/Datasets/Dataset.cs b/Rdmp.Core/Curation/Data/Datasets/Dataset.cs index 475975b747..bd2597e504 100644 --- a/Rdmp.Core/Curation/Data/Datasets/Dataset.cs +++ b/Rdmp.Core/Curation/Data/Datasets/Dataset.cs @@ -16,7 +16,7 @@ namespace Rdmp.Core.Curation.Data.Datasets; /// -public sealed class Dataset : DatabaseEntity, IDataset, IHasFolder +public class Dataset : DatabaseEntity, IDataset, IHasFolder { private string _name; private string _digitalObjectIdentifier; @@ -24,6 +24,7 @@ public sealed class Dataset : DatabaseEntity, IDataset, IHasFolder private string _folder = FolderHelper.Root; private string _type = null; private string _url = null; + private int? _providerId = null; /// [DoNotImportDescriptions] @@ -66,6 +67,13 @@ public string Url get => _url; set => SetField(ref _url, value); } + + public int? Provider_ID + { + get => _providerId; + set => SetField(ref _providerId, value); + } + public override string ToString() => Name; @@ -88,5 +96,11 @@ internal Dataset(ICatalogueRepository repository, DbDataReader r) DigitalObjectIdentifier = r["DigitalObjectIdentifier"].ToString(); if (r["Source"] != DBNull.Value) Source = r["Source"].ToString(); + if (r["Type"] != DBNull.Value) + Type = r["Type"].ToString(); + if (r["Url"] != DBNull.Value) + Url = r["Url"].ToString(); + if (r["Provider_ID"] != DBNull.Value) + Provider_ID= int.Parse(r["Provider_ID"].ToString()); } } \ No newline at end of file diff --git a/Rdmp.Core/Curation/Data/Datasets/DatasetProviderConfiguration.cs b/Rdmp.Core/Curation/Data/Datasets/DatasetProviderConfiguration.cs index 2c6ec34740..7f334ba18e 100644 --- a/Rdmp.Core/Curation/Data/Datasets/DatasetProviderConfiguration.cs +++ b/Rdmp.Core/Curation/Data/Datasets/DatasetProviderConfiguration.cs @@ -28,7 +28,7 @@ public DatasetProviderConfiguration(ICatalogueRepository repository,string name, {"Type",type}, {"URL",url}, {"DataAccessCredentials_ID",dataAccessCredentialsID }, - {"Oranisation_ID", organisationId } + {"Organisation_ID", organisationId } }); } diff --git a/Rdmp.Core/Databases/CatalogueDatabase/up/086_updateDataset.sql b/Rdmp.Core/Databases/CatalogueDatabase/up/086_updateDataset.sql index 9f84d462a1..008d2b3aa7 100644 --- a/Rdmp.Core/Databases/CatalogueDatabase/up/086_updateDataset.sql +++ b/Rdmp.Core/Databases/CatalogueDatabase/up/086_updateDataset.sql @@ -26,6 +26,10 @@ ADD [Provider] [int] NULL; ALTER TABLE [dbo].[Dataset] ADD [Url] [varchar](500) NULL; ALTER TABLE [dbo].[Dataset] -ADD FOREIGN KEY (Provider) REFERENCES DatasetProviderConfiguration(ID); +ADD [Type] [varchar](500) NULL; +ALTER TABLE [dbo].[Dataset] +ADD [Provider_ID] [int] NULL; +ALTER TABLE [dbo].[Dataset] +ADD FOREIGN KEY (Provider_ID) REFERENCES DatasetProviderConfiguration(ID); END GO \ No newline at end of file diff --git a/Rdmp.Core/Datasets/PluginDataset.cs b/Rdmp.Core/Datasets/PluginDataset.cs new file mode 100644 index 0000000000..3115714c04 --- /dev/null +++ b/Rdmp.Core/Datasets/PluginDataset.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Datasets +{ + public class PluginDataset: Curation.Data.Datasets.Dataset + { + + public PluginDataset() { } + } +} diff --git a/Rdmp.Core/Datasets/PluginDatasetProvider.cs b/Rdmp.Core/Datasets/PluginDatasetProvider.cs index abbbd00a4b..14c17968bc 100644 --- a/Rdmp.Core/Datasets/PluginDatasetProvider.cs +++ b/Rdmp.Core/Datasets/PluginDatasetProvider.cs @@ -1,4 +1,5 @@ -using Rdmp.Core.Curation.Data.Datasets; +using Rdmp.Core.CommandExecution; +using Rdmp.Core.Curation.Data.Datasets; using Rdmp.Core.Repositories; using System; using System.Collections.Generic; @@ -12,11 +13,13 @@ public abstract class PluginDatasetProvider : IDatasetProvider { protected DatasetProviderConfiguration Configuration { get; } protected ICatalogueRepository Repository { get; } + protected IBasicActivateItems Activator { get; } - protected PluginDatasetProvider(ICatalogueRepository repository, DatasetProviderConfiguration configuration) + protected PluginDatasetProvider(IBasicActivateItems activator, DatasetProviderConfiguration configuration) { Configuration = configuration; - Repository = repository; + Activator = activator; + Repository = activator.RepositoryLocator.CatalogueRepository; } public abstract Curation.Data.Datasets.Dataset FetchDatasetByID(int id); diff --git a/Rdmp.Core/Datasets/PureDataset.cs b/Rdmp.Core/Datasets/PureDataset.cs new file mode 100644 index 0000000000..3d7598b353 --- /dev/null +++ b/Rdmp.Core/Datasets/PureDataset.cs @@ -0,0 +1,25 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Datasets; + +public class PureDataset: PluginDataset +{ + public int PureID { get; set; } + public string UUID { get; set; } + public string CreatedDate { get; set; } + public string CreatedBy { get; set; } + public string ModifiedDate { get; set; } + public string ModifiedBy { get; set; } + + public string PortalURL { get; set; } + + public string Version { get; set; } + + + + public PureDataset() { } +} diff --git a/Rdmp.Core/Datasets/PureDatasetProvider.cs b/Rdmp.Core/Datasets/PureDatasetProvider.cs index 6a9b392190..6867bd1525 100644 --- a/Rdmp.Core/Datasets/PureDatasetProvider.cs +++ b/Rdmp.Core/Datasets/PureDatasetProvider.cs @@ -1,10 +1,14 @@ using Amazon.Runtime.Internal.Endpoints.StandardLibrary; +using Newtonsoft.Json; using NPOI.SS.Formula.Functions; +using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.Data.Datasets; +using Rdmp.Core.Curation.Data.Serialization; using Rdmp.Core.Repositories; using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Net.Http; using System.Text; @@ -17,13 +21,12 @@ public class PureDatasetProvider : PluginDatasetProvider { private HttpClient _client; - public PureDatasetProvider(ICatalogueRepository repository,DatasetProviderConfiguration configuration) : base(repository, configuration) + public PureDatasetProvider(IBasicActivateItems activator, DatasetProviderConfiguration configuration) : base(activator, configuration) { _client = new HttpClient(); - var credentials = repository.GetAllObjectsWhere("ID", Configuration.DataAccessCredentials_ID).First(); + var credentials = Repository.GetAllObjectsWhere("ID", Configuration.DataAccessCredentials_ID).First(); var apiKey = credentials.GetDecryptedPassword(); _client.DefaultRequestHeaders.Add("api-key", apiKey); - //FetchDatasets(); } private string UrltoUUID(string url) => url.Split("/").Last(); @@ -42,9 +45,11 @@ public override void AddExistingDataset(string name, string url) var dataset = new Curation.Data.Datasets.Dataset(Repository, name) { Url = url, - Type = this.ToString() + Type = this.ToString(), + Provider_ID = Configuration.ID }; dataset.SaveToDatabase(); + Activator.Publish(dataset); } else { @@ -57,9 +62,50 @@ public override Curation.Data.Datasets.Dataset Create() throw new NotImplementedException(); } + private PureDataset FetchPureDataset(Curation.Data.Datasets.Dataset dataset) + { + var uri = $"{Configuration.Url}/data-sets/{UrltoUUID(dataset.Url)}"; + var response = Task.Run(async () => await _client.GetAsync(uri)).Result; + if (response.StatusCode != System.Net.HttpStatusCode.OK) + { + throw new Exception("Error"); + } + var detailsString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; + PureDataset pd = JsonConvert.DeserializeObject(detailsString); + return pd; + } + + public override Curation.Data.Datasets.Dataset FetchDatasetByID(int id) { - return null; + var dataset = Repository.GetAllObjectsWhere("ID", id).FirstOrDefault(); + //if (dataset is null) + //{ + // throw new Exception("Unable to find local dataset with ID"); + //} + //if (dataset.Type != this.ToString()) + //{ + // var x = this.ToString(); + // throw new Exception("Dataset is of the wrong type"); + //} + //if (dataset.Provider_ID is null) + //{ + // throw new Exception("No provider specified"); + //} + //var provider = Repository.GetAllObjectsWhere("ID", dataset.Provider_ID).FirstOrDefault(); + //if (provider is null) + //{ + // throw new Exception("Unable to locate provider"); + //} + //var uri = $"{Configuration.Url}/data-sets/{UrltoUUID(dataset.Url)}"; + //var response = Task.Run(async () => await _client.GetAsync(uri)).Result; + //if (response.StatusCode != System.Net.HttpStatusCode.OK) + //{ + // throw new Exception("Error"); + //} + //var detailsString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; + //PureDataset pd = JsonConvert.DeserializeObject(detailsString); + return dataset; } public override List FetchDatasets() diff --git a/Rdmp.Core/Providers/CatalogueChildProvider.cs b/Rdmp.Core/Providers/CatalogueChildProvider.cs index 3a39f178ed..eb766f0df6 100644 --- a/Rdmp.Core/Providers/CatalogueChildProvider.cs +++ b/Rdmp.Core/Providers/CatalogueChildProvider.cs @@ -271,12 +271,6 @@ public CatalogueChildProvider(ICatalogueRepository repository, IChildProvider[] AllDataAccessCredentialsNode = new AllDataAccessCredentialsNode(); AddChildren(AllDataAccessCredentialsNode); - - //TESTING - var x = new DatasetProviderConfiguration(repository, "pure", "TODO", "https://dundee-staging.elsevierpure.com/ws/api",AllDataAccessCredentials.First().ID, "ea4354b7-2253-4ca6-add0-e78220794087"); - x.SaveToDatabase(); - var y = new PureDatasetProvider(repository,repository.GetAllObjects().First()); - y.AddExistingDataset("Test", "https://dundee-staging.elsevierpure.com/en/datasets/8ae76754-3fcb-467b-b269-675869470719"); AllConnectionStringKeywordsNode = new AllConnectionStringKeywordsNode(); AllConnectionStringKeywords = GetAllObjects(repository).ToArray(); AddToDictionaries(new HashSet(AllConnectionStringKeywords), From fcefc68291f4181fd1d3ab376de564860586909e Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 24 Sep 2024 17:27:19 +0100 Subject: [PATCH 056/106] upate to pure class --- Rdmp.Core/Datasets/PureDataset.cs | 97 +++++++++++++++++++++-- Rdmp.Core/Datasets/PureDatasetProvider.cs | 7 +- 2 files changed, 96 insertions(+), 8 deletions(-) diff --git a/Rdmp.Core/Datasets/PureDataset.cs b/Rdmp.Core/Datasets/PureDataset.cs index 3d7598b353..4d7cbb7c80 100644 --- a/Rdmp.Core/Datasets/PureDataset.cs +++ b/Rdmp.Core/Datasets/PureDataset.cs @@ -1,12 +1,73 @@ -using System; +using Amazon.Auth.AccessControlPolicy.ActionIdentifiers; +using SynthEHR; using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace Rdmp.Core.Datasets; -public class PureDataset: PluginDataset +public class ENGBWrapper +{ + public string en_GB { get; set; } +} + +public class URITerm +{ + public string URI { get; set; } + public ENGBWrapper term { get; set; } +} +public class PureDescription +{ + public string PureID { get; set; } + public ENGBWrapper Value { get; set; } +} + +public class System +{ + public string SystemName { get; set; } + public string UUID { get; set; } +} + +public class Name() +{ + public string FirstName { get; set; } + public string LastName { get; set; } +} + +public class PurePerson +{ + public string TypeDiscriminator { get; set; } + public string PureID { get; set; } + + public Name Name { get; set; } + public URITerm Role { get; set; } + + public List Organizations { get; set; } +} + +public class PureDate +{ + public int Year { get; set; } + public int Month { get; set; } + public int Day { get; set; } +} + +public class Visability +{ + public string Key { get; set; } + public ENGBWrapper Description { get; set; } +} + +public class Workflow +{ + public string Step { get; set; } + public ENGBWrapper Description { get; set; } +} + +public class Geolocation +{ + public ENGBWrapper GeographicalCoverage { get; set; } +} + +public class PureDataset : PluginDataset { public int PureID { get; set; } public string UUID { get; set; } @@ -19,7 +80,33 @@ public class PureDataset: PluginDataset public string Version { get; set; } + public ENGBWrapper Title { get; set; } + + public List Descriptions { get; set; } + public System ManagingOrganisation { get; set; } + + public new URITerm Type { get; set; } + + public System Publisher { get; set; } + + public Geolocation Geolocation { get; set; } + + public List Persons { get; set; } + + public List Organizations { get; set; } + public PureDate PublicationAvailableDate { get; set; } + + public bool OpenAireCompliant { get; set; } + + public Visability Visability { get; set; } + + //TODO custom defined fields + + public Workflow Workflow { get; set; } + + public string SystemName { get; set; } public PureDataset() { } + } diff --git a/Rdmp.Core/Datasets/PureDatasetProvider.cs b/Rdmp.Core/Datasets/PureDatasetProvider.cs index 6867bd1525..24f5816dbd 100644 --- a/Rdmp.Core/Datasets/PureDatasetProvider.cs +++ b/Rdmp.Core/Datasets/PureDatasetProvider.cs @@ -14,7 +14,7 @@ using System.Text; using System.Text.Json.Nodes; using System.Threading.Tasks; - +using System.Net; namespace Rdmp.Core.Datasets { public class PureDatasetProvider : PluginDatasetProvider @@ -35,7 +35,7 @@ private bool CheckDatasetExistsAtURL(string url) { var uri = $"{Configuration.Url}/data-sets/{UrltoUUID(url)}"; var response = Task.Run(async () => await _client.GetAsync(uri)).Result; - return response.StatusCode == System.Net.HttpStatusCode.OK; + return response.StatusCode == HttpStatusCode.OK; } public override void AddExistingDataset(string name, string url) @@ -66,7 +66,7 @@ private PureDataset FetchPureDataset(Curation.Data.Datasets.Dataset dataset) { var uri = $"{Configuration.Url}/data-sets/{UrltoUUID(dataset.Url)}"; var response = Task.Run(async () => await _client.GetAsync(uri)).Result; - if (response.StatusCode != System.Net.HttpStatusCode.OK) + if (response.StatusCode != HttpStatusCode.OK) { throw new Exception("Error"); } @@ -79,6 +79,7 @@ private PureDataset FetchPureDataset(Curation.Data.Datasets.Dataset dataset) public override Curation.Data.Datasets.Dataset FetchDatasetByID(int id) { var dataset = Repository.GetAllObjectsWhere("ID", id).FirstOrDefault(); + FetchPureDataset(dataset); //if (dataset is null) //{ // throw new Exception("Unable to find local dataset with ID"); From 2a516bf15bc863ffbf3b5f92b0d6952b068e9911 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 25 Sep 2024 14:05:03 +0100 Subject: [PATCH 057/106] add changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c4ac89757..172ecdb1dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [8.4.0] - Unreleased + ## [8.3.1] - Unreleased - Improve Performance of regenerating problems with child providers From 01928421da152d716eb886917c9ac3b699a827a9 Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 26 Sep 2024 09:30:12 +0100 Subject: [PATCH 058/106] add basic ui --- .../HomePane/HomeBoxUI.Designer.cs | 13 +- .../WindowManagement/HomePane/HomeBoxUI.cs | 12 +- Rdmp.Core/Datasets/PureDatasetProvider.cs | 4 +- .../ProposeExecutionWhenTargetIsDataset.cs | 11 ++ .../PureDatasetConfigurationUI.Designer.cs | 90 +++++++++++++ .../PureDatasetConfigurationUI.cs | 63 +++++++++ .../PureDatasetConfigurationUI.resx | 123 ++++++++++++++++++ 7 files changed, 299 insertions(+), 17 deletions(-) create mode 100644 Rdmp.UI/SubComponents/PureDatasetConfigurationUI.Designer.cs create mode 100644 Rdmp.UI/SubComponents/PureDatasetConfigurationUI.cs create mode 100644 Rdmp.UI/SubComponents/PureDatasetConfigurationUI.resx diff --git a/Application/ResearchDataManagementPlatform/WindowManagement/HomePane/HomeBoxUI.Designer.cs b/Application/ResearchDataManagementPlatform/WindowManagement/HomePane/HomeBoxUI.Designer.cs index f04524082b..44cc866c45 100644 --- a/Application/ResearchDataManagementPlatform/WindowManagement/HomePane/HomeBoxUI.Designer.cs +++ b/Application/ResearchDataManagementPlatform/WindowManagement/HomePane/HomeBoxUI.Designer.cs @@ -49,6 +49,7 @@ private void InitializeComponent() lblTitle.Font = new System.Drawing.Font("Microsoft Sans Serif", 21.75F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, 0); lblTitle.ForeColor = System.Drawing.SystemColors.Desktop; lblTitle.Location = new System.Drawing.Point(0, 0); + lblTitle.Margin = new System.Windows.Forms.Padding(4, 0, 4, 0); lblTitle.Name = "lblTitle"; lblTitle.Size = new System.Drawing.Size(71, 33); lblTitle.TabIndex = 1; @@ -59,7 +60,7 @@ private void InitializeComponent() toolStrip1.Items.AddRange(new System.Windows.Forms.ToolStripItem[] { btnNew, btnNewDropdown, toolStripSeparator1, btnOpen }); toolStrip1.Location = new System.Drawing.Point(0, 33); toolStrip1.Name = "toolStrip1"; - toolStrip1.Size = new System.Drawing.Size(480, 25); + toolStrip1.Size = new System.Drawing.Size(560, 25); toolStrip1.TabIndex = 2; toolStrip1.Text = "toolStrip1"; // @@ -100,15 +101,14 @@ private void InitializeComponent() olvRecent.AllColumns.Add(olvName); olvRecent.CellEditUseWholeCell = false; olvRecent.Columns.AddRange(new System.Windows.Forms.ColumnHeader[] { olvName }); - olvRecent.Cursor = System.Windows.Forms.Cursors.Default; olvRecent.Dock = System.Windows.Forms.DockStyle.Fill; olvRecent.FullRowSelect = true; - olvRecent.HideSelection = false; olvRecent.Location = new System.Drawing.Point(0, 58); + olvRecent.Margin = new System.Windows.Forms.Padding(4, 3, 4, 3); olvRecent.Name = "olvRecent"; olvRecent.RowHeight = 19; olvRecent.ShowGroups = false; - olvRecent.Size = new System.Drawing.Size(480, 441); + olvRecent.Size = new System.Drawing.Size(560, 518); olvRecent.TabIndex = 3; olvRecent.UseCompatibleStateImageBehavior = false; olvRecent.View = System.Windows.Forms.View.Details; @@ -122,13 +122,14 @@ private void InitializeComponent() // // HomeBoxUI // - AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F); + AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; Controls.Add(olvRecent); Controls.Add(toolStrip1); Controls.Add(lblTitle); + Margin = new System.Windows.Forms.Padding(4, 3, 4, 3); Name = "HomeBoxUI"; - Size = new System.Drawing.Size(480, 499); + Size = new System.Drawing.Size(560, 576); toolStrip1.ResumeLayout(false); toolStrip1.PerformLayout(); ((System.ComponentModel.ISupportInitialize)olvRecent).EndInit(); diff --git a/Application/ResearchDataManagementPlatform/WindowManagement/HomePane/HomeBoxUI.cs b/Application/ResearchDataManagementPlatform/WindowManagement/HomePane/HomeBoxUI.cs index 1cd196b288..28b5d354de 100644 --- a/Application/ResearchDataManagementPlatform/WindowManagement/HomePane/HomeBoxUI.cs +++ b/Application/ResearchDataManagementPlatform/WindowManagement/HomePane/HomeBoxUI.cs @@ -46,12 +46,6 @@ public void SetUp(IActivateItems activator, string title, Type openType, AtomicC if (!_doneSetup) { _activator = activator; - //TESTING - //var x = new DatasetProviderConfiguration(repository, "pure", "TODO", "https://dundee-staging.elsevierpure.com/ws/api",AllDataAccessCredentials.First().ID, "ea4354b7-2253-4ca6-add0-e78220794087"); - //x.SaveToDatabase(); - var y = new PureDatasetProvider(_activator, _activator.RepositoryLocator.CatalogueRepository.GetAllObjects().First()); - y.FetchDatasetByID(10); - //y.AddExistingDataset("Test", "https://dundee-staging.elsevierpure.com/en/datasets/8ae76754-3fcb-467b-b269-675869470719"); lblTitle.Text = title; btnNew.Image = FamFamFamIcons.add.ImageToBitmap(); @@ -67,9 +61,9 @@ public void SetUp(IActivateItems activator, string title, Type openType, AtomicC btnOpen.Click += (s, e) => { if (activator.SelectObject(new DialogArgs - { - WindowTitle = "Open" - }, activator.GetAll(openType).ToArray(), out var selected)) + { + WindowTitle = "Open" + }, activator.GetAll(openType).ToArray(), out var selected)) Open(selected); }; diff --git a/Rdmp.Core/Datasets/PureDatasetProvider.cs b/Rdmp.Core/Datasets/PureDatasetProvider.cs index 24f5816dbd..f2d65b31eb 100644 --- a/Rdmp.Core/Datasets/PureDatasetProvider.cs +++ b/Rdmp.Core/Datasets/PureDatasetProvider.cs @@ -29,7 +29,7 @@ public PureDatasetProvider(IBasicActivateItems activator, DatasetProviderConfigu _client.DefaultRequestHeaders.Add("api-key", apiKey); } - private string UrltoUUID(string url) => url.Split("/").Last(); + public static string UrltoUUID(string url) => url.Split("/").Last(); private bool CheckDatasetExistsAtURL(string url) { @@ -62,7 +62,7 @@ public override Curation.Data.Datasets.Dataset Create() throw new NotImplementedException(); } - private PureDataset FetchPureDataset(Curation.Data.Datasets.Dataset dataset) + public PureDataset FetchPureDataset(Curation.Data.Datasets.Dataset dataset) { var uri = $"{Configuration.Url}/data-sets/{UrltoUUID(dataset.Url)}"; var response = Task.Run(async () => await _client.GetAsync(uri)).Result; diff --git a/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsDataset.cs b/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsDataset.cs index 22f816d991..4dd1e01798 100644 --- a/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsDataset.cs +++ b/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsDataset.cs @@ -4,8 +4,10 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . +using Org.BouncyCastle.Asn1.X509.Qualified; using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation.Data.Datasets; +using Rdmp.Core.Datasets; using Rdmp.Core.ReusableLibraryCode.Annotations; using Rdmp.UI.ItemActivation; using Rdmp.UI.SubComponents; @@ -25,9 +27,18 @@ public ProposeExecutionWhenTargetIsDataset(IActivateItems itemActivator) : base( public override void Activate(Dataset target) { + string PureAssembly = typeof(PureDatasetProvider).ToString(); + + if(target.Type == PureAssembly) + { + ItemActivator.Activate(target); + return; + } + ItemActivator.Activate(target); } + [CanBeNull] public override ICommandExecution ProposeExecution(ICombineToMakeCommand cmd, Dataset target, diff --git a/Rdmp.UI/SubComponents/PureDatasetConfigurationUI.Designer.cs b/Rdmp.UI/SubComponents/PureDatasetConfigurationUI.Designer.cs new file mode 100644 index 0000000000..56be63815b --- /dev/null +++ b/Rdmp.UI/SubComponents/PureDatasetConfigurationUI.Designer.cs @@ -0,0 +1,90 @@ +namespace Rdmp.UI.SubComponents +{ + partial class PureDatasetConfigurationUI + { + /// + /// Required designer variable. + /// + private System.ComponentModel.IContainer components = null; + + /// + /// Clean up any resources being used. + /// + /// true if managed resources should be disposed; otherwise, false. + protected override void Dispose(bool disposing) + { + if (disposing && (components != null)) + { + components.Dispose(); + } + base.Dispose(disposing); + } + + #region Component Designer generated code + + /// + /// Required method for Designer support - do not modify + /// the contents of this method with the code editor. + /// + private void InitializeComponent() + { + label1 = new System.Windows.Forms.Label(); + columnButtonRenderer1 = new BrightIdeasSoftware.ColumnButtonRenderer(); + btnViewOnPure = new System.Windows.Forms.Button(); + lblUrl = new System.Windows.Forms.Label(); + SuspendLayout(); + // + // label1 + // + label1.AutoSize = true; + label1.Location = new System.Drawing.Point(42, 80); + label1.Name = "label1"; + label1.Size = new System.Drawing.Size(542, 15); + label1.TabIndex = 0; + label1.Text = "Pure Datasets are not currently managable via RDMP. Please visit the following link to edit this dataset"; + label1.Click += label1_Click; + // + // columnButtonRenderer1 + // + columnButtonRenderer1.ButtonPadding = new System.Drawing.Size(10, 10); + // + // btnViewOnPure + // + btnViewOnPure.Enabled = false; + btnViewOnPure.Location = new System.Drawing.Point(42, 40); + btnViewOnPure.Name = "btnViewOnPure"; + btnViewOnPure.Size = new System.Drawing.Size(121, 23); + btnViewOnPure.TabIndex = 1; + btnViewOnPure.Text = "View on Pure"; + btnViewOnPure.UseVisualStyleBackColor = true; + btnViewOnPure.Click += btnViewOnPure_Click; + // + // lblUrl + // + lblUrl.AutoSize = true; + lblUrl.Location = new System.Drawing.Point(47, 104); + lblUrl.Name = "lblUrl"; + lblUrl.Size = new System.Drawing.Size(0, 15); + lblUrl.TabIndex = 2; + // + // PureDatasetConfigurationUI + // + AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); + AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; + Controls.Add(lblUrl); + Controls.Add(btnViewOnPure); + Controls.Add(label1); + Name = "PureDatasetConfigurationUI"; + Size = new System.Drawing.Size(955, 513); + ResumeLayout(false); + PerformLayout(); + } + + #endregion + + private System.Windows.Forms.Label label1; + private BrightIdeasSoftware.ColumnButtonRenderer columnButtonRenderer1; + private System.Windows.Forms.Button btnViewOnPure; + private System.Windows.Forms.Label lblUrl; + } +} diff --git a/Rdmp.UI/SubComponents/PureDatasetConfigurationUI.cs b/Rdmp.UI/SubComponents/PureDatasetConfigurationUI.cs new file mode 100644 index 0000000000..fe52d827b9 --- /dev/null +++ b/Rdmp.UI/SubComponents/PureDatasetConfigurationUI.cs @@ -0,0 +1,63 @@ +using Rdmp.Core.Curation.Data.Datasets; +using Rdmp.Core.Datasets; +using Rdmp.Core.ReusableLibraryCode; +using Rdmp.UI.ItemActivation; +using Rdmp.UI.Refreshing; +using Rdmp.UI.TestsAndSetup.ServicePropogation; +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Data; +using System.Diagnostics; +using System.Drawing; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Windows.Forms; + +namespace Rdmp.UI.SubComponents; + +public partial class PureDatasetConfigurationUI : DatsetConfigurationUI_Design, IRefreshBusSubscriber +{ + + private PureDataset _dataset; + public PureDatasetConfigurationUI() + { + InitializeComponent(); + } + + public override void SetDatabaseObject(IActivateItems activator, Dataset databaseObject) + { + base.SetDatabaseObject(activator, databaseObject); + var provider = new PureDatasetProvider(activator, activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("ID", databaseObject.Provider_ID).First()); + _dataset = provider.FetchPureDataset(databaseObject); + lblUrl.Text = _dataset.PortalURL; + if (_dataset.PortalURL is not null) + { + btnViewOnPure.Enabled = true; + } + + } + + public void RefreshBus_RefreshObject(object sender, RefreshObjectEventArgs e) + { + throw new NotImplementedException(); + } + + private void label1_Click(object sender, EventArgs e) + { + + } + + private void btnViewOnPure_Click(object sender, EventArgs e) + { + UsefulStuff.OpenUrl(_dataset.PortalURL); + } +} + +[TypeDescriptionProvider( + typeof(AbstractControlDescriptionProvider))] +public abstract class + PureDatsetConfigurationUI_Design : RDMPSingleDatabaseObjectControl +{ +} \ No newline at end of file diff --git a/Rdmp.UI/SubComponents/PureDatasetConfigurationUI.resx b/Rdmp.UI/SubComponents/PureDatasetConfigurationUI.resx new file mode 100644 index 0000000000..5fab7db9b7 --- /dev/null +++ b/Rdmp.UI/SubComponents/PureDatasetConfigurationUI.resx @@ -0,0 +1,123 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + 17, 17 + + \ No newline at end of file From 87dd4ea3e9dd7b6df251e42623a4f9f9191e1062 Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 26 Sep 2024 09:41:37 +0100 Subject: [PATCH 059/106] add read dataset --- .../WindowManagement/HomePane/HomeBoxUI.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Application/ResearchDataManagementPlatform/WindowManagement/HomePane/HomeBoxUI.cs b/Application/ResearchDataManagementPlatform/WindowManagement/HomePane/HomeBoxUI.cs index 28b5d354de..f5ebbcd8f2 100644 --- a/Application/ResearchDataManagementPlatform/WindowManagement/HomePane/HomeBoxUI.cs +++ b/Application/ResearchDataManagementPlatform/WindowManagement/HomePane/HomeBoxUI.cs @@ -41,7 +41,8 @@ public void SetUp(IActivateItems activator, string title, Type openType, AtomicC params IAtomicCommand[] newCommands) { _openType = openType; - + var x = new PureDatasetProvider(activator, activator.RepositoryLocator.CatalogueRepository.GetAllObjects().First()); + x.AddExistingDataset("Real", "https://dundee-staging.elsevierpure.com/en/datasets/2e44c728-6076-49de-8cdb-5c35c1b586a6"); if (!_doneSetup) { From 53596b0457088e55eac543d8f70b5fe4a5849e1a Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 26 Sep 2024 10:21:29 +0100 Subject: [PATCH 060/106] Bugfix/rdmp 253 filter ordering (#2007) * add start * partial * tidy up code * filter updates * update db migrations * update correct db * update changelog * add base creation sql * fix create * tidy up --- CHANGELOG.md | 3 + .../Data/Aggregation/AggregateFilter.cs | 6 +- Rdmp.Core/Curation/Data/ConcreteFilter.cs | 5 +- Rdmp.Core/Curation/Data/ExtractionFilter.cs | 6 +- .../SpontaneouslyInventedFilter.cs | 5 +- .../Data/DeployedExtractionFilter.cs | 6 +- .../CreateCatalogue.sql | 2 + .../up/087_AddAggregateFilterOrdering.sql | 13 ++++ .../CreateDataExportManager.sql | Bin 207702 -> 207772 bytes .../up/026_AddFilterOrder.sql | 8 +++ Rdmp.Core/Rdmp.Core.csproj | 4 ++ .../ExecuteCommandReorderFilter.cs | 62 ++++++++++++++++++ ...poseExecutionWhenTargetIsConcreteFilter.cs | 18 +++-- SharedAssemblyInfo.cs | 6 +- 14 files changed, 131 insertions(+), 13 deletions(-) create mode 100644 Rdmp.Core/Databases/CatalogueDatabase/up/087_AddAggregateFilterOrdering.sql create mode 100644 Rdmp.Core/Databases/DataExportDatabase/up/026_AddFilterOrder.sql create mode 100644 Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandReorderFilter.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 172ecdb1dd..3bd8a30350 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ + # Changelog All notable changes to this project will be documented in this file. @@ -7,6 +8,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [8.4.0] - Unreleased +- Add Ordering to Filters + ## [8.3.1] - Unreleased - Improve Performance of regenerating problems with child providers diff --git a/Rdmp.Core/Curation/Data/Aggregation/AggregateFilter.cs b/Rdmp.Core/Curation/Data/Aggregation/AggregateFilter.cs index 7b23404346..307a7ef8aa 100644 --- a/Rdmp.Core/Curation/Data/Aggregation/AggregateFilter.cs +++ b/Rdmp.Core/Curation/Data/Aggregation/AggregateFilter.cs @@ -1,4 +1,4 @@ -// Copyright (c) The University of Dundee 2018-2019 +// Copyright (c) The University of Dundee 2018-2024 // This file is part of the Research Data Management Platform (RDMP). // RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. @@ -35,6 +35,7 @@ public class AggregateFilter : ConcreteFilter, IDisableable private int? _clonedFromExtractionFilterID; private int? _associatedColumnInfoID; private bool _isDisabled; + private int _order; /// public override int? ClonedFromExtractionFilter_ID @@ -90,6 +91,8 @@ public IEnumerable AggregateFilterParameters ? Repository.GetObjectByID(FilterContainer_ID.Value) : null; + public override int Order { get => _order; set => SetField(ref _order, value); } + #endregion public AggregateFilter() @@ -121,6 +124,7 @@ internal AggregateFilter(ICatalogueRepository repository, DbDataReader r) : base Name = r["Name"] as string; IsMandatory = (bool)r["IsMandatory"]; ClonedFromExtractionFilter_ID = ObjectToNullableInt(r["ClonedFromExtractionFilter_ID"]); + Order = int.Parse(r["Order"].ToString()); var associatedColumnInfo_ID = r["AssociatedColumnInfo_ID"]; if (associatedColumnInfo_ID != DBNull.Value) diff --git a/Rdmp.Core/Curation/Data/ConcreteFilter.cs b/Rdmp.Core/Curation/Data/ConcreteFilter.cs index c29ebbdeca..b8fe1f49de 100644 --- a/Rdmp.Core/Curation/Data/ConcreteFilter.cs +++ b/Rdmp.Core/Curation/Data/ConcreteFilter.cs @@ -1,4 +1,4 @@ -// Copyright (c) The University of Dundee 2018-2019 +// Copyright (c) The University of Dundee 2018-2024 // This file is part of the Research Data Management Platform (RDMP). // RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. @@ -25,7 +25,7 @@ namespace Rdmp.Core.Curation.Data; /// ConcreteFilter is used to provide UI editing of an IFilter without having to add persistence / DatabaseEntity logic to IFilter (which would break /// SpontaneouslyInventedFilters) /// -public abstract class ConcreteFilter : DatabaseEntity, IFilter, ICheckable +public abstract class ConcreteFilter : DatabaseEntity, IFilter, ICheckable, IOrderable { /// protected ConcreteFilter(IRepository repository, DbDataReader r) : base(repository, r) @@ -100,6 +100,7 @@ public bool IsMandatory /// [NoMappingToDatabase] public abstract IContainer FilterContainer { get; } + public abstract int Order { get; set; } #endregion diff --git a/Rdmp.Core/Curation/Data/ExtractionFilter.cs b/Rdmp.Core/Curation/Data/ExtractionFilter.cs index f6ce6f8d4c..febd12b629 100644 --- a/Rdmp.Core/Curation/Data/ExtractionFilter.cs +++ b/Rdmp.Core/Curation/Data/ExtractionFilter.cs @@ -1,4 +1,4 @@ -// Copyright (c) The University of Dundee 2018-2019 +// Copyright (c) The University of Dundee 2018-2024 // This file is part of the Research Data Management Platform (RDMP). // RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. @@ -40,6 +40,7 @@ public class ExtractionFilter : ConcreteFilter, IHasDependencies, IInjectKnown _knownExtractionFilterParameterSets; + private int _order; /// /// The column in the which is best/most associated with this filter. A filter can query any column in any of the table(s) under @@ -133,6 +134,8 @@ internal ExtractionFilter(ICatalogueRepository repository, DbDataReader r) Description = r["Description"] as string; Name = r["Name"] as string; IsMandatory = (bool)r["IsMandatory"]; + Order = int.Parse(r["Order"].ToString()); + ClearAllInjections(); } @@ -154,6 +157,7 @@ public override int? ClonedFromExtractionFilter_ID set => throw new NotSupportedException( "ClonedFromExtractionFilter_ID is only supported on lower level filters e.g. DeployedExtractionFilter and AggregateFilter"); } + public override int Order { get => _order; set => SetField(ref _order,value); } /// public IHasDependencies[] GetObjectsThisDependsOn() diff --git a/Rdmp.Core/Curation/Data/Spontaneous/SpontaneouslyInventedFilter.cs b/Rdmp.Core/Curation/Data/Spontaneous/SpontaneouslyInventedFilter.cs index 8bbadc4f85..dd8458426f 100644 --- a/Rdmp.Core/Curation/Data/Spontaneous/SpontaneouslyInventedFilter.cs +++ b/Rdmp.Core/Curation/Data/Spontaneous/SpontaneouslyInventedFilter.cs @@ -1,4 +1,4 @@ -// Copyright (c) The University of Dundee 2018-2019 +// Copyright (c) The University of Dundee 2018-2024 // This file is part of the Research Data Management Platform (RDMP). // RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. @@ -22,6 +22,7 @@ public class SpontaneouslyInventedFilter : ConcreteFilter { private readonly MemoryCatalogueRepository _repo; private readonly ISqlParameter[] _filterParametersIfAny; + private int _order =0; /// /// Creates a new temporary (unsaveable) filter in the given memory @@ -68,6 +69,8 @@ public SpontaneouslyInventedFilter(MemoryCatalogueRepository repo, IFilter copyF ? _repo.GetObjectByID(FilterContainer_ID.Value) : null; + public override int Order { get => _order; set => SetField(ref _order, value); } + public override ColumnInfo GetColumnInfoIfExists() => null; public override IFilterFactory GetFilterFactory() => null; diff --git a/Rdmp.Core/DataExport/Data/DeployedExtractionFilter.cs b/Rdmp.Core/DataExport/Data/DeployedExtractionFilter.cs index d1e7960ade..122eece307 100644 --- a/Rdmp.Core/DataExport/Data/DeployedExtractionFilter.cs +++ b/Rdmp.Core/DataExport/Data/DeployedExtractionFilter.cs @@ -1,4 +1,4 @@ -// Copyright (c) The University of Dundee 2018-2019 +// Copyright (c) The University of Dundee 2018-2024 // This file is part of the Research Data Management Platform (RDMP). // RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. @@ -36,6 +36,7 @@ public class DeployedExtractionFilter : ConcreteFilter private int? _clonedFromExtractionFilterID; private int? _filterContainerID; + private int _order; /// public override int? ClonedFromExtractionFilter_ID @@ -69,6 +70,8 @@ public override int? FilterContainer_ID ? Repository.GetObjectByID(FilterContainer_ID.Value) : null; + public override int Order { get => _order; set => SetField(ref _order, value); } + #endregion /// @@ -138,6 +141,7 @@ internal DeployedExtractionFilter(IDataExportRepository repository, DbDataReader FilterContainer_ID = null; ClonedFromExtractionFilter_ID = ObjectToNullableInt(r["ClonedFromExtractionFilter_ID"]); + Order = int.Parse(r["Order"].ToString()); } /// diff --git a/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql b/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql index e794b1e35c..418fcf7e9d 100644 --- a/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql +++ b/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql @@ -141,6 +141,7 @@ CREATE TABLE [dbo].[AggregateFilter]( [AssociatedColumnInfo_ID] [int] NULL, [ID] [int] IDENTITY(1,1) NOT NULL, [SoftwareVersion] [nvarchar](50) NOT NULL, + [Order] [int] NOT NULL DEFAULT 0 CONSTRAINT [PK_AggregateFilter] PRIMARY KEY CLUSTERED ( [ID] ASC @@ -464,6 +465,7 @@ CREATE TABLE [dbo].[ExtractionFilter]( [Name] [varchar](100) NOT NULL, [IsMandatory] [bit] NOT NULL, [SoftwareVersion] [nvarchar](50) NOT NULL, + [Order] [int] NOT NULL DEFAULT 0 CONSTRAINT [PK_ExtractionFilter] PRIMARY KEY CLUSTERED ( [ID] ASC diff --git a/Rdmp.Core/Databases/CatalogueDatabase/up/087_AddAggregateFilterOrdering.sql b/Rdmp.Core/Databases/CatalogueDatabase/up/087_AddAggregateFilterOrdering.sql new file mode 100644 index 0000000000..0d3e473416 --- /dev/null +++ b/Rdmp.Core/Databases/CatalogueDatabase/up/087_AddAggregateFilterOrdering.sql @@ -0,0 +1,13 @@ +----Version: 8.4.0 +----Description: Add Order to Aggregate Filters + +if not exists (select 1 from sys.columns where name = 'Order' and OBJECT_NAME(object_id) = 'AggregateFilter') +BEGIN +ALTER TABLE [dbo].[AggregateFilter] +ADD [Order] [int] NOT NULL DEFAULT 0 WITH VALUES +END +if not exists (select 1 from sys.columns where name = 'Order' and OBJECT_NAME(object_id) = 'ExtractionFilter') +BEGIN +ALTER TABLE [dbo].[ExtractionFilter] +ADD [Order] [int] NOT NULL DEFAULT 0 WITH VALUES +END \ No newline at end of file diff --git a/Rdmp.Core/Databases/DataExportDatabase/runAfterCreateDatabase/CreateDataExportManager.sql b/Rdmp.Core/Databases/DataExportDatabase/runAfterCreateDatabase/CreateDataExportManager.sql index 15f75c2cc478241235068c0ad4aa43f49a778571..ac1c03a92acf841ff12f997ebad6538dc418bdb8 100644 GIT binary patch delta 56 zcmcb1jc3kvo(&l?lM7tgCND6Sn%uz8H#sO-h|yp&BdhAV!Z delta 30 lcmbPpo#)y$o(&l?lM})fn$O9!pOaw(Vy5lqWSGzK0RYrq45$DA diff --git a/Rdmp.Core/Databases/DataExportDatabase/up/026_AddFilterOrder.sql b/Rdmp.Core/Databases/DataExportDatabase/up/026_AddFilterOrder.sql new file mode 100644 index 0000000000..1181d65c9b --- /dev/null +++ b/Rdmp.Core/Databases/DataExportDatabase/up/026_AddFilterOrder.sql @@ -0,0 +1,8 @@ +----Version: 8.4.0 +----Description: Add Order to Aggregate Filters + +if not exists (select 1 from sys.columns where name = 'Order' and OBJECT_NAME(object_id) = 'DeployedExtractionFilter') +BEGIN +ALTER TABLE [dbo].[DeployedExtractionFilter] +ADD [Order] [int] NOT NULL DEFAULT 0 WITH VALUES +END \ No newline at end of file diff --git a/Rdmp.Core/Rdmp.Core.csproj b/Rdmp.Core/Rdmp.Core.csproj index 449cc6abdb..3f2209d40d 100644 --- a/Rdmp.Core/Rdmp.Core.csproj +++ b/Rdmp.Core/Rdmp.Core.csproj @@ -129,6 +129,7 @@ + @@ -157,6 +158,7 @@ + @@ -255,6 +257,7 @@ + @@ -297,6 +300,7 @@ + diff --git a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandReorderFilter.cs b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandReorderFilter.cs new file mode 100644 index 0000000000..e6483fa5a1 --- /dev/null +++ b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandReorderFilter.cs @@ -0,0 +1,62 @@ +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using Rdmp.Core.Curation.Data; +using Rdmp.Core.MapsDirectlyToDatabaseTable; +using Rdmp.UI.ItemActivation; +using System; +using System.Linq; + +namespace Rdmp.UI.CommandExecution.AtomicCommands; + +public class ExecuteCommandReorderFilter : BasicUICommandExecution +{ + private ConcreteFilter _source; + private ConcreteFilter _target; + private InsertOption _insertOption; + + public ExecuteCommandReorderFilter(IActivateItems activator, ConcreteFilter source, ConcreteFilter destination, InsertOption insertOption) : base(activator) + { + _source = source; + _target = destination; + _insertOption = insertOption; + if (_source.FilterContainer_ID is null || _target.FilterContainer_ID is null) + { + SetImpossible("Both filters must exist within some container in order to be orderable"); + } + if (_source.FilterContainer_ID != _target.FilterContainer_ID) + { + SetImpossible("Cannot reorder filters as they do not share a parent"); + } + } + + public override void Execute() + { + var order = _target.Order; + + var filters = _target.FilterContainer.GetFilters().Where(f => f is ConcreteFilter).Select(f => (ConcreteFilter)f).ToArray(); + Array.Sort( + filters, + delegate (ConcreteFilter a, ConcreteFilter b) { return a.Order.CompareTo(b.Order); } + ); + if (!filters.All(c => c.Order != order)) + { + foreach (var orderable in filters) + { + if (orderable.Order < order) + orderable.Order--; + else if (orderable.Order > order) + orderable.Order++; + else //collision on order + orderable.Order += _insertOption == InsertOption.InsertAbove ? 1 : -1; + ((ISaveable)orderable).SaveToDatabase(); + } + } + _source.Order = order; + _source.SaveToDatabase(); + Publish(_target.FilterContainer); + } +} diff --git a/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsConcreteFilter.cs b/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsConcreteFilter.cs index 93f6fb3c1c..98d290b1b0 100644 --- a/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsConcreteFilter.cs +++ b/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsConcreteFilter.cs @@ -5,7 +5,9 @@ // You should have received a copy of the GNU General Public License along with RDMP. If not, see . using Rdmp.Core.CommandExecution; +using Rdmp.Core.CommandExecution.Combining; using Rdmp.Core.Curation.Data; +using Rdmp.UI.CommandExecution.AtomicCommands; using Rdmp.UI.ExtractionUIs.FilterUIs; using Rdmp.UI.ItemActivation; @@ -24,8 +26,16 @@ public override void Activate(ConcreteFilter target) ItemActivator.Activate(target); } - public override ICommandExecution ProposeExecution(ICombineToMakeCommand cmd, ConcreteFilter target, - InsertOption insertOption = InsertOption.Default) => - //currently nothing can be dropped onto a filter - null; + public override ICommandExecution ProposeExecution(ICombineToMakeCommand cmd, ConcreteFilter targetFilter, + InsertOption insertOption = InsertOption.Default) + { + return cmd switch + { + FilterCombineable sourceFilterCommand => + !sourceFilterCommand.Filter.Equals(targetFilter) && sourceFilterCommand.Filter is ConcreteFilter && sourceFilterCommand.Filter.FilterContainer_ID == targetFilter.FilterContainer_ID ? + new ExecuteCommandReorderFilter(ItemActivator, (ConcreteFilter)sourceFilterCommand.Filter, targetFilter, insertOption) + : null, + _ => null + }; + } } \ No newline at end of file diff --git a/SharedAssemblyInfo.cs b/SharedAssemblyInfo.cs index 559a541653..dd904c2b97 100644 --- a/SharedAssemblyInfo.cs +++ b/SharedAssemblyInfo.cs @@ -10,6 +10,6 @@ [assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] -[assembly: AssemblyVersion("8.3.0")] -[assembly: AssemblyFileVersion("8.3.0")] -[assembly: AssemblyInformationalVersion("8.3.0")] \ No newline at end of file +[assembly: AssemblyVersion("8.4.0")] +[assembly: AssemblyFileVersion("8.4.0")] +[assembly: AssemblyInformationalVersion("8.4.0")] \ No newline at end of file From e94faa38ec8baf235ae77ca4bfaaefa82a0392bd Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 26 Sep 2024 11:55:56 +0100 Subject: [PATCH 061/106] workign create --- .../WindowManagement/HomePane/HomeBoxUI.cs | 11 +- Rdmp.Core/Datasets/PureDataset.cs | 136 ++++++++++++------ Rdmp.Core/Datasets/PureDatasetProvider.cs | 52 ++++++- 3 files changed, 150 insertions(+), 49 deletions(-) diff --git a/Application/ResearchDataManagementPlatform/WindowManagement/HomePane/HomeBoxUI.cs b/Application/ResearchDataManagementPlatform/WindowManagement/HomePane/HomeBoxUI.cs index f5ebbcd8f2..74b9d66beb 100644 --- a/Application/ResearchDataManagementPlatform/WindowManagement/HomePane/HomeBoxUI.cs +++ b/Application/ResearchDataManagementPlatform/WindowManagement/HomePane/HomeBoxUI.cs @@ -41,11 +41,12 @@ public void SetUp(IActivateItems activator, string title, Type openType, AtomicC params IAtomicCommand[] newCommands) { _openType = openType; - var x = new PureDatasetProvider(activator, activator.RepositoryLocator.CatalogueRepository.GetAllObjects().First()); - x.AddExistingDataset("Real", "https://dundee-staging.elsevierpure.com/en/datasets/2e44c728-6076-49de-8cdb-5c35c1b586a6"); + if (!_doneSetup) { + var x = new PureDatasetProvider(activator, activator.RepositoryLocator.CatalogueRepository.GetAllObjects().First()); + x.Create(); _activator = activator; lblTitle.Text = title; @@ -62,9 +63,9 @@ public void SetUp(IActivateItems activator, string title, Type openType, AtomicC btnOpen.Click += (s, e) => { if (activator.SelectObject(new DialogArgs - { - WindowTitle = "Open" - }, activator.GetAll(openType).ToArray(), out var selected)) + { + WindowTitle = "Open" + }, activator.GetAll(openType).ToArray(), out var selected)) Open(selected); }; diff --git a/Rdmp.Core/Datasets/PureDataset.cs b/Rdmp.Core/Datasets/PureDataset.cs index 4d7cbb7c80..efba52f59d 100644 --- a/Rdmp.Core/Datasets/PureDataset.cs +++ b/Rdmp.Core/Datasets/PureDataset.cs @@ -1,110 +1,164 @@ using Amazon.Auth.AccessControlPolicy.ActionIdentifiers; using SynthEHR; using System.Collections.Generic; +using System.Text.Json.Serialization; namespace Rdmp.Core.Datasets; +#nullable enable public class ENGBWrapper { - public string en_GB { get; set; } + public ENGBWrapper(string? text) { en_GB = text; } + public string? en_GB { get; set; } } public class URITerm { - public string URI { get; set; } + public URITerm(string? uri, ENGBWrapper enGBWrapper) + { + URI = uri; + term = enGBWrapper; + } + public string? URI { get; set; } public ENGBWrapper term { get; set; } } public class PureDescription { - public string PureID { get; set; } - public ENGBWrapper Value { get; set; } + public int? PureID { get; set; } + public ENGBWrapper? Value { get; set; } } public class System { - public string SystemName { get; set; } - public string UUID { get; set; } + public System(string? uuid, string? systemName) + { + UUID = uuid; + SystemName = systemName; + } + public string? SystemName { get; set; } + public string? UUID { get; set; } } public class Name() { - public string FirstName { get; set; } - public string LastName { get; set; } + public string? FirstName { get; set; } + public string? LastName { get; set; } } public class PurePerson { - public string TypeDiscriminator { get; set; } - public string PureID { get; set; } + public string? TypeDiscriminator { get; set; } + public int? PureID { get; set; } - public Name Name { get; set; } - public URITerm Role { get; set; } + public Name? Name { get; set; } + public URITerm? Role { get; set; } - public List Organizations { get; set; } + public List? Organizations { get; set; } } public class PureDate { + public PureDate(int year, int? month = null, int? day = null) + { + Year = year; + if (month != null) Month = month; + if (day != null) Day = day; + } public int Year { get; set; } - public int Month { get; set; } - public int Day { get; set; } + public int? Month { get; set; } + public int? Day { get; set; } } -public class Visability +public class Visibility { - public string Key { get; set; } + public Visibility(string? key, ENGBWrapper description) + { + Key = key; + Description = description; + } + public string? Key { get; set; } public ENGBWrapper Description { get; set; } } public class Workflow { - public string Step { get; set; } - public ENGBWrapper Description { get; set; } + public string? Step { get; set; } + public ENGBWrapper? Description { get; set; } } public class Geolocation { - public ENGBWrapper GeographicalCoverage { get; set; } + public ENGBWrapper? GeographicalCoverage { get; set; } } public class PureDataset : PluginDataset { - public int PureID { get; set; } - public string UUID { get; set; } - public string CreatedDate { get; set; } - public string CreatedBy { get; set; } - public string ModifiedDate { get; set; } - public string ModifiedBy { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? PureID { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? UUID { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? CreatedDate { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + + public string? CreatedBy { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + + public string? ModifiedDate { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + + public string? ModifiedBy { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? PortalURL { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Version { get; set; } - public string PortalURL { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public ENGBWrapper? Title { get; set; } - public string Version { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? Descriptions { get; set; } - public ENGBWrapper Title { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public System? ManagingOrganization { get; set; } - public List Descriptions { get; set; } - public System ManagingOrganisation { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public new URITerm? Type { get; set; } - public new URITerm Type { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public System? Publisher { get; set; } - public System Publisher { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public Geolocation? Geolocation { get; set; } - public Geolocation Geolocation { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? Persons { get; set; } - public List Persons { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? Organizations { get; set; } - public List Organizations { get; set; } - public PureDate PublicationAvailableDate { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public PureDate? PublicationAvailableDate { get; set; } - public bool OpenAireCompliant { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? OpenAireCompliant { get; set; } - public Visability Visability { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public Visibility? Visibility { get; set; } //TODO custom defined fields - public Workflow Workflow { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public Workflow? Workflow { get; set; } - public string SystemName { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? SystemName { get; set; } +#nullable disable public PureDataset() { } diff --git a/Rdmp.Core/Datasets/PureDatasetProvider.cs b/Rdmp.Core/Datasets/PureDatasetProvider.cs index f2d65b31eb..3f8fb03088 100644 --- a/Rdmp.Core/Datasets/PureDatasetProvider.cs +++ b/Rdmp.Core/Datasets/PureDatasetProvider.cs @@ -11,10 +11,11 @@ using System.IO; using System.Linq; using System.Net.Http; -using System.Text; -using System.Text.Json.Nodes; using System.Threading.Tasks; +using System.Text.Json; using System.Net; +using JsonSerializer = System.Text.Json.JsonSerializer; +using System.Text; namespace Rdmp.Core.Datasets { public class PureDatasetProvider : PluginDatasetProvider @@ -59,7 +60,52 @@ public override void AddExistingDataset(string name, string url) public override Curation.Data.Datasets.Dataset Create() { - throw new NotImplementedException(); + //this works but i'm not sure it's linking up with the person correctly + + var pureDataset = new PureDataset(); + pureDataset.Title = new ENGBWrapper("Test from API create"); + pureDataset.ManagingOrganization = new System(Configuration.Organisation_ID, "Organization"); + pureDataset.Type = new URITerm("/dk/atira/pure/dataset/datasettypes/dataset/dataset", new ENGBWrapper("Dataset")); + pureDataset.Publisher = new System("6c6a0126-6ad7-45dc-8a94-7619a384f92e", "Publisher"); + + var person = new PurePerson(); + person.TypeDiscriminator = "InternalDataSetPersonAssociation"; + person.PureID = 135523720; + var name = new Name(); + name.FirstName = "James"; + name.LastName = "Friel"; + person.Name = name; + person.Role = new URITerm("/dk/atira/pure/dataset/roles/dataset/creator", new ENGBWrapper("Creator")); + person.Organizations = new List() { new System(Configuration.Organisation_ID, "Organization") }; + + pureDataset.Persons = [person]; + pureDataset.Visibility = new Visibility("FREE", new ENGBWrapper("Public - No restriction")); + pureDataset.Organizations = [new System(Configuration.Organisation_ID, "Organization")]; + pureDataset.PublicationAvailableDate = new PureDate(2024, null, null); + + var serializeOptions = new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = true + }; + + var jsonString = JsonSerializer.Serialize(pureDataset, serializeOptions); + + var uri = $"{Configuration.Url}/data-sets"; + var httpContent = new StringContent(jsonString, Encoding.UTF8, "application/json"); + + var response = Task.Run(async () => await _client.PutAsync(uri, httpContent)).Result; + if (response.StatusCode != HttpStatusCode.Created) + { + throw new Exception("Error"); + } + var detailsString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; + PureDataset pd = JsonConvert.DeserializeObject(detailsString); + var dataset = new Curation.Data.Datasets.Dataset(Repository, pureDataset.Title.en_GB); + dataset.Provider_ID = Configuration.ID; + dataset.Url = pd.PortalURL; + dataset.SaveToDatabase(); + return dataset; } public PureDataset FetchPureDataset(Curation.Data.Datasets.Dataset dataset) From 061e75ecb09a142b34b5f5159b9b53f9c9933584 Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 26 Sep 2024 15:53:18 +0100 Subject: [PATCH 062/106] start of ui --- .../WindowManagement/HomePane/HomeBoxUI.cs | 2 - .../Datasets/DatasetProviderConfiguration.cs | 15 +- Rdmp.Core/Datasets/PureDataset.cs | 15 +- Rdmp.Core/Datasets/PureDatasetProvider.cs | 95 ++++------ Rdmp.UI/Collections/DatasetsCollectionUI.cs | 4 +- .../ExecuteCommandCreateNewPureDatasetUI.cs | 30 +++ .../CreateNewPureDatasetUI.Designer.cs | 177 ++++++++++++++++++ .../Datasets/CreateNewPureDatasetUI.cs | 46 +++++ .../Datasets/CreateNewPureDatasetUI.resx | 120 ++++++++++++ 9 files changed, 434 insertions(+), 70 deletions(-) create mode 100644 Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandCreateNewPureDatasetUI.cs create mode 100644 Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureDatasetUI.Designer.cs create mode 100644 Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureDatasetUI.cs create mode 100644 Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureDatasetUI.resx diff --git a/Application/ResearchDataManagementPlatform/WindowManagement/HomePane/HomeBoxUI.cs b/Application/ResearchDataManagementPlatform/WindowManagement/HomePane/HomeBoxUI.cs index 74b9d66beb..9f0c900d68 100644 --- a/Application/ResearchDataManagementPlatform/WindowManagement/HomePane/HomeBoxUI.cs +++ b/Application/ResearchDataManagementPlatform/WindowManagement/HomePane/HomeBoxUI.cs @@ -45,8 +45,6 @@ public void SetUp(IActivateItems activator, string title, Type openType, AtomicC if (!_doneSetup) { - var x = new PureDatasetProvider(activator, activator.RepositoryLocator.CatalogueRepository.GetAllObjects().First()); - x.Create(); _activator = activator; lblTitle.Text = title; diff --git a/Rdmp.Core/Curation/Data/Datasets/DatasetProviderConfiguration.cs b/Rdmp.Core/Curation/Data/Datasets/DatasetProviderConfiguration.cs index 7f334ba18e..55734e6f4c 100644 --- a/Rdmp.Core/Curation/Data/Datasets/DatasetProviderConfiguration.cs +++ b/Rdmp.Core/Curation/Data/Datasets/DatasetProviderConfiguration.cs @@ -1,4 +1,5 @@ using MongoDB.Driver; +using Rdmp.Core.MapsDirectlyToDatabaseTable.Attributes; using Rdmp.Core.Repositories; using System; using System.Collections.Generic; @@ -41,10 +42,10 @@ internal DatasetProviderConfiguration(ICatalogueRepository repository, DbDataRea Organisation_ID = r["Organisation_ID"].ToString(); } - public string Type + + public override string ToString() { - get => _type; - set => SetField(ref _type, value); + return _name; } public string Name @@ -53,6 +54,14 @@ public string Name set => SetField(ref _name, value); } + + public string Type + { + get => _type; + set => SetField(ref _type, value); + } + + public string Url { get => _url; diff --git a/Rdmp.Core/Datasets/PureDataset.cs b/Rdmp.Core/Datasets/PureDataset.cs index efba52f59d..c7667bccf0 100644 --- a/Rdmp.Core/Datasets/PureDataset.cs +++ b/Rdmp.Core/Datasets/PureDataset.cs @@ -89,6 +89,15 @@ public class Workflow public class Geolocation { public ENGBWrapper? GeographicalCoverage { get; set; } + public string? Point { get; set; } + + public string? Polygon { get; set; } +} + +public class TemporalcoveragePeriod +{ + public PureDate? StartDate { get; set; } + public PureDate? EndDate { get; set; } } public class PureDataset : PluginDataset @@ -158,8 +167,12 @@ public class PureDataset : PluginDataset [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? SystemName { get; set; } -#nullable disable + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + + public TemporalcoveragePeriod? TemporalcoveragePeriod { get; set; } + +#nullable disable public PureDataset() { } diff --git a/Rdmp.Core/Datasets/PureDatasetProvider.cs b/Rdmp.Core/Datasets/PureDatasetProvider.cs index 3f8fb03088..c343ae2cf1 100644 --- a/Rdmp.Core/Datasets/PureDatasetProvider.cs +++ b/Rdmp.Core/Datasets/PureDatasetProvider.cs @@ -58,30 +58,20 @@ public override void AddExistingDataset(string name, string url) } } - public override Curation.Data.Datasets.Dataset Create() + public Curation.Data.Datasets.Dataset Create(string title, string publisherUid, List people, Visibility visibility, PureDate publicationDate) { //this works but i'm not sure it's linking up with the person correctly var pureDataset = new PureDataset(); - pureDataset.Title = new ENGBWrapper("Test from API create"); + pureDataset.Title = new ENGBWrapper(title); pureDataset.ManagingOrganization = new System(Configuration.Organisation_ID, "Organization"); pureDataset.Type = new URITerm("/dk/atira/pure/dataset/datasettypes/dataset/dataset", new ENGBWrapper("Dataset")); - pureDataset.Publisher = new System("6c6a0126-6ad7-45dc-8a94-7619a384f92e", "Publisher"); - - var person = new PurePerson(); - person.TypeDiscriminator = "InternalDataSetPersonAssociation"; - person.PureID = 135523720; - var name = new Name(); - name.FirstName = "James"; - name.LastName = "Friel"; - person.Name = name; - person.Role = new URITerm("/dk/atira/pure/dataset/roles/dataset/creator", new ENGBWrapper("Creator")); - person.Organizations = new List() { new System(Configuration.Organisation_ID, "Organization") }; - - pureDataset.Persons = [person]; - pureDataset.Visibility = new Visibility("FREE", new ENGBWrapper("Public - No restriction")); + pureDataset.Publisher = new System(publisherUid, "Publisher"); + + pureDataset.Persons = people; + pureDataset.Visibility = visibility; pureDataset.Organizations = [new System(Configuration.Organisation_ID, "Organization")]; - pureDataset.PublicationAvailableDate = new PureDate(2024, null, null); + pureDataset.PublicationAvailableDate = publicationDate; var serializeOptions = new JsonSerializerOptions { @@ -108,6 +98,25 @@ public override Curation.Data.Datasets.Dataset Create() return dataset; } + public void Update(string uuid, PureDataset datasetUpdates) + { + var serializeOptions = new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = true + }; + + var jsonString = JsonSerializer.Serialize(datasetUpdates, serializeOptions); + var uri = $"{Configuration.Url}/data-sets/{uuid}"; + var httpContent = new StringContent(jsonString, Encoding.UTF8, "application/json"); + + var response = Task.Run(async () => await _client.PutAsync(uri, httpContent)).Result; + if (response.StatusCode != HttpStatusCode.Created) + { + throw new Exception("Error"); + } + } + public PureDataset FetchPureDataset(Curation.Data.Datasets.Dataset dataset) { var uri = $"{Configuration.Url}/data-sets/{UrltoUUID(dataset.Url)}"; @@ -125,62 +134,22 @@ public PureDataset FetchPureDataset(Curation.Data.Datasets.Dataset dataset) public override Curation.Data.Datasets.Dataset FetchDatasetByID(int id) { var dataset = Repository.GetAllObjectsWhere("ID", id).FirstOrDefault(); - FetchPureDataset(dataset); - //if (dataset is null) - //{ - // throw new Exception("Unable to find local dataset with ID"); - //} - //if (dataset.Type != this.ToString()) - //{ - // var x = this.ToString(); - // throw new Exception("Dataset is of the wrong type"); - //} - //if (dataset.Provider_ID is null) - //{ - // throw new Exception("No provider specified"); - //} - //var provider = Repository.GetAllObjectsWhere("ID", dataset.Provider_ID).FirstOrDefault(); - //if (provider is null) - //{ - // throw new Exception("Unable to locate provider"); - //} - //var uri = $"{Configuration.Url}/data-sets/{UrltoUUID(dataset.Url)}"; - //var response = Task.Run(async () => await _client.GetAsync(uri)).Result; - //if (response.StatusCode != System.Net.HttpStatusCode.OK) - //{ - // throw new Exception("Error"); - //} - //var detailsString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; - //PureDataset pd = JsonConvert.DeserializeObject(detailsString); return dataset; } public override List FetchDatasets() { - ////TODO will have to think about paginations - //var uri = $"{Configuration.Url}/data-sets"; - //var response = Task.Run(async () => await _client.GetAsync(uri)).Result; - //if(response.StatusCode != System.Net.HttpStatusCode.OK) - //{ - // throw new Exception("Error"); - //} - //var detailsString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; - //var jsonObjects = JsonObject.Parse(detailsString); - //foreach(var item in jsonObjects.AsObject()) { - - // if (item.Key == "items") - // { - // Console.WriteLine(item.ToString()); - // //the values here are the datasets - // } - //} - return new List(); - + return Repository.GetAllObjectsWhere("Provider_ID", Configuration.ID).ToList(); } public override Curation.Data.Datasets.Dataset Update() { throw new NotImplementedException(); } + + public override Curation.Data.Datasets.Dataset Create() + { + throw new NotImplementedException(); + } } } diff --git a/Rdmp.UI/Collections/DatasetsCollectionUI.cs b/Rdmp.UI/Collections/DatasetsCollectionUI.cs index 6c82090542..05bf2dfc88 100644 --- a/Rdmp.UI/Collections/DatasetsCollectionUI.cs +++ b/Rdmp.UI/Collections/DatasetsCollectionUI.cs @@ -34,7 +34,9 @@ public override void SetItemActivator(IActivateItems activator) a => new IAtomicCommand[] { new ExecuteCommandCreateNewDatasetUI(Activator) - { OverrideCommandName = "Add New Dataset", Weight = -50.9f } + { OverrideCommandName = "Add New Dataset", Weight = -50.9f }, + new ExecuteCommandCreateNewPureDatasetUI(Activator) + { OverrideCommandName = "Add New Pure Dataset", Weight = -50.9f } }; Activator.RefreshBus.EstablishLifetimeSubscription(this); tlvDatasets.AddObject(activator.CoreChildProvider.DatasetRootFolder); diff --git a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandCreateNewPureDatasetUI.cs b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandCreateNewPureDatasetUI.cs new file mode 100644 index 0000000000..cc117e2719 --- /dev/null +++ b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandCreateNewPureDatasetUI.cs @@ -0,0 +1,30 @@ +// Copyright (c) The University of Dundee 2018-2019 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using Rdmp.Core.CommandExecution.AtomicCommands; +using Rdmp.UI.ItemActivation; +using Rdmp.UI.SimpleDialogs.Datasets; +using System; + + +namespace Rdmp.UI.CommandExecution.AtomicCommands; + +public class ExecuteCommandCreateNewPureDatasetUI : ExecuteCommandCreateDataset +{ + private readonly IActivateItems _activator; + + public ExecuteCommandCreateNewPureDatasetUI(IActivateItems activator) : base( + activator, "New Dataset") + { + _activator = activator; + } + + public override void Execute() + { + var ui = new CreateNewPureDatasetUI(_activator, this); + ui.ShowDialog(); + } +} diff --git a/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureDatasetUI.Designer.cs b/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureDatasetUI.Designer.cs new file mode 100644 index 0000000000..da55014f34 --- /dev/null +++ b/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureDatasetUI.Designer.cs @@ -0,0 +1,177 @@ +namespace Rdmp.UI.SimpleDialogs.Datasets +{ + partial class CreateNewPureDatasetUI + { + /// + /// Required designer variable. + /// + private System.ComponentModel.IContainer components = null; + + /// + /// Clean up any resources being used. + /// + /// true if managed resources should be disposed; otherwise, false. + protected override void Dispose(bool disposing) + { + if (disposing && (components != null)) + { + components.Dispose(); + } + base.Dispose(disposing); + } + + #region Windows Form Designer generated code + + /// + /// Required method for Designer support - do not modify + /// the contents of this method with the code editor. + /// + private void InitializeComponent() + { + label1 = new System.Windows.Forms.Label(); + cbDatasetProvider = new System.Windows.Forms.ComboBox(); + tbName = new System.Windows.Forms.TextBox(); + label2 = new System.Windows.Forms.Label(); + label3 = new System.Windows.Forms.Label(); + tbDescription = new System.Windows.Forms.TextBox(); + cbPublisher = new System.Windows.Forms.ComboBox(); + label4 = new System.Windows.Forms.Label(); + label5 = new System.Windows.Forms.Label(); + label6 = new System.Windows.Forms.Label(); + label7 = new System.Windows.Forms.Label(); + SuspendLayout(); + // + // label1 + // + label1.AutoSize = true; + label1.Location = new System.Drawing.Point(12, 21); + label1.Name = "label1"; + label1.Size = new System.Drawing.Size(96, 15); + label1.TabIndex = 0; + label1.Text = "Dataset Provider:"; + label1.Click += label1_Click; + // + // cbDatasetProvider + // + cbDatasetProvider.FormattingEnabled = true; + cbDatasetProvider.Location = new System.Drawing.Point(114, 18); + cbDatasetProvider.Name = "cbDatasetProvider"; + cbDatasetProvider.Size = new System.Drawing.Size(252, 23); + cbDatasetProvider.TabIndex = 1; + // + // tbName + // + tbName.Location = new System.Drawing.Point(116, 62); + tbName.Name = "tbName"; + tbName.Size = new System.Drawing.Size(250, 23); + tbName.TabIndex = 2; + // + // label2 + // + label2.AutoSize = true; + label2.Location = new System.Drawing.Point(69, 65); + label2.Name = "label2"; + label2.Size = new System.Drawing.Size(42, 15); + label2.TabIndex = 3; + label2.Text = "Name:"; + // + // label3 + // + label3.AutoSize = true; + label3.Location = new System.Drawing.Point(41, 107); + label3.Name = "label3"; + label3.Size = new System.Drawing.Size(70, 15); + label3.TabIndex = 5; + label3.Text = "Description:"; + label3.Click += this.label3_Click; + // + // tbDescription + // + tbDescription.Location = new System.Drawing.Point(116, 104); + tbDescription.Name = "tbDescription"; + tbDescription.Size = new System.Drawing.Size(250, 23); + tbDescription.TabIndex = 4; + tbDescription.TextChanged += textBox2_TextChanged; + // + // cbPublisher + // + cbPublisher.FormattingEnabled = true; + cbPublisher.Location = new System.Drawing.Point(114, 147); + cbPublisher.Name = "cbPublisher"; + cbPublisher.Size = new System.Drawing.Size(252, 23); + cbPublisher.TabIndex = 7; + // + // label4 + // + label4.AutoSize = true; + label4.Location = new System.Drawing.Point(53, 150); + label4.Name = "label4"; + label4.Size = new System.Drawing.Size(59, 15); + label4.TabIndex = 6; + label4.Text = "Publisher:"; + // + // label5 + // + label5.AutoSize = true; + label5.Location = new System.Drawing.Point(49, 197); + label5.Name = "label5"; + label5.Size = new System.Drawing.Size(46, 15); + label5.TabIndex = 8; + label5.Text = "People:"; + // + // label6 + // + label6.AutoSize = true; + label6.Location = new System.Drawing.Point(52, 229); + label6.Name = "label6"; + label6.Size = new System.Drawing.Size(54, 15); + label6.TabIndex = 9; + label6.Text = "Visibility:"; + // + // label7 + // + label7.AutoSize = true; + label7.Location = new System.Drawing.Point(2, 261); + label7.Name = "label7"; + label7.Size = new System.Drawing.Size(116, 15); + label7.TabIndex = 10; + label7.Text = "Date made available:"; + // + // CreateNewPureDatasetUI + // + AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); + AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; + ClientSize = new System.Drawing.Size(800, 450); + Controls.Add(label7); + Controls.Add(label6); + Controls.Add(label5); + Controls.Add(cbPublisher); + Controls.Add(label4); + Controls.Add(label3); + Controls.Add(tbDescription); + Controls.Add(label2); + Controls.Add(tbName); + Controls.Add(cbDatasetProvider); + Controls.Add(label1); + Name = "CreateNewPureDatasetUI"; + Text = "Create New Pure Dataset"; + Load += CreateNewPureDatasetUI_Load; + ResumeLayout(false); + PerformLayout(); + } + + #endregion + + private System.Windows.Forms.Label label1; + private System.Windows.Forms.ComboBox cbDatasetProvider; + private System.Windows.Forms.TextBox tbName; + private System.Windows.Forms.Label label2; + private System.Windows.Forms.Label label3; + private System.Windows.Forms.TextBox tbDescription; + private System.Windows.Forms.ComboBox cbPublisher; + private System.Windows.Forms.Label label4; + private System.Windows.Forms.Label label5; + private System.Windows.Forms.Label label6; + private System.Windows.Forms.Label label7; + } +} \ No newline at end of file diff --git a/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureDatasetUI.cs b/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureDatasetUI.cs new file mode 100644 index 0000000000..6b43e60281 --- /dev/null +++ b/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureDatasetUI.cs @@ -0,0 +1,46 @@ +using Amazon.Auth.AccessControlPolicy; +using Rdmp.Core.CommandExecution.AtomicCommands; +using Rdmp.Core.Curation.Data.Datasets; +using Rdmp.Core.Datasets; +using Rdmp.UI.CommandExecution.AtomicCommands; +using Rdmp.UI.ItemActivation; +using Rdmp.UI.TestsAndSetup.ServicePropogation; +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Data; +using System.Drawing; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Windows.Forms; + +namespace Rdmp.UI.SimpleDialogs.Datasets; + +public partial class CreateNewPureDatasetUI : RDMPForm +{ + private readonly IActivateItems _activator; + + public CreateNewPureDatasetUI(IActivateItems activator, ExecuteCommandCreateNewPureDatasetUI command) : base(activator) + { + _activator = activator; + InitializeComponent(); + var configs = _activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Type", typeof(PureDatasetProvider).ToString()); + cbDatasetProvider.Items.AddRange(configs); + } + + private void CreateNewPureDatasetUI_Load(object sender, EventArgs e) + { + + } + + private void label1_Click(object sender, EventArgs e) + { + + } + + private void textBox2_TextChanged(object sender, EventArgs e) + { + + } +} diff --git a/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureDatasetUI.resx b/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureDatasetUI.resx new file mode 100644 index 0000000000..af32865ec1 --- /dev/null +++ b/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureDatasetUI.resx @@ -0,0 +1,120 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + \ No newline at end of file From 30def9da547963e00f8f5c0b59006e0b4598e0a1 Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 26 Sep 2024 15:55:42 +0100 Subject: [PATCH 063/106] update migration number --- ...nfiguration.sql => 088_AddRegexRedactionConfiguration.sql} | 2 +- Rdmp.Core/Rdmp.Core.csproj | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) rename Rdmp.Core/Databases/CatalogueDatabase/up/{086_AddRegexRedactionConfiguration.sql => 088_AddRegexRedactionConfiguration.sql} (99%) diff --git a/Rdmp.Core/Databases/CatalogueDatabase/up/086_AddRegexRedactionConfiguration.sql b/Rdmp.Core/Databases/CatalogueDatabase/up/088_AddRegexRedactionConfiguration.sql similarity index 99% rename from Rdmp.Core/Databases/CatalogueDatabase/up/086_AddRegexRedactionConfiguration.sql rename to Rdmp.Core/Databases/CatalogueDatabase/up/088_AddRegexRedactionConfiguration.sql index 21b8cf216a..ff45dfcc95 100644 --- a/Rdmp.Core/Databases/CatalogueDatabase/up/086_AddRegexRedactionConfiguration.sql +++ b/Rdmp.Core/Databases/CatalogueDatabase/up/088_AddRegexRedactionConfiguration.sql @@ -1,4 +1,4 @@ ---Version: 8.2.4 +--Version: 8.4.0 --Description: Add configuration for regex redaction diff --git a/Rdmp.Core/Rdmp.Core.csproj b/Rdmp.Core/Rdmp.Core.csproj index d3d6578be5..3fbf307c66 100644 --- a/Rdmp.Core/Rdmp.Core.csproj +++ b/Rdmp.Core/Rdmp.Core.csproj @@ -128,9 +128,9 @@ - + @@ -257,7 +257,7 @@ - + From ab2644894b03e74930d7f70535cd0c0015aece11 Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 26 Sep 2024 15:57:06 +0100 Subject: [PATCH 064/106] add changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bd8a30350..22d807ce1f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,4 @@ - # Changelog All notable changes to this project will be documented in this file. @@ -9,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [8.4.0] - Unreleased - Add Ordering to Filters +- [MSSQL ONLY] Add ability to perform Regex redactions on data loads and existing catalogues ## [8.3.1] - Unreleased From e7e9e98158b32c94c101f6a87e91b704b39ec1a9 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 30 Sep 2024 10:00:13 +0100 Subject: [PATCH 065/106] improved create --- Rdmp.Core/Datasets/PluginDatasetProvider.cs | 2 +- Rdmp.Core/Datasets/PureDatasetProvider.cs | 17 ++--- Rdmp.UI/Collections/DatasetsCollectionUI.cs | 4 +- .../CreateNewPureDatasetUI.Designer.cs | 65 ++++++++++++++++++- .../Datasets/CreateNewPureDatasetUI.cs | 46 ++++++++++++- 5 files changed, 119 insertions(+), 15 deletions(-) diff --git a/Rdmp.Core/Datasets/PluginDatasetProvider.cs b/Rdmp.Core/Datasets/PluginDatasetProvider.cs index 14c17968bc..8380eab0fa 100644 --- a/Rdmp.Core/Datasets/PluginDatasetProvider.cs +++ b/Rdmp.Core/Datasets/PluginDatasetProvider.cs @@ -30,6 +30,6 @@ protected PluginDatasetProvider(IBasicActivateItems activator, DatasetProviderCo public abstract Curation.Data.Datasets.Dataset Create(); - public abstract Curation.Data.Datasets.Dataset Update(); + public abstract void Update(string uuid, PluginDataset datasetUpdates); } diff --git a/Rdmp.Core/Datasets/PureDatasetProvider.cs b/Rdmp.Core/Datasets/PureDatasetProvider.cs index c343ae2cf1..f8c44332b4 100644 --- a/Rdmp.Core/Datasets/PureDatasetProvider.cs +++ b/Rdmp.Core/Datasets/PureDatasetProvider.cs @@ -41,13 +41,19 @@ private bool CheckDatasetExistsAtURL(string url) public override void AddExistingDataset(string name, string url) { - if (CheckDatasetExistsAtURL(url)) + var uri = $"{Configuration.Url}/data-sets/{UrltoUUID(url)}"; + var response = Task.Run(async () => await _client.GetAsync(uri)).Result; + if (response.StatusCode == HttpStatusCode.OK) { + var detailsString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; + PureDataset pd = JsonConvert.DeserializeObject(detailsString); var dataset = new Curation.Data.Datasets.Dataset(Repository, name) { Url = url, Type = this.ToString(), - Provider_ID = Configuration.ID + Provider_ID = Configuration.ID, + DigitalObjectIdentifier = pd.DigitalObjectIdentifier, + Folder= Configuration.Name, }; dataset.SaveToDatabase(); Activator.Publish(dataset); @@ -98,7 +104,7 @@ public Curation.Data.Datasets.Dataset Create(string title, string publisherUid, return dataset; } - public void Update(string uuid, PureDataset datasetUpdates) + public override void Update(string uuid, PluginDataset datasetUpdates) { var serializeOptions = new JsonSerializerOptions { @@ -142,11 +148,6 @@ public override Curation.Data.Datasets.Dataset FetchDatasetByID(int id) return Repository.GetAllObjectsWhere("Provider_ID", Configuration.ID).ToList(); } - public override Curation.Data.Datasets.Dataset Update() - { - throw new NotImplementedException(); - } - public override Curation.Data.Datasets.Dataset Create() { throw new NotImplementedException(); diff --git a/Rdmp.UI/Collections/DatasetsCollectionUI.cs b/Rdmp.UI/Collections/DatasetsCollectionUI.cs index 05bf2dfc88..14737a9cf9 100644 --- a/Rdmp.UI/Collections/DatasetsCollectionUI.cs +++ b/Rdmp.UI/Collections/DatasetsCollectionUI.cs @@ -36,7 +36,9 @@ public override void SetItemActivator(IActivateItems activator) new ExecuteCommandCreateNewDatasetUI(Activator) { OverrideCommandName = "Add New Dataset", Weight = -50.9f }, new ExecuteCommandCreateNewPureDatasetUI(Activator) - { OverrideCommandName = "Add New Pure Dataset", Weight = -50.9f } + { OverrideCommandName = "Create New Pure Dataset", SuggestedCategory="Pure Datasets" ,Weight = -50.9f }, + //new ExecuteCommandCreateNewPureDatasetUI(Activator) + // { OverrideCommandName = "Import Existing Pure Dataset", SuggestedCategory="Pure Datasets" ,Weight = -50.9f } }; Activator.RefreshBus.EstablishLifetimeSubscription(this); tlvDatasets.AddObject(activator.CoreChildProvider.DatasetRootFolder); diff --git a/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureDatasetUI.Designer.cs b/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureDatasetUI.Designer.cs index da55014f34..ed6aebad22 100644 --- a/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureDatasetUI.Designer.cs +++ b/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureDatasetUI.Designer.cs @@ -39,6 +39,11 @@ private void InitializeComponent() label5 = new System.Windows.Forms.Label(); label6 = new System.Windows.Forms.Label(); label7 = new System.Windows.Forms.Label(); + cbVisibility = new System.Windows.Forms.ComboBox(); + tbDateMadeAvailable = new System.Windows.Forms.TextBox(); + lbPeople = new System.Windows.Forms.ListBox(); + btnCreate = new System.Windows.Forms.Button(); + btnCancel = new System.Windows.Forms.Button(); SuspendLayout(); // // label1 @@ -49,7 +54,6 @@ private void InitializeComponent() label1.Size = new System.Drawing.Size(96, 15); label1.TabIndex = 0; label1.Text = "Dataset Provider:"; - label1.Click += label1_Click; // // cbDatasetProvider // @@ -58,6 +62,7 @@ private void InitializeComponent() cbDatasetProvider.Name = "cbDatasetProvider"; cbDatasetProvider.Size = new System.Drawing.Size(252, 23); cbDatasetProvider.TabIndex = 1; + cbDatasetProvider.SelectedIndexChanged += cbDatasetProvider_SelectedIndexChanged; // // tbName // @@ -83,7 +88,6 @@ private void InitializeComponent() label3.Size = new System.Drawing.Size(70, 15); label3.TabIndex = 5; label3.Text = "Description:"; - label3.Click += this.label3_Click; // // tbDescription // @@ -91,10 +95,10 @@ private void InitializeComponent() tbDescription.Name = "tbDescription"; tbDescription.Size = new System.Drawing.Size(250, 23); tbDescription.TabIndex = 4; - tbDescription.TextChanged += textBox2_TextChanged; // // cbPublisher // + cbPublisher.Enabled = false; cbPublisher.FormattingEnabled = true; cbPublisher.Location = new System.Drawing.Point(114, 147); cbPublisher.Name = "cbPublisher"; @@ -137,11 +141,61 @@ private void InitializeComponent() label7.TabIndex = 10; label7.Text = "Date made available:"; // + // cbVisibility + // + cbVisibility.FormattingEnabled = true; + cbVisibility.Location = new System.Drawing.Point(114, 226); + cbVisibility.Name = "cbVisibility"; + cbVisibility.Size = new System.Drawing.Size(252, 23); + cbVisibility.TabIndex = 11; + // + // tbDateMadeAvailable + // + tbDateMadeAvailable.Location = new System.Drawing.Point(116, 261); + tbDateMadeAvailable.Name = "tbDateMadeAvailable"; + tbDateMadeAvailable.Size = new System.Drawing.Size(250, 23); + tbDateMadeAvailable.TabIndex = 12; + // + // lbPeople + // + lbPeople.Enabled = false; + lbPeople.FormattingEnabled = true; + lbPeople.ItemHeight = 15; + lbPeople.Location = new System.Drawing.Point(116, 190); + lbPeople.Name = "lbPeople"; + lbPeople.Size = new System.Drawing.Size(250, 34); + lbPeople.TabIndex = 13; + // + // btnCreate + // + btnCreate.Location = new System.Drawing.Point(708, 406); + btnCreate.Name = "btnCreate"; + btnCreate.Size = new System.Drawing.Size(75, 23); + btnCreate.TabIndex = 14; + btnCreate.Text = "Create"; + btnCreate.UseVisualStyleBackColor = true; + btnCreate.Click += btnCreate_Click; + // + // btnCancel + // + btnCancel.Location = new System.Drawing.Point(627, 406); + btnCancel.Name = "btnCancel"; + btnCancel.Size = new System.Drawing.Size(75, 23); + btnCancel.TabIndex = 15; + btnCancel.Text = "Cancel"; + btnCancel.UseVisualStyleBackColor = true; + btnCancel.Click += btnCancel_Click; + // // CreateNewPureDatasetUI // AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; ClientSize = new System.Drawing.Size(800, 450); + Controls.Add(btnCancel); + Controls.Add(btnCreate); + Controls.Add(lbPeople); + Controls.Add(tbDateMadeAvailable); + Controls.Add(cbVisibility); Controls.Add(label7); Controls.Add(label6); Controls.Add(label5); @@ -173,5 +227,10 @@ private void InitializeComponent() private System.Windows.Forms.Label label5; private System.Windows.Forms.Label label6; private System.Windows.Forms.Label label7; + private System.Windows.Forms.ComboBox cbVisibility; + private System.Windows.Forms.TextBox tbDateMadeAvailable; + private System.Windows.Forms.ListBox lbPeople; + private System.Windows.Forms.Button btnCreate; + private System.Windows.Forms.Button btnCancel; } } \ No newline at end of file diff --git a/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureDatasetUI.cs b/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureDatasetUI.cs index 6b43e60281..0cff917051 100644 --- a/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureDatasetUI.cs +++ b/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureDatasetUI.cs @@ -1,5 +1,6 @@ using Amazon.Auth.AccessControlPolicy; using Rdmp.Core.CommandExecution.AtomicCommands; +using Rdmp.Core.Curation.Data.Aggregation; using Rdmp.Core.Curation.Data.Datasets; using Rdmp.Core.Datasets; using Rdmp.UI.CommandExecution.AtomicCommands; @@ -21,12 +22,19 @@ public partial class CreateNewPureDatasetUI : RDMPForm { private readonly IActivateItems _activator; + private readonly string[] _visibilities = { "FREE", "CAMPUS", "BACKEND", "CONFIDENTIAL" }; + + private DatasetProviderConfiguration _providerConfiguration; + private PureDatasetProvider _datasetProvider; + public CreateNewPureDatasetUI(IActivateItems activator, ExecuteCommandCreateNewPureDatasetUI command) : base(activator) { _activator = activator; InitializeComponent(); var configs = _activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Type", typeof(PureDatasetProvider).ToString()); cbDatasetProvider.Items.AddRange(configs); + cbVisibility.Items.AddRange(_visibilities); + tbDateMadeAvailable.Text = DateTime.Now.ToString(); } private void CreateNewPureDatasetUI_Load(object sender, EventArgs e) @@ -34,13 +42,47 @@ private void CreateNewPureDatasetUI_Load(object sender, EventArgs e) } - private void label1_Click(object sender, EventArgs e) + private void btnCancel_Click(object sender, EventArgs e) + { + Close(); + } + + private void btnCreate_Click(object sender, EventArgs e) { + if (_activator.YesNo("Are you sure?", "Create dataset?")) + { + //var provider = new PureDatasetProvider(_activator, (DatasetProviderConfiguration)cbDatasetProvider.SelectedItem); + //var date = new PureDate(2024);//TODO + //provider.Create(tbName.Text,((PureDataset.System)cbPublisher.SelectedItem).UUID,lbPeople.SelectedItems,cbVisibility.SelectedItem,date); + } + } + private void Reset() + { + cbPublisher.Items.Clear(); + cbPublisher.Enabled = false; + lbPeople.Items.Clear(); + lbPeople.Enabled = false; } - private void textBox2_TextChanged(object sender, EventArgs e) + private void cbDatasetProvider_SelectedIndexChanged(object sender, EventArgs e) { + if (cbDatasetProvider.SelectedItem is DatasetProviderConfiguration config) + { + _providerConfiguration = config; + _datasetProvider = new PureDatasetProvider(_activator,_providerConfiguration); + //fetch people + //add people + lbPeople.Enabled = true; + //fetch publishers + //add publishers + cbPublisher.Enabled = true; + + } + else + { + Reset(); + } } } From 14725dc8451ef676161ed203293c2f2f92e75895 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 30 Sep 2024 10:45:45 +0100 Subject: [PATCH 066/106] add import --- Rdmp.Core/Datasets/PureDatasetProvider.cs | 7 +- Rdmp.UI/Collections/DatasetsCollectionUI.cs | 7 +- ...ecuteCommandImportExistingPureDatasetUI.cs | 30 ++++ Rdmp.UI/Rdmp.UI.csproj | 3 + .../ImportExistingPureDatasetUI.Designer.cs | 131 ++++++++++++++++++ .../Datasets/ImportExistingPureDatasetUI.cs | 75 ++++++++++ .../Datasets/ImportExistingPureDatasetUI.resx | 120 ++++++++++++++++ 7 files changed, 367 insertions(+), 6 deletions(-) create mode 100644 Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandImportExistingPureDatasetUI.cs create mode 100644 Rdmp.UI/SimpleDialogs/Datasets/ImportExistingPureDatasetUI.Designer.cs create mode 100644 Rdmp.UI/SimpleDialogs/Datasets/ImportExistingPureDatasetUI.cs create mode 100644 Rdmp.UI/SimpleDialogs/Datasets/ImportExistingPureDatasetUI.resx diff --git a/Rdmp.Core/Datasets/PureDatasetProvider.cs b/Rdmp.Core/Datasets/PureDatasetProvider.cs index f8c44332b4..ab4fe9f824 100644 --- a/Rdmp.Core/Datasets/PureDatasetProvider.cs +++ b/Rdmp.Core/Datasets/PureDatasetProvider.cs @@ -32,7 +32,7 @@ public PureDatasetProvider(IBasicActivateItems activator, DatasetProviderConfigu public static string UrltoUUID(string url) => url.Split("/").Last(); - private bool CheckDatasetExistsAtURL(string url) + public bool CheckDatasetExistsAtURL(string url) { var uri = $"{Configuration.Url}/data-sets/{UrltoUUID(url)}"; var response = Task.Run(async () => await _client.GetAsync(uri)).Result; @@ -47,13 +47,14 @@ public override void AddExistingDataset(string name, string url) { var detailsString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; PureDataset pd = JsonConvert.DeserializeObject(detailsString); - var dataset = new Curation.Data.Datasets.Dataset(Repository, name) + var datasetName = string.IsNullOrWhiteSpace(name) ? pd.Title.en_GB : name; + var dataset = new Curation.Data.Datasets.Dataset(Repository, datasetName) { Url = url, Type = this.ToString(), Provider_ID = Configuration.ID, DigitalObjectIdentifier = pd.DigitalObjectIdentifier, - Folder= Configuration.Name, + Folder = $"\\{Configuration.Name}", }; dataset.SaveToDatabase(); Activator.Publish(dataset); diff --git a/Rdmp.UI/Collections/DatasetsCollectionUI.cs b/Rdmp.UI/Collections/DatasetsCollectionUI.cs index 14737a9cf9..b8ccbba0b8 100644 --- a/Rdmp.UI/Collections/DatasetsCollectionUI.cs +++ b/Rdmp.UI/Collections/DatasetsCollectionUI.cs @@ -37,8 +37,8 @@ public override void SetItemActivator(IActivateItems activator) { OverrideCommandName = "Add New Dataset", Weight = -50.9f }, new ExecuteCommandCreateNewPureDatasetUI(Activator) { OverrideCommandName = "Create New Pure Dataset", SuggestedCategory="Pure Datasets" ,Weight = -50.9f }, - //new ExecuteCommandCreateNewPureDatasetUI(Activator) - // { OverrideCommandName = "Import Existing Pure Dataset", SuggestedCategory="Pure Datasets" ,Weight = -50.9f } + new ExecuteCommandImportExistingPureDatasetUI(Activator) + { OverrideCommandName = "Import Existing Pure Dataset", SuggestedCategory="Pure Datasets" ,Weight = -50.9f } }; Activator.RefreshBus.EstablishLifetimeSubscription(this); tlvDatasets.AddObject(activator.CoreChildProvider.DatasetRootFolder); @@ -56,7 +56,8 @@ public override void SetItemActivator(IActivateItems activator) Alignment = ToolStripItemAlignment.Right, ToolTipText = "Refresh Object" }; - _refresh.Click += delegate (object sender, EventArgs e) { + _refresh.Click += delegate (object sender, EventArgs e) + { var dataset = Activator.CoreChildProvider.AllDatasets.First(); if (dataset is not null) { diff --git a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandImportExistingPureDatasetUI.cs b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandImportExistingPureDatasetUI.cs new file mode 100644 index 0000000000..4cc7a6cfe1 --- /dev/null +++ b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandImportExistingPureDatasetUI.cs @@ -0,0 +1,30 @@ +// Copyright (c) The University of Dundee 2018-2019 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using Rdmp.Core.CommandExecution.AtomicCommands; +using Rdmp.UI.ItemActivation; +using Rdmp.UI.SimpleDialogs.Datasets; +using System; + + +namespace Rdmp.UI.CommandExecution.AtomicCommands; + +public class ExecuteCommandImportExistingPureDatasetUI : ExecuteCommandCreateDataset +{ + private readonly IActivateItems _activator; + + public ExecuteCommandImportExistingPureDatasetUI(IActivateItems activator) : base( + activator, "New Dataset") + { + _activator = activator; + } + + public override void Execute() + { + var ui = new ImportExistingPureDatasetUI(_activator, this); + ui.ShowDialog(); + } +} diff --git a/Rdmp.UI/Rdmp.UI.csproj b/Rdmp.UI/Rdmp.UI.csproj index ad5df71f36..5bb1bbb3d9 100644 --- a/Rdmp.UI/Rdmp.UI.csproj +++ b/Rdmp.UI/Rdmp.UI.csproj @@ -133,6 +133,9 @@ DatabaseTypeUI.cs + + Form + UserControl diff --git a/Rdmp.UI/SimpleDialogs/Datasets/ImportExistingPureDatasetUI.Designer.cs b/Rdmp.UI/SimpleDialogs/Datasets/ImportExistingPureDatasetUI.Designer.cs new file mode 100644 index 0000000000..ab89db7c99 --- /dev/null +++ b/Rdmp.UI/SimpleDialogs/Datasets/ImportExistingPureDatasetUI.Designer.cs @@ -0,0 +1,131 @@ +namespace Rdmp.UI.SimpleDialogs.Datasets +{ + partial class ImportExistingPureDatasetUI + { + /// + /// Required designer variable. + /// + private System.ComponentModel.IContainer components = null; + + /// + /// Clean up any resources being used. + /// + /// true if managed resources should be disposed; otherwise, false. + protected override void Dispose(bool disposing) + { + if (disposing && (components != null)) + { + components.Dispose(); + } + base.Dispose(disposing); + } + + #region Windows Form Designer generated code + + /// + /// Required method for Designer support - do not modify + /// the contents of this method with the code editor. + /// + private void InitializeComponent() + { + btnCreate = new System.Windows.Forms.Button(); + btnCancel = new System.Windows.Forms.Button(); + label1 = new System.Windows.Forms.Label(); + tbUrl = new System.Windows.Forms.TextBox(); + lblError = new System.Windows.Forms.Label(); + label2 = new System.Windows.Forms.Label(); + cbProviders = new System.Windows.Forms.ComboBox(); + SuspendLayout(); + // + // btnCreate + // + btnCreate.Location = new System.Drawing.Point(550, 108); + btnCreate.Name = "btnCreate"; + btnCreate.Size = new System.Drawing.Size(75, 23); + btnCreate.TabIndex = 14; + btnCreate.Text = "Create"; + btnCreate.UseVisualStyleBackColor = true; + btnCreate.Click += btnCreate_Click; + // + // btnCancel + // + btnCancel.Location = new System.Drawing.Point(459, 108); + btnCancel.Name = "btnCancel"; + btnCancel.Size = new System.Drawing.Size(75, 23); + btnCancel.TabIndex = 15; + btnCancel.Text = "Cancel"; + btnCancel.UseVisualStyleBackColor = true; + btnCancel.Click += btnCancel_Click; + // + // label1 + // + label1.AutoSize = true; + label1.Location = new System.Drawing.Point(37, 59); + label1.Name = "label1"; + label1.Size = new System.Drawing.Size(31, 15); + label1.TabIndex = 16; + label1.Text = "URL:"; + // + // tbUrl + // + tbUrl.Location = new System.Drawing.Point(74, 56); + tbUrl.Name = "tbUrl"; + tbUrl.Size = new System.Drawing.Size(551, 23); + tbUrl.TabIndex = 17; + // + // lblError + // + lblError.AutoSize = true; + lblError.ForeColor = System.Drawing.Color.Red; + lblError.Location = new System.Drawing.Point(23, 95); + lblError.Name = "lblError"; + lblError.Size = new System.Drawing.Size(0, 15); + lblError.TabIndex = 18; + // + // label2 + // + label2.AutoSize = true; + label2.Location = new System.Drawing.Point(14, 20); + label2.Name = "label2"; + label2.Size = new System.Drawing.Size(54, 15); + label2.TabIndex = 19; + label2.Text = "Provider:"; + // + // cbProviders + // + cbProviders.FormattingEnabled = true; + cbProviders.Location = new System.Drawing.Point(74, 17); + cbProviders.Name = "cbProviders"; + cbProviders.Size = new System.Drawing.Size(345, 23); + cbProviders.TabIndex = 20; + cbProviders.SelectedIndexChanged += cbProviders_SelectedIndexChanged; + // + // ImportExistingPureDatasetUI + // + AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); + AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; + ClientSize = new System.Drawing.Size(667, 158); + Controls.Add(cbProviders); + Controls.Add(label2); + Controls.Add(lblError); + Controls.Add(tbUrl); + Controls.Add(label1); + Controls.Add(btnCancel); + Controls.Add(btnCreate); + Name = "ImportExistingPureDatasetUI"; + Text = "Import Existing Pure Dataset"; + Load += ImportExistingPureDatasetUI_Load; + ResumeLayout(false); + PerformLayout(); + } + + #endregion + private System.Windows.Forms.Button btnCreate; + private System.Windows.Forms.Button btnCancel; + private System.Windows.Forms.Label label1; + private System.Windows.Forms.TextBox tbUrl; + private System.Windows.Forms.Label lblError; + private System.Windows.Forms.Label label2; + private System.Windows.Forms.ComboBox cbProviders; + } +} \ No newline at end of file diff --git a/Rdmp.UI/SimpleDialogs/Datasets/ImportExistingPureDatasetUI.cs b/Rdmp.UI/SimpleDialogs/Datasets/ImportExistingPureDatasetUI.cs new file mode 100644 index 0000000000..7ae1c6321b --- /dev/null +++ b/Rdmp.UI/SimpleDialogs/Datasets/ImportExistingPureDatasetUI.cs @@ -0,0 +1,75 @@ +using Amazon.Auth.AccessControlPolicy; +using Rdmp.Core.CommandExecution.AtomicCommands; +using Rdmp.Core.Curation.Data.Aggregation; +using Rdmp.Core.Curation.Data.Datasets; +using Rdmp.Core.Datasets; +using Rdmp.UI.CommandExecution.AtomicCommands; +using Rdmp.UI.ItemActivation; +using Rdmp.UI.TestsAndSetup.ServicePropogation; +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Data; +using System.Drawing; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Windows.Forms; + +namespace Rdmp.UI.SimpleDialogs.Datasets; + +public partial class ImportExistingPureDatasetUI : RDMPForm +{ + private readonly IActivateItems _activator; + + private readonly string[] _visibilities = { "FREE", "CAMPUS", "BACKEND", "CONFIDENTIAL" }; + + private DatasetProviderConfiguration _providerConfiguration; + private PureDatasetProvider _datasetProvider; + + public ImportExistingPureDatasetUI(IActivateItems activator, ExecuteCommandImportExistingPureDatasetUI command) : base(activator) + { + _activator = activator; + InitializeComponent(); + var configs = _activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Type", typeof(PureDatasetProvider).ToString()); + cbProviders.Items.AddRange(configs); + } + + private void ImportExistingPureDatasetUI_Load(object sender, EventArgs e) + { + + } + + private void btnCancel_Click(object sender, EventArgs e) + { + Dispose(); + } + + private void btnCreate_Click(object sender, EventArgs e) + { + if (_activator.YesNo("Please confirm you wish to import this Dataset", "Import Dataset")) + { + if (!_datasetProvider.CheckDatasetExistsAtURL(tbUrl.Text)) + { + lblError.Text = "Unable to locate dataset. Ensure UUID url is used"; + lblError.Visible = true; + return; + } + else + { + _datasetProvider.AddExistingDataset(null, tbUrl.Text); + Close(); + Dispose(); + } + } + } + + private void cbProviders_SelectedIndexChanged(object sender, EventArgs e) + { + if (cbProviders.SelectedItem is DatasetProviderConfiguration config) + { + _providerConfiguration = config; + _datasetProvider = new PureDatasetProvider(_activator, _providerConfiguration); + } + } +} diff --git a/Rdmp.UI/SimpleDialogs/Datasets/ImportExistingPureDatasetUI.resx b/Rdmp.UI/SimpleDialogs/Datasets/ImportExistingPureDatasetUI.resx new file mode 100644 index 0000000000..af32865ec1 --- /dev/null +++ b/Rdmp.UI/SimpleDialogs/Datasets/ImportExistingPureDatasetUI.resx @@ -0,0 +1,120 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + \ No newline at end of file From e35731b8fd9d7ba1ab939ba69b69e35f8f8e71a6 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 30 Sep 2024 15:19:03 +0100 Subject: [PATCH 067/106] partially working data load --- Rdmp.Core/Curation/Data/DataLoad/Argument.cs | 3 +- .../PostLoad/PureDatasetPostLoadUpdater.cs | 162 ++++++++++++++++++ Rdmp.Core/Datasets/PureDataset.cs | 37 +++- Rdmp.Core/Datasets/PureDatasetProvider.cs | 11 +- 4 files changed, 205 insertions(+), 8 deletions(-) create mode 100644 Rdmp.Core/DataLoad/Modules/Mutilators/PostLoad/PureDatasetPostLoadUpdater.cs diff --git a/Rdmp.Core/Curation/Data/DataLoad/Argument.cs b/Rdmp.Core/Curation/Data/DataLoad/Argument.cs index 141f960991..062074ffd9 100644 --- a/Rdmp.Core/Curation/Data/DataLoad/Argument.cs +++ b/Rdmp.Core/Curation/Data/DataLoad/Argument.cs @@ -71,7 +71,8 @@ public abstract class Argument : DatabaseEntity, IArgument typeof(CatalogueRepository), //user must be IDemandToUseAPipeline - typeof(Pipeline) + typeof(Pipeline), + typeof(Datasets.Dataset) }; #region Database Properties diff --git a/Rdmp.Core/DataLoad/Modules/Mutilators/PostLoad/PureDatasetPostLoadUpdater.cs b/Rdmp.Core/DataLoad/Modules/Mutilators/PostLoad/PureDatasetPostLoadUpdater.cs new file mode 100644 index 0000000000..ce957a2e61 --- /dev/null +++ b/Rdmp.Core/DataLoad/Modules/Mutilators/PostLoad/PureDatasetPostLoadUpdater.cs @@ -0,0 +1,162 @@ +using FAnsi.Discovery; +using MongoDB.Driver.Linq; +using NPOI.SS.Formula.Functions; +using Org.BouncyCastle.Asn1.X509.Qualified; +using Rdmp.Core.CommandExecution; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.DataLoad; +using Rdmp.Core.Curation.Data.Datasets; +using Rdmp.Core.DataLoad.Engine.Job; +using Rdmp.Core.DataLoad.Engine.Mutilators; +using Rdmp.Core.DataLoad.Triggers; +using Rdmp.Core.Datasets; +using Rdmp.Core.MapsDirectlyToDatabaseTable; +using Rdmp.Core.QueryBuilding; +using Rdmp.Core.ReusableLibraryCode.Checks; +using Rdmp.Core.ReusableLibraryCode.Progress; +using System; +using System.Collections.Generic; +using System.Data; +using System.DirectoryServices.Protocols; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.DataLoad.Modules.Mutilators.PostLoad +{ + public class PureDatasetPostLoadUpdater : IMutilateDataTables + { + private readonly string _type = typeof(PureDatasetProvider).ToString(); + private PureDatasetProvider _provider; + private PureDataset _pureDataset; + private int _newRecordsCount; + private PureDate _earliestDate; + private PureDate _latestDate; + + + [DemandsInitialization("The Dataset to update")] + public Curation.Data.Datasets.Dataset Dataset { get; set; } + + + [DemandsInitialization("Update the time period associated with the dataset", DefaultValue = true)] + public bool UpdateTimePeriods { get; set; } + + [DemandsInitialization("Update the description of the dataset", DefaultValue = true)] + public bool UpdateDescription { get; set; } + + [DemandsInitialization("The Column within the data to use for temporal updates")] + public string DateColumn { get; set; } + + [DemandsInitialization(@"The Text added to the description to denote the update. +Some variables are available: +%d - Todays date +%c - The number of new records +%s - The earliest date in the new records +%l - The latest date in the new records +", DefaultValue = "Update %d: Added %c Records with dates between %s and %l.")] + public string DescriptionUpdateText { get; set; } + + public PureDatasetPostLoadUpdater() { } + + public void Check(ICheckNotifier notifier) + { + if (Dataset is null) notifier.OnCheckPerformed(new CheckEventArgs("No Dataset was selected", CheckResult.Fail)); + if (Dataset.Type != _type) notifier.OnCheckPerformed(new CheckEventArgs("Dataset is not a Pure Dataset.", CheckResult.Fail)); + } + + public void Initialize(DiscoveredDatabase dbInfo, LoadStage loadStage) + { + if (loadStage != LoadStage.PostLoad) + { + throw new Exception("Dataset Updater can only be done in the PostLoad stage."); + } + } + + public void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventsListener) + { + } + + public ExitCodeType Mutilate(IDataLoadJob job) + { + var catalogues = job.LoadMetadata.GetAllCatalogues(); + if (catalogues.Count() != 1) + { + job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error, "Can only update a pure dataset when the load affects a single catalogue.")); + return ExitCodeType.Error; + } + var catalogue = catalogues.First(); + var column = catalogue.CatalogueItems.Where(ci => ci.ColumnInfo.GetRuntimeName() == DateColumn).First(); + // get latest dataload id and use that to run a query + var dataLoadID = job.JobID; + + var qb = new QueryBuilder(null, null); + qb.AddColumn(new ColumnInfoToIColumn(new MemoryRepository(), column.ColumnInfo)); + qb.AddCustomLine($"{SpecialFieldNames.DataLoadRunID} = {dataLoadID}", FAnsi.Discovery.QuerySyntax.QueryComponent.WHERE); + var sql = qb.SQL; + + var dt = new DataTable(); + dt.BeginLoadData(); + var server = catalogue.GetDistinctLiveDatabaseServer(ReusableLibraryCode.DataAccess.DataAccessContext.DataLoad,false); + var con = server.GetConnection(); + con.Open(); + using (var cmd = server.GetCommand(sql, con)) + { + using var da = server.GetDataAdapter(cmd); + da.Fill(dt); + } + dt.EndLoadData(); + if (dt.Rows.Count ==0) return ExitCodeType.OperationNotRequired; + _newRecordsCount = dt.Rows.Count; + _earliestDate = new PureDate(dt.AsEnumerable().Min(row => DateTime.Parse(row[0].ToString()))); + _latestDate = new PureDate(dt.AsEnumerable().Max(row => DateTime.Parse(row[0].ToString()))); + + _provider = new PureDatasetProvider(new ThrowImmediatelyActivator(job.RepositoryLocator), job.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("ID", Dataset.Provider_ID).First()); + _pureDataset = _provider.FetchPureDataset(Dataset); + var datasetUpdate = new PureDataset(); + if (UpdateDescription) + { + datasetUpdate.Descriptions = GetDescriptions(); + } + //dont think the temportal stuff is updating correctly... + if (UpdateTimePeriods) + { + datasetUpdate.TemporalCoveragePeriod = GetTemporalCoveragePeriod(); + } + _provider.Update(_pureDataset.UUID, datasetUpdate); + + return ExitCodeType.Success; + } + + private TemporalCoveragePeriod GetTemporalCoveragePeriod() + { + var coverage = _pureDataset.TemporalCoveragePeriod ?? new TemporalCoveragePeriod(); + if(_pureDataset.TemporalCoveragePeriod is null || _pureDataset.TemporalCoveragePeriod.StartDate is null || _earliestDate.IsBefore(_pureDataset.TemporalCoveragePeriod.StartDate)) + coverage.StartDate = _earliestDate; + + if(_pureDataset.TemporalCoveragePeriod is null || _pureDataset.TemporalCoveragePeriod.EndDate is null || _pureDataset.TemporalCoveragePeriod.EndDate.IsBefore(_latestDate)) + coverage.EndDate = _latestDate; + return coverage; + } + + private string GetUpdateText() + { + + return DescriptionUpdateText.Replace("%d", DateTime.Now.ToString("dd/MM/yyyy")).Replace("%c", $"{_newRecordsCount}").Replace("%s", $"{_earliestDate.Day}/{_earliestDate.Month}/{_earliestDate.Year}").Replace("%l", $"{_latestDate.Day}/{_latestDate.Month}/{_latestDate.Year}"); + } + + private List GetDescriptions() + { + var descriptions = new List(); + foreach (var description in _pureDataset.Descriptions) + { + description.Value.en_GB += @$" +{GetUpdateText()}"; + descriptions.Add(description); + } + + return descriptions; + } + + } +} diff --git a/Rdmp.Core/Datasets/PureDataset.cs b/Rdmp.Core/Datasets/PureDataset.cs index c7667bccf0..5dc1a963df 100644 --- a/Rdmp.Core/Datasets/PureDataset.cs +++ b/Rdmp.Core/Datasets/PureDataset.cs @@ -1,5 +1,6 @@ using Amazon.Auth.AccessControlPolicy.ActionIdentifiers; using SynthEHR; +using System; using System.Collections.Generic; using System.Text.Json.Serialization; @@ -24,8 +25,10 @@ public URITerm(string? uri, ENGBWrapper enGBWrapper) } public class PureDescription { - public int? PureID { get; set; } + public int? PureId { get; set; } public ENGBWrapper? Value { get; set; } + + public URITerm Term { get => new URITerm("/dk/atira/pure/dataset/descriptions/datasetdescription", new ENGBWrapper("Description")); } } public class System @@ -48,7 +51,7 @@ public class Name() public class PurePerson { public string? TypeDiscriminator { get; set; } - public int? PureID { get; set; } + public int? PureId { get; set; } public Name? Name { get; set; } public URITerm? Role { get; set; } @@ -58,6 +61,30 @@ public class PurePerson public class PureDate { + public PureDate(DateTime dateTime) + { + Year = dateTime.Year; + Month = dateTime.Month; + Day = dateTime.Day; + } + + public PureDate() { } + + public bool IsBefore(PureDate date) + { + if (Year < date.Year) return true; + if (Year == date.Year) + { + if (Month < date.Month) return true; + if (Month == date.Month) + { + return Day < date.Day; + } + } + + return false; + } + public PureDate(int year, int? month = null, int? day = null) { Year = year; @@ -94,7 +121,7 @@ public class Geolocation public string? Polygon { get; set; } } -public class TemporalcoveragePeriod +public class TemporalCoveragePeriod { public PureDate? StartDate { get; set; } public PureDate? EndDate { get; set; } @@ -103,7 +130,7 @@ public class TemporalcoveragePeriod public class PureDataset : PluginDataset { [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public int? PureID { get; set; } + public int? PureId { get; set; } [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? UUID { get; set; } @@ -170,7 +197,7 @@ public class PureDataset : PluginDataset [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public TemporalcoveragePeriod? TemporalcoveragePeriod { get; set; } + public TemporalCoveragePeriod? TemporalCoveragePeriod { get; set; } #nullable disable diff --git a/Rdmp.Core/Datasets/PureDatasetProvider.cs b/Rdmp.Core/Datasets/PureDatasetProvider.cs index ab4fe9f824..f30cafdc58 100644 --- a/Rdmp.Core/Datasets/PureDatasetProvider.cs +++ b/Rdmp.Core/Datasets/PureDatasetProvider.cs @@ -113,12 +113,19 @@ public override void Update(string uuid, PluginDataset datasetUpdates) WriteIndented = true }; - var jsonString = JsonSerializer.Serialize(datasetUpdates, serializeOptions); + var options = new JsonWriterOptions + { + Indented = true + }; + + using var stream = new MemoryStream(); + JsonSerializer.Serialize(stream, (PureDataset)datasetUpdates, serializeOptions); + var jsonString = Encoding.UTF8.GetString(stream.ToArray()); var uri = $"{Configuration.Url}/data-sets/{uuid}"; var httpContent = new StringContent(jsonString, Encoding.UTF8, "application/json"); var response = Task.Run(async () => await _client.PutAsync(uri, httpContent)).Result; - if (response.StatusCode != HttpStatusCode.Created) + if (response.StatusCode != HttpStatusCode.OK) { throw new Exception("Error"); } From ae40017b6c4dcdf4bee808230dfa5a7b57bfdf19 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 30 Sep 2024 15:26:33 +0100 Subject: [PATCH 068/106] remove comment --- .../Modules/Mutilators/PostLoad/PureDatasetPostLoadUpdater.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/Rdmp.Core/DataLoad/Modules/Mutilators/PostLoad/PureDatasetPostLoadUpdater.cs b/Rdmp.Core/DataLoad/Modules/Mutilators/PostLoad/PureDatasetPostLoadUpdater.cs index ce957a2e61..67255b24ee 100644 --- a/Rdmp.Core/DataLoad/Modules/Mutilators/PostLoad/PureDatasetPostLoadUpdater.cs +++ b/Rdmp.Core/DataLoad/Modules/Mutilators/PostLoad/PureDatasetPostLoadUpdater.cs @@ -118,7 +118,6 @@ public ExitCodeType Mutilate(IDataLoadJob job) { datasetUpdate.Descriptions = GetDescriptions(); } - //dont think the temportal stuff is updating correctly... if (UpdateTimePeriods) { datasetUpdate.TemporalCoveragePeriod = GetTemporalCoveragePeriod(); From 40c712da5de1a83faafdffcccb8be28c728120f9 Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 3 Oct 2024 09:27:03 +0100 Subject: [PATCH 069/106] fetch providers --- Rdmp.Core/Datasets/PureDatasetProvider.cs | 16 ++++++++++++++++ .../Datasets/CreateNewPureDatasetUI.cs | 6 ++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/Rdmp.Core/Datasets/PureDatasetProvider.cs b/Rdmp.Core/Datasets/PureDatasetProvider.cs index f30cafdc58..44dafde0ef 100644 --- a/Rdmp.Core/Datasets/PureDatasetProvider.cs +++ b/Rdmp.Core/Datasets/PureDatasetProvider.cs @@ -16,6 +16,7 @@ using System.Net; using JsonSerializer = System.Text.Json.JsonSerializer; using System.Text; +using System.Data; namespace Rdmp.Core.Datasets { public class PureDatasetProvider : PluginDatasetProvider @@ -156,6 +157,21 @@ public override Curation.Data.Datasets.Dataset FetchDatasetByID(int id) return Repository.GetAllObjectsWhere("Provider_ID", Configuration.ID).ToList(); } + + public List FetchPublishers() + { + var uri = $"{Configuration.Url}/publishers"; + var response = Task.Run(async () => await _client.GetAsync(uri)).Result; + if (response.StatusCode != HttpStatusCode.OK) + { + throw new Exception("Error"); + } + var detailsString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; + List publishers = JsonConvert.DeserializeObject>(detailsString); + return publishers; + + } + public override Curation.Data.Datasets.Dataset Create() { throw new NotImplementedException(); diff --git a/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureDatasetUI.cs b/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureDatasetUI.cs index 0cff917051..197788368d 100644 --- a/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureDatasetUI.cs +++ b/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureDatasetUI.cs @@ -26,6 +26,7 @@ public partial class CreateNewPureDatasetUI : RDMPForm private DatasetProviderConfiguration _providerConfiguration; private PureDatasetProvider _datasetProvider; + private List _publishers; public CreateNewPureDatasetUI(IActivateItems activator, ExecuteCommandCreateNewPureDatasetUI command) : base(activator) { @@ -74,8 +75,9 @@ private void cbDatasetProvider_SelectedIndexChanged(object sender, EventArgs e) //fetch people //add people lbPeople.Enabled = true; - //fetch publishers - //add publishers + + _publishers = _datasetProvider.FetchPublishers(); + cbPublisher.Items.Add(_publishers); cbPublisher.Enabled = true; } From bd547a2be01f79924b0f7578feff5a348a3c15fa Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 8 Oct 2024 07:47:13 +0100 Subject: [PATCH 070/106] add timeout --- .../ExecuteCommandPerformRegexRedactionOnCatalogue.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs index ca10d5c672..f5186e42ff 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs @@ -87,6 +87,7 @@ public override void Execute() conn.Open(); using (var cmd = _server.GetCommand(sql, conn)) { + cmd.CommandTimeout = 60000; using var da = _server.GetDataAdapter(cmd); da.Fill(dt); } From bfdc6442b59e6385755db10cfc104989db30f1f2 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 8 Oct 2024 15:34:12 +0100 Subject: [PATCH 071/106] update insert --- .../OverwriteMigrationStrategy.cs | 25 +++++++++++++------ SharedAssemblyInfo.cs | 2 +- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/Rdmp.Core/DataLoad/Engine/Migration/QueryBuilding/OverwriteMigrationStrategy.cs b/Rdmp.Core/DataLoad/Engine/Migration/QueryBuilding/OverwriteMigrationStrategy.cs index d7b696231e..5fb129f5b5 100644 --- a/Rdmp.Core/DataLoad/Engine/Migration/QueryBuilding/OverwriteMigrationStrategy.cs +++ b/Rdmp.Core/DataLoad/Engine/Migration/QueryBuilding/OverwriteMigrationStrategy.cs @@ -20,9 +20,11 @@ namespace Rdmp.Core.DataLoad.Engine.Migration.QueryBuilding; /// -/// Migrates from STAGING to LIVE a single table (with a MigrationColumnSet). This is an UPSERT (new replaces old) operation achieved (in SQL) with MERGE and -/// UPDATE (based on primary key). Both tables must be on the same server. A MERGE sql statement will be created using LiveMigrationQueryHelper and executed -/// within a transaction. +/// Migrates from STAGING to LIVE a single table (with a MigrationColumnSet). This is an UPSERT (new replaces old) +/// operation achieved (in SQL) with MERGE and +/// UPDATE (based on primary key). Both tables must be on the same server. A MERGE sql statement will be created +/// using LiveMigrationQueryHelper and executed +/// within a transaction. /// public class OverwriteMigrationStrategy : DatabaseMigrationStrategy { @@ -97,8 +99,12 @@ CrossDatabaseMergeCommandTo..ToTable.Age is null columnsToMigrate.DestinationTable.GetFullyQualifiedName())))); sbInsert.AppendLine("WHERE"); - sbInsert.AppendLine( - $"{columnsToMigrate.DestinationTable.GetFullyQualifiedName()}.{syntax.EnsureWrapped(columnsToMigrate.PrimaryKeys.First().GetRuntimeName())} IS NULL"); + sbInsert.AppendLine(string.Join(" AND ", + columnsToMigrate.PrimaryKeys.Select(pk => + $"{columnsToMigrate.DestinationTable.GetFullyQualifiedName()}.{syntax.EnsureWrapped(pk.GetRuntimeName())} IS NULL"))); + + //sbInsert.AppendLine( + // $"{columnsToMigrate.DestinationTable.GetFullyQualifiedName()}.{syntax.EnsureWrapped(columnsToMigrate.PrimaryKeys.First().GetRuntimeName())} IS NULL"); //right at the end of the SELECT if (columnsToMigrate.DestinationTable.Database.Server.DatabaseType == DatabaseType.MySql) @@ -196,7 +202,10 @@ CrossDatabaseMergeCommandTo..ToTable.Age is null } } - private static string GetORLine(DiscoveredColumn c, IQuerySyntaxHelper syntax) => string.Format( - "(t1.{0} <> t2.{0} OR (t1.{0} is null AND t2.{0} is not null) OR (t2.{0} is null AND t1.{0} is not null))", - syntax.EnsureWrapped(c.GetRuntimeName())); + private static string GetORLine(DiscoveredColumn c, IQuerySyntaxHelper syntax) + { + return string.Format( + "(t1.{0} <> t2.{0} OR (t1.{0} is null AND t2.{0} is not null) OR (t2.{0} is null AND t1.{0} is not null))", + syntax.EnsureWrapped(c.GetRuntimeName())); + } } \ No newline at end of file diff --git a/SharedAssemblyInfo.cs b/SharedAssemblyInfo.cs index 559a541653..80ff38559e 100644 --- a/SharedAssemblyInfo.cs +++ b/SharedAssemblyInfo.cs @@ -12,4 +12,4 @@ [assembly: AssemblyVersion("8.3.0")] [assembly: AssemblyFileVersion("8.3.0")] -[assembly: AssemblyInformationalVersion("8.3.0")] \ No newline at end of file +[assembly: AssemblyInformationalVersion("8.3.1-guneet-pk-migrate")] \ No newline at end of file From ca5a0b78e7c66da0047dfd893e7870f3fca1daf7 Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 10 Oct 2024 08:23:02 +0100 Subject: [PATCH 072/106] tidy up --- CHANGELOG.md | 1 + .../Migration/QueryBuilding/OverwriteMigrationStrategy.cs | 3 --- SharedAssemblyInfo.cs | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9c5930f28..f51780771f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Improve Performance of regenerating problems with child providers - Update UI Tab opening Logic - Add Filter to Left-Hand Tree View +- Update Migration strategy to account for all Primary Keys when moving from staging -> live ## [8.3.0] - 2024-09-23 diff --git a/Rdmp.Core/DataLoad/Engine/Migration/QueryBuilding/OverwriteMigrationStrategy.cs b/Rdmp.Core/DataLoad/Engine/Migration/QueryBuilding/OverwriteMigrationStrategy.cs index 5fb129f5b5..b476af145f 100644 --- a/Rdmp.Core/DataLoad/Engine/Migration/QueryBuilding/OverwriteMigrationStrategy.cs +++ b/Rdmp.Core/DataLoad/Engine/Migration/QueryBuilding/OverwriteMigrationStrategy.cs @@ -103,9 +103,6 @@ CrossDatabaseMergeCommandTo..ToTable.Age is null columnsToMigrate.PrimaryKeys.Select(pk => $"{columnsToMigrate.DestinationTable.GetFullyQualifiedName()}.{syntax.EnsureWrapped(pk.GetRuntimeName())} IS NULL"))); - //sbInsert.AppendLine( - // $"{columnsToMigrate.DestinationTable.GetFullyQualifiedName()}.{syntax.EnsureWrapped(columnsToMigrate.PrimaryKeys.First().GetRuntimeName())} IS NULL"); - //right at the end of the SELECT if (columnsToMigrate.DestinationTable.Database.Server.DatabaseType == DatabaseType.MySql) sbInsert.Append(" FOR UPDATE"); diff --git a/SharedAssemblyInfo.cs b/SharedAssemblyInfo.cs index 80ff38559e..069388dd64 100644 --- a/SharedAssemblyInfo.cs +++ b/SharedAssemblyInfo.cs @@ -12,4 +12,4 @@ [assembly: AssemblyVersion("8.3.0")] [assembly: AssemblyFileVersion("8.3.0")] -[assembly: AssemblyInformationalVersion("8.3.1-guneet-pk-migrate")] \ No newline at end of file +[assembly: AssemblyInformationalVersion("8.3.1")] \ No newline at end of file From f2febf05081b1a1c8743b3ff3f8e15b56cb71e79 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 15 Oct 2024 14:47:50 +0100 Subject: [PATCH 073/106] add creation provider --- .../ExecuteCommandDeleteDatasetTest.cs | 2 +- Rdmp.Core/Datasets/PureDataset.cs | 31 ++-- Rdmp.Core/Datasets/PureDatasetProvider.cs | 10 +- Rdmp.UI/Collections/DatasetsCollectionUI.cs | 4 +- ...cuteCommandCreateNewPureConfigurationUI.cs | 28 ++++ .../CreateNewPureConfigurationUI.Designer.cs | 154 ++++++++++++++++++ .../Datasets/CreateNewPureConfigurationUI.cs | 70 ++++++++ .../CreateNewPureConfigurationUI.resx | 120 ++++++++++++++ .../Datasets/CreateNewPureDatasetUI.cs | 3 +- Rdmp.UI/SimpleDialogs/SelectDialog`1.resx | 63 ------- 10 files changed, 397 insertions(+), 88 deletions(-) create mode 100644 Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandCreateNewPureConfigurationUI.cs create mode 100644 Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureConfigurationUI.Designer.cs create mode 100644 Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureConfigurationUI.cs create mode 100644 Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureConfigurationUI.resx delete mode 100644 Rdmp.UI/SimpleDialogs/SelectDialog`1.resx diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandDeleteDatasetTest.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandDeleteDatasetTest.cs index 155e2442e0..8b09efdb32 100644 --- a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandDeleteDatasetTest.cs +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandDeleteDatasetTest.cs @@ -24,7 +24,7 @@ public void TestDeleteExistingDataset() public void TestDeleteNonExistantDataset() { Assert.That(GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects(), Is.Empty); - var founddataset = new Core.Curation.Data.Dataset(); + var founddataset = new Core.Curation.Data.Datasets.Dataset(); var delCmd = new ExecuteCommandDeleteDataset(GetMockActivator(), founddataset); Assert.Throws(() => delCmd.Execute()); Assert.That(GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects(), Is.Empty); diff --git a/Rdmp.Core/Datasets/PureDataset.cs b/Rdmp.Core/Datasets/PureDataset.cs index 5dc1a963df..ed0c00db68 100644 --- a/Rdmp.Core/Datasets/PureDataset.cs +++ b/Rdmp.Core/Datasets/PureDataset.cs @@ -31,17 +31,6 @@ public class PureDescription public URITerm Term { get => new URITerm("/dk/atira/pure/dataset/descriptions/datasetdescription", new ENGBWrapper("Description")); } } -public class System -{ - public System(string? uuid, string? systemName) - { - UUID = uuid; - SystemName = systemName; - } - public string? SystemName { get; set; } - public string? UUID { get; set; } -} - public class Name() { public string? FirstName { get; set; } @@ -56,7 +45,7 @@ public class PurePerson public Name? Name { get; set; } public URITerm? Role { get; set; } - public List? Organizations { get; set; } + public List? Organizations { get; set; } } public class PureDate @@ -127,6 +116,17 @@ public class TemporalCoveragePeriod public PureDate? EndDate { get; set; } } +public class PureSystem +{ + public PureSystem(string? uuid, string? systemName) + { + UUID = uuid; + SystemName = systemName; + } + public string? SystemName { get; set; } + public string? UUID { get; set; } +} + public class PureDataset : PluginDataset { [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] @@ -161,13 +161,13 @@ public class PureDataset : PluginDataset public List? Descriptions { get; set; } [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public System? ManagingOrganization { get; set; } + public PureSystem? ManagingOrganization { get; set; } [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public new URITerm? Type { get; set; } [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public System? Publisher { get; set; } + public PureSystem? Publisher { get; set; } [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public Geolocation? Geolocation { get; set; } @@ -176,7 +176,7 @@ public class PureDataset : PluginDataset public List? Persons { get; set; } [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public List? Organizations { get; set; } + public List? Organizations { get; set; } [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public PureDate? PublicationAvailableDate { get; set; } @@ -202,5 +202,4 @@ public class PureDataset : PluginDataset #nullable disable public PureDataset() { } - } diff --git a/Rdmp.Core/Datasets/PureDatasetProvider.cs b/Rdmp.Core/Datasets/PureDatasetProvider.cs index 44dafde0ef..6744369e81 100644 --- a/Rdmp.Core/Datasets/PureDatasetProvider.cs +++ b/Rdmp.Core/Datasets/PureDatasetProvider.cs @@ -72,13 +72,13 @@ public Curation.Data.Datasets.Dataset Create(string title, string publisherUid, var pureDataset = new PureDataset(); pureDataset.Title = new ENGBWrapper(title); - pureDataset.ManagingOrganization = new System(Configuration.Organisation_ID, "Organization"); + pureDataset.ManagingOrganization = new PureSystem(Configuration.Organisation_ID, "Organization"); pureDataset.Type = new URITerm("/dk/atira/pure/dataset/datasettypes/dataset/dataset", new ENGBWrapper("Dataset")); - pureDataset.Publisher = new System(publisherUid, "Publisher"); + pureDataset.Publisher = new PureSystem(publisherUid, "Publisher"); pureDataset.Persons = people; pureDataset.Visibility = visibility; - pureDataset.Organizations = [new System(Configuration.Organisation_ID, "Organization")]; + pureDataset.Organizations = [new PureSystem(Configuration.Organisation_ID, "Organization")]; pureDataset.PublicationAvailableDate = publicationDate; var serializeOptions = new JsonSerializerOptions @@ -158,7 +158,7 @@ public override Curation.Data.Datasets.Dataset FetchDatasetByID(int id) } - public List FetchPublishers() + public List FetchPublishers() { var uri = $"{Configuration.Url}/publishers"; var response = Task.Run(async () => await _client.GetAsync(uri)).Result; @@ -167,7 +167,7 @@ public List FetchPublishers() throw new Exception("Error"); } var detailsString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; - List publishers = JsonConvert.DeserializeObject>(detailsString); + List publishers = JsonConvert.DeserializeObject>(detailsString); return publishers; } diff --git a/Rdmp.UI/Collections/DatasetsCollectionUI.cs b/Rdmp.UI/Collections/DatasetsCollectionUI.cs index 6a300639f9..578ad6d2f8 100644 --- a/Rdmp.UI/Collections/DatasetsCollectionUI.cs +++ b/Rdmp.UI/Collections/DatasetsCollectionUI.cs @@ -38,7 +38,9 @@ public override void SetItemActivator(IActivateItems activator) new ExecuteCommandCreateNewPureDatasetUI(Activator) { OverrideCommandName = "Create New Pure Dataset", SuggestedCategory="Pure Datasets" ,Weight = -50.9f }, new ExecuteCommandImportExistingPureDatasetUI(Activator) - { OverrideCommandName = "Import Existing Pure Dataset", SuggestedCategory="Pure Datasets" ,Weight = -50.9f } + { OverrideCommandName = "Import Existing Pure Dataset", SuggestedCategory="Pure Datasets" ,Weight = -50.9f }, + new ExecuteCommandCreateNewPureConfigurationUI(Activator) + { OverrideCommandName = "Configure Pure Dataset Provider", SuggestedCategory="Pure Datasets" ,Weight = -50.9f } }; Activator.RefreshBus.EstablishLifetimeSubscription(this); tlvDatasets.AddObject(activator.CoreChildProvider.DatasetRootFolder); diff --git a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandCreateNewPureConfigurationUI.cs b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandCreateNewPureConfigurationUI.cs new file mode 100644 index 0000000000..4d3eaefdd9 --- /dev/null +++ b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandCreateNewPureConfigurationUI.cs @@ -0,0 +1,28 @@ +using Rdmp.Core.CommandExecution; +using Rdmp.Core.CommandExecution.AtomicCommands; +using Rdmp.UI.ItemActivation; +using Rdmp.UI.SimpleDialogs.Datasets; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.UI.CommandExecution.AtomicCommands; + +public class ExecuteCommandCreateNewPureConfigurationUI : BasicCommandExecution, IAtomicCommand +{ + private readonly IActivateItems _activator; + + public ExecuteCommandCreateNewPureConfigurationUI(IActivateItems activator) : base(activator) + { + _activator = activator; + } + + public override void Execute() + { + base.Execute(); + var ui = new CreateNewPureConfigurationUI(_activator); + ui.ShowDialog(); + } +} diff --git a/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureConfigurationUI.Designer.cs b/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureConfigurationUI.Designer.cs new file mode 100644 index 0000000000..262ed25ef6 --- /dev/null +++ b/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureConfigurationUI.Designer.cs @@ -0,0 +1,154 @@ +namespace Rdmp.UI.SimpleDialogs.Datasets +{ + partial class CreateNewPureConfigurationUI + { + /// + /// Required designer variable. + /// + private System.ComponentModel.IContainer components = null; + + /// + /// Clean up any resources being used. + /// + /// true if managed resources should be disposed; otherwise, false. + protected override void Dispose(bool disposing) + { + if (disposing && (components != null)) + { + components.Dispose(); + } + base.Dispose(disposing); + } + + #region Windows Form Designer generated code + + /// + /// Required method for Designer support - do not modify + /// the contents of this method with the code editor. + /// + private void InitializeComponent() + { + label1 = new System.Windows.Forms.Label(); + label2 = new System.Windows.Forms.Label(); + label3 = new System.Windows.Forms.Label(); + label4 = new System.Windows.Forms.Label(); + tbName = new System.Windows.Forms.TextBox(); + tbUrl = new System.Windows.Forms.TextBox(); + tbOrganisationId = new System.Windows.Forms.TextBox(); + cbCredentials = new System.Windows.Forms.ComboBox(); + btnSave = new System.Windows.Forms.Button(); + SuspendLayout(); + // + // label1 + // + label1.AutoSize = true; + label1.Location = new System.Drawing.Point(136, 76); + label1.Name = "label1"; + label1.Size = new System.Drawing.Size(25, 15); + label1.TabIndex = 0; + label1.Text = "Url:"; + // + // label2 + // + label2.AutoSize = true; + label2.Location = new System.Drawing.Point(119, 37); + label2.Name = "label2"; + label2.Size = new System.Drawing.Size(42, 15); + label2.TabIndex = 1; + label2.Text = "Name:"; + // + // label3 + // + label3.AutoSize = true; + label3.Location = new System.Drawing.Point(26, 116); + label3.Name = "label3"; + label3.Size = new System.Drawing.Size(135, 15); + label3.TabIndex = 2; + label3.Text = "Data Access Credentials:"; + // + // label4 + // + label4.AutoSize = true; + label4.Location = new System.Drawing.Point(69, 152); + label4.Name = "label4"; + label4.Size = new System.Drawing.Size(92, 15); + label4.TabIndex = 3; + label4.Text = "Organisation ID:"; + // + // tbName + // + tbName.Location = new System.Drawing.Point(169, 34); + tbName.Name = "tbName"; + tbName.Size = new System.Drawing.Size(370, 23); + tbName.TabIndex = 4; + tbName.TextChanged += ValidateForm; + // + // tbUrl + // + tbUrl.Location = new System.Drawing.Point(169, 73); + tbUrl.Name = "tbUrl"; + tbUrl.Size = new System.Drawing.Size(370, 23); + tbUrl.TabIndex = 5; + tbUrl.TextChanged += ValidateForm; + // + // tbOrganisationId + // + tbOrganisationId.Location = new System.Drawing.Point(169, 149); + tbOrganisationId.Name = "tbOrganisationId"; + tbOrganisationId.Size = new System.Drawing.Size(370, 23); + tbOrganisationId.TabIndex = 6; + tbOrganisationId.TextChanged += ValidateForm; + // + // cbCredentials + // + cbCredentials.FormattingEnabled = true; + cbCredentials.Location = new System.Drawing.Point(169, 108); + cbCredentials.Name = "cbCredentials"; + cbCredentials.Size = new System.Drawing.Size(208, 23); + cbCredentials.TabIndex = 7; + cbCredentials.SelectedIndexChanged += ValidateForm; + // + // btnSave + // + btnSave.Enabled = false; + btnSave.Location = new System.Drawing.Point(464, 192); + btnSave.Name = "btnSave"; + btnSave.Size = new System.Drawing.Size(75, 23); + btnSave.TabIndex = 8; + btnSave.Text = "Save"; + btnSave.UseVisualStyleBackColor = true; + btnSave.Click += Save; + // + // CreateNewPureConfigurationUI + // + AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); + AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; + ClientSize = new System.Drawing.Size(800, 450); + Controls.Add(btnSave); + Controls.Add(cbCredentials); + Controls.Add(tbOrganisationId); + Controls.Add(tbUrl); + Controls.Add(tbName); + Controls.Add(label4); + Controls.Add(label3); + Controls.Add(label2); + Controls.Add(label1); + Name = "CreateNewPureConfigurationUI"; + Text = "Create Pure Configuration"; + ResumeLayout(false); + PerformLayout(); + } + + #endregion + + private System.Windows.Forms.Label label1; + private System.Windows.Forms.Label label2; + private System.Windows.Forms.Label label3; + private System.Windows.Forms.Label label4; + private System.Windows.Forms.TextBox tbName; + private System.Windows.Forms.TextBox tbUrl; + private System.Windows.Forms.TextBox tbOrganisationId; + private System.Windows.Forms.ComboBox cbCredentials; + private System.Windows.Forms.Button btnSave; + } +} \ No newline at end of file diff --git a/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureConfigurationUI.cs b/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureConfigurationUI.cs new file mode 100644 index 0000000000..3761d45c0b --- /dev/null +++ b/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureConfigurationUI.cs @@ -0,0 +1,70 @@ +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.Datasets; +using Rdmp.Core.Datasets; +using Rdmp.UI.ItemActivation; +using Rdmp.UI.TestsAndSetup.ServicePropogation; +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Data; +using System.Drawing; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Windows.Forms; + +namespace Rdmp.UI.SimpleDialogs.Datasets +{ + public partial class CreateNewPureConfigurationUI : RDMPForm + { + + private readonly IActivateItems _activator; + public CreateNewPureConfigurationUI(IActivateItems activator) + { + InitializeComponent(); + _activator = activator; + var dataAccessCredentials = _activator.RepositoryLocator.CatalogueRepository.GetAllObjects(); + cbCredentials.Items.Clear(); + cbCredentials.Items.AddRange(dataAccessCredentials); + } + + private void DisableSave() + { + btnSave.Enabled = false; + } + + private void ValidateForm(object sender, EventArgs e) + { + + if (cbCredentials.SelectedItem is null) + { + DisableSave(); + return; + } + if (string.IsNullOrWhiteSpace(tbName.Text)) + { + DisableSave(); + return; + } + if (string.IsNullOrWhiteSpace(tbUrl.Text)) + { + DisableSave(); + return; + } + if (string.IsNullOrWhiteSpace(tbOrganisationId.Text)) + { + DisableSave(); + return; + } + btnSave.Enabled = true; + } + + private void Save(object sender, EventArgs e) + { + var config = new DatasetProviderConfiguration(_activator.RepositoryLocator.CatalogueRepository, tbName.Text, typeof(PureDatasetProvider).ToString(), tbUrl.Text, ((DataAccessCredentials)cbCredentials.SelectedItem).ID, tbOrganisationId.Text); + config.SaveToDatabase(); + Close(); + _activator.Show($"Dataset Provider '{tbName.Text}' has successfully been created"); + } + } +} diff --git a/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureConfigurationUI.resx b/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureConfigurationUI.resx new file mode 100644 index 0000000000..af32865ec1 --- /dev/null +++ b/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureConfigurationUI.resx @@ -0,0 +1,120 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + \ No newline at end of file diff --git a/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureDatasetUI.cs b/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureDatasetUI.cs index 197788368d..a1b9af3146 100644 --- a/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureDatasetUI.cs +++ b/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureDatasetUI.cs @@ -15,7 +15,6 @@ using System.Text; using System.Threading.Tasks; using System.Windows.Forms; - namespace Rdmp.UI.SimpleDialogs.Datasets; public partial class CreateNewPureDatasetUI : RDMPForm @@ -26,7 +25,7 @@ public partial class CreateNewPureDatasetUI : RDMPForm private DatasetProviderConfiguration _providerConfiguration; private PureDatasetProvider _datasetProvider; - private List _publishers; + private List _publishers; public CreateNewPureDatasetUI(IActivateItems activator, ExecuteCommandCreateNewPureDatasetUI command) : base(activator) { diff --git a/Rdmp.UI/SimpleDialogs/SelectDialog`1.resx b/Rdmp.UI/SimpleDialogs/SelectDialog`1.resx deleted file mode 100644 index 6252b1ef07..0000000000 --- a/Rdmp.UI/SimpleDialogs/SelectDialog`1.resx +++ /dev/null @@ -1,63 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - text/microsoft-resx - - - 2.0 - - - System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - True - - \ No newline at end of file From 7f92d138d82b701687cb6603e312f0f4288b7a37 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 15 Oct 2024 15:56:01 +0100 Subject: [PATCH 074/106] nice name --- Rdmp.UI/Menus/DatasetMenu.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rdmp.UI/Menus/DatasetMenu.cs b/Rdmp.UI/Menus/DatasetMenu.cs index 5abca0c0fe..fd51236a5c 100644 --- a/Rdmp.UI/Menus/DatasetMenu.cs +++ b/Rdmp.UI/Menus/DatasetMenu.cs @@ -15,6 +15,6 @@ public class DatasetMenu: RDMPContextMenuStrip public DatasetMenu(RDMPContextMenuStripArgs args, Dataset dataset): base(args, dataset) { - Add(new ExecuteCommandDeleteDatasetUI(_activator,dataset)); + Add(new ExecuteCommandDeleteDatasetUI(_activator, dataset) { OverrideCommandName="Delete Dataset"}); } } From 028540bf1998a108c96cb4afd74e7d7396a78ff9 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 16 Oct 2024 12:56:31 +0100 Subject: [PATCH 075/106] update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8f56ae99d..ea9e79f7e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ - +ha # Changelog All notable changes to this project will be documented in this file. From 9ae81fc6799aa1d3c2c9041782098c957bc291d0 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 16 Oct 2024 14:44:34 +0100 Subject: [PATCH 076/106] partial ui --- CHANGELOG.md | 2 +- ...DistinctAgainstCatalogueMutilationTests.cs | 79 ++++++++ Rdmp.Core/Curation/Data/Datasets/Dataset.cs | 2 +- .../AllDatasetProviderConfigurationsNode.png | Bin 0 -> 233 bytes .../Icons/IconProvision/CatalogueIcons.resx | 3 + Rdmp.Core/Providers/CatalogueChildProvider.cs | 12 ++ Rdmp.Core/Providers/ICoreChildProvider.cs | 1 + .../AllDatasetProviderConfigurationsNode.cs | 6 + .../ConfigurationsCollectionUI.Designer.cs | 1 + .../Collections/ConfigurationsCollectionUI.cs | 18 +- Rdmp.UI/Collections/DatasetsCollectionUI.cs | 136 -------------- ...henTargetIsDatasetProviderConfiguration.cs | 33 ++++ Rdmp.UI/Menus/DatasetMenu.cs | 2 +- ...DatasetProviderConfigurationUI.Designer.cs | 173 ++++++++++++++++++ .../DatasetProviderConfigurationUI.cs | 73 ++++++++ .../DatasetProviderConfigurationUI.resx | 120 ++++++++++++ 16 files changed, 519 insertions(+), 142 deletions(-) create mode 100644 Rdmp.Core.Tests/DataLoad/Engine/Integration/DistinctAgainstCatalogueMutilationTests.cs create mode 100644 Rdmp.Core/Icons/AllDatasetProviderConfigurationsNode.png create mode 100644 Rdmp.Core/Providers/Nodes/AllDatasetProviderConfigurationsNode.cs delete mode 100644 Rdmp.UI/Collections/DatasetsCollectionUI.cs create mode 100644 Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsDatasetProviderConfiguration.cs create mode 100644 Rdmp.UI/SubComponents/DatasetProviderConfigurationUI.Designer.cs create mode 100644 Rdmp.UI/SubComponents/DatasetProviderConfigurationUI.cs create mode 100644 Rdmp.UI/SubComponents/DatasetProviderConfigurationUI.resx diff --git a/CHANGELOG.md b/CHANGELOG.md index b3d4ebb86d..e1d0a79246 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ - +ataset # Changelog All notable changes to this project will be documented in this file. diff --git a/Rdmp.Core.Tests/DataLoad/Engine/Integration/DistinctAgainstCatalogueMutilationTests.cs b/Rdmp.Core.Tests/DataLoad/Engine/Integration/DistinctAgainstCatalogueMutilationTests.cs new file mode 100644 index 0000000000..1c978e2723 --- /dev/null +++ b/Rdmp.Core.Tests/DataLoad/Engine/Integration/DistinctAgainstCatalogueMutilationTests.cs @@ -0,0 +1,79 @@ +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using FAnsi; +using NSubstitute.Routing.Handlers; +using NUnit.Framework; +using System.Data; +using System; +using Rdmp.Core.CommandExecution.AtomicCommands.CatalogueCreationCommands; +using Tests.Common; +using Rdmp.Core.Curation; +using System.Linq; +using Rdmp.Core.CommandExecution; +using Rdmp.Core.DataLoad.Modules.Mutilators; + +namespace Rdmp.Core.Tests.DataLoad.Engine.Integration; + +public class DistinctAgainstCatalogueMutilationTests: DatabaseTests +{ + + [Test] + [TestCase(DatabaseType.MicrosoftSQLServer)] + [TestCase(DatabaseType.MySql)] + [TestCase(DatabaseType.PostgreSql)] + [TestCase(DatabaseType.Oracle)] + public void TestDistinctAgainstCatalogueMutilation_New(DatabaseType type) + { + var db = GetCleanedServer(type, "TestDistinctAgainstCatalogueMutilation"); + const int batchCount = 1000; + + using var dt = new DataTable("TestDistinctAgainstCatalogueMutilation_New"); + dt.BeginLoadData(); + dt.Columns.Add("pk"); + dt.Columns.Add("f1"); + dt.Columns.Add("f2"); + dt.Columns.Add("f3"); + dt.Columns.Add("f4"); + + var r = new Random(123); + + for (var i = 0; i < batchCount; i++) + { + var randInt = r.Next(int.MaxValue); + + dt.Rows.Add(new object[] { randInt, randInt, randInt, randInt, randInt }); + dt.Rows.Add(new object[] { randInt, randInt, randInt, randInt, randInt + 1 }); + } + + dt.EndLoadData(); + + dt.EndLoadData(); + var tbl = db.CreateTable(dt.TableName, dt); + + var importer = new TableInfoImporter(CatalogueRepository, tbl); + importer.DoImport(out var tableInfo, out var colInfos); + + //lie about what hte primary key is because this component is designed to run in the RAW environment and we are simulating a LIVE TableInfo (correctly) + var pk = colInfos.Single(c => c.GetRuntimeName().Equals("pk")); + pk.IsPrimaryKey = true; + pk.SaveToDatabase(); + var cmd = new ExecuteCommandCreateNewCatalogueByImportingExistingDataTable(new ThrowImmediatelyActivator(RepositoryLocator), tbl, null); + cmd.Execute(); + + //var distincter = new DistinctAgainstCatalogueMutilation(); + } + + public void TestDistinctAgainstCatalogueMutilation_Update(DatabaseType type) + { + + } + + public void TestDistinctAgainstCatalogueMutilation_NewAndUpdat(DatabaseType type) + { + + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/Dataset.cs b/Rdmp.Core/Curation/Data/Datasets/Dataset.cs index 60d9ca1146..74b863ae88 100644 --- a/Rdmp.Core/Curation/Data/Datasets/Dataset.cs +++ b/Rdmp.Core/Curation/Data/Datasets/Dataset.cs @@ -15,7 +15,7 @@ namespace Rdmp.Core.Curation.Data.Datasets; /// -public sealed class Dataset : DatabaseEntity, IDataset, IHasFolder +public class Dataset : DatabaseEntity, IDataset, IHasFolder { private string _name; private string _digitalObjectIdentifier; diff --git a/Rdmp.Core/Icons/AllDatasetProviderConfigurationsNode.png b/Rdmp.Core/Icons/AllDatasetProviderConfigurationsNode.png new file mode 100644 index 0000000000000000000000000000000000000000..3e9d6ed41b4901ee2e3811ef6e85df0978acf1d2 GIT binary patch literal 233 zcmeAS@N?(olHy`uVBq!ia0vp^1|ZDA1SD@H!x2AA6$+S6}hN-9%vmdKI;Vst04O9%8~^|S literal 0 HcmV?d00001 diff --git a/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx b/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx index 6abf0c1c75..8c95363cbb 100644 --- a/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx +++ b/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx @@ -781,4 +781,7 @@ ..\StandardRegex31.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + ..\AllDatasetProviderConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + \ No newline at end of file diff --git a/Rdmp.Core/Providers/CatalogueChildProvider.cs b/Rdmp.Core/Providers/CatalogueChildProvider.cs index 6a10fdc83f..762cb1b1cd 100644 --- a/Rdmp.Core/Providers/CatalogueChildProvider.cs +++ b/Rdmp.Core/Providers/CatalogueChildProvider.cs @@ -214,6 +214,8 @@ public class CatalogueChildProvider : ICoreChildProvider public HashSet OrphanAggregateConfigurations; public AggregateConfiguration[] TemplateAggregateConfigurations; + public AllDatasetConfigurationsNode AllDatasetConfigurationsNode { get; set; } + public DatasetProviderConfiguration[] DatasetProviderConfigurations { get; set; } protected Stopwatch ProgressStopwatch = Stopwatch.StartNew(); private int _progress; @@ -457,6 +459,9 @@ public CatalogueChildProvider(ICatalogueRepository repository, IChildProvider[] AllRegexRedactionConfigurationsNode = new AllRegexRedactionConfigurationsNode(); AddChildren(AllRegexRedactionConfigurationsNode); + AllDatasetConfigurationsNode = new AllDatasetConfigurationsNode(); + DatasetProviderConfigurations = GetAllObjects(repository); + AddChildren(AllDatasetConfigurationsNode); AllDatasets = GetAllObjects(repository); AllDatasetsNode = new AllDatasetsNode(); @@ -623,6 +628,13 @@ private void AddChildren(AllDatasetsNode allDatasetsNode) AddToDictionaries(children, descendancy); } + private void AddChildren(AllDatasetConfigurationsNode allDatasetConfigurationNode) + { + var children = new HashSet(DatasetProviderConfigurations); + var descendancy = new DescendancyList(allDatasetConfigurationNode); + AddToDictionaries(children, descendancy); + } + private void AddChildren(AllGovernanceNode allGovernanceNode) { var children = new HashSet(); diff --git a/Rdmp.Core/Providers/ICoreChildProvider.cs b/Rdmp.Core/Providers/ICoreChildProvider.cs index 1ac7e16b5b..5c6b472169 100644 --- a/Rdmp.Core/Providers/ICoreChildProvider.cs +++ b/Rdmp.Core/Providers/ICoreChildProvider.cs @@ -84,6 +84,7 @@ public interface ICoreChildProvider : IChildProvider AllRegexRedactionConfigurationsNode AllRegexRedactionConfigurationsNode { get; } + AllDatasetConfigurationsNode AllDatasetConfigurationsNode { get; } AllObjectSharingNode AllObjectSharingNode { get; } ObjectImport[] AllImports { get; } ObjectExport[] AllExports { get; } diff --git a/Rdmp.Core/Providers/Nodes/AllDatasetProviderConfigurationsNode.cs b/Rdmp.Core/Providers/Nodes/AllDatasetProviderConfigurationsNode.cs new file mode 100644 index 0000000000..40c49d249b --- /dev/null +++ b/Rdmp.Core/Providers/Nodes/AllDatasetProviderConfigurationsNode.cs @@ -0,0 +1,6 @@ +namespace Rdmp.Core.Providers.Nodes; + +public class AllDatasetConfigurationsNode : SingletonNode +{ + public AllDatasetConfigurationsNode():base("Dataset Provider Configurations") { } +} diff --git a/Rdmp.UI/Collections/ConfigurationsCollectionUI.Designer.cs b/Rdmp.UI/Collections/ConfigurationsCollectionUI.Designer.cs index c2fd81e05c..0b261da08d 100644 --- a/Rdmp.UI/Collections/ConfigurationsCollectionUI.Designer.cs +++ b/Rdmp.UI/Collections/ConfigurationsCollectionUI.Designer.cs @@ -32,6 +32,7 @@ protected override void Dispose(bool disposing) private void InitializeComponent() { this.components = new System.ComponentModel.Container(); + this.tbFilter = new TextBox(); this.tlvConfigurations = new BrightIdeasSoftware.TreeListView(); this.olvName = ((BrightIdeasSoftware.OLVColumn)(new BrightIdeasSoftware.OLVColumn())); ((System.ComponentModel.ISupportInitialize)(this.tlvConfigurations)).BeginInit(); diff --git a/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs b/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs index 41a7a32adf..5dbd5461ac 100644 --- a/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs +++ b/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs @@ -6,6 +6,7 @@ using System.Linq; using Rdmp.Core.Curation.Data; using Rdmp.Core.Providers.Nodes; +using Rdmp.Core.Curation.Data.Datasets; namespace Rdmp.UI.Collections; @@ -24,11 +25,21 @@ private IAtomicCommand[] GetWhitespaceRightClickMenu() return new IAtomicCommand[] { new ExecuteCommandCreateNewDatasetUI(_activator){ - OverrideCommandName="Add New Dataset" + OverrideCommandName="Add New Dataset", SuggestedCategory="Datasets" + }, + new ExecuteCommandCreateNewPureDatasetUI(_activator){ + OverrideCommandName="Create New Pure Dataset", SuggestedCategory="Pure Datasets" + }, + new ExecuteCommandImportExistingPureDatasetUI(_activator) + { + OverrideCommandName="Import Existing Pure Dataset", SuggestedCategory="Pure Datasets" + }, + new ExecuteCommandCreateNewPureConfigurationUI(_activator){ + OverrideCommandName="Create New Pure Configuration", SuggestedCategory="Pure Datasets" }, new ExecuteCommandAddNewRegexRedactionConfigurationUI(_activator) { - OverrideCommandName="Add New Regex Redaction Configuration" + OverrideCommandName="Add New Regex Redaction Configuration", SuggestedCategory="Regex" } }; } @@ -38,11 +49,12 @@ public override void SetItemActivator(IActivateItems activator) base.SetItemActivator(activator); _activator = activator; CommonTreeFunctionality.SetUp(RDMPCollection.Configurations, tlvConfigurations, activator, olvName, olvName, - new RDMPCollectionCommonFunctionalitySettings()); + new RDMPCollectionCommonFunctionalitySettings(),tbFilter); CommonTreeFunctionality.WhitespaceRightClickMenuCommandsGetter = e => GetWhitespaceRightClickMenu(); Activator.RefreshBus.EstablishLifetimeSubscription(this); tlvConfigurations.AddObject(Activator.CoreChildProvider.AllDatasetsNode); tlvConfigurations.AddObject(Activator.CoreChildProvider.AllRegexRedactionConfigurationsNode); + tlvConfigurations.AddObject(Activator.CoreChildProvider.AllDatasetConfigurationsNode); tlvConfigurations.Refresh(); } diff --git a/Rdmp.UI/Collections/DatasetsCollectionUI.cs b/Rdmp.UI/Collections/DatasetsCollectionUI.cs deleted file mode 100644 index 578ad6d2f8..0000000000 --- a/Rdmp.UI/Collections/DatasetsCollectionUI.cs +++ /dev/null @@ -1,136 +0,0 @@ -using Rdmp.Core.CommandExecution.AtomicCommands; -using Rdmp.Core; -using Rdmp.Core.MapsDirectlyToDatabaseTable; -using Rdmp.UI.CommandExecution.AtomicCommands; -using Rdmp.UI.ItemActivation; -using Rdmp.UI.Refreshing; -using System; -using System.Collections.Generic; -using System.Linq; -using Rdmp.Core.Icons.IconProvision; -using System.Windows.Forms; -using Rdmp.Core.Curation.Data.Datasets; - -namespace Rdmp.UI.Collections; - -public partial class DatasetsCollectionUI : RDMPCollectionUI, ILifetimeSubscriber -{ - - private Dataset[] _datasets; - private bool _firstTime = true; - - public DatasetsCollectionUI() - { - InitializeComponent(); - } - - public override void SetItemActivator(IActivateItems activator) - { - base.SetItemActivator(activator); - - CommonTreeFunctionality.SetUp(RDMPCollection.Datasets, tlvDatasets, Activator, olvName, olvName, - new RDMPCollectionCommonFunctionalitySettings(),tbFilter); - CommonTreeFunctionality.WhitespaceRightClickMenuCommandsGetter = - a => new IAtomicCommand[] - { - new ExecuteCommandCreateNewDatasetUI(Activator) - { OverrideCommandName = "Add New Dataset", Weight = -50.9f }, - new ExecuteCommandCreateNewPureDatasetUI(Activator) - { OverrideCommandName = "Create New Pure Dataset", SuggestedCategory="Pure Datasets" ,Weight = -50.9f }, - new ExecuteCommandImportExistingPureDatasetUI(Activator) - { OverrideCommandName = "Import Existing Pure Dataset", SuggestedCategory="Pure Datasets" ,Weight = -50.9f }, - new ExecuteCommandCreateNewPureConfigurationUI(Activator) - { OverrideCommandName = "Configure Pure Dataset Provider", SuggestedCategory="Pure Datasets" ,Weight = -50.9f } - }; - Activator.RefreshBus.EstablishLifetimeSubscription(this); - tlvDatasets.AddObject(activator.CoreChildProvider.DatasetRootFolder); - - RefreshDatasets(Activator.CoreChildProvider.DatasetRootFolder); - - if (_firstTime) - { - CommonTreeFunctionality.SetupColumnTracking(olvName, new Guid("f8b0481e-378c-4996-9400-cb039c2efc5c")); - _firstTime = false; - var _refresh = new ToolStripMenuItem - { - Visible = true, - Image = FamFamFamIcons.arrow_refresh.ImageToBitmap(), - Alignment = ToolStripItemAlignment.Right, - ToolTipText = "Refresh Object" - }; - _refresh.Click += delegate (object sender, EventArgs e) - { - var dataset = Activator.CoreChildProvider.AllDatasets.First(); - if (dataset is not null) - { - var cmd = new ExecuteCommandRefreshObject(Activator, dataset); - cmd.Execute(); - } - }; - CommonFunctionality.Add(_refresh); - } - } - - - - public void RefreshBus_RefreshObject(object sender, RefreshObjectEventArgs e) - { - RefreshDatasets(Activator.CoreChildProvider.DatasetRootFolder); - } - - private void RefreshDatasets(object oRefreshFrom) - { - var rootFolder = Activator.CoreChildProvider.DatasetRootFolder; - if (_datasets != null) - { - var newCatalogues = CommonTreeFunctionality.CoreChildProvider.AllDatasets.Except(_datasets); - if (newCatalogues.Any()) - { - oRefreshFrom = rootFolder; //refresh from the root instead - tlvDatasets.RefreshObject(oRefreshFrom); - } - } - _datasets = CommonTreeFunctionality.CoreChildProvider.AllDatasets; - if (_firstTime || Equals(oRefreshFrom, rootFolder)) - { - tlvDatasets.RefreshObject(rootFolder); - tlvDatasets.Expand(rootFolder); - _firstTime = false; - } - - } - - - /// - /// Returns all root objects in RDMP that match the . Handles unpicking tree collisions e.g. where matches 2 objects with one being the child of the other - /// - /// - /// - /// - public static List FindRootObjects(IActivateItems activator, - Func condition) - { - var datasets = - activator.RepositoryLocator.CatalogueRepository.GetAllObjects(); - - var actualRootFavourites = new List(); - - foreach (var currentFavourite in datasets) - actualRootFavourites.Add(currentFavourite); - - return actualRootFavourites; - } - - - /// - /// Return true if the object should be displayed in this pane - /// - /// - /// - protected virtual bool IncludeObject(IMapsDirectlyToDatabaseTable key) => - Activator.RepositoryLocator.CatalogueRepository.GetAllObjects().Contains(key); - - public static bool IsRootObject(IActivateItems activator, object root) => - //never favourite - false; -} \ No newline at end of file diff --git a/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsDatasetProviderConfiguration.cs b/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsDatasetProviderConfiguration.cs new file mode 100644 index 0000000000..8b00da0d45 --- /dev/null +++ b/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsDatasetProviderConfiguration.cs @@ -0,0 +1,33 @@ +using Rdmp.Core.CommandExecution; +using Rdmp.Core.Curation.Data.Datasets; +using Rdmp.UI.ItemActivation; +using Rdmp.UI.SubComponents; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.UI.CommandExecution.Proposals; + +public class ProposeExecutionWhenTargetIsDatasetProviderConfiguration : RDMPCommandExecutionProposal +{ + public ProposeExecutionWhenTargetIsDatasetProviderConfiguration(IActivateItems itemActivator) : base(itemActivator) + { + } + + public override void Activate(DatasetProviderConfiguration target) + { + ItemActivator.Activate(target); + } + + public override bool CanActivate(DatasetProviderConfiguration target) + { + return true; + } + + public override ICommandExecution ProposeExecution(ICombineToMakeCommand cmd, DatasetProviderConfiguration target, InsertOption insertOption = InsertOption.Default) + { + return null; + } +} diff --git a/Rdmp.UI/Menus/DatasetMenu.cs b/Rdmp.UI/Menus/DatasetMenu.cs index efe95afcc7..a8d734ce86 100644 --- a/Rdmp.UI/Menus/DatasetMenu.cs +++ b/Rdmp.UI/Menus/DatasetMenu.cs @@ -15,6 +15,6 @@ public class DatasetMenu : RDMPContextMenuStrip public DatasetMenu(RDMPContextMenuStripArgs args, Dataset dataset) : base(args, dataset) { - Add(new ExecuteCommandDeleteDatasetUI(_activator, dataset) { OverrideCommandName="Delete Dataset"}); + Add(new ExecuteCommandDeleteDatasetUI(_activator, dataset) { OverrideCommandName = "Delete Dataset" }); } } diff --git a/Rdmp.UI/SubComponents/DatasetProviderConfigurationUI.Designer.cs b/Rdmp.UI/SubComponents/DatasetProviderConfigurationUI.Designer.cs new file mode 100644 index 0000000000..1bd650fa52 --- /dev/null +++ b/Rdmp.UI/SubComponents/DatasetProviderConfigurationUI.Designer.cs @@ -0,0 +1,173 @@ +namespace Rdmp.UI.SubComponents +{ + partial class DatasetProviderConfigurationUI + { + /// + /// Required designer variable. + /// + private System.ComponentModel.IContainer components = null; + + /// + /// Clean up any resources being used. + /// + /// true if managed resources should be disposed; otherwise, false. + protected override void Dispose(bool disposing) + { + if (disposing && (components != null)) + { + components.Dispose(); + } + base.Dispose(disposing); + } + + #region Component Designer generated code + + /// + /// Required method for Designer support - do not modify + /// the contents of this method with the code editor. + /// + private void InitializeComponent() + { + label1 = new System.Windows.Forms.Label(); + tbName = new System.Windows.Forms.TextBox(); + btnSave = new System.Windows.Forms.Button(); + tbType = new System.Windows.Forms.TextBox(); + label2 = new System.Windows.Forms.Label(); + tbUrl = new System.Windows.Forms.TextBox(); + label3 = new System.Windows.Forms.Label(); + tbOrgId = new System.Windows.Forms.TextBox(); + label4 = new System.Windows.Forms.Label(); + cbAccessCredentials = new System.Windows.Forms.ComboBox(); + label5 = new System.Windows.Forms.Label(); + SuspendLayout(); + // + // label1 + // + label1.AutoSize = true; + label1.Location = new System.Drawing.Point(102, 75); + label1.Name = "label1"; + label1.Size = new System.Drawing.Size(42, 15); + label1.TabIndex = 0; + label1.Text = "Name:"; + label1.Click += label1_Click; + // + // tbName + // + tbName.Location = new System.Drawing.Point(146, 72); + tbName.Name = "tbName"; + tbName.Size = new System.Drawing.Size(252, 23); + tbName.TabIndex = 1; + // + // btnSave + // + btnSave.Location = new System.Drawing.Point(323, 279); + btnSave.Name = "btnSave"; + btnSave.Size = new System.Drawing.Size(75, 23); + btnSave.TabIndex = 2; + btnSave.Text = "Save"; + btnSave.UseVisualStyleBackColor = true; + btnSave.Click += btnSave_Click; + // + // tbType + // + tbType.Enabled = false; + tbType.Location = new System.Drawing.Point(146, 115); + tbType.Name = "tbType"; + tbType.Size = new System.Drawing.Size(252, 23); + tbType.TabIndex = 4; + // + // label2 + // + label2.AutoSize = true; + label2.Location = new System.Drawing.Point(102, 118); + label2.Name = "label2"; + label2.Size = new System.Drawing.Size(34, 15); + label2.TabIndex = 3; + label2.Text = "Type:"; + // + // tbUrl + // + tbUrl.Location = new System.Drawing.Point(146, 154); + tbUrl.Name = "tbUrl"; + tbUrl.Size = new System.Drawing.Size(252, 23); + tbUrl.TabIndex = 6; + // + // label3 + // + label3.AutoSize = true; + label3.Location = new System.Drawing.Point(102, 157); + label3.Name = "label3"; + label3.Size = new System.Drawing.Size(31, 15); + label3.TabIndex = 5; + label3.Text = "URL:"; + label3.Click += label3_Click; + // + // tbOrgId + // + tbOrgId.Location = new System.Drawing.Point(146, 194); + tbOrgId.Name = "tbOrgId"; + tbOrgId.Size = new System.Drawing.Size(252, 23); + tbOrgId.TabIndex = 8; + // + // label4 + // + label4.AutoSize = true; + label4.Location = new System.Drawing.Point(47, 197); + label4.Name = "label4"; + label4.Size = new System.Drawing.Size(92, 15); + label4.TabIndex = 7; + label4.Text = "Organisation ID:"; + // + // cbAccessCredentials + // + cbAccessCredentials.FormattingEnabled = true; + cbAccessCredentials.Location = new System.Drawing.Point(146, 236); + cbAccessCredentials.Name = "cbAccessCredentials"; + cbAccessCredentials.Size = new System.Drawing.Size(252, 23); + cbAccessCredentials.TabIndex = 9; + // + // label5 + // + label5.AutoSize = true; + label5.Location = new System.Drawing.Point(1, 239); + label5.Name = "label5"; + label5.Size = new System.Drawing.Size(135, 15); + label5.TabIndex = 10; + label5.Text = "Data Access Credentials:"; + // + // DatasetProviderConfigurationUI + // + AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); + AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; + Controls.Add(label5); + Controls.Add(cbAccessCredentials); + Controls.Add(tbOrgId); + Controls.Add(label4); + Controls.Add(tbUrl); + Controls.Add(label3); + Controls.Add(tbType); + Controls.Add(label2); + Controls.Add(btnSave); + Controls.Add(tbName); + Controls.Add(label1); + Name = "DatasetProviderConfigurationUI"; + Size = new System.Drawing.Size(1003, 530); + ResumeLayout(false); + PerformLayout(); + } + + #endregion + + private System.Windows.Forms.Label label1; + private System.Windows.Forms.TextBox tbName; + private System.Windows.Forms.Button btnSave; + private System.Windows.Forms.TextBox tbType; + private System.Windows.Forms.Label label2; + private System.Windows.Forms.TextBox tbUrl; + private System.Windows.Forms.Label label3; + private System.Windows.Forms.TextBox tbOrgId; + private System.Windows.Forms.Label label4; + private System.Windows.Forms.ComboBox cbAccessCredentials; + private System.Windows.Forms.Label label5; + } +} diff --git a/Rdmp.UI/SubComponents/DatasetProviderConfigurationUI.cs b/Rdmp.UI/SubComponents/DatasetProviderConfigurationUI.cs new file mode 100644 index 0000000000..c2c24a99dd --- /dev/null +++ b/Rdmp.UI/SubComponents/DatasetProviderConfigurationUI.cs @@ -0,0 +1,73 @@ +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.Datasets; +using Rdmp.UI.ItemActivation; +using Rdmp.UI.Refreshing; +using Rdmp.UI.TestsAndSetup.ServicePropogation; +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Data; +using System.Drawing; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Windows.Forms; + +namespace Rdmp.UI.SubComponents; + +public partial class DatasetProviderConfigurationUI : DatasetProviderConfigurationUI_Design, IRefreshBusSubscriber +{ + private DatasetProviderConfiguration _configuration; + private IActivateItems _activator; + public DatasetProviderConfigurationUI() + { + InitializeComponent(); + } + + public override void SetDatabaseObject(IActivateItems activator, DatasetProviderConfiguration databaseObject) + { + _activator = activator; + base.SetDatabaseObject(activator, databaseObject); + _configuration = databaseObject; + tbName.Text = _configuration.Name; + tbOrgId.Text = _configuration.Organisation_ID; + tbType.Text = _configuration.Type; + tbUrl.Text = _configuration.Url; + cbAccessCredentials.Items.Clear(); + var credentials = _activator.RepositoryLocator.CatalogueRepository.GetAllObjects().ToArray(); + cbAccessCredentials.Items.AddRange(credentials); + cbAccessCredentials.SelectedIndex = Array.FindIndex(credentials, c => c.ID == _configuration.DataAccessCredentials_ID); + } + + private void label1_Click(object sender, EventArgs e) + { + + } + + private void label3_Click(object sender, EventArgs e) + { + + } + + private void btnSave_Click(object sender, EventArgs e) + { + _configuration.Name = tbName.Text; + _configuration.Url = tbUrl.Text; + _configuration.DataAccessCredentials_ID = ((DataAccessCredentials)cbAccessCredentials.SelectedItem).ID; + _configuration.Organisation_ID = tbOrgId.Text; + _configuration.SaveToDatabase(); + Publish(_configuration); + _activator.Show("Updated Dataset Provider Configuration"); + } + + public void RefreshBus_RefreshObject(object sender, RefreshObjectEventArgs e) + { + } +} + +[TypeDescriptionProvider( + typeof(AbstractControlDescriptionProvider))] +public abstract class + DatasetProviderConfigurationUI_Design : RDMPSingleDatabaseObjectControl +{ +} \ No newline at end of file diff --git a/Rdmp.UI/SubComponents/DatasetProviderConfigurationUI.resx b/Rdmp.UI/SubComponents/DatasetProviderConfigurationUI.resx new file mode 100644 index 0000000000..af32865ec1 --- /dev/null +++ b/Rdmp.UI/SubComponents/DatasetProviderConfigurationUI.resx @@ -0,0 +1,120 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + \ No newline at end of file From 14a16c03bc4a864a17a1023d64cdac182c812583 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 16 Oct 2024 14:56:36 +0100 Subject: [PATCH 077/106] tidy up --- Rdmp.Core/Providers/CatalogueChildProvider.cs | 4 ++-- Rdmp.UI/Collections/ConfigurationsCollectionUI.cs | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Rdmp.Core/Providers/CatalogueChildProvider.cs b/Rdmp.Core/Providers/CatalogueChildProvider.cs index 762cb1b1cd..210f3c8cf1 100644 --- a/Rdmp.Core/Providers/CatalogueChildProvider.cs +++ b/Rdmp.Core/Providers/CatalogueChildProvider.cs @@ -68,6 +68,7 @@ public class CatalogueChildProvider : ICoreChildProvider //Catalogue side of things public Catalogue[] AllCatalogues { get; set; } public Curation.Data.Datasets.Dataset[] AllDatasets { get; set; } + public Dictionary AllCataloguesDictionary { get; private set; } public SupportingDocument[] AllSupportingDocuments { get; set; } @@ -217,6 +218,7 @@ public class CatalogueChildProvider : ICoreChildProvider public AllDatasetConfigurationsNode AllDatasetConfigurationsNode { get; set; } public DatasetProviderConfiguration[] DatasetProviderConfigurations { get; set; } + protected Stopwatch ProgressStopwatch = Stopwatch.StartNew(); private int _progress; @@ -396,8 +398,6 @@ public CatalogueChildProvider(ICatalogueRepository repository, IChildProvider[] DatasetRootFolder = FolderHelper.BuildFolderTree(AllDatasets); - - AddChildren(DatasetRootFolder, new DescendancyList(DatasetRootFolder)); ReportProgress("Build Catalogue Folder Root"); diff --git a/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs b/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs index 5dbd5461ac..c98e62f2c0 100644 --- a/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs +++ b/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs @@ -52,6 +52,7 @@ public override void SetItemActivator(IActivateItems activator) new RDMPCollectionCommonFunctionalitySettings(),tbFilter); CommonTreeFunctionality.WhitespaceRightClickMenuCommandsGetter = e => GetWhitespaceRightClickMenu(); Activator.RefreshBus.EstablishLifetimeSubscription(this); + //tlvConfigurations.AddObject(Activator.CoreChildProvider.DatasetRootFolder); tlvConfigurations.AddObject(Activator.CoreChildProvider.AllDatasetsNode); tlvConfigurations.AddObject(Activator.CoreChildProvider.AllRegexRedactionConfigurationsNode); tlvConfigurations.AddObject(Activator.CoreChildProvider.AllDatasetConfigurationsNode); From c7770f0d774cb1945cc30ea644e3e723e76209be Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 17 Oct 2024 14:04:42 +0100 Subject: [PATCH 078/106] improved update --- .../WindowManagement/HomePane/HomeBoxUI.cs | 4 +- CHANGELOG.md | 1 - Rdmp.Core/Datasets/PureDataset.cs | 8 + Rdmp.Core/Datasets/PureDatasetProvider.cs | 102 ++++---- .../Icons/IconProvision/CatalogueIcons.resx | 5 +- Rdmp.Core/Icons/IconProvision/RDMPConcept.cs | 4 +- Rdmp.Core/Providers/CatalogueChildProvider.cs | 8 +- .../Providers/DataExportChildProvider.cs | 2 + Rdmp.Core/Providers/ICoreChildProvider.cs | 2 +- .../AllDatasetProviderConfigurationsNode.cs | 4 +- .../Collections/ConfigurationsCollectionUI.cs | 5 +- .../ExecuteCommandCreateNewPureDatasetUI.cs | 30 --- .../CreateNewPureDatasetUI.Designer.cs | 236 ------------------ .../Datasets/CreateNewPureDatasetUI.cs | 89 ------- .../Datasets/CreateNewPureDatasetUI.resx | 120 --------- .../PureDatasetConfigurationUI.Designer.cs | 129 ++++++++-- .../PureDatasetConfigurationUI.cs | 63 ++++- 17 files changed, 241 insertions(+), 571 deletions(-) delete mode 100644 Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandCreateNewPureDatasetUI.cs delete mode 100644 Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureDatasetUI.Designer.cs delete mode 100644 Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureDatasetUI.cs delete mode 100644 Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureDatasetUI.resx diff --git a/Application/ResearchDataManagementPlatform/WindowManagement/HomePane/HomeBoxUI.cs b/Application/ResearchDataManagementPlatform/WindowManagement/HomePane/HomeBoxUI.cs index 9f0c900d68..1c6e5c42a0 100644 --- a/Application/ResearchDataManagementPlatform/WindowManagement/HomePane/HomeBoxUI.cs +++ b/Application/ResearchDataManagementPlatform/WindowManagement/HomePane/HomeBoxUI.cs @@ -27,7 +27,7 @@ public partial class HomeBoxUI : UserControl { private IActivateItems _activator; private bool _doneSetup; - private Type _openType; + private System.Type _openType; private RDMPCollectionCommonFunctionality CommonTreeFunctionality { get; } = new(); @@ -37,7 +37,7 @@ public HomeBoxUI() olvRecent.ItemActivate += OlvRecent_ItemActivate; } - public void SetUp(IActivateItems activator, string title, Type openType, AtomicCommandUIFactory factory, + public void SetUp(IActivateItems activator, string title, System.Type openType, AtomicCommandUIFactory factory, params IAtomicCommand[] newCommands) { _openType = openType; diff --git a/CHANGELOG.md b/CHANGELOG.md index e1d0a79246..ae6d76c4fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,3 @@ -ataset # Changelog All notable changes to this project will be documented in this file. diff --git a/Rdmp.Core/Datasets/PureDataset.cs b/Rdmp.Core/Datasets/PureDataset.cs index ed0c00db68..a586327683 100644 --- a/Rdmp.Core/Datasets/PureDataset.cs +++ b/Rdmp.Core/Datasets/PureDataset.cs @@ -1,4 +1,6 @@ using Amazon.Auth.AccessControlPolicy.ActionIdentifiers; +using NPOI.SS.Formula.Atp; +using NPOI.SS.Formula.Functions; using SynthEHR; using System; using System.Collections.Generic; @@ -59,6 +61,12 @@ public PureDate(DateTime dateTime) public PureDate() { } + + public DateTime ToDateTime() + { + return new DateTime(Year, Month??1, Day??1, 0, 0, 0); + } + public bool IsBefore(PureDate date) { if (Year < date.Year) return true; diff --git a/Rdmp.Core/Datasets/PureDatasetProvider.cs b/Rdmp.Core/Datasets/PureDatasetProvider.cs index 6744369e81..a34ffeea9a 100644 --- a/Rdmp.Core/Datasets/PureDatasetProvider.cs +++ b/Rdmp.Core/Datasets/PureDatasetProvider.cs @@ -66,45 +66,45 @@ public override void AddExistingDataset(string name, string url) } } - public Curation.Data.Datasets.Dataset Create(string title, string publisherUid, List people, Visibility visibility, PureDate publicationDate) - { - //this works but i'm not sure it's linking up with the person correctly - - var pureDataset = new PureDataset(); - pureDataset.Title = new ENGBWrapper(title); - pureDataset.ManagingOrganization = new PureSystem(Configuration.Organisation_ID, "Organization"); - pureDataset.Type = new URITerm("/dk/atira/pure/dataset/datasettypes/dataset/dataset", new ENGBWrapper("Dataset")); - pureDataset.Publisher = new PureSystem(publisherUid, "Publisher"); - - pureDataset.Persons = people; - pureDataset.Visibility = visibility; - pureDataset.Organizations = [new PureSystem(Configuration.Organisation_ID, "Organization")]; - pureDataset.PublicationAvailableDate = publicationDate; - - var serializeOptions = new JsonSerializerOptions - { - PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - WriteIndented = true - }; - - var jsonString = JsonSerializer.Serialize(pureDataset, serializeOptions); - - var uri = $"{Configuration.Url}/data-sets"; - var httpContent = new StringContent(jsonString, Encoding.UTF8, "application/json"); - - var response = Task.Run(async () => await _client.PutAsync(uri, httpContent)).Result; - if (response.StatusCode != HttpStatusCode.Created) - { - throw new Exception("Error"); - } - var detailsString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; - PureDataset pd = JsonConvert.DeserializeObject(detailsString); - var dataset = new Curation.Data.Datasets.Dataset(Repository, pureDataset.Title.en_GB); - dataset.Provider_ID = Configuration.ID; - dataset.Url = pd.PortalURL; - dataset.SaveToDatabase(); - return dataset; - } + //public Curation.Data.Datasets.Dataset Create(string title, string publisherUid, List people, Visibility visibility, PureDate publicationDate) + //{ + // //this works but i'm not sure it's linking up with the person correctly + + // var pureDataset = new PureDataset(); + // pureDataset.Title = new ENGBWrapper(title); + // pureDataset.ManagingOrganization = new PureSystem(Configuration.Organisation_ID, "Organization"); + // pureDataset.Type = new URITerm("/dk/atira/pure/dataset/datasettypes/dataset/dataset", new ENGBWrapper("Dataset")); + // pureDataset.Publisher = new PureSystem(publisherUid, "Publisher"); + + // pureDataset.Persons = people; + // pureDataset.Visibility = visibility; + // pureDataset.Organizations = [new PureSystem(Configuration.Organisation_ID, "Organization")]; + // pureDataset.PublicationAvailableDate = publicationDate; + + // var serializeOptions = new JsonSerializerOptions + // { + // PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + // WriteIndented = true + // }; + + // var jsonString = JsonSerializer.Serialize(pureDataset, serializeOptions); + + // var uri = $"{Configuration.Url}/data-sets"; + // var httpContent = new StringContent(jsonString, Encoding.UTF8, "application/json"); + + // var response = Task.Run(async () => await _client.PutAsync(uri, httpContent)).Result; + // if (response.StatusCode != HttpStatusCode.Created) + // { + // throw new Exception("Error"); + // } + // var detailsString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; + // PureDataset pd = JsonConvert.DeserializeObject(detailsString); + // var dataset = new Curation.Data.Datasets.Dataset(Repository, pureDataset.Title.en_GB); + // dataset.Provider_ID = Configuration.ID; + // dataset.Url = pd.PortalURL; + // dataset.SaveToDatabase(); + // return dataset; + //} public override void Update(string uuid, PluginDataset datasetUpdates) { @@ -158,19 +158,19 @@ public override Curation.Data.Datasets.Dataset FetchDatasetByID(int id) } - public List FetchPublishers() - { - var uri = $"{Configuration.Url}/publishers"; - var response = Task.Run(async () => await _client.GetAsync(uri)).Result; - if (response.StatusCode != HttpStatusCode.OK) - { - throw new Exception("Error"); - } - var detailsString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; - List publishers = JsonConvert.DeserializeObject>(detailsString); - return publishers; + //public List FetchPublishers() + //{ + // var uri = $"{Configuration.Url}/publishers"; + // var response = Task.Run(async () => await _client.GetAsync(uri)).Result; + // if (response.StatusCode != HttpStatusCode.OK) + // { + // throw new Exception("Error"); + // } + // var detailsString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; + // List publishers = JsonConvert.DeserializeObject>(detailsString); + // return publishers; - } + //} public override Curation.Data.Datasets.Dataset Create() { diff --git a/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx b/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx index 8c95363cbb..856c935696 100644 --- a/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx +++ b/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx @@ -782,6 +782,9 @@ ..\StandardRegex31.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - ..\AllDatasetProviderConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + ..\famfamfam\folder_table.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\famfamfam\cog.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 \ No newline at end of file diff --git a/Rdmp.Core/Icons/IconProvision/RDMPConcept.cs b/Rdmp.Core/Icons/IconProvision/RDMPConcept.cs index 3fc3fbb0d4..3bab8f26ca 100644 --- a/Rdmp.Core/Icons/IconProvision/RDMPConcept.cs +++ b/Rdmp.Core/Icons/IconProvision/RDMPConcept.cs @@ -212,5 +212,7 @@ public enum RDMPConcept RegexRedaction, RegexRedactionConfiguration, RegexRedactionKey, - AllRegexRedactionConfigurationsNode + AllRegexRedactionConfigurationsNode, + DatasetProviderConfiguration, + AllDatasetProviderConfigurationsNode } \ No newline at end of file diff --git a/Rdmp.Core/Providers/CatalogueChildProvider.cs b/Rdmp.Core/Providers/CatalogueChildProvider.cs index 210f3c8cf1..dfdaa0da38 100644 --- a/Rdmp.Core/Providers/CatalogueChildProvider.cs +++ b/Rdmp.Core/Providers/CatalogueChildProvider.cs @@ -215,7 +215,7 @@ public class CatalogueChildProvider : ICoreChildProvider public HashSet OrphanAggregateConfigurations; public AggregateConfiguration[] TemplateAggregateConfigurations; - public AllDatasetConfigurationsNode AllDatasetConfigurationsNode { get; set; } + public AllDatasetProviderConfigurationsNode AllDatasetProviderConfigurationsNode { get; set; } public DatasetProviderConfiguration[] DatasetProviderConfigurations { get; set; } @@ -459,9 +459,9 @@ public CatalogueChildProvider(ICatalogueRepository repository, IChildProvider[] AllRegexRedactionConfigurationsNode = new AllRegexRedactionConfigurationsNode(); AddChildren(AllRegexRedactionConfigurationsNode); - AllDatasetConfigurationsNode = new AllDatasetConfigurationsNode(); + AllDatasetProviderConfigurationsNode = new AllDatasetProviderConfigurationsNode(); DatasetProviderConfigurations = GetAllObjects(repository); - AddChildren(AllDatasetConfigurationsNode); + AddChildren(AllDatasetProviderConfigurationsNode); AllDatasets = GetAllObjects(repository); AllDatasetsNode = new AllDatasetsNode(); @@ -628,7 +628,7 @@ private void AddChildren(AllDatasetsNode allDatasetsNode) AddToDictionaries(children, descendancy); } - private void AddChildren(AllDatasetConfigurationsNode allDatasetConfigurationNode) + private void AddChildren(AllDatasetProviderConfigurationsNode allDatasetConfigurationNode) { var children = new HashSet(DatasetProviderConfigurations); var descendancy = new DescendancyList(allDatasetConfigurationNode); diff --git a/Rdmp.Core/Providers/DataExportChildProvider.cs b/Rdmp.Core/Providers/DataExportChildProvider.cs index 62236ab882..f818f88ee5 100644 --- a/Rdmp.Core/Providers/DataExportChildProvider.cs +++ b/Rdmp.Core/Providers/DataExportChildProvider.cs @@ -820,6 +820,8 @@ public override void UpdateTo(ICoreChildProvider other) AllDatasetsNode = dxOther.AllDatasetsNode; AllRegexRedactionConfigurations = dxOther.AllRegexRedactionConfigurations; AllRegexRedactionConfigurationsNode = dxOther.AllRegexRedactionConfigurationsNode; + AllDatasetProviderConfigurationsNode = dxOther.AllDatasetProviderConfigurationsNode; + AllDataAccessCredentialsNode = dxOther.AllDataAccessCredentialsNode; } } diff --git a/Rdmp.Core/Providers/ICoreChildProvider.cs b/Rdmp.Core/Providers/ICoreChildProvider.cs index 5c6b472169..46d2469c98 100644 --- a/Rdmp.Core/Providers/ICoreChildProvider.cs +++ b/Rdmp.Core/Providers/ICoreChildProvider.cs @@ -84,7 +84,7 @@ public interface ICoreChildProvider : IChildProvider AllRegexRedactionConfigurationsNode AllRegexRedactionConfigurationsNode { get; } - AllDatasetConfigurationsNode AllDatasetConfigurationsNode { get; } + AllDatasetProviderConfigurationsNode AllDatasetProviderConfigurationsNode { get; } AllObjectSharingNode AllObjectSharingNode { get; } ObjectImport[] AllImports { get; } ObjectExport[] AllExports { get; } diff --git a/Rdmp.Core/Providers/Nodes/AllDatasetProviderConfigurationsNode.cs b/Rdmp.Core/Providers/Nodes/AllDatasetProviderConfigurationsNode.cs index 40c49d249b..48081a6419 100644 --- a/Rdmp.Core/Providers/Nodes/AllDatasetProviderConfigurationsNode.cs +++ b/Rdmp.Core/Providers/Nodes/AllDatasetProviderConfigurationsNode.cs @@ -1,6 +1,6 @@ namespace Rdmp.Core.Providers.Nodes; -public class AllDatasetConfigurationsNode : SingletonNode +public class AllDatasetProviderConfigurationsNode : SingletonNode { - public AllDatasetConfigurationsNode():base("Dataset Provider Configurations") { } + public AllDatasetProviderConfigurationsNode():base("Dataset Provider Configurations") { } } diff --git a/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs b/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs index c98e62f2c0..72df7df53b 100644 --- a/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs +++ b/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs @@ -27,9 +27,6 @@ private IAtomicCommand[] GetWhitespaceRightClickMenu() new ExecuteCommandCreateNewDatasetUI(_activator){ OverrideCommandName="Add New Dataset", SuggestedCategory="Datasets" }, - new ExecuteCommandCreateNewPureDatasetUI(_activator){ - OverrideCommandName="Create New Pure Dataset", SuggestedCategory="Pure Datasets" - }, new ExecuteCommandImportExistingPureDatasetUI(_activator) { OverrideCommandName="Import Existing Pure Dataset", SuggestedCategory="Pure Datasets" @@ -55,7 +52,7 @@ public override void SetItemActivator(IActivateItems activator) //tlvConfigurations.AddObject(Activator.CoreChildProvider.DatasetRootFolder); tlvConfigurations.AddObject(Activator.CoreChildProvider.AllDatasetsNode); tlvConfigurations.AddObject(Activator.CoreChildProvider.AllRegexRedactionConfigurationsNode); - tlvConfigurations.AddObject(Activator.CoreChildProvider.AllDatasetConfigurationsNode); + tlvConfigurations.AddObject(Activator.CoreChildProvider.AllDatasetProviderConfigurationsNode); tlvConfigurations.Refresh(); } diff --git a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandCreateNewPureDatasetUI.cs b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandCreateNewPureDatasetUI.cs deleted file mode 100644 index cc117e2719..0000000000 --- a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandCreateNewPureDatasetUI.cs +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) The University of Dundee 2018-2019 -// This file is part of the Research Data Management Platform (RDMP). -// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. -// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. -// You should have received a copy of the GNU General Public License along with RDMP. If not, see . - -using Rdmp.Core.CommandExecution.AtomicCommands; -using Rdmp.UI.ItemActivation; -using Rdmp.UI.SimpleDialogs.Datasets; -using System; - - -namespace Rdmp.UI.CommandExecution.AtomicCommands; - -public class ExecuteCommandCreateNewPureDatasetUI : ExecuteCommandCreateDataset -{ - private readonly IActivateItems _activator; - - public ExecuteCommandCreateNewPureDatasetUI(IActivateItems activator) : base( - activator, "New Dataset") - { - _activator = activator; - } - - public override void Execute() - { - var ui = new CreateNewPureDatasetUI(_activator, this); - ui.ShowDialog(); - } -} diff --git a/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureDatasetUI.Designer.cs b/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureDatasetUI.Designer.cs deleted file mode 100644 index ed6aebad22..0000000000 --- a/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureDatasetUI.Designer.cs +++ /dev/null @@ -1,236 +0,0 @@ -namespace Rdmp.UI.SimpleDialogs.Datasets -{ - partial class CreateNewPureDatasetUI - { - /// - /// Required designer variable. - /// - private System.ComponentModel.IContainer components = null; - - /// - /// Clean up any resources being used. - /// - /// true if managed resources should be disposed; otherwise, false. - protected override void Dispose(bool disposing) - { - if (disposing && (components != null)) - { - components.Dispose(); - } - base.Dispose(disposing); - } - - #region Windows Form Designer generated code - - /// - /// Required method for Designer support - do not modify - /// the contents of this method with the code editor. - /// - private void InitializeComponent() - { - label1 = new System.Windows.Forms.Label(); - cbDatasetProvider = new System.Windows.Forms.ComboBox(); - tbName = new System.Windows.Forms.TextBox(); - label2 = new System.Windows.Forms.Label(); - label3 = new System.Windows.Forms.Label(); - tbDescription = new System.Windows.Forms.TextBox(); - cbPublisher = new System.Windows.Forms.ComboBox(); - label4 = new System.Windows.Forms.Label(); - label5 = new System.Windows.Forms.Label(); - label6 = new System.Windows.Forms.Label(); - label7 = new System.Windows.Forms.Label(); - cbVisibility = new System.Windows.Forms.ComboBox(); - tbDateMadeAvailable = new System.Windows.Forms.TextBox(); - lbPeople = new System.Windows.Forms.ListBox(); - btnCreate = new System.Windows.Forms.Button(); - btnCancel = new System.Windows.Forms.Button(); - SuspendLayout(); - // - // label1 - // - label1.AutoSize = true; - label1.Location = new System.Drawing.Point(12, 21); - label1.Name = "label1"; - label1.Size = new System.Drawing.Size(96, 15); - label1.TabIndex = 0; - label1.Text = "Dataset Provider:"; - // - // cbDatasetProvider - // - cbDatasetProvider.FormattingEnabled = true; - cbDatasetProvider.Location = new System.Drawing.Point(114, 18); - cbDatasetProvider.Name = "cbDatasetProvider"; - cbDatasetProvider.Size = new System.Drawing.Size(252, 23); - cbDatasetProvider.TabIndex = 1; - cbDatasetProvider.SelectedIndexChanged += cbDatasetProvider_SelectedIndexChanged; - // - // tbName - // - tbName.Location = new System.Drawing.Point(116, 62); - tbName.Name = "tbName"; - tbName.Size = new System.Drawing.Size(250, 23); - tbName.TabIndex = 2; - // - // label2 - // - label2.AutoSize = true; - label2.Location = new System.Drawing.Point(69, 65); - label2.Name = "label2"; - label2.Size = new System.Drawing.Size(42, 15); - label2.TabIndex = 3; - label2.Text = "Name:"; - // - // label3 - // - label3.AutoSize = true; - label3.Location = new System.Drawing.Point(41, 107); - label3.Name = "label3"; - label3.Size = new System.Drawing.Size(70, 15); - label3.TabIndex = 5; - label3.Text = "Description:"; - // - // tbDescription - // - tbDescription.Location = new System.Drawing.Point(116, 104); - tbDescription.Name = "tbDescription"; - tbDescription.Size = new System.Drawing.Size(250, 23); - tbDescription.TabIndex = 4; - // - // cbPublisher - // - cbPublisher.Enabled = false; - cbPublisher.FormattingEnabled = true; - cbPublisher.Location = new System.Drawing.Point(114, 147); - cbPublisher.Name = "cbPublisher"; - cbPublisher.Size = new System.Drawing.Size(252, 23); - cbPublisher.TabIndex = 7; - // - // label4 - // - label4.AutoSize = true; - label4.Location = new System.Drawing.Point(53, 150); - label4.Name = "label4"; - label4.Size = new System.Drawing.Size(59, 15); - label4.TabIndex = 6; - label4.Text = "Publisher:"; - // - // label5 - // - label5.AutoSize = true; - label5.Location = new System.Drawing.Point(49, 197); - label5.Name = "label5"; - label5.Size = new System.Drawing.Size(46, 15); - label5.TabIndex = 8; - label5.Text = "People:"; - // - // label6 - // - label6.AutoSize = true; - label6.Location = new System.Drawing.Point(52, 229); - label6.Name = "label6"; - label6.Size = new System.Drawing.Size(54, 15); - label6.TabIndex = 9; - label6.Text = "Visibility:"; - // - // label7 - // - label7.AutoSize = true; - label7.Location = new System.Drawing.Point(2, 261); - label7.Name = "label7"; - label7.Size = new System.Drawing.Size(116, 15); - label7.TabIndex = 10; - label7.Text = "Date made available:"; - // - // cbVisibility - // - cbVisibility.FormattingEnabled = true; - cbVisibility.Location = new System.Drawing.Point(114, 226); - cbVisibility.Name = "cbVisibility"; - cbVisibility.Size = new System.Drawing.Size(252, 23); - cbVisibility.TabIndex = 11; - // - // tbDateMadeAvailable - // - tbDateMadeAvailable.Location = new System.Drawing.Point(116, 261); - tbDateMadeAvailable.Name = "tbDateMadeAvailable"; - tbDateMadeAvailable.Size = new System.Drawing.Size(250, 23); - tbDateMadeAvailable.TabIndex = 12; - // - // lbPeople - // - lbPeople.Enabled = false; - lbPeople.FormattingEnabled = true; - lbPeople.ItemHeight = 15; - lbPeople.Location = new System.Drawing.Point(116, 190); - lbPeople.Name = "lbPeople"; - lbPeople.Size = new System.Drawing.Size(250, 34); - lbPeople.TabIndex = 13; - // - // btnCreate - // - btnCreate.Location = new System.Drawing.Point(708, 406); - btnCreate.Name = "btnCreate"; - btnCreate.Size = new System.Drawing.Size(75, 23); - btnCreate.TabIndex = 14; - btnCreate.Text = "Create"; - btnCreate.UseVisualStyleBackColor = true; - btnCreate.Click += btnCreate_Click; - // - // btnCancel - // - btnCancel.Location = new System.Drawing.Point(627, 406); - btnCancel.Name = "btnCancel"; - btnCancel.Size = new System.Drawing.Size(75, 23); - btnCancel.TabIndex = 15; - btnCancel.Text = "Cancel"; - btnCancel.UseVisualStyleBackColor = true; - btnCancel.Click += btnCancel_Click; - // - // CreateNewPureDatasetUI - // - AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); - AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; - ClientSize = new System.Drawing.Size(800, 450); - Controls.Add(btnCancel); - Controls.Add(btnCreate); - Controls.Add(lbPeople); - Controls.Add(tbDateMadeAvailable); - Controls.Add(cbVisibility); - Controls.Add(label7); - Controls.Add(label6); - Controls.Add(label5); - Controls.Add(cbPublisher); - Controls.Add(label4); - Controls.Add(label3); - Controls.Add(tbDescription); - Controls.Add(label2); - Controls.Add(tbName); - Controls.Add(cbDatasetProvider); - Controls.Add(label1); - Name = "CreateNewPureDatasetUI"; - Text = "Create New Pure Dataset"; - Load += CreateNewPureDatasetUI_Load; - ResumeLayout(false); - PerformLayout(); - } - - #endregion - - private System.Windows.Forms.Label label1; - private System.Windows.Forms.ComboBox cbDatasetProvider; - private System.Windows.Forms.TextBox tbName; - private System.Windows.Forms.Label label2; - private System.Windows.Forms.Label label3; - private System.Windows.Forms.TextBox tbDescription; - private System.Windows.Forms.ComboBox cbPublisher; - private System.Windows.Forms.Label label4; - private System.Windows.Forms.Label label5; - private System.Windows.Forms.Label label6; - private System.Windows.Forms.Label label7; - private System.Windows.Forms.ComboBox cbVisibility; - private System.Windows.Forms.TextBox tbDateMadeAvailable; - private System.Windows.Forms.ListBox lbPeople; - private System.Windows.Forms.Button btnCreate; - private System.Windows.Forms.Button btnCancel; - } -} \ No newline at end of file diff --git a/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureDatasetUI.cs b/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureDatasetUI.cs deleted file mode 100644 index a1b9af3146..0000000000 --- a/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureDatasetUI.cs +++ /dev/null @@ -1,89 +0,0 @@ -using Amazon.Auth.AccessControlPolicy; -using Rdmp.Core.CommandExecution.AtomicCommands; -using Rdmp.Core.Curation.Data.Aggregation; -using Rdmp.Core.Curation.Data.Datasets; -using Rdmp.Core.Datasets; -using Rdmp.UI.CommandExecution.AtomicCommands; -using Rdmp.UI.ItemActivation; -using Rdmp.UI.TestsAndSetup.ServicePropogation; -using System; -using System.Collections.Generic; -using System.ComponentModel; -using System.Data; -using System.Drawing; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using System.Windows.Forms; -namespace Rdmp.UI.SimpleDialogs.Datasets; - -public partial class CreateNewPureDatasetUI : RDMPForm -{ - private readonly IActivateItems _activator; - - private readonly string[] _visibilities = { "FREE", "CAMPUS", "BACKEND", "CONFIDENTIAL" }; - - private DatasetProviderConfiguration _providerConfiguration; - private PureDatasetProvider _datasetProvider; - private List _publishers; - - public CreateNewPureDatasetUI(IActivateItems activator, ExecuteCommandCreateNewPureDatasetUI command) : base(activator) - { - _activator = activator; - InitializeComponent(); - var configs = _activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Type", typeof(PureDatasetProvider).ToString()); - cbDatasetProvider.Items.AddRange(configs); - cbVisibility.Items.AddRange(_visibilities); - tbDateMadeAvailable.Text = DateTime.Now.ToString(); - } - - private void CreateNewPureDatasetUI_Load(object sender, EventArgs e) - { - - } - - private void btnCancel_Click(object sender, EventArgs e) - { - Close(); - } - - private void btnCreate_Click(object sender, EventArgs e) - { - if (_activator.YesNo("Are you sure?", "Create dataset?")) - { - //var provider = new PureDatasetProvider(_activator, (DatasetProviderConfiguration)cbDatasetProvider.SelectedItem); - //var date = new PureDate(2024);//TODO - //provider.Create(tbName.Text,((PureDataset.System)cbPublisher.SelectedItem).UUID,lbPeople.SelectedItems,cbVisibility.SelectedItem,date); - } - } - - private void Reset() - { - cbPublisher.Items.Clear(); - cbPublisher.Enabled = false; - lbPeople.Items.Clear(); - lbPeople.Enabled = false; - } - - private void cbDatasetProvider_SelectedIndexChanged(object sender, EventArgs e) - { - if (cbDatasetProvider.SelectedItem is DatasetProviderConfiguration config) - { - _providerConfiguration = config; - _datasetProvider = new PureDatasetProvider(_activator,_providerConfiguration); - //fetch people - //add people - lbPeople.Enabled = true; - - _publishers = _datasetProvider.FetchPublishers(); - cbPublisher.Items.Add(_publishers); - cbPublisher.Enabled = true; - - } - else - { - Reset(); - } - - } -} diff --git a/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureDatasetUI.resx b/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureDatasetUI.resx deleted file mode 100644 index af32865ec1..0000000000 --- a/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureDatasetUI.resx +++ /dev/null @@ -1,120 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - text/microsoft-resx - - - 2.0 - - - System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - \ No newline at end of file diff --git a/Rdmp.UI/SubComponents/PureDatasetConfigurationUI.Designer.cs b/Rdmp.UI/SubComponents/PureDatasetConfigurationUI.Designer.cs index 56be63815b..604522394d 100644 --- a/Rdmp.UI/SubComponents/PureDatasetConfigurationUI.Designer.cs +++ b/Rdmp.UI/SubComponents/PureDatasetConfigurationUI.Designer.cs @@ -28,22 +28,19 @@ protected override void Dispose(bool disposing) /// private void InitializeComponent() { - label1 = new System.Windows.Forms.Label(); columnButtonRenderer1 = new BrightIdeasSoftware.ColumnButtonRenderer(); btnViewOnPure = new System.Windows.Forms.Button(); - lblUrl = new System.Windows.Forms.Label(); + label1 = new System.Windows.Forms.Label(); + tbName = new System.Windows.Forms.TextBox(); + tbDescription = new System.Windows.Forms.TextBox(); + label2 = new System.Windows.Forms.Label(); + label3 = new System.Windows.Forms.Label(); + tbTemporalStart = new System.Windows.Forms.TextBox(); + tbTemporalEnd = new System.Windows.Forms.TextBox(); + label4 = new System.Windows.Forms.Label(); + btnSave = new System.Windows.Forms.Button(); SuspendLayout(); // - // label1 - // - label1.AutoSize = true; - label1.Location = new System.Drawing.Point(42, 80); - label1.Name = "label1"; - label1.Size = new System.Drawing.Size(542, 15); - label1.TabIndex = 0; - label1.Text = "Pure Datasets are not currently managable via RDMP. Please visit the following link to edit this dataset"; - label1.Click += label1_Click; - // // columnButtonRenderer1 // columnButtonRenderer1.ButtonPadding = new System.Drawing.Size(10, 10); @@ -51,7 +48,7 @@ private void InitializeComponent() // btnViewOnPure // btnViewOnPure.Enabled = false; - btnViewOnPure.Location = new System.Drawing.Point(42, 40); + btnViewOnPure.Location = new System.Drawing.Point(29, 26); btnViewOnPure.Name = "btnViewOnPure"; btnViewOnPure.Size = new System.Drawing.Size(121, 23); btnViewOnPure.TabIndex = 1; @@ -59,32 +56,114 @@ private void InitializeComponent() btnViewOnPure.UseVisualStyleBackColor = true; btnViewOnPure.Click += btnViewOnPure_Click; // - // lblUrl + // label1 + // + label1.AutoSize = true; + label1.Location = new System.Drawing.Point(122, 72); + label1.Name = "label1"; + label1.Size = new System.Drawing.Size(42, 15); + label1.TabIndex = 2; + label1.Text = "Name:"; + // + // tbName + // + tbName.Location = new System.Drawing.Point(170, 69); + tbName.Name = "tbName"; + tbName.Size = new System.Drawing.Size(712, 23); + tbName.TabIndex = 3; + // + // tbDescription + // + tbDescription.Location = new System.Drawing.Point(170, 111); + tbDescription.Multiline = true; + tbDescription.Name = "tbDescription"; + tbDescription.Size = new System.Drawing.Size(712, 295); + tbDescription.TabIndex = 5; + // + // label2 // - lblUrl.AutoSize = true; - lblUrl.Location = new System.Drawing.Point(47, 104); - lblUrl.Name = "lblUrl"; - lblUrl.Size = new System.Drawing.Size(0, 15); - lblUrl.TabIndex = 2; + label2.AutoSize = true; + label2.Location = new System.Drawing.Point(97, 114); + label2.Name = "label2"; + label2.Size = new System.Drawing.Size(70, 15); + label2.TabIndex = 4; + label2.Text = "Description:"; + // + // label3 + // + label3.AutoSize = true; + label3.Location = new System.Drawing.Point(68, 428); + label3.Name = "label3"; + label3.Size = new System.Drawing.Size(96, 15); + label3.TabIndex = 6; + label3.Text = "Temporal Period:"; + // + // tbTemporalStart + // + tbTemporalStart.Location = new System.Drawing.Point(170, 425); + tbTemporalStart.Name = "tbTemporalStart"; + tbTemporalStart.Size = new System.Drawing.Size(181, 23); + tbTemporalStart.TabIndex = 7; + // + // tbTemporalEnd + // + tbTemporalEnd.Location = new System.Drawing.Point(375, 425); + tbTemporalEnd.Name = "tbTemporalEnd"; + tbTemporalEnd.Size = new System.Drawing.Size(181, 23); + tbTemporalEnd.TabIndex = 8; + // + // label4 + // + label4.AutoSize = true; + label4.Location = new System.Drawing.Point(357, 428); + label4.Name = "label4"; + label4.Size = new System.Drawing.Size(12, 15); + label4.TabIndex = 9; + label4.Text = "-"; + label4.Click += label4_Click; + // + // btnSave + // + btnSave.Location = new System.Drawing.Point(807, 466); + btnSave.Name = "btnSave"; + btnSave.Size = new System.Drawing.Size(75, 23); + btnSave.TabIndex = 14; + btnSave.Text = "Save"; + btnSave.UseVisualStyleBackColor = true; + btnSave.Click += btnSave_Click; // // PureDatasetConfigurationUI // AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; - Controls.Add(lblUrl); - Controls.Add(btnViewOnPure); + Controls.Add(btnSave); + Controls.Add(label4); + Controls.Add(tbTemporalEnd); + Controls.Add(tbTemporalStart); + Controls.Add(label3); + Controls.Add(tbDescription); + Controls.Add(label2); + Controls.Add(tbName); Controls.Add(label1); + Controls.Add(btnViewOnPure); Name = "PureDatasetConfigurationUI"; - Size = new System.Drawing.Size(955, 513); + Size = new System.Drawing.Size(955, 648); + Load += PureDatasetConfigurationUI_Load; ResumeLayout(false); PerformLayout(); } #endregion - - private System.Windows.Forms.Label label1; private BrightIdeasSoftware.ColumnButtonRenderer columnButtonRenderer1; private System.Windows.Forms.Button btnViewOnPure; - private System.Windows.Forms.Label lblUrl; + private System.Windows.Forms.Label label1; + private System.Windows.Forms.TextBox tbName; + private System.Windows.Forms.TextBox tbDescription; + private System.Windows.Forms.Label label2; + private System.Windows.Forms.Label label3; + private System.Windows.Forms.TextBox tbTemporalStart; + private System.Windows.Forms.TextBox tbTemporalEnd; + private System.Windows.Forms.Label label4; + private System.Windows.Forms.Button btnSave; } } diff --git a/Rdmp.UI/SubComponents/PureDatasetConfigurationUI.cs b/Rdmp.UI/SubComponents/PureDatasetConfigurationUI.cs index fe52d827b9..60d7bc9715 100644 --- a/Rdmp.UI/SubComponents/PureDatasetConfigurationUI.cs +++ b/Rdmp.UI/SubComponents/PureDatasetConfigurationUI.cs @@ -21,6 +21,9 @@ public partial class PureDatasetConfigurationUI : DatsetConfigurationUI_Design, { private PureDataset _dataset; + private IActivateItems _activator; + private PureDatasetProvider _provider; + private Dataset _dbObject; public PureDatasetConfigurationUI() { InitializeComponent(); @@ -28,15 +31,22 @@ public PureDatasetConfigurationUI() public override void SetDatabaseObject(IActivateItems activator, Dataset databaseObject) { + _activator = activator; + _dbObject = databaseObject; base.SetDatabaseObject(activator, databaseObject); - var provider = new PureDatasetProvider(activator, activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("ID", databaseObject.Provider_ID).First()); - _dataset = provider.FetchPureDataset(databaseObject); - lblUrl.Text = _dataset.PortalURL; + _provider = new PureDatasetProvider(activator, activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("ID", databaseObject.Provider_ID).First()); + _dataset = _provider.FetchPureDataset(databaseObject); if (_dataset.PortalURL is not null) { btnViewOnPure.Enabled = true; } - + tbName.Text = _dataset.Title.en_GB; + if (_dataset.Descriptions.Any(d => d.Value is not null)) + tbDescription.Text = _dataset.Descriptions.Where(d => d.Value is not null).First().Value.en_GB; + if (_dataset.TemporalCoveragePeriod is not null && _dataset.TemporalCoveragePeriod.StartDate is not null) + tbTemporalStart.Text = _dataset.TemporalCoveragePeriod.StartDate.ToDateTime().ToString(); + if (_dataset.TemporalCoveragePeriod is not null && _dataset.TemporalCoveragePeriod.EndDate is not null) + tbTemporalEnd.Text = _dataset.TemporalCoveragePeriod.EndDate.ToDateTime().ToString(); } public void RefreshBus_RefreshObject(object sender, RefreshObjectEventArgs e) @@ -53,6 +63,51 @@ private void btnViewOnPure_Click(object sender, EventArgs e) { UsefulStuff.OpenUrl(_dataset.PortalURL); } + + private void PureDatasetConfigurationUI_Load(object sender, EventArgs e) + { + + } + + private void btnSave_Click(object sender, EventArgs e) + { + if (_activator.YesNo("Are you sure?", "Save Changes")) + { + var datasetUpdate = new PureDataset(); + _dataset.Title.en_GB = tbName.Text; + datasetUpdate.Title = _dataset.Title; + var datasetDescriptionTerm = "/dk/atira/pure/dataset/descriptions/datasetdescription"; + var description = _dataset.Descriptions.Where(d => d.Term.URI == datasetDescriptionTerm).FirstOrDefault(); + if (description is null) + { + //error + Console.WriteLine("No known description!"); + } + else + { + description.Value = new ENGBWrapper(tbDescription.Text); + datasetUpdate.Descriptions = new List { description }; + } + if (!string.IsNullOrWhiteSpace(tbTemporalStart.Text)) + { + _dataset.TemporalCoveragePeriod.StartDate = new PureDate(DateTime.Parse(tbTemporalStart.Text)); + } + if (!string.IsNullOrWhiteSpace(tbTemporalEnd.Text)) + { + _dataset.TemporalCoveragePeriod.EndDate = new PureDate(DateTime.Parse(tbTemporalEnd.Text)); + } + if (!string.IsNullOrWhiteSpace(tbTemporalStart.Text) || !string.IsNullOrWhiteSpace(tbTemporalEnd.Text)) + datasetUpdate.TemporalCoveragePeriod = _dataset.TemporalCoveragePeriod; + _provider.Update(_dataset.UUID, datasetUpdate); + SetDatabaseObject(_activator, _dbObject); + _activator.Show("Successfully updated Pure dataset"); + } + } + + private void label4_Click(object sender, EventArgs e) + { + + } } [TypeDescriptionProvider( From 8352c0d471010809c0285586a9c25a91df7518a4 Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 18 Oct 2024 08:15:37 +0100 Subject: [PATCH 079/106] add updates --- .../PostLoad/PureDatasetPostLoadUpdater.cs | 5 ++- Rdmp.Core/Datasets/PurePublisher.cs | 45 +++++++++++++++++++ .../ImportExistingPureDatasetUI.Designer.cs | 22 ++++----- .../Datasets/ImportExistingPureDatasetUI.cs | 7 +-- 4 files changed, 63 insertions(+), 16 deletions(-) create mode 100644 Rdmp.Core/Datasets/PurePublisher.cs diff --git a/Rdmp.Core/DataLoad/Modules/Mutilators/PostLoad/PureDatasetPostLoadUpdater.cs b/Rdmp.Core/DataLoad/Modules/Mutilators/PostLoad/PureDatasetPostLoadUpdater.cs index 67255b24ee..ee1a09ffa0 100644 --- a/Rdmp.Core/DataLoad/Modules/Mutilators/PostLoad/PureDatasetPostLoadUpdater.cs +++ b/Rdmp.Core/DataLoad/Modules/Mutilators/PostLoad/PureDatasetPostLoadUpdater.cs @@ -62,7 +62,7 @@ public PureDatasetPostLoadUpdater() { } public void Check(ICheckNotifier notifier) { if (Dataset is null) notifier.OnCheckPerformed(new CheckEventArgs("No Dataset was selected", CheckResult.Fail)); - if (Dataset.Type != _type) notifier.OnCheckPerformed(new CheckEventArgs("Dataset is not a Pure Dataset.", CheckResult.Fail)); + if (Dataset is not null && Dataset.Type != _type) notifier.OnCheckPerformed(new CheckEventArgs("Dataset is not a Pure Dataset.", CheckResult.Fail)); } public void Initialize(DiscoveredDatabase dbInfo, LoadStage loadStage) @@ -147,7 +147,8 @@ private string GetUpdateText() private List GetDescriptions() { var descriptions = new List(); - foreach (var description in _pureDataset.Descriptions) + var datasetDescriptionTerm = "/dk/atira/pure/dataset/descriptions/datasetdescription"; + foreach (var description in _pureDataset.Descriptions.Where(d => d.Term.URI == datasetDescriptionTerm)) { description.Value.en_GB += @$" {GetUpdateText()}"; diff --git a/Rdmp.Core/Datasets/PurePublisher.cs b/Rdmp.Core/Datasets/PurePublisher.cs new file mode 100644 index 0000000000..db983c1190 --- /dev/null +++ b/Rdmp.Core/Datasets/PurePublisher.cs @@ -0,0 +1,45 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Datasets; +// Root myDeserializedClass = JsonConvert.DeserializeObject(myJsonResponse); +public class Description +{ + public string en_GB { get; set; } +} + +public class PurePublisher +{ + public int pureId { get; set; } + public string uuid { get; set; } + public string createdBy { get; set; } + public DateTime createdDate { get; set; } + public string modifiedBy { get; set; } + public DateTime modifiedDate { get; set; } + public string version { get; set; } + public string name { get; set; } + public Type type { get; set; } + public PublisherWorkflow workflow { get; set; } + public string systemName { get; set; } +} + +public class Term +{ + public string en_GB { get; set; } +} + +public class Type +{ + public string uri { get; set; } + public Term term { get; set; } +} + +public class PublisherWorkflow +{ + public string step { get; set; } + public Description description { get; set; } +} + diff --git a/Rdmp.UI/SimpleDialogs/Datasets/ImportExistingPureDatasetUI.Designer.cs b/Rdmp.UI/SimpleDialogs/Datasets/ImportExistingPureDatasetUI.Designer.cs index ab89db7c99..48e50fbafb 100644 --- a/Rdmp.UI/SimpleDialogs/Datasets/ImportExistingPureDatasetUI.Designer.cs +++ b/Rdmp.UI/SimpleDialogs/Datasets/ImportExistingPureDatasetUI.Designer.cs @@ -30,11 +30,11 @@ private void InitializeComponent() { btnCreate = new System.Windows.Forms.Button(); btnCancel = new System.Windows.Forms.Button(); - label1 = new System.Windows.Forms.Label(); tbUrl = new System.Windows.Forms.TextBox(); lblError = new System.Windows.Forms.Label(); label2 = new System.Windows.Forms.Label(); cbProviders = new System.Windows.Forms.ComboBox(); + label1 = new System.Windows.Forms.Label(); SuspendLayout(); // // btnCreate @@ -57,15 +57,6 @@ private void InitializeComponent() btnCancel.UseVisualStyleBackColor = true; btnCancel.Click += btnCancel_Click; // - // label1 - // - label1.AutoSize = true; - label1.Location = new System.Drawing.Point(37, 59); - label1.Name = "label1"; - label1.Size = new System.Drawing.Size(31, 15); - label1.TabIndex = 16; - label1.Text = "URL:"; - // // tbUrl // tbUrl.Location = new System.Drawing.Point(74, 56); @@ -100,6 +91,15 @@ private void InitializeComponent() cbProviders.TabIndex = 20; cbProviders.SelectedIndexChanged += cbProviders_SelectedIndexChanged; // + // label1 + // + label1.AutoSize = true; + label1.Location = new System.Drawing.Point(37, 59); + label1.Name = "label1"; + label1.Size = new System.Drawing.Size(37, 15); + label1.TabIndex = 16; + label1.Text = "UUID:"; + // // ImportExistingPureDatasetUI // AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); @@ -122,10 +122,10 @@ private void InitializeComponent() #endregion private System.Windows.Forms.Button btnCreate; private System.Windows.Forms.Button btnCancel; - private System.Windows.Forms.Label label1; private System.Windows.Forms.TextBox tbUrl; private System.Windows.Forms.Label lblError; private System.Windows.Forms.Label label2; private System.Windows.Forms.ComboBox cbProviders; + private System.Windows.Forms.Label label1; } } \ No newline at end of file diff --git a/Rdmp.UI/SimpleDialogs/Datasets/ImportExistingPureDatasetUI.cs b/Rdmp.UI/SimpleDialogs/Datasets/ImportExistingPureDatasetUI.cs index 7ae1c6321b..77065d5dcf 100644 --- a/Rdmp.UI/SimpleDialogs/Datasets/ImportExistingPureDatasetUI.cs +++ b/Rdmp.UI/SimpleDialogs/Datasets/ImportExistingPureDatasetUI.cs @@ -49,15 +49,16 @@ private void btnCreate_Click(object sender, EventArgs e) { if (_activator.YesNo("Please confirm you wish to import this Dataset", "Import Dataset")) { - if (!_datasetProvider.CheckDatasetExistsAtURL(tbUrl.Text)) + var url = _providerConfiguration.Url.Split("/ws/api").First() + "/en/datasets/"+tbUrl.Text; + if (!_datasetProvider.CheckDatasetExistsAtURL(url)) { - lblError.Text = "Unable to locate dataset. Ensure UUID url is used"; + lblError.Text = "Unable to locate dataset. Ensure UUID is correct."; lblError.Visible = true; return; } else { - _datasetProvider.AddExistingDataset(null, tbUrl.Text); + _datasetProvider.AddExistingDataset(null, url); Close(); Dispose(); } From f962319201ea1dc1bf03e6a5efc6ea2313221d0f Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 18 Oct 2024 08:33:22 +0100 Subject: [PATCH 080/106] add update sql --- .../CreateCatalogue.sql | 19 +++++++++++++++++++ .../089_AddDatasetProviderConfiguration.sql | 17 +++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 Rdmp.Core/Databases/CatalogueDatabase/up/089_AddDatasetProviderConfiguration.sql diff --git a/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql b/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql index 3d3c8ce877..2cc307be7b 100644 --- a/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql +++ b/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql @@ -1400,6 +1400,25 @@ CONSTRAINT [PK_RegexRedactionKey] PRIMARY KEY CLUSTERED END GO +if not exists (select 1 from sys.tables where name = 'DatasetProviderConfiguration') +BEGIN +CREATE TABLE [dbo].[DatasetProviderConfiguration]( + [ID] [int] IDENTITY(1,1) NOT NULL, + Type [varchar(256)] NOT NULL, + Url [varchar(500)] NOT NULL, + Name [varchar(256)] NOT NULL, + DataAccessCredentials_ID [int] NOT NULL, + Organisation_ID [varchar(256)] NOT NULL + CONSTRAINT FK_DatasetProviderConfiguration_DataAccessCredentials FOREIGN KEY (DataAccessCredentials_ID) REFERENCES DataAccessCredentials(ID) ON DELETE CASCADE, +CONSTRAINT [PK_RegexRedactionKey] PRIMARY KEY CLUSTERED +( + [ID] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +END +GO + + EXEC sys.sp_addextendedproperty @name=N'MS_Description', @value=N'Table ID' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'Catalogue', @level2type=N'COLUMN',@level2name=N'ID' GO diff --git a/Rdmp.Core/Databases/CatalogueDatabase/up/089_AddDatasetProviderConfiguration.sql b/Rdmp.Core/Databases/CatalogueDatabase/up/089_AddDatasetProviderConfiguration.sql new file mode 100644 index 0000000000..74a3779fb2 --- /dev/null +++ b/Rdmp.Core/Databases/CatalogueDatabase/up/089_AddDatasetProviderConfiguration.sql @@ -0,0 +1,17 @@ +if not exists (select 1 from sys.tables where name = 'DatasetProviderConfiguration') +BEGIN +CREATE TABLE [dbo].[DatasetProviderConfiguration]( + [ID] [int] IDENTITY(1,1) NOT NULL, + Type [varchar(256)] NOT NULL, + Url [varchar(500)] NOT NULL, + Name [varchar(256)] NOT NULL, + DataAccessCredentials_ID [int] NOT NULL, + Organisation_ID [varchar(256)] NOT NULL + CONSTRAINT FK_DatasetProviderConfiguration_DataAccessCredentials FOREIGN KEY (DataAccessCredentials_ID) REFERENCES DataAccessCredentials(ID) ON DELETE CASCADE, +CONSTRAINT [PK_RegexRedactionKey] PRIMARY KEY CLUSTERED +( + [ID] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +END +GO From 5728604be6fd71917ee8d25d7066a38a048b506f Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 18 Oct 2024 08:34:09 +0100 Subject: [PATCH 081/106] include scripts --- Rdmp.Core/Rdmp.Core.csproj | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/Rdmp.Core/Rdmp.Core.csproj b/Rdmp.Core/Rdmp.Core.csproj index 3fbf307c66..ec882bb0fb 100644 --- a/Rdmp.Core/Rdmp.Core.csproj +++ b/Rdmp.Core/Rdmp.Core.csproj @@ -131,6 +131,7 @@ + @@ -258,6 +259,7 @@ + @@ -352,6 +354,18 @@ + + True + True + 089_AddDatasetProviderConfiguration.sql + + + + + True + True + 089_AddDatasetProviderConfiguration.sql + True True @@ -399,4 +413,13 @@ PublicResXFileCodeGenerator + + + TextTemplatingFileGenerator + 089_AddDatasetProviderConfiguration.cs + + + + + From c2e7857ccc80c6b1004c92b70c589606dc41a38d Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 18 Oct 2024 08:39:29 +0100 Subject: [PATCH 082/106] update sql --- .../runAfterCreateDatabase/CreateCatalogue.sql | 8 ++++---- .../up/089_AddDatasetProviderConfiguration.sql | 8 ++++---- Rdmp.Core/Rdmp.Core.csproj | 5 ----- 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql b/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql index 2cc307be7b..7f10d280be 100644 --- a/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql +++ b/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql @@ -1404,11 +1404,11 @@ if not exists (select 1 from sys.tables where name = 'DatasetProviderConfigurati BEGIN CREATE TABLE [dbo].[DatasetProviderConfiguration]( [ID] [int] IDENTITY(1,1) NOT NULL, - Type [varchar(256)] NOT NULL, - Url [varchar(500)] NOT NULL, - Name [varchar(256)] NOT NULL, + Type [nvarchar](256) NOT NULL, + Url [nvarchar](500) NOT NULL, + Name [nvarchar](256) NOT NULL, DataAccessCredentials_ID [int] NOT NULL, - Organisation_ID [varchar(256)] NOT NULL + Organisation_ID [nvarchar](256) NOT NULL CONSTRAINT FK_DatasetProviderConfiguration_DataAccessCredentials FOREIGN KEY (DataAccessCredentials_ID) REFERENCES DataAccessCredentials(ID) ON DELETE CASCADE, CONSTRAINT [PK_RegexRedactionKey] PRIMARY KEY CLUSTERED ( diff --git a/Rdmp.Core/Databases/CatalogueDatabase/up/089_AddDatasetProviderConfiguration.sql b/Rdmp.Core/Databases/CatalogueDatabase/up/089_AddDatasetProviderConfiguration.sql index 74a3779fb2..57cfc0a172 100644 --- a/Rdmp.Core/Databases/CatalogueDatabase/up/089_AddDatasetProviderConfiguration.sql +++ b/Rdmp.Core/Databases/CatalogueDatabase/up/089_AddDatasetProviderConfiguration.sql @@ -2,11 +2,11 @@ BEGIN CREATE TABLE [dbo].[DatasetProviderConfiguration]( [ID] [int] IDENTITY(1,1) NOT NULL, - Type [varchar(256)] NOT NULL, - Url [varchar(500)] NOT NULL, - Name [varchar(256)] NOT NULL, + Type [nvarchar](256) NOT NULL, + Url [nvarchar](500) NOT NULL, + Name [nvarchar](256) NOT NULL, DataAccessCredentials_ID [int] NOT NULL, - Organisation_ID [varchar(256)] NOT NULL + Organisation_ID [nvarchar](256) NOT NULL CONSTRAINT FK_DatasetProviderConfiguration_DataAccessCredentials FOREIGN KEY (DataAccessCredentials_ID) REFERENCES DataAccessCredentials(ID) ON DELETE CASCADE, CONSTRAINT [PK_RegexRedactionKey] PRIMARY KEY CLUSTERED ( diff --git a/Rdmp.Core/Rdmp.Core.csproj b/Rdmp.Core/Rdmp.Core.csproj index ec882bb0fb..f2357f2dd1 100644 --- a/Rdmp.Core/Rdmp.Core.csproj +++ b/Rdmp.Core/Rdmp.Core.csproj @@ -361,11 +361,6 @@ - - True - True - 089_AddDatasetProviderConfiguration.sql - True True From 07b20c5281bd53a8773b98f906e71398132a0aef Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 18 Oct 2024 08:43:50 +0100 Subject: [PATCH 083/106] update sql --- .../runAfterCreateDatabase/CreateCatalogue.sql | 2 +- .../up/089_AddDatasetProviderConfiguration.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql b/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql index 7f10d280be..ede034806b 100644 --- a/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql +++ b/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql @@ -1410,7 +1410,7 @@ CREATE TABLE [dbo].[DatasetProviderConfiguration]( DataAccessCredentials_ID [int] NOT NULL, Organisation_ID [nvarchar](256) NOT NULL CONSTRAINT FK_DatasetProviderConfiguration_DataAccessCredentials FOREIGN KEY (DataAccessCredentials_ID) REFERENCES DataAccessCredentials(ID) ON DELETE CASCADE, -CONSTRAINT [PK_RegexRedactionKey] PRIMARY KEY CLUSTERED +CONSTRAINT [PK_DatasetProviderConfiguration] PRIMARY KEY CLUSTERED ( [ID] ASC )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] diff --git a/Rdmp.Core/Databases/CatalogueDatabase/up/089_AddDatasetProviderConfiguration.sql b/Rdmp.Core/Databases/CatalogueDatabase/up/089_AddDatasetProviderConfiguration.sql index 57cfc0a172..a2eaac297f 100644 --- a/Rdmp.Core/Databases/CatalogueDatabase/up/089_AddDatasetProviderConfiguration.sql +++ b/Rdmp.Core/Databases/CatalogueDatabase/up/089_AddDatasetProviderConfiguration.sql @@ -8,7 +8,7 @@ CREATE TABLE [dbo].[DatasetProviderConfiguration]( DataAccessCredentials_ID [int] NOT NULL, Organisation_ID [nvarchar](256) NOT NULL CONSTRAINT FK_DatasetProviderConfiguration_DataAccessCredentials FOREIGN KEY (DataAccessCredentials_ID) REFERENCES DataAccessCredentials(ID) ON DELETE CASCADE, -CONSTRAINT [PK_RegexRedactionKey] PRIMARY KEY CLUSTERED +CONSTRAINT [PK_DatasetProviderConfiguration] PRIMARY KEY CLUSTERED ( [ID] ASC )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] From dc4ea40612116f722488c5c56976335f6f497707 Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 18 Oct 2024 08:56:58 +0100 Subject: [PATCH 084/106] update sql --- .../CreateCatalogue.sql | 34 +++++++++---------- .../089_AddDatasetProviderConfiguration.sql | 5 ++- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql b/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql index ede034806b..041f55be50 100644 --- a/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql +++ b/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql @@ -1400,23 +1400,23 @@ CONSTRAINT [PK_RegexRedactionKey] PRIMARY KEY CLUSTERED END GO -if not exists (select 1 from sys.tables where name = 'DatasetProviderConfiguration') -BEGIN -CREATE TABLE [dbo].[DatasetProviderConfiguration]( - [ID] [int] IDENTITY(1,1) NOT NULL, - Type [nvarchar](256) NOT NULL, - Url [nvarchar](500) NOT NULL, - Name [nvarchar](256) NOT NULL, - DataAccessCredentials_ID [int] NOT NULL, - Organisation_ID [nvarchar](256) NOT NULL - CONSTRAINT FK_DatasetProviderConfiguration_DataAccessCredentials FOREIGN KEY (DataAccessCredentials_ID) REFERENCES DataAccessCredentials(ID) ON DELETE CASCADE, -CONSTRAINT [PK_DatasetProviderConfiguration] PRIMARY KEY CLUSTERED -( - [ID] ASC -)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] -) ON [PRIMARY] -END -GO +--if not exists (select 1 from sys.tables where name = 'DatasetProviderConfiguration') +--BEGIN +--CREATE TABLE [dbo].[DatasetProviderConfiguration]( +-- [ID] [int] IDENTITY(1,1) NOT NULL, +-- Type [nvarchar](256) NOT NULL, +-- Url [nvarchar](500) NOT NULL, +-- Name [nvarchar](256) NOT NULL, +-- DataAccessCredentials_ID [int] NOT NULL, +-- Organisation_ID [nvarchar](256) NOT NULL +-- CONSTRAINT FK_DatasetProviderConfiguration_DataAccessCredentials FOREIGN KEY (DataAccessCredentials_ID) REFERENCES DataAccessCredentials(ID) ON DELETE CASCADE, +--CONSTRAINT [PK_DatasetProviderConfiguration] PRIMARY KEY CLUSTERED +--( +-- [ID] ASC +--)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +--) ON [PRIMARY] +--END +--GO diff --git a/Rdmp.Core/Databases/CatalogueDatabase/up/089_AddDatasetProviderConfiguration.sql b/Rdmp.Core/Databases/CatalogueDatabase/up/089_AddDatasetProviderConfiguration.sql index a2eaac297f..23a226b60c 100644 --- a/Rdmp.Core/Databases/CatalogueDatabase/up/089_AddDatasetProviderConfiguration.sql +++ b/Rdmp.Core/Databases/CatalogueDatabase/up/089_AddDatasetProviderConfiguration.sql @@ -1,4 +1,7 @@ -if not exists (select 1 from sys.tables where name = 'DatasetProviderConfiguration') +--Version: 8.4.0.0 +--Description: Adds the ability to store configuration to talk to remote dataset stores + +if not exists (select 1 from sys.tables where name = 'DatasetProviderConfiguration') BEGIN CREATE TABLE [dbo].[DatasetProviderConfiguration]( [ID] [int] IDENTITY(1,1) NOT NULL, From 43d200812920d8d8494a12912ea72bd920e86077 Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 18 Oct 2024 09:54:08 +0100 Subject: [PATCH 085/106] add docs --- .../Datasets/DatasetProviderConfiguration.cs | 1 + .../Datasets/IDatasetProviderConfiguration.cs | 30 +- .../PostLoad/PureDatasetPostLoadUpdater.cs | 15 +- .../Datasets/DatasetConfigurationUICommon.cs | 2 +- Rdmp.Core/Datasets/IDatasetProvider.cs | 22 +- Rdmp.Core/Datasets/InternalDatasetProvider.cs | 8 +- Rdmp.Core/Datasets/PluginDataset.cs | 12 +- Rdmp.Core/Datasets/PureDataset.cs | 141 +-------- Rdmp.Core/Datasets/PureDatasetProvider.cs | 270 +++++++++--------- Rdmp.Core/Datasets/PurePublisher.cs | 65 ++--- .../SubComponents/DatasetConfigurationUI.cs | 2 +- .../PureDatasetConfigurationUI.cs | 7 +- 12 files changed, 239 insertions(+), 336 deletions(-) diff --git a/Rdmp.Core/Curation/Data/Datasets/DatasetProviderConfiguration.cs b/Rdmp.Core/Curation/Data/Datasets/DatasetProviderConfiguration.cs index 55734e6f4c..eebcf90ba3 100644 --- a/Rdmp.Core/Curation/Data/Datasets/DatasetProviderConfiguration.cs +++ b/Rdmp.Core/Curation/Data/Datasets/DatasetProviderConfiguration.cs @@ -10,6 +10,7 @@ namespace Rdmp.Core.Curation.Data.Datasets; +/// public class DatasetProviderConfiguration : DatabaseEntity, IDatasetProviderConfiguration { diff --git a/Rdmp.Core/Curation/Data/Datasets/IDatasetProviderConfiguration.cs b/Rdmp.Core/Curation/Data/Datasets/IDatasetProviderConfiguration.cs index 117ec3eb81..2c09bd2463 100644 --- a/Rdmp.Core/Curation/Data/Datasets/IDatasetProviderConfiguration.cs +++ b/Rdmp.Core/Curation/Data/Datasets/IDatasetProviderConfiguration.cs @@ -1,20 +1,36 @@ -using Org.BouncyCastle.Bcpg.OpenPgp; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; +using Rdmp.Core.MapsDirectlyToDatabaseTable; namespace Rdmp.Core.Curation.Data.Datasets; -public interface IDatasetProviderConfiguration + +/// +/// Remords configuration for accessing external dataset providers, such as PURE +/// +public interface IDatasetProviderConfiguration: IMapsDirectlyToDatabaseTable { + /// + /// The assembly name of the provider this configuration is used for + /// public string Type { get; } + + /// + /// A friendly name for the configuration + /// public string Name { get; } + + /// + /// The API url + /// public string Url { get; } + /// + /// Reference to which credentials should be used to access the API + /// public int DataAccessCredentials_ID { get; } + /// + /// The organisation ID to use with the remote provider + /// public string Organisation_ID { get; } diff --git a/Rdmp.Core/DataLoad/Modules/Mutilators/PostLoad/PureDatasetPostLoadUpdater.cs b/Rdmp.Core/DataLoad/Modules/Mutilators/PostLoad/PureDatasetPostLoadUpdater.cs index ee1a09ffa0..c76eb21f85 100644 --- a/Rdmp.Core/DataLoad/Modules/Mutilators/PostLoad/PureDatasetPostLoadUpdater.cs +++ b/Rdmp.Core/DataLoad/Modules/Mutilators/PostLoad/PureDatasetPostLoadUpdater.cs @@ -10,6 +10,7 @@ using Rdmp.Core.DataLoad.Engine.Mutilators; using Rdmp.Core.DataLoad.Triggers; using Rdmp.Core.Datasets; +using Rdmp.Core.Datasets.PureItems; using Rdmp.Core.MapsDirectlyToDatabaseTable; using Rdmp.Core.QueryBuilding; using Rdmp.Core.ReusableLibraryCode.Checks; @@ -25,6 +26,10 @@ namespace Rdmp.Core.DataLoad.Modules.Mutilators.PostLoad { + /// + /// Used to update a Pure dataset with updated temporal data to include the new records + /// and update the description of the dataset + /// public class PureDatasetPostLoadUpdater : IMutilateDataTables { private readonly string _type = typeof(PureDatasetProvider).ToString(); @@ -97,7 +102,7 @@ public ExitCodeType Mutilate(IDataLoadJob job) var dt = new DataTable(); dt.BeginLoadData(); - var server = catalogue.GetDistinctLiveDatabaseServer(ReusableLibraryCode.DataAccess.DataAccessContext.DataLoad,false); + var server = catalogue.GetDistinctLiveDatabaseServer(ReusableLibraryCode.DataAccess.DataAccessContext.DataLoad, false); var con = server.GetConnection(); con.Open(); using (var cmd = server.GetCommand(sql, con)) @@ -106,7 +111,7 @@ public ExitCodeType Mutilate(IDataLoadJob job) da.Fill(dt); } dt.EndLoadData(); - if (dt.Rows.Count ==0) return ExitCodeType.OperationNotRequired; + if (dt.Rows.Count == 0) return ExitCodeType.OperationNotRequired; _newRecordsCount = dt.Rows.Count; _earliestDate = new PureDate(dt.AsEnumerable().Min(row => DateTime.Parse(row[0].ToString()))); _latestDate = new PureDate(dt.AsEnumerable().Max(row => DateTime.Parse(row[0].ToString()))); @@ -130,10 +135,10 @@ public ExitCodeType Mutilate(IDataLoadJob job) private TemporalCoveragePeriod GetTemporalCoveragePeriod() { var coverage = _pureDataset.TemporalCoveragePeriod ?? new TemporalCoveragePeriod(); - if(_pureDataset.TemporalCoveragePeriod is null || _pureDataset.TemporalCoveragePeriod.StartDate is null || _earliestDate.IsBefore(_pureDataset.TemporalCoveragePeriod.StartDate)) + if (_pureDataset.TemporalCoveragePeriod is null || _pureDataset.TemporalCoveragePeriod.StartDate is null || _earliestDate.IsBefore(_pureDataset.TemporalCoveragePeriod.StartDate)) coverage.StartDate = _earliestDate; - if(_pureDataset.TemporalCoveragePeriod is null || _pureDataset.TemporalCoveragePeriod.EndDate is null || _pureDataset.TemporalCoveragePeriod.EndDate.IsBefore(_latestDate)) + if (_pureDataset.TemporalCoveragePeriod is null || _pureDataset.TemporalCoveragePeriod.EndDate is null || _pureDataset.TemporalCoveragePeriod.EndDate.IsBefore(_latestDate)) coverage.EndDate = _latestDate; return coverage; } @@ -150,7 +155,7 @@ private List GetDescriptions() var datasetDescriptionTerm = "/dk/atira/pure/dataset/descriptions/datasetdescription"; foreach (var description in _pureDataset.Descriptions.Where(d => d.Term.URI == datasetDescriptionTerm)) { - description.Value.en_GB += @$" + description.Value.En_GB += @$" {GetUpdateText()}"; descriptions.Add(description); } diff --git a/Rdmp.Core/Datasets/DatasetConfigurationUICommon.cs b/Rdmp.Core/Datasets/DatasetConfigurationUICommon.cs index 07ee42c59f..9e7e3472e3 100644 --- a/Rdmp.Core/Datasets/DatasetConfigurationUICommon.cs +++ b/Rdmp.Core/Datasets/DatasetConfigurationUICommon.cs @@ -6,7 +6,7 @@ using Rdmp.Core.CommandExecution; -namespace Rdmp.Core.Dataset; +namespace Rdmp.Core.Datasets; /// /// Common methods used by Cohort Builder UI implementations. Eliminates diff --git a/Rdmp.Core/Datasets/IDatasetProvider.cs b/Rdmp.Core/Datasets/IDatasetProvider.cs index e294c1acb4..a39c4de499 100644 --- a/Rdmp.Core/Datasets/IDatasetProvider.cs +++ b/Rdmp.Core/Datasets/IDatasetProvider.cs @@ -1,17 +1,25 @@ using Rdmp.Core.Curation.Data.Datasets; -using System; using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace Rdmp.Core.Datasets; - +/// +/// Base interface for dataset providers to impliment +/// public interface IDatasetProvider { - List FetchDatasets(); + /// + /// Fetch known datasets + /// + /// + List FetchDatasets(); + - Curation.Data.Datasets.Dataset FetchDatasetByID(int id); + /// + /// Fetch a specific dataset by its ID + /// + /// + /// + Dataset FetchDatasetByID(int id); } diff --git a/Rdmp.Core/Datasets/InternalDatasetProvider.cs b/Rdmp.Core/Datasets/InternalDatasetProvider.cs index 615985a48c..04740666d7 100644 --- a/Rdmp.Core/Datasets/InternalDatasetProvider.cs +++ b/Rdmp.Core/Datasets/InternalDatasetProvider.cs @@ -1,12 +1,12 @@ using Rdmp.Core.CommandExecution; -using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace Rdmp.Core.Datasets; +/// +/// Provider for internal datasets +/// public class InternalDatasetProvider : IDatasetProvider { private readonly IBasicActivateItems _activator; @@ -15,11 +15,13 @@ public InternalDatasetProvider(IBasicActivateItems activator) _activator = activator; } + /// public Curation.Data.Datasets.Dataset FetchDatasetByID(int id) { return _activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("ID", id).FirstOrDefault(); } + /// public List FetchDatasets() { return _activator.RepositoryLocator.CatalogueRepository.GetAllObjects().ToList(); diff --git a/Rdmp.Core/Datasets/PluginDataset.cs b/Rdmp.Core/Datasets/PluginDataset.cs index 3115714c04..91673de888 100644 --- a/Rdmp.Core/Datasets/PluginDataset.cs +++ b/Rdmp.Core/Datasets/PluginDataset.cs @@ -1,12 +1,12 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; +using Rdmp.Core.Curation.Data.Datasets; + namespace Rdmp.Core.Datasets { - public class PluginDataset: Curation.Data.Datasets.Dataset + /// + /// Base class to allow all plugin dataset types to be based off + /// + public class PluginDataset: Dataset { public PluginDataset() { } diff --git a/Rdmp.Core/Datasets/PureDataset.cs b/Rdmp.Core/Datasets/PureDataset.cs index a586327683..dd6dfc1ce8 100644 --- a/Rdmp.Core/Datasets/PureDataset.cs +++ b/Rdmp.Core/Datasets/PureDataset.cs @@ -1,7 +1,11 @@ -using Amazon.Auth.AccessControlPolicy.ActionIdentifiers; -using NPOI.SS.Formula.Atp; -using NPOI.SS.Formula.Functions; -using SynthEHR; +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + + +using Rdmp.Core.Datasets.PureItems; using System; using System.Collections.Generic; using System.Text.Json.Serialization; @@ -9,132 +13,9 @@ namespace Rdmp.Core.Datasets; #nullable enable -public class ENGBWrapper -{ - public ENGBWrapper(string? text) { en_GB = text; } - public string? en_GB { get; set; } -} - -public class URITerm -{ - public URITerm(string? uri, ENGBWrapper enGBWrapper) - { - URI = uri; - term = enGBWrapper; - } - public string? URI { get; set; } - public ENGBWrapper term { get; set; } -} -public class PureDescription -{ - public int? PureId { get; set; } - public ENGBWrapper? Value { get; set; } - - public URITerm Term { get => new URITerm("/dk/atira/pure/dataset/descriptions/datasetdescription", new ENGBWrapper("Description")); } -} - -public class Name() -{ - public string? FirstName { get; set; } - public string? LastName { get; set; } -} - -public class PurePerson -{ - public string? TypeDiscriminator { get; set; } - public int? PureId { get; set; } - - public Name? Name { get; set; } - public URITerm? Role { get; set; } - - public List? Organizations { get; set; } -} - -public class PureDate -{ - public PureDate(DateTime dateTime) - { - Year = dateTime.Year; - Month = dateTime.Month; - Day = dateTime.Day; - } - - public PureDate() { } - - - public DateTime ToDateTime() - { - return new DateTime(Year, Month??1, Day??1, 0, 0, 0); - } - - public bool IsBefore(PureDate date) - { - if (Year < date.Year) return true; - if (Year == date.Year) - { - if (Month < date.Month) return true; - if (Month == date.Month) - { - return Day < date.Day; - } - } - - return false; - } - - public PureDate(int year, int? month = null, int? day = null) - { - Year = year; - if (month != null) Month = month; - if (day != null) Day = day; - } - public int Year { get; set; } - public int? Month { get; set; } - public int? Day { get; set; } -} - -public class Visibility -{ - public Visibility(string? key, ENGBWrapper description) - { - Key = key; - Description = description; - } - public string? Key { get; set; } - public ENGBWrapper Description { get; set; } -} - -public class Workflow -{ - public string? Step { get; set; } - public ENGBWrapper? Description { get; set; } -} - -public class Geolocation -{ - public ENGBWrapper? GeographicalCoverage { get; set; } - public string? Point { get; set; } - - public string? Polygon { get; set; } -} - -public class TemporalCoveragePeriod -{ - public PureDate? StartDate { get; set; } - public PureDate? EndDate { get; set; } -} - -public class PureSystem -{ - public PureSystem(string? uuid, string? systemName) - { - UUID = uuid; - SystemName = systemName; - } - public string? SystemName { get; set; } - public string? UUID { get; set; } -} - +/// +/// Used for mapping Pure datasets from the API into a C# object. +/// public class PureDataset : PluginDataset { [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] diff --git a/Rdmp.Core/Datasets/PureDatasetProvider.cs b/Rdmp.Core/Datasets/PureDatasetProvider.cs index a34ffeea9a..b85e2f6c81 100644 --- a/Rdmp.Core/Datasets/PureDatasetProvider.cs +++ b/Rdmp.Core/Datasets/PureDatasetProvider.cs @@ -17,164 +17,166 @@ using JsonSerializer = System.Text.Json.JsonSerializer; using System.Text; using System.Data; -namespace Rdmp.Core.Datasets +namespace Rdmp.Core.Datasets; + +/// +/// Provides access to a remore PURE system to access datasets +/// +public class PureDatasetProvider : PluginDatasetProvider { - public class PureDatasetProvider : PluginDatasetProvider + private HttpClient _client; + + public PureDatasetProvider(IBasicActivateItems activator, DatasetProviderConfiguration configuration) : base(activator, configuration) { - private HttpClient _client; + _client = new HttpClient(); + var credentials = Repository.GetAllObjectsWhere("ID", Configuration.DataAccessCredentials_ID).First(); + var apiKey = credentials.GetDecryptedPassword(); + _client.DefaultRequestHeaders.Add("api-key", apiKey); + } - public PureDatasetProvider(IBasicActivateItems activator, DatasetProviderConfiguration configuration) : base(activator, configuration) - { - _client = new HttpClient(); - var credentials = Repository.GetAllObjectsWhere("ID", Configuration.DataAccessCredentials_ID).First(); - var apiKey = credentials.GetDecryptedPassword(); - _client.DefaultRequestHeaders.Add("api-key", apiKey); - } + public static string UrltoUUID(string url) => url.Split("/").Last(); - public static string UrltoUUID(string url) => url.Split("/").Last(); + public bool CheckDatasetExistsAtURL(string url) + { + var uri = $"{Configuration.Url}/data-sets/{UrltoUUID(url)}"; + var response = Task.Run(async () => await _client.GetAsync(uri)).Result; + return response.StatusCode == HttpStatusCode.OK; + } - public bool CheckDatasetExistsAtURL(string url) + public override void AddExistingDataset(string name, string url) + { + var uri = $"{Configuration.Url}/data-sets/{UrltoUUID(url)}"; + var response = Task.Run(async () => await _client.GetAsync(uri)).Result; + if (response.StatusCode == HttpStatusCode.OK) { - var uri = $"{Configuration.Url}/data-sets/{UrltoUUID(url)}"; - var response = Task.Run(async () => await _client.GetAsync(uri)).Result; - return response.StatusCode == HttpStatusCode.OK; + var detailsString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; + PureDataset pd = JsonConvert.DeserializeObject(detailsString); + var datasetName = string.IsNullOrWhiteSpace(name) ? pd.Title.En_GB : name; + var dataset = new Curation.Data.Datasets.Dataset(Repository, datasetName) + { + Url = url, + Type = this.ToString(), + Provider_ID = Configuration.ID, + DigitalObjectIdentifier = pd.DigitalObjectIdentifier, + Folder = $"\\{Configuration.Name}", + }; + dataset.SaveToDatabase(); + Activator.Publish(dataset); } - - public override void AddExistingDataset(string name, string url) + else { - var uri = $"{Configuration.Url}/data-sets/{UrltoUUID(url)}"; - var response = Task.Run(async () => await _client.GetAsync(uri)).Result; - if (response.StatusCode == HttpStatusCode.OK) - { - var detailsString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; - PureDataset pd = JsonConvert.DeserializeObject(detailsString); - var datasetName = string.IsNullOrWhiteSpace(name) ? pd.Title.en_GB : name; - var dataset = new Curation.Data.Datasets.Dataset(Repository, datasetName) - { - Url = url, - Type = this.ToString(), - Provider_ID = Configuration.ID, - DigitalObjectIdentifier = pd.DigitalObjectIdentifier, - Folder = $"\\{Configuration.Name}", - }; - dataset.SaveToDatabase(); - Activator.Publish(dataset); - } - else - { - throw new Exception("Cannot access dataset at provided url"); - } + throw new Exception("Cannot access dataset at provided url"); } + } - //public Curation.Data.Datasets.Dataset Create(string title, string publisherUid, List people, Visibility visibility, PureDate publicationDate) - //{ - // //this works but i'm not sure it's linking up with the person correctly - - // var pureDataset = new PureDataset(); - // pureDataset.Title = new ENGBWrapper(title); - // pureDataset.ManagingOrganization = new PureSystem(Configuration.Organisation_ID, "Organization"); - // pureDataset.Type = new URITerm("/dk/atira/pure/dataset/datasettypes/dataset/dataset", new ENGBWrapper("Dataset")); - // pureDataset.Publisher = new PureSystem(publisherUid, "Publisher"); - - // pureDataset.Persons = people; - // pureDataset.Visibility = visibility; - // pureDataset.Organizations = [new PureSystem(Configuration.Organisation_ID, "Organization")]; - // pureDataset.PublicationAvailableDate = publicationDate; - - // var serializeOptions = new JsonSerializerOptions - // { - // PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - // WriteIndented = true - // }; - - // var jsonString = JsonSerializer.Serialize(pureDataset, serializeOptions); - - // var uri = $"{Configuration.Url}/data-sets"; - // var httpContent = new StringContent(jsonString, Encoding.UTF8, "application/json"); - - // var response = Task.Run(async () => await _client.PutAsync(uri, httpContent)).Result; - // if (response.StatusCode != HttpStatusCode.Created) - // { - // throw new Exception("Error"); - // } - // var detailsString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; - // PureDataset pd = JsonConvert.DeserializeObject(detailsString); - // var dataset = new Curation.Data.Datasets.Dataset(Repository, pureDataset.Title.en_GB); - // dataset.Provider_ID = Configuration.ID; - // dataset.Url = pd.PortalURL; - // dataset.SaveToDatabase(); - // return dataset; - //} - - public override void Update(string uuid, PluginDataset datasetUpdates) + //public Curation.Data.Datasets.Dataset Create(string title, string publisherUid, List people, Visibility visibility, PureDate publicationDate) + //{ + // //this works but i'm not sure it's linking up with the person correctly + + // var pureDataset = new PureDataset(); + // pureDataset.Title = new ENGBWrapper(title); + // pureDataset.ManagingOrganization = new PureSystem(Configuration.Organisation_ID, "Organization"); + // pureDataset.Type = new URITerm("/dk/atira/pure/dataset/datasettypes/dataset/dataset", new ENGBWrapper("Dataset")); + // pureDataset.Publisher = new PureSystem(publisherUid, "Publisher"); + + // pureDataset.Persons = people; + // pureDataset.Visibility = visibility; + // pureDataset.Organizations = [new PureSystem(Configuration.Organisation_ID, "Organization")]; + // pureDataset.PublicationAvailableDate = publicationDate; + + // var serializeOptions = new JsonSerializerOptions + // { + // PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + // WriteIndented = true + // }; + + // var jsonString = JsonSerializer.Serialize(pureDataset, serializeOptions); + + // var uri = $"{Configuration.Url}/data-sets"; + // var httpContent = new StringContent(jsonString, Encoding.UTF8, "application/json"); + + // var response = Task.Run(async () => await _client.PutAsync(uri, httpContent)).Result; + // if (response.StatusCode != HttpStatusCode.Created) + // { + // throw new Exception("Error"); + // } + // var detailsString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; + // PureDataset pd = JsonConvert.DeserializeObject(detailsString); + // var dataset = new Curation.Data.Datasets.Dataset(Repository, pureDataset.Title.en_GB); + // dataset.Provider_ID = Configuration.ID; + // dataset.Url = pd.PortalURL; + // dataset.SaveToDatabase(); + // return dataset; + //} + + public override void Update(string uuid, PluginDataset datasetUpdates) + { + var serializeOptions = new JsonSerializerOptions { - var serializeOptions = new JsonSerializerOptions - { - PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - WriteIndented = true - }; + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = true + }; - var options = new JsonWriterOptions - { - Indented = true - }; + var options = new JsonWriterOptions + { + Indented = true + }; - using var stream = new MemoryStream(); - JsonSerializer.Serialize(stream, (PureDataset)datasetUpdates, serializeOptions); - var jsonString = Encoding.UTF8.GetString(stream.ToArray()); - var uri = $"{Configuration.Url}/data-sets/{uuid}"; - var httpContent = new StringContent(jsonString, Encoding.UTF8, "application/json"); + using var stream = new MemoryStream(); + JsonSerializer.Serialize(stream, (PureDataset)datasetUpdates, serializeOptions); + var jsonString = Encoding.UTF8.GetString(stream.ToArray()); + var uri = $"{Configuration.Url}/data-sets/{uuid}"; + var httpContent = new StringContent(jsonString, Encoding.UTF8, "application/json"); - var response = Task.Run(async () => await _client.PutAsync(uri, httpContent)).Result; - if (response.StatusCode != HttpStatusCode.OK) - { - throw new Exception("Error"); - } + var response = Task.Run(async () => await _client.PutAsync(uri, httpContent)).Result; + if (response.StatusCode != HttpStatusCode.OK) + { + throw new Exception("Error"); } + } - public PureDataset FetchPureDataset(Curation.Data.Datasets.Dataset dataset) + public PureDataset FetchPureDataset(Curation.Data.Datasets.Dataset dataset) + { + var uri = $"{Configuration.Url}/data-sets/{UrltoUUID(dataset.Url)}"; + var response = Task.Run(async () => await _client.GetAsync(uri)).Result; + if (response.StatusCode != HttpStatusCode.OK) { - var uri = $"{Configuration.Url}/data-sets/{UrltoUUID(dataset.Url)}"; - var response = Task.Run(async () => await _client.GetAsync(uri)).Result; - if (response.StatusCode != HttpStatusCode.OK) - { - throw new Exception("Error"); - } - var detailsString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; - PureDataset pd = JsonConvert.DeserializeObject(detailsString); - return pd; + throw new Exception("Error"); } + var detailsString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; + PureDataset pd = JsonConvert.DeserializeObject(detailsString); + return pd; + } - public override Curation.Data.Datasets.Dataset FetchDatasetByID(int id) - { - var dataset = Repository.GetAllObjectsWhere("ID", id).FirstOrDefault(); - return dataset; - } + public override Curation.Data.Datasets.Dataset FetchDatasetByID(int id) + { + var dataset = Repository.GetAllObjectsWhere("ID", id).FirstOrDefault(); + return dataset; + } - public override List FetchDatasets() - { - return Repository.GetAllObjectsWhere("Provider_ID", Configuration.ID).ToList(); - } + public override List FetchDatasets() + { + return Repository.GetAllObjectsWhere("Provider_ID", Configuration.ID).ToList(); + } - //public List FetchPublishers() - //{ - // var uri = $"{Configuration.Url}/publishers"; - // var response = Task.Run(async () => await _client.GetAsync(uri)).Result; - // if (response.StatusCode != HttpStatusCode.OK) - // { - // throw new Exception("Error"); - // } - // var detailsString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; - // List publishers = JsonConvert.DeserializeObject>(detailsString); - // return publishers; + //public List FetchPublishers() + //{ + // var uri = $"{Configuration.Url}/publishers"; + // var response = Task.Run(async () => await _client.GetAsync(uri)).Result; + // if (response.StatusCode != HttpStatusCode.OK) + // { + // throw new Exception("Error"); + // } + // var detailsString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; + // List publishers = JsonConvert.DeserializeObject>(detailsString); + // return publishers; - //} + //} - public override Curation.Data.Datasets.Dataset Create() - { - throw new NotImplementedException(); - } + public override Curation.Data.Datasets.Dataset Create() + { + throw new NotImplementedException(); } } diff --git a/Rdmp.Core/Datasets/PurePublisher.cs b/Rdmp.Core/Datasets/PurePublisher.cs index db983c1190..aae53d236f 100644 --- a/Rdmp.Core/Datasets/PurePublisher.cs +++ b/Rdmp.Core/Datasets/PurePublisher.cs @@ -1,45 +1,32 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using Rdmp.Core.Datasets.PureItems; +using System; +#nullable enable namespace Rdmp.Core.Datasets; -// Root myDeserializedClass = JsonConvert.DeserializeObject(myJsonResponse); -public class Description -{ - public string en_GB { get; set; } -} +/// +/// Used to mapping Pure publishers from the API into a C# Object. +/// This is not indended for modification within RDMP. +/// public class PurePublisher { - public int pureId { get; set; } - public string uuid { get; set; } - public string createdBy { get; set; } - public DateTime createdDate { get; set; } - public string modifiedBy { get; set; } - public DateTime modifiedDate { get; set; } - public string version { get; set; } - public string name { get; set; } - public Type type { get; set; } - public PublisherWorkflow workflow { get; set; } - public string systemName { get; set; } -} - -public class Term -{ - public string en_GB { get; set; } -} - -public class Type -{ - public string uri { get; set; } - public Term term { get; set; } -} - -public class PublisherWorkflow -{ - public string step { get; set; } - public Description description { get; set; } -} + public int? pureId { get; set; } + public string? uuid { get; set; } + public string? createdBy { get; set; } + public DateTime? createdDate { get; set; } + public string? modifiedBy { get; set; } + public DateTime? modifiedDate { get; set; } + public string? version { get; set; } + public string? name { get; set; } + public URITerm? type { get; set; } + public Workflow? workflow { get; set; } + public string? systemName { get; set; } + public PurePublisher() { } +} \ No newline at end of file diff --git a/Rdmp.UI/SubComponents/DatasetConfigurationUI.cs b/Rdmp.UI/SubComponents/DatasetConfigurationUI.cs index aa44e9aab7..6f4fa4320c 100644 --- a/Rdmp.UI/SubComponents/DatasetConfigurationUI.cs +++ b/Rdmp.UI/SubComponents/DatasetConfigurationUI.cs @@ -4,7 +4,7 @@ using System.Linq; using System.Windows.Forms; using Rdmp.UI.Refreshing; -using Rdmp.Core.Dataset; +using Rdmp.Core.Datasets; using Rdmp.Core.Curation.Data; using Rdmp.UI.ItemActivation; using Rdmp.Core.Curation.Data.Datasets; diff --git a/Rdmp.UI/SubComponents/PureDatasetConfigurationUI.cs b/Rdmp.UI/SubComponents/PureDatasetConfigurationUI.cs index 60d7bc9715..ea158dcc4b 100644 --- a/Rdmp.UI/SubComponents/PureDatasetConfigurationUI.cs +++ b/Rdmp.UI/SubComponents/PureDatasetConfigurationUI.cs @@ -1,5 +1,6 @@ using Rdmp.Core.Curation.Data.Datasets; using Rdmp.Core.Datasets; +using Rdmp.Core.Datasets.PureItems; using Rdmp.Core.ReusableLibraryCode; using Rdmp.UI.ItemActivation; using Rdmp.UI.Refreshing; @@ -40,9 +41,9 @@ public override void SetDatabaseObject(IActivateItems activator, Dataset databas { btnViewOnPure.Enabled = true; } - tbName.Text = _dataset.Title.en_GB; + tbName.Text = _dataset.Title.En_GB; if (_dataset.Descriptions.Any(d => d.Value is not null)) - tbDescription.Text = _dataset.Descriptions.Where(d => d.Value is not null).First().Value.en_GB; + tbDescription.Text = _dataset.Descriptions.Where(d => d.Value is not null).First().Value.En_GB; if (_dataset.TemporalCoveragePeriod is not null && _dataset.TemporalCoveragePeriod.StartDate is not null) tbTemporalStart.Text = _dataset.TemporalCoveragePeriod.StartDate.ToDateTime().ToString(); if (_dataset.TemporalCoveragePeriod is not null && _dataset.TemporalCoveragePeriod.EndDate is not null) @@ -74,7 +75,7 @@ private void btnSave_Click(object sender, EventArgs e) if (_activator.YesNo("Are you sure?", "Save Changes")) { var datasetUpdate = new PureDataset(); - _dataset.Title.en_GB = tbName.Text; + _dataset.Title.En_GB = tbName.Text; datasetUpdate.Title = _dataset.Title; var datasetDescriptionTerm = "/dk/atira/pure/dataset/descriptions/datasetdescription"; var description = _dataset.Descriptions.Where(d => d.Term.URI == datasetDescriptionTerm).FirstOrDefault(); From ba998b3543a974b6932d897f545d841c10dc0244 Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 18 Oct 2024 09:58:35 +0100 Subject: [PATCH 086/106] add missing files --- Rdmp.Core/Datasets/PureItems/ENGBWrapper.cs | 16 ++++++ Rdmp.Core/Datasets/PureItems/Geolocation.cs | 18 ++++++ Rdmp.Core/Datasets/PureItems/Name.cs | 16 ++++++ Rdmp.Core/Datasets/PureItems/PureDate.cs | 56 +++++++++++++++++++ .../Datasets/PureItems/PureDescription.cs | 19 +++++++ Rdmp.Core/Datasets/PureItems/PurePerson.cs | 23 ++++++++ Rdmp.Core/Datasets/PureItems/PureSystem.cs | 21 +++++++ .../PureItems/TemporalCoveragePeriod.cs | 18 ++++++ Rdmp.Core/Datasets/PureItems/URITerm.cs | 21 +++++++ Rdmp.Core/Datasets/PureItems/Visibility.cs | 21 +++++++ Rdmp.Core/Datasets/PureItems/Workflow.cs | 16 ++++++ 11 files changed, 245 insertions(+) create mode 100644 Rdmp.Core/Datasets/PureItems/ENGBWrapper.cs create mode 100644 Rdmp.Core/Datasets/PureItems/Geolocation.cs create mode 100644 Rdmp.Core/Datasets/PureItems/Name.cs create mode 100644 Rdmp.Core/Datasets/PureItems/PureDate.cs create mode 100644 Rdmp.Core/Datasets/PureItems/PureDescription.cs create mode 100644 Rdmp.Core/Datasets/PureItems/PurePerson.cs create mode 100644 Rdmp.Core/Datasets/PureItems/PureSystem.cs create mode 100644 Rdmp.Core/Datasets/PureItems/TemporalCoveragePeriod.cs create mode 100644 Rdmp.Core/Datasets/PureItems/URITerm.cs create mode 100644 Rdmp.Core/Datasets/PureItems/Visibility.cs create mode 100644 Rdmp.Core/Datasets/PureItems/Workflow.cs diff --git a/Rdmp.Core/Datasets/PureItems/ENGBWrapper.cs b/Rdmp.Core/Datasets/PureItems/ENGBWrapper.cs new file mode 100644 index 0000000000..a283ff1129 --- /dev/null +++ b/Rdmp.Core/Datasets/PureItems/ENGBWrapper.cs @@ -0,0 +1,16 @@ +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +namespace Rdmp.Core.Datasets.PureItems; +#nullable enable +/// +/// Internal PURE system class +/// +public class ENGBWrapper +{ + public ENGBWrapper(string? text) { En_GB = text; } + public string? En_GB { get; set; } +} diff --git a/Rdmp.Core/Datasets/PureItems/Geolocation.cs b/Rdmp.Core/Datasets/PureItems/Geolocation.cs new file mode 100644 index 0000000000..e1b6fde5f1 --- /dev/null +++ b/Rdmp.Core/Datasets/PureItems/Geolocation.cs @@ -0,0 +1,18 @@ +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . +#nullable enable + +namespace Rdmp.Core.Datasets.PureItems; +/// +/// Internal PURE system class +/// +public class Geolocation +{ + public ENGBWrapper? GeographicalCoverage { get; set; } + public string? Point { get; set; } + + public string? Polygon { get; set; } +} \ No newline at end of file diff --git a/Rdmp.Core/Datasets/PureItems/Name.cs b/Rdmp.Core/Datasets/PureItems/Name.cs new file mode 100644 index 0000000000..2c24eb7758 --- /dev/null +++ b/Rdmp.Core/Datasets/PureItems/Name.cs @@ -0,0 +1,16 @@ +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . +#nullable enable + +namespace Rdmp.Core.Datasets.PureItems; +/// +/// Internal PURE system class +/// +public class Name() +{ + public string? FirstName { get; set; } + public string? LastName { get; set; } +} \ No newline at end of file diff --git a/Rdmp.Core/Datasets/PureItems/PureDate.cs b/Rdmp.Core/Datasets/PureItems/PureDate.cs new file mode 100644 index 0000000000..68dc14a029 --- /dev/null +++ b/Rdmp.Core/Datasets/PureItems/PureDate.cs @@ -0,0 +1,56 @@ +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . +#nullable enable + +using System; + +namespace Rdmp.Core.Datasets.PureItems; + +/// +/// Internal PURE system class +/// +public class PureDate +{ + public PureDate(DateTime dateTime) + { + Year = dateTime.Year; + Month = dateTime.Month; + Day = dateTime.Day; + } + + public PureDate() { } + + + public DateTime ToDateTime() + { + return new DateTime(Year, Month ?? 1, Day ?? 1, 0, 0, 0); + } + + public bool IsBefore(PureDate date) + { + if (Year < date.Year) return true; + if (Year == date.Year) + { + if (Month < date.Month) return true; + if (Month == date.Month) + { + return Day < date.Day; + } + } + + return false; + } + + public PureDate(int year, int? month = null, int? day = null) + { + Year = year; + if (month != null) Month = month; + if (day != null) Day = day; + } + public int Year { get; set; } + public int? Month { get; set; } + public int? Day { get; set; } +} diff --git a/Rdmp.Core/Datasets/PureItems/PureDescription.cs b/Rdmp.Core/Datasets/PureItems/PureDescription.cs new file mode 100644 index 0000000000..eaa6dd75d5 --- /dev/null +++ b/Rdmp.Core/Datasets/PureItems/PureDescription.cs @@ -0,0 +1,19 @@ +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . +#nullable enable + + +namespace Rdmp.Core.Datasets.PureItems; +/// +/// Internal PURE system class +/// +public class PureDescription +{ + public int? PureId { get; set; } + public ENGBWrapper? Value { get; set; } + + public URITerm Term { get => new URITerm("/dk/atira/pure/dataset/descriptions/datasetdescription", new ENGBWrapper("Description")); } +} \ No newline at end of file diff --git a/Rdmp.Core/Datasets/PureItems/PurePerson.cs b/Rdmp.Core/Datasets/PureItems/PurePerson.cs new file mode 100644 index 0000000000..cadc8f5ab1 --- /dev/null +++ b/Rdmp.Core/Datasets/PureItems/PurePerson.cs @@ -0,0 +1,23 @@ +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . +#nullable enable + +using System.Collections.Generic; + +namespace Rdmp.Core.Datasets.PureItems; +/// +/// Internal PURE system class +/// +public class PurePerson +{ + public string? TypeDiscriminator { get; set; } + public int? PureId { get; set; } + + public Name? Name { get; set; } + public URITerm? Role { get; set; } + + public List? Organizations { get; set; } +} \ No newline at end of file diff --git a/Rdmp.Core/Datasets/PureItems/PureSystem.cs b/Rdmp.Core/Datasets/PureItems/PureSystem.cs new file mode 100644 index 0000000000..a780b769f1 --- /dev/null +++ b/Rdmp.Core/Datasets/PureItems/PureSystem.cs @@ -0,0 +1,21 @@ +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . +#nullable enable + +namespace Rdmp.Core.Datasets.PureItems; +/// +/// Internal PURE system class +/// +public class PureSystem +{ + public PureSystem(string? uuid, string? systemName) + { + UUID = uuid; + SystemName = systemName; + } + public string? SystemName { get; set; } + public string? UUID { get; set; } +} \ No newline at end of file diff --git a/Rdmp.Core/Datasets/PureItems/TemporalCoveragePeriod.cs b/Rdmp.Core/Datasets/PureItems/TemporalCoveragePeriod.cs new file mode 100644 index 0000000000..3485b84827 --- /dev/null +++ b/Rdmp.Core/Datasets/PureItems/TemporalCoveragePeriod.cs @@ -0,0 +1,18 @@ +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . +#nullable enable + + + +namespace Rdmp.Core.Datasets.PureItems; +/// +/// Internal PURE system class +/// +public class TemporalCoveragePeriod +{ + public PureDate? StartDate { get; set; } + public PureDate? EndDate { get; set; } +} diff --git a/Rdmp.Core/Datasets/PureItems/URITerm.cs b/Rdmp.Core/Datasets/PureItems/URITerm.cs new file mode 100644 index 0000000000..46d71cd296 --- /dev/null +++ b/Rdmp.Core/Datasets/PureItems/URITerm.cs @@ -0,0 +1,21 @@ +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . +#nullable enable + +namespace Rdmp.Core.Datasets.PureItems; +/// +/// Internal PURE system class +/// +public class URITerm +{ + public URITerm(string? uri, ENGBWrapper enGBWrapper) + { + URI = uri; + Term = enGBWrapper; + } + public string? URI { get; set; } + public ENGBWrapper Term { get; set; } +} \ No newline at end of file diff --git a/Rdmp.Core/Datasets/PureItems/Visibility.cs b/Rdmp.Core/Datasets/PureItems/Visibility.cs new file mode 100644 index 0000000000..ec90cd7c15 --- /dev/null +++ b/Rdmp.Core/Datasets/PureItems/Visibility.cs @@ -0,0 +1,21 @@ +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . +#nullable enable + +namespace Rdmp.Core.Datasets.PureItems; +/// +/// Internal PURE system class +/// +public class Visibility +{ + public Visibility(string? key, ENGBWrapper description) + { + Key = key; + Description = description; + } + public string? Key { get; set; } + public ENGBWrapper Description { get; set; } +} \ No newline at end of file diff --git a/Rdmp.Core/Datasets/PureItems/Workflow.cs b/Rdmp.Core/Datasets/PureItems/Workflow.cs new file mode 100644 index 0000000000..e0f5c68da8 --- /dev/null +++ b/Rdmp.Core/Datasets/PureItems/Workflow.cs @@ -0,0 +1,16 @@ +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . +#nullable enable + +namespace Rdmp.Core.Datasets.PureItems; +/// +/// Internal PURE system class +/// +public class Workflow +{ + public string? Step { get; set; } + public ENGBWrapper? Description { get; set; } +} \ No newline at end of file From 0b5596aa157b422bfae96497ec5b0463942dfe63 Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 18 Oct 2024 10:19:59 +0100 Subject: [PATCH 087/106] update tests --- .../Icons/IconProvision/CatalogueIcons.resx | 1464 +++++++++-------- Rdmp.Core/Icons/IconProvision/RDMPConcept.cs | 4 +- Tests.Common/UnitTests.cs | 20 +- 3 files changed, 755 insertions(+), 733 deletions(-) diff --git a/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx b/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx index 856c935696..d702f41750 100644 --- a/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx +++ b/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx @@ -1,6 +1,6 @@  - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - text/microsoft-resx - - - 2.0 - - - System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - - ..\..\Icons\columninfo.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\DQE.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\Lookup.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\ExecuteArrow.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\executeArrowWithTickOverlay.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\spanner.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\tableinfo.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\backup.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\sync.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\StarHollow.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\loadMetadata.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\logging.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\importFile.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\Catalogue.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\Filter.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\ExtractionInformation.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\FilterContainer.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\cohortAggregate.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\greenIssue.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\orangeIssue.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\redIssue.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\Clipboard.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\patientIndexTable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\expandAllNodes.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\ExtractionInformation_SpecialApproval.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\ExtractionInformation_Supplemental.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\AggregatesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\catalogueItemsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\BigCohort.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\BigGraph.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\BigPatientIndexTable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\cohortIdentificationConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\frozenCohortIdentificationConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\supportingDocument.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\documentationNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\supportingDocumentExtractable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\EXCEPT.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\INTERSECT.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\supportingDocumentExtractableGlobal.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\supportingDocumentGlobal.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\supportingSqlExtractableGlobal.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\supportingSqlGlobal.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\UNION.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\collisionResolution.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\AllServersNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\..\Icons\tableinfoTableValuedFunction.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AggregateGraph.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CatalogueFolder.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CatalogueItem.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CatalogueItemTransform.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CatalogueItemIssue.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\DataAccessCredentials.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllDataAccessCredentialsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Database.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractionDirectoryNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\SQL.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\supportingSqlExtractable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ANOColumnInfo.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CloneExtractionConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\EmptyProject.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractableDataSetDisabled.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractionConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Failed.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\FrozenExtractionConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Project.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Release.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\SqlThenVSNow.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ANOTable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllANOTablesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractableCohort.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractableDataSet.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\JoinableCollectionNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ParametersNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\SupportingSQLTable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\TableInfoServerNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\TimeCoverageField.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExternalCohortTable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractableDataSetPackage.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractableDataSetsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\PivotField.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ProjectsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AggregateDimension.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\DashboardControl.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\DashboardLayout.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ReOrder.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\JoinInfo.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\StandardRegex.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Favourite.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractionFilterParameterSet.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AutomationServiceSlot.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CacheProgress.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExternalDatabaseServer.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExternalDatabaseServer_ANO.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExternalDatabaseServer_Cache.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExternalDatabaseServer_DQE.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExternalDatabaseServer_IdentifierDump.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExternalDatabaseServer_Logging.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Plugin.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Kill.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\EXCEPTCohortAggregate.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\INTERSECTCohortAggregate.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\UNIONCohortAggregate.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\RowCounts_Ignore.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\RowCounts_Respect.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\NoIconAvailable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\File.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Help.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\UnfreezeExtractionConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllCataloguesUsedByLoadMetadataNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllProcessTasksUsedByLoadMetadataNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\LoadDirectoryNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\LoadBubble.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\LoadFinalDatabase.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Exe.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\LoadMetadataScheduleNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\LoadPeriodically.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\LoadProgress.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\GetFilesStage.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\LoadBubbleMounting.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\PreLoadDiscardedColumn.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\famfamfam\Tick.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Warning.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllExternalServersNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\DecryptionPrivateKeyNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\PreLoadDiscardedColumnsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\PermissionWindow.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Pipeline.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractionConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllCohortsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ProjectCohortIdentificationConfigurationAssociationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ProjectSavedCohortsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CommittedCohortIdentificationNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\collapseAllNodes.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllAutomationServerSlotsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllRDMPRemotesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\RemoteRDMP.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AutomateablePipeline.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\PipelineComponent.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\PipelineComponentSource.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\PipelineComponentDestination.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\PipelineComponentArgument.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AggregateTopX.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AutomationServiceException.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllObjectExportsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllObjectImportsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ObjectExport.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ObjectImport.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllObjectSharingNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CohortCustomColumn.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractableColumn.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ProjectCohortsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\FrozenExtractionConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\TinyGreen.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\TinyRed.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\TinyYellow.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractionInformation_ProjectSpecific.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ProjectCatalogue.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ProjectCataloguesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\DropHere.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\MakeProjectSpecificCatalogueNormalAgain.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllPermissionWindowsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Waiting.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\WaitingForDatabase.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Writing.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\FileMissing.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllConnectionStringKeywordsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ConnectionStringKeyword.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllStandardRegexesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllPipelinesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\StandardPipelineUseCaseNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\OtherPipelinesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllGovernanceNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\GovernancePeriod.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\GovernanceDocument.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\LookupCompositeJoinInfo.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllFreeCohortIdentificationConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllProjectCohortIdentificationConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Menu.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CatalogueLookupsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\DatabaseRefresh.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Diff.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CohortAggregateContainer.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\WhatIsACohort.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllPluginsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Graph.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Main.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ProcessTask.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CacheFetchFailure.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\CumulativeExtractionResults.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\DQEGraphAnnotation.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Evaluation.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\LoadModuleAssembly.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ProcessTaskArgument.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ProjectCohortIdentificationConfigurationAssociation.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ReleaseLog.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\SelectedDataSetsForcedJoin.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\SpontaneouslyInventedColumn.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\SupplementalExtractionResults.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\TicketingSystemConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\WindowLayout.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllDashboardsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllOrphanAggregateConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtendedProperty.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\ExtractionProgress.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Attacher.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\DataProvider.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\MutilateDataTables.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllTemplateAggregateConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Commit.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Memento.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\TableInfoDatabaseNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Dataset.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a - - - ..\AggregateContinuousDateAxis.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\LoadMetadataCatalogueLinkage.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\famfamfam\cog.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\TicketingSystemReleaseStatus.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\Dataset1.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\AllStandardRegexesNode1.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\StandardRegex3.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\StandardRegex311.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\StandardRegex31.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\famfamfam\folder_table.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - ..\famfamfam\cog.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + + ..\..\Icons\columninfo.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\DQE.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\Lookup.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\ExecuteArrow.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\executeArrowWithTickOverlay.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\spanner.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\tableinfo.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\backup.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\sync.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\StarHollow.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\loadMetadata.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\logging.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\importFile.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\Catalogue.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\Filter.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\ExtractionInformation.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\FilterContainer.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\cohortAggregate.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\greenIssue.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\orangeIssue.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\redIssue.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\Clipboard.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\patientIndexTable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\expandAllNodes.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\ExtractionInformation_SpecialApproval.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\ExtractionInformation_Supplemental.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\AggregatesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\catalogueItemsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\BigCohort.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\BigGraph.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\BigPatientIndexTable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\cohortIdentificationConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\frozenCohortIdentificationConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\supportingDocument.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\documentationNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\supportingDocumentExtractable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\EXCEPT.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\INTERSECT.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\supportingDocumentExtractableGlobal.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\supportingDocumentGlobal.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\supportingSqlExtractableGlobal.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\supportingSqlGlobal.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\UNION.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\collisionResolution.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\AllServersNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\..\Icons\tableinfoTableValuedFunction.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AggregateGraph.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\CatalogueFolder.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\CatalogueItem.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\CatalogueItemTransform.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\CatalogueItemIssue.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\DataAccessCredentials.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllDataAccessCredentialsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Database.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ExtractionDirectoryNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\SQL.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\supportingSqlExtractable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ANOColumnInfo.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\CloneExtractionConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\EmptyProject.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ExtractableDataSetDisabled.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ExtractionConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Failed.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\FrozenExtractionConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Project.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Release.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\SqlThenVSNow.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ANOTable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllANOTablesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ExtractableCohort.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ExtractableDataSet.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\JoinableCollectionNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ParametersNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\SupportingSQLTable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\TableInfoServerNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\TimeCoverageField.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ExternalCohortTable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ExtractableDataSetPackage.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ExtractableDataSetsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\PivotField.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ProjectsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AggregateDimension.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\DashboardControl.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\DashboardLayout.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ReOrder.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\JoinInfo.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\StandardRegex.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Favourite.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ExtractionFilterParameterSet.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AutomationServiceSlot.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\CacheProgress.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ExternalDatabaseServer.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ExternalDatabaseServer_ANO.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ExternalDatabaseServer_Cache.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ExternalDatabaseServer_DQE.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ExternalDatabaseServer_IdentifierDump.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ExternalDatabaseServer_Logging.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Plugin.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Kill.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\EXCEPTCohortAggregate.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\INTERSECTCohortAggregate.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\UNIONCohortAggregate.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\RowCounts_Ignore.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\RowCounts_Respect.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\NoIconAvailable.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\File.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Help.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\UnfreezeExtractionConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllCataloguesUsedByLoadMetadataNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllProcessTasksUsedByLoadMetadataNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\LoadDirectoryNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\LoadBubble.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\LoadFinalDatabase.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Exe.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\LoadMetadataScheduleNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\LoadPeriodically.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\LoadProgress.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\GetFilesStage.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\LoadBubbleMounting.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\PreLoadDiscardedColumn.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\famfamfam\Tick.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Warning.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllExternalServersNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\DecryptionPrivateKeyNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\PreLoadDiscardedColumnsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\PermissionWindow.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Pipeline.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ExtractionConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllCohortsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ProjectCohortIdentificationConfigurationAssociationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ProjectSavedCohortsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\CommittedCohortIdentificationNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\collapseAllNodes.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllAutomationServerSlotsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllRDMPRemotesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\RemoteRDMP.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AutomateablePipeline.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\PipelineComponent.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\PipelineComponentSource.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\PipelineComponentDestination.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\PipelineComponentArgument.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AggregateTopX.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AutomationServiceException.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllObjectExportsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllObjectImportsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ObjectExport.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ObjectImport.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllObjectSharingNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\CohortCustomColumn.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ExtractableColumn.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ProjectCohortsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\FrozenExtractionConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\TinyGreen.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\TinyRed.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\TinyYellow.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ExtractionInformation_ProjectSpecific.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ProjectCatalogue.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ProjectCataloguesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\DropHere.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\MakeProjectSpecificCatalogueNormalAgain.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllPermissionWindowsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Waiting.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\WaitingForDatabase.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Writing.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\FileMissing.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllConnectionStringKeywordsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ConnectionStringKeyword.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllStandardRegexesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllPipelinesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\StandardPipelineUseCaseNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\OtherPipelinesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllGovernanceNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\GovernancePeriod.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\GovernanceDocument.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\LookupCompositeJoinInfo.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllFreeCohortIdentificationConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllProjectCohortIdentificationConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Menu.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\CatalogueLookupsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\DatabaseRefresh.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Diff.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\CohortAggregateContainer.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\WhatIsACohort.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllPluginsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Graph.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Main.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ProcessTask.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\CacheFetchFailure.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\CumulativeExtractionResults.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\DQEGraphAnnotation.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Evaluation.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\LoadModuleAssembly.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ProcessTaskArgument.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ProjectCohortIdentificationConfigurationAssociation.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ReleaseLog.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\SelectedDataSetsForcedJoin.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\SpontaneouslyInventedColumn.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\SupplementalExtractionResults.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\TicketingSystemConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\WindowLayout.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllDashboardsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllOrphanAggregateConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ExtendedProperty.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\ExtractionProgress.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Attacher.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\DataProvider.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\MutilateDataTables.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllTemplateAggregateConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Commit.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Memento.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\TableInfoDatabaseNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Dataset.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a + + + ..\AggregateContinuousDateAxis.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\LoadMetadataCatalogueLinkage.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\famfamfam\cog.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\TicketingSystemReleaseStatus.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Dataset1.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllStandardRegexesNode1.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\StandardRegex3.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\StandardRegex311.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\StandardRegex31.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\famfamfam\folder_table.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\famfamfam\cog.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Dataset.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a + + + ..\Dataset.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a + \ No newline at end of file diff --git a/Rdmp.Core/Icons/IconProvision/RDMPConcept.cs b/Rdmp.Core/Icons/IconProvision/RDMPConcept.cs index 3bab8f26ca..0e1761808d 100644 --- a/Rdmp.Core/Icons/IconProvision/RDMPConcept.cs +++ b/Rdmp.Core/Icons/IconProvision/RDMPConcept.cs @@ -214,5 +214,7 @@ public enum RDMPConcept RegexRedactionKey, AllRegexRedactionConfigurationsNode, DatasetProviderConfiguration, - AllDatasetProviderConfigurationsNode + AllDatasetProviderConfigurationsNode, + PluginDataset, + PureDataset } \ No newline at end of file diff --git a/Tests.Common/UnitTests.cs b/Tests.Common/UnitTests.cs index 4853068808..69b7dc6a55 100644 --- a/Tests.Common/UnitTests.cs +++ b/Tests.Common/UnitTests.cs @@ -27,6 +27,7 @@ using Rdmp.Core.Curation.Data.Cohort.Joinables; using Rdmp.Core.Curation.Data.Dashboarding; using Rdmp.Core.Curation.Data.DataLoad; +using Rdmp.Core.Curation.Data.Datasets; using Rdmp.Core.Curation.Data.Governance; using Rdmp.Core.Curation.Data.ImportExport; using Rdmp.Core.Curation.Data.Pipelines; @@ -39,6 +40,7 @@ using Rdmp.Core.DataExport.DataRelease.Audit; using Rdmp.Core.DataExport.DataRelease.Potential; using Rdmp.Core.DataLoad.Modules.DataFlowOperations; +using Rdmp.Core.Datasets; using Rdmp.Core.MapsDirectlyToDatabaseTable; using Rdmp.Core.Repositories; using Rdmp.Core.ReusableLibraryCode.Checks; @@ -592,17 +594,29 @@ public static T WhenIHaveA(MemoryDataExportRepository repository) where T : D return (T)(object)new Setting(repository.CatalogueRepository, "", ""); } - if(typeof(T) == typeof(RegexRedaction)) + if (typeof(T) == typeof(RegexRedaction)) { return (T)(object)new RegexRedaction(repository.CatalogueRepository, 0, 0, "", "", 0, new Dictionary()); } if (typeof(T) == typeof(RegexRedactionConfiguration)) { - return (T)(object)new RegexRedactionConfiguration(repository.CatalogueRepository,"name",new System.Text.RegularExpressions.Regex(".*"),"T"); + return (T)(object)new RegexRedactionConfiguration(repository.CatalogueRepository, "name", new System.Text.RegularExpressions.Regex(".*"), "T"); } if (typeof(T) == typeof(RegexRedactionKey)) { - return (T)(object)new RegexRedactionKey(repository.CatalogueRepository,WhenIHaveA(repository),WhenIHaveA(repository),"PK"); + return (T)(object)new RegexRedactionKey(repository.CatalogueRepository, WhenIHaveA(repository), WhenIHaveA(repository), "PK"); + } + if (typeof(T) == typeof(PluginDataset)) + { + return (T)(object)new PluginDataset(); + } + if (typeof(T) == typeof(PureDataset)) + { + return (T)(object)new PureDataset(); + } + if (typeof(T) == typeof(DatasetProviderConfiguration)) + { + return (T)(object)new DatasetProviderConfiguration(); } From 515b2e2b1a3f9b31b074dc199bb9f4a045f829a8 Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 18 Oct 2024 11:12:20 +0100 Subject: [PATCH 088/106] update for tests --- .../Data/Datasets/DatasetProviderConfiguration.cs | 2 +- Rdmp.Core/Datasets/PluginDataset.cs | 8 ++++++-- Rdmp.Core/Datasets/PureDataset.cs | 5 +++++ Rdmp.Core/Datasets/PureItems/{Name.cs => PureName.cs} | 2 +- Rdmp.Core/Datasets/PureItems/PurePerson.cs | 2 +- Tests.Common/UnitTests.cs | 9 ++++++--- 6 files changed, 20 insertions(+), 8 deletions(-) rename Rdmp.Core/Datasets/PureItems/{Name.cs => PureName.cs} (97%) diff --git a/Rdmp.Core/Curation/Data/Datasets/DatasetProviderConfiguration.cs b/Rdmp.Core/Curation/Data/Datasets/DatasetProviderConfiguration.cs index eebcf90ba3..b4a23a5620 100644 --- a/Rdmp.Core/Curation/Data/Datasets/DatasetProviderConfiguration.cs +++ b/Rdmp.Core/Curation/Data/Datasets/DatasetProviderConfiguration.cs @@ -28,7 +28,7 @@ public DatasetProviderConfiguration(ICatalogueRepository repository,string name, { {"Name",name }, {"Type",type}, - {"URL",url}, + {"Url",url}, {"DataAccessCredentials_ID",dataAccessCredentialsID }, {"Organisation_ID", organisationId } }); diff --git a/Rdmp.Core/Datasets/PluginDataset.cs b/Rdmp.Core/Datasets/PluginDataset.cs index 91673de888..7d45a48dbc 100644 --- a/Rdmp.Core/Datasets/PluginDataset.cs +++ b/Rdmp.Core/Datasets/PluginDataset.cs @@ -1,4 +1,6 @@ using Rdmp.Core.Curation.Data.Datasets; +using Rdmp.Core.Repositories; +using System.Data.Common; namespace Rdmp.Core.Datasets @@ -6,9 +8,11 @@ namespace Rdmp.Core.Datasets /// /// Base class to allow all plugin dataset types to be based off /// - public class PluginDataset: Dataset + public class PluginDataset : Dataset { - + public PluginDataset(ICatalogueRepository catalogueRepository, string name) : base(catalogueRepository, name) { } public PluginDataset() { } + + public PluginDataset(ICatalogueRepository repository, DbDataReader r) : base(repository, r) { } } } diff --git a/Rdmp.Core/Datasets/PureDataset.cs b/Rdmp.Core/Datasets/PureDataset.cs index dd6dfc1ce8..b5a653b877 100644 --- a/Rdmp.Core/Datasets/PureDataset.cs +++ b/Rdmp.Core/Datasets/PureDataset.cs @@ -6,8 +6,10 @@ using Rdmp.Core.Datasets.PureItems; +using Rdmp.Core.Repositories; using System; using System.Collections.Generic; +using System.Data.Common; using System.Text.Json.Serialization; namespace Rdmp.Core.Datasets; @@ -91,4 +93,7 @@ public class PureDataset : PluginDataset #nullable disable public PureDataset() { } + public PureDataset(ICatalogueRepository catalogueRepository, string name) : base(catalogueRepository, name) { } + public PureDataset(ICatalogueRepository repository, DbDataReader r) : base(repository, r) { } + } diff --git a/Rdmp.Core/Datasets/PureItems/Name.cs b/Rdmp.Core/Datasets/PureItems/PureName.cs similarity index 97% rename from Rdmp.Core/Datasets/PureItems/Name.cs rename to Rdmp.Core/Datasets/PureItems/PureName.cs index 2c24eb7758..768081dcde 100644 --- a/Rdmp.Core/Datasets/PureItems/Name.cs +++ b/Rdmp.Core/Datasets/PureItems/PureName.cs @@ -9,7 +9,7 @@ namespace Rdmp.Core.Datasets.PureItems; /// /// Internal PURE system class /// -public class Name() +public class PureName() { public string? FirstName { get; set; } public string? LastName { get; set; } diff --git a/Rdmp.Core/Datasets/PureItems/PurePerson.cs b/Rdmp.Core/Datasets/PureItems/PurePerson.cs index cadc8f5ab1..f874c9da36 100644 --- a/Rdmp.Core/Datasets/PureItems/PurePerson.cs +++ b/Rdmp.Core/Datasets/PureItems/PurePerson.cs @@ -16,7 +16,7 @@ public class PurePerson public string? TypeDiscriminator { get; set; } public int? PureId { get; set; } - public Name? Name { get; set; } + public PureName? Name { get; set; } public URITerm? Role { get; set; } public List? Organizations { get; set; } diff --git a/Tests.Common/UnitTests.cs b/Tests.Common/UnitTests.cs index 69b7dc6a55..c297c2c147 100644 --- a/Tests.Common/UnitTests.cs +++ b/Tests.Common/UnitTests.cs @@ -45,6 +45,7 @@ using Rdmp.Core.Repositories; using Rdmp.Core.ReusableLibraryCode.Checks; using Rdmp.Core.Setting; +using Spectre.Console; namespace Tests.Common; @@ -608,15 +609,17 @@ public static T WhenIHaveA(MemoryDataExportRepository repository) where T : D } if (typeof(T) == typeof(PluginDataset)) { - return (T)(object)new PluginDataset(); + return (T)(object)new PluginDataset(repository,"Plugin Dataset"); } if (typeof(T) == typeof(PureDataset)) { - return (T)(object)new PureDataset(); + return (T)(object) new PureDataset(repository,"Pure Dataset"); } if (typeof(T) == typeof(DatasetProviderConfiguration)) { - return (T)(object)new DatasetProviderConfiguration(); + var creds = WhenIHaveA(repository); + creds.SaveToDatabase(); + return (T)(object)new DatasetProviderConfiguration(repository, "test configuration", "RDMP.Unkown.Provider", "google.com", creds.ID, "unknown"); } From bd6ca5652ceb62cfc360ebd0b1fbb02f180f44cb Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 22 Oct 2024 09:55:46 +0100 Subject: [PATCH 089/106] fix build --- Rdmp.UI/Collections/DatasetsCollectionUI.cs | 129 -------------------- 1 file changed, 129 deletions(-) delete mode 100644 Rdmp.UI/Collections/DatasetsCollectionUI.cs diff --git a/Rdmp.UI/Collections/DatasetsCollectionUI.cs b/Rdmp.UI/Collections/DatasetsCollectionUI.cs deleted file mode 100644 index 5e5d8aaccf..0000000000 --- a/Rdmp.UI/Collections/DatasetsCollectionUI.cs +++ /dev/null @@ -1,129 +0,0 @@ -using Rdmp.Core.CommandExecution.AtomicCommands; -using Rdmp.Core; -using Rdmp.Core.MapsDirectlyToDatabaseTable; -using Rdmp.UI.CommandExecution.AtomicCommands; -using Rdmp.UI.ItemActivation; -using Rdmp.UI.Refreshing; -using System; -using System.Collections.Generic; -using System.Linq; -using Rdmp.Core.Curation.Data; -using Rdmp.Core.Icons.IconProvision; -using System.Windows.Forms; - -namespace Rdmp.UI.Collections; - -public partial class DatasetsCollectionUI : RDMPCollectionUI, ILifetimeSubscriber -{ - - private Dataset[] _datasets; - private bool _firstTime = true; - - public DatasetsCollectionUI() - { - InitializeComponent(); - } - - public override void SetItemActivator(IActivateItems activator) - { - base.SetItemActivator(activator); - - CommonTreeFunctionality.SetUp(RDMPCollection.Datasets, tlvDatasets, Activator, olvName, olvName, - new RDMPCollectionCommonFunctionalitySettings(),tbFilter); - CommonTreeFunctionality.WhitespaceRightClickMenuCommandsGetter = - a => new IAtomicCommand[] - { - new ExecuteCommandCreateNewDatasetUI(Activator) - { OverrideCommandName = "Add New Dataset", Weight = -50.9f } - }; - Activator.RefreshBus.EstablishLifetimeSubscription(this); - tlvDatasets.AddObject(activator.CoreChildProvider.DatasetRootFolder); - - RefreshDatasets(Activator.CoreChildProvider.DatasetRootFolder); - - if (_firstTime) - { - CommonTreeFunctionality.SetupColumnTracking(olvName, new Guid("f8b0481e-378c-4996-9400-cb039c2efc5c")); - _firstTime = false; - var _refresh = new ToolStripMenuItem - { - Visible = true, - Image = FamFamFamIcons.arrow_refresh.ImageToBitmap(), - Alignment = ToolStripItemAlignment.Right, - ToolTipText = "Refresh Object" - }; - _refresh.Click += delegate (object sender, EventArgs e) { - var dataset = Activator.CoreChildProvider.AllDatasets.First(); - if (dataset is not null) - { - var cmd = new ExecuteCommandRefreshObject(Activator, dataset); - cmd.Execute(); - } - }; - CommonFunctionality.Add(_refresh); - } - } - - - - public void RefreshBus_RefreshObject(object sender, RefreshObjectEventArgs e) - { - RefreshDatasets(Activator.CoreChildProvider.DatasetRootFolder); - } - - private void RefreshDatasets(object oRefreshFrom) - { - var rootFolder = Activator.CoreChildProvider.DatasetRootFolder; - if (_datasets != null) - { - var newCatalogues = CommonTreeFunctionality.CoreChildProvider.AllDatasets.Except(_datasets); - if (newCatalogues.Any()) - { - oRefreshFrom = rootFolder; //refresh from the root instead - tlvDatasets.RefreshObject(oRefreshFrom); - } - } - _datasets = CommonTreeFunctionality.CoreChildProvider.AllDatasets; - if (_firstTime || Equals(oRefreshFrom, rootFolder)) - { - tlvDatasets.RefreshObject(rootFolder); - tlvDatasets.Expand(rootFolder); - _firstTime = false; - } - - } - - - /// - /// Returns all root objects in RDMP that match the . Handles unpicking tree collisions e.g. where matches 2 objects with one being the child of the other - /// - /// - /// - /// - public static List FindRootObjects(IActivateItems activator, - Func condition) - { - var datasets = - activator.RepositoryLocator.CatalogueRepository.GetAllObjects(); - - var actualRootFavourites = new List(); - - foreach (var currentFavourite in datasets) - actualRootFavourites.Add(currentFavourite); - - return actualRootFavourites; - } - - - /// - /// Return true if the object should be displayed in this pane - /// - /// - /// - protected virtual bool IncludeObject(IMapsDirectlyToDatabaseTable key) => - Activator.RepositoryLocator.CatalogueRepository.GetAllObjects().Contains(key); - - public static bool IsRootObject(IActivateItems activator, object root) => - //never favourite - false; -} \ No newline at end of file From a6c36a58a89829adabb31403c62e068d5ae3bdf1 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 22 Oct 2024 10:28:28 +0100 Subject: [PATCH 090/106] interim --- .../ExecuteCommandPerformRegexRedactionOnCatalogue.cs | 5 +++++ Rdmp.UI/SimpleDialogs/RedactCatalogueUI.cs | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs index f5186e42ff..5543c1faf2 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs @@ -99,6 +99,11 @@ public override void Execute() RegexRedactionHelper.Redact(columnInfo, row, _cataloguePKs, _redactionConfiguration, redactionsToSaveTable, pksToSave, redactionUpates); } redactionUpates.EndLoadData(); + if(pksToSave.Rows.Count == 0) + { + _activator.Show("Unable to find any matching Redactions"); + return; + } for (int i = 0; i < pksToSave.Rows.Count; i++) { pksToSave.Rows[i][nameof(RegexRedactionHelper.Constants.ID)] = i + 1; diff --git a/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.cs b/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.cs index 80cfe67e3a..ef19d8181e 100644 --- a/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.cs +++ b/Rdmp.UI/SimpleDialogs/RedactCatalogueUI.cs @@ -41,6 +41,11 @@ public RedactCatalogueUI() } + public override string GetTabName() + { + return $"{base.GetTabName()} - Regex Redaction"; + } + public void HandleNewRegex(object sender, EventArgs e) { //_activator From 9de73e745e09efbf3c312bb8ea998c96d2f733dd Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 22 Oct 2024 11:03:02 +0100 Subject: [PATCH 091/106] tidy up --- .../ExecuteCommandPerformRegexRedactionOnCatalogue.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs index 5543c1faf2..c5bb0cb846 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs @@ -60,11 +60,14 @@ public override void Execute() { redactionsToSaveTable = RegexRedactionHelper.GenerateRedactionsDataTable(); pksToSave = RegexRedactionHelper.GeneratePKDataTable(); + var columnName = columnInfo.Name; var table = columnInfo.TableInfo.Name; + _discoveredTable = columnInfo.TableInfo.Discover(DataAccessContext.InternalDataProcessing); DiscoveredColumn[] discoveredColumns = _discoveredTable.DiscoverColumns(); _discoveredPKColumns = discoveredColumns.Where(c => c.IsPrimaryKey).ToArray(); + if (_discoveredPKColumns.Length != 0) { _cataloguePKs = _catalogue.CatalogueItems.Where(c => c.ColumnInfo.IsPrimaryKey).ToList(); @@ -81,6 +84,7 @@ public override void Execute() qb.TopX = (int)_readLimit; } var sql = qb.SQL; + var dt = new DataTable(); dt.BeginLoadData(); var conn = _server.GetConnection(); @@ -92,6 +96,7 @@ public override void Execute() da.Fill(dt); } conn.Close(); + redactionUpates = dt.Clone(); redactionUpates.BeginLoadData(); foreach (DataRow row in dt.Rows) @@ -134,7 +139,6 @@ public override void Execute() throw new Exception($"Unable to identify any primary keys in table '{table}'. Redactions cannot be performed on tables without primary keys"); } resultCount += redactionUpates.Rows.Count; - } } } From c3c3a1eea2920b63bf40020828924dd63df073b9 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 22 Oct 2024 11:58:02 +0100 Subject: [PATCH 092/106] tidy up code --- ...ommandRestoreRegexRedactedValueInCatalogue.cs | 2 +- .../RegexRedaction/RegexRedactionHelper.cs | 2 -- .../Mutilators/RegexRedactionMutilator.cs | 2 +- .../Collections/ConfigurationsCollectionUI.cs | 4 ++++ ...CommandAddNewRegexRedactionConfigurationUI.cs | 16 +++++++--------- .../ExecuteCommandRegexRedaction.cs | 8 +++++++- ...ionWhenTargetIsRegexRedactionConfiguration.cs | 8 +++++++- .../CreateNewRegexRedactionConfigurationUI.cs | 12 ++---------- 8 files changed, 29 insertions(+), 25 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreRegexRedactedValueInCatalogue.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreRegexRedactedValueInCatalogue.cs index 4caffc644f..2e8a8feb69 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreRegexRedactedValueInCatalogue.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreRegexRedactedValueInCatalogue.cs @@ -69,7 +69,7 @@ public override void Execute() var updateHelper = server.GetQuerySyntaxHelper().UpdateHelper; var sqlLines = new List { - new CustomLine($"t1.{columnInfo.GetRuntimeName()} = '{newValue}'", QueryComponent.SET) + new($"t1.{columnInfo.GetRuntimeName()} = '{newValue}'", QueryComponent.SET) }; foreach (var rk in _redaction.RedactionKeys) { diff --git a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs index 104797ad5a..c8491c55ad 100644 --- a/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs +++ b/Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs @@ -113,8 +113,6 @@ public static void Redact(ColumnInfo column, DataRow match, List public static void SaveRedactions(ICatalogueRepository catalogueRepo, DiscoveredTable pksToSave, DiscoveredTable redactionsToSaveTable, DiscoveredServer _server, int timeout = 30000) { - //the update isn't working? and do we need the +1? - var sql = $@" DECLARE @output TABLE (id1 int, inc int IDENTITY(1,1)) INSERT INTO RegexRedaction(RedactionConfiguration_ID,ColumnInfo_ID,startingIndex,ReplacementValue,RedactedValue) OUTPUT inserted.id as id1 INTO @output diff --git a/Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs b/Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs index 630174e6fe..e128a73ba3 100644 --- a/Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs +++ b/Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs @@ -73,7 +73,7 @@ protected override void MutilateTable(IDataLoadJob job, ITableInfo tableInfo, Di { if (ColumnMatches(column)) { - var pkSeparator = pkColumnInfos.Count() > 0 ? "," : ""; + var pkSeparator = pkColumnInfos.Any() ? "," : ""; var sql = @$" SELECT {column.GetRuntimeName()} {pkSeparator} {string.Join(", ", pkColumnInfos.Select(c => c.GetRuntimeName()))} FROM {table.GetRuntimeName()} diff --git a/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs b/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs index 41a7a32adf..60bf6b0a6f 100644 --- a/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs +++ b/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs @@ -6,6 +6,7 @@ using System.Linq; using Rdmp.Core.Curation.Data; using Rdmp.Core.Providers.Nodes; +using Rdmp.Core.Curation.DataHelper.RegexRedaction; namespace Rdmp.UI.Collections; @@ -53,6 +54,9 @@ public void RefreshBus_RefreshObject(object sender, RefreshObjectEventArgs e) case Dataset: tlvConfigurations.RefreshObject(tlvConfigurations.Objects.OfType()); break; + case RegexRedactionConfiguration: + tlvConfigurations.RefreshObject(tlvConfigurations.Objects.OfType()); + break; } } diff --git a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandAddNewRegexRedactionConfigurationUI.cs b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandAddNewRegexRedactionConfigurationUI.cs index 4ed68a9510..494c41ee9a 100644 --- a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandAddNewRegexRedactionConfigurationUI.cs +++ b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandAddNewRegexRedactionConfigurationUI.cs @@ -1,14 +1,12 @@ -using Rdmp.Core.CommandExecution; -using Rdmp.Core.CommandExecution.AtomicCommands; +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using Rdmp.Core.CommandExecution; using Rdmp.UI.ItemActivation; -using Rdmp.UI.SimpleDialogs.Datasets; using Rdmp.UI.SimpleDialogs.RegexRedactionConfigurationForm; -using Rdmp.UI.TestsAndSetup.ServicePropogation; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace Rdmp.UI.CommandExecution.AtomicCommands { diff --git a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandRegexRedaction.cs b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandRegexRedaction.cs index d780cd57a8..0d27a28e2d 100644 --- a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandRegexRedaction.cs +++ b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandRegexRedaction.cs @@ -1,4 +1,10 @@ -using Rdmp.Core.CommandExecution.AtomicCommands; +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using Rdmp.Core.CommandExecution.AtomicCommands; using Rdmp.Core.Curation.Data; using Rdmp.Core.Icons.IconProvision; using Rdmp.Core.ReusableLibraryCode.Icons.IconProvision; diff --git a/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsRegexRedactionConfiguration.cs b/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsRegexRedactionConfiguration.cs index fbab8356be..e38b591d92 100644 --- a/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsRegexRedactionConfiguration.cs +++ b/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsRegexRedactionConfiguration.cs @@ -1,4 +1,10 @@ -using Rdmp.Core.CommandExecution; +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation.DataHelper.RegexRedaction; using Rdmp.Core.ReusableLibraryCode.Annotations; using Rdmp.UI.ItemActivation; diff --git a/Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationForm/CreateNewRegexRedactionConfigurationUI.cs b/Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationForm/CreateNewRegexRedactionConfigurationUI.cs index 064e88ad77..0f2118d49f 100644 --- a/Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationForm/CreateNewRegexRedactionConfigurationUI.cs +++ b/Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationForm/CreateNewRegexRedactionConfigurationUI.cs @@ -1,18 +1,10 @@ -using Rdmp.Core.CommandExecution.AtomicCommands; -using Rdmp.Core.Curation.DataHelper.RegexRedaction; -using Rdmp.UI.CommandExecution.AtomicCommands; +using Rdmp.Core.Curation.DataHelper.RegexRedaction; using Rdmp.UI.ItemActivation; using Rdmp.UI.TestsAndSetup.ServicePropogation; using System; -using System.Collections.Generic; -using System.ComponentModel; -using System.Data; using System.Drawing; -using System.Linq; -using System.Text; using System.Text.RegularExpressions; -using System.Threading.Tasks; -using System.Windows.Forms; + namespace Rdmp.UI.SimpleDialogs.RegexRedactionConfigurationForm { From 716b0fc604aae522f47e54c9472784300fedaa06 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 23 Oct 2024 08:35:59 +0100 Subject: [PATCH 093/106] add missing file --- Rdmp.UI/SimpleDialogs/SelectDialog`1.resx | 63 +++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 Rdmp.UI/SimpleDialogs/SelectDialog`1.resx diff --git a/Rdmp.UI/SimpleDialogs/SelectDialog`1.resx b/Rdmp.UI/SimpleDialogs/SelectDialog`1.resx new file mode 100644 index 0000000000..6252b1ef07 --- /dev/null +++ b/Rdmp.UI/SimpleDialogs/SelectDialog`1.resx @@ -0,0 +1,63 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + True + + \ No newline at end of file From 2bf9cebf49c8e3d5b03451e48be1fba8fc1ec6fb Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 24 Oct 2024 10:15:00 +0100 Subject: [PATCH 094/106] add changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac12fd6ed8..e767d0b328 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [8.4.0] - Unreleased - Add Ordering to Filters +- Update to use all primary keys when migrating data from staging -> live ## [8.3.1] - 2024-10-22 From d8201c503ca41d50c61e40f7266603695ad42314 Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 24 Oct 2024 10:18:10 +0100 Subject: [PATCH 095/106] update changelog --- CHANGELOG.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e767d0b328..2e78679b6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,14 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [8.4.0] - Unreleased - Add Ordering to Filters -- Update to use all primary keys when migrating data from staging -> live +- Update Migration strategy to account for all Primary Keys when moving from staging -> live ## [8.3.1] - 2024-10-22 - Improve Performance of regenerating problems with child providers - Update UI Tab opening Logic - Add Filter to Left-Hand Tree View -- Update Migration strategy to account for all Primary Keys when moving from staging -> live ## [8.3.0] - 2024-09-23 From f20c627de02ff41c57f6e36fcf537541cd1bc640 Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 31 Oct 2024 09:11:34 +0000 Subject: [PATCH 096/106] fix db update --- .../Databases/CatalogueDatabase/up/086_updateDataset.sql | 4 ++-- .../SimpleDialogs/Datasets/CreateNewPureConfigurationUI.cs | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Rdmp.Core/Databases/CatalogueDatabase/up/086_updateDataset.sql b/Rdmp.Core/Databases/CatalogueDatabase/up/086_updateDataset.sql index 008d2b3aa7..2d383b5643 100644 --- a/Rdmp.Core/Databases/CatalogueDatabase/up/086_updateDataset.sql +++ b/Rdmp.Core/Databases/CatalogueDatabase/up/086_updateDataset.sql @@ -1,4 +1,4 @@ ---Version: 8.3.1 +--Version: 8.4.0 --Description: Add new fields to the Dataset object @@ -19,7 +19,7 @@ END GO -if exists(select 1 from INFORMATION_SCHEMA.COLUMNS where TABLE_NAME='Dataset' and COLUMN_NAME='Url') +if not exists(select 1 from INFORMATION_SCHEMA.COLUMNS where TABLE_NAME='Dataset' and COLUMN_NAME='Provider_ID') BEGIN ALTER TABLE [dbo].[Dataset] ADD [Provider] [int] NULL; diff --git a/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureConfigurationUI.cs b/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureConfigurationUI.cs index 3761d45c0b..8c4384ae03 100644 --- a/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureConfigurationUI.cs +++ b/Rdmp.UI/SimpleDialogs/Datasets/CreateNewPureConfigurationUI.cs @@ -63,6 +63,7 @@ private void Save(object sender, EventArgs e) { var config = new DatasetProviderConfiguration(_activator.RepositoryLocator.CatalogueRepository, tbName.Text, typeof(PureDatasetProvider).ToString(), tbUrl.Text, ((DataAccessCredentials)cbCredentials.SelectedItem).ID, tbOrganisationId.Text); config.SaveToDatabase(); + _activator.Publish(config); Close(); _activator.Show($"Dataset Provider '{tbName.Text}' has successfully been created"); } From 7f4223d49e269b13bd3ff18d3c302ec21fbc15e6 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 6 Nov 2024 11:55:07 +0000 Subject: [PATCH 097/106] Bugfix/rdmp 259 delta load off by one issue (#2024) * fix off by one issue * update tests * update changelog --- CHANGELOG.md | 1 + .../Engine/Integration/RemoteAttacherTests.cs | 10 +++++----- .../DataLoad/Modules/Attachers/RemoteAttacher.cs | 12 ++++++------ 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e78679b6d..5fc793dce2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [8.4.0] - Unreleased - Add Ordering to Filters +- Fix Delta Load off by one issue - Update Migration strategy to account for all Primary Keys when moving from staging -> live ## [8.3.1] - 2024-10-22 diff --git a/Rdmp.Core.Tests/DataLoad/Engine/Integration/RemoteAttacherTests.cs b/Rdmp.Core.Tests/DataLoad/Engine/Integration/RemoteAttacherTests.cs index 726312b35e..ac771182e7 100644 --- a/Rdmp.Core.Tests/DataLoad/Engine/Integration/RemoteAttacherTests.cs +++ b/Rdmp.Core.Tests/DataLoad/Engine/Integration/RemoteAttacherTests.cs @@ -28,7 +28,7 @@ public void TestRemoteAttacherParameter(AttacherHistoricalDurations duration, st RemoteTableDateColumn = "date" }; var lmd = new LoadMetadata(); - Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo($" WHERE CAST(date as Date) >= DATEADD({convertTime}, -1, GETDATE())")); + Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo($" WHERE CAST(date as Date) > DATEADD({convertTime}, -1, GETDATE())")); } [Test] public void TestRemoteAttacherParameterSinceLastUse() @@ -42,7 +42,7 @@ public void TestRemoteAttacherParameterSinceLastUse() { LastLoadTime = DateTime.Now }; - Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo($" WHERE CAST(date as Date) >= convert(Date,'{lmd.LastLoadTime.GetValueOrDefault():yyyy-MM-dd HH:mm:ss.fff}')")); + Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo($" WHERE CAST(date as Date) > convert(Date,'{lmd.LastLoadTime.GetValueOrDefault():yyyy-MM-dd HH:mm:ss.fff}')")); } [Test] public void TestRemoteAttacherParameterSinceLastUse_NULL() @@ -130,7 +130,7 @@ public void TestRemoteAttacherParameterDeltaReading() { LastLoadTime = DateTime.Now }; - Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo($" WHERE CAST(date as Date) >= convert(Date,'{attacher.DeltaReadingStartDate.AddDays(-attacher.DeltaReadingLookBackDays):yyyy-MM-dd HH:mm:ss.fff}') AND CAST(date as Date) <= convert(Date,'{attacher.DeltaReadingStartDate.AddDays(attacher.DeltaReadingLookForwardDays):yyyy-MM-dd HH:mm:ss.fff}')")); + Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo($" WHERE CAST(date as Date) >= convert(Date,'{attacher.DeltaReadingStartDate.AddDays(-attacher.DeltaReadingLookBackDays):yyyy-MM-dd HH:mm:ss.fff}') AND CAST(date as Date) < convert(Date,'{attacher.DeltaReadingStartDate.AddDays(attacher.DeltaReadingLookForwardDays):yyyy-MM-dd HH:mm:ss.fff}')")); } [Test] public void TestRemoteAttacherParameterDeltaReading_NoLookBack() @@ -146,7 +146,7 @@ public void TestRemoteAttacherParameterDeltaReading_NoLookBack() { LastLoadTime = DateTime.Now }; - Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo($" WHERE CAST(date as Date) >= convert(Date,'{attacher.DeltaReadingStartDate:yyyy-MM-dd HH:mm:ss.fff}') AND CAST(date as Date) <= convert(Date,'{attacher.DeltaReadingStartDate.AddDays(attacher.DeltaReadingLookForwardDays):yyyy-MM-dd HH:mm:ss.fff}')")); + Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo($" WHERE CAST(date as Date) >= convert(Date,'{attacher.DeltaReadingStartDate:yyyy-MM-dd HH:mm:ss.fff}') AND CAST(date as Date) < convert(Date,'{attacher.DeltaReadingStartDate.AddDays(attacher.DeltaReadingLookForwardDays):yyyy-MM-dd HH:mm:ss.fff}')")); } [Test] public void TestRemoteAttacherParameterDeltaReading_NoLookForward() @@ -162,7 +162,7 @@ public void TestRemoteAttacherParameterDeltaReading_NoLookForward() { LastLoadTime = DateTime.Now }; - Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo($" WHERE CAST(date as Date) >= convert(Date,'{attacher.DeltaReadingStartDate.AddDays(-attacher.DeltaReadingLookBackDays):yyyy-MM-dd HH:mm:ss.fff}') AND CAST(date as Date) <= convert(Date,'{attacher.DeltaReadingStartDate:yyyy-MM-dd HH:mm:ss.fff}')")); + Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo($" WHERE CAST(date as Date) >= convert(Date,'{attacher.DeltaReadingStartDate.AddDays(-attacher.DeltaReadingLookBackDays):yyyy-MM-dd HH:mm:ss.fff}') AND CAST(date as Date) < convert(Date,'{attacher.DeltaReadingStartDate:yyyy-MM-dd HH:mm:ss.fff}')")); } [Test] public void TestRemoteAttacherParameterDeltaReadingNoDates() diff --git a/Rdmp.Core/DataLoad/Modules/Attachers/RemoteAttacher.cs b/Rdmp.Core/DataLoad/Modules/Attachers/RemoteAttacher.cs index 2c39517b77..64a4ef0788 100644 --- a/Rdmp.Core/DataLoad/Modules/Attachers/RemoteAttacher.cs +++ b/Rdmp.Core/DataLoad/Modules/Attachers/RemoteAttacher.cs @@ -100,15 +100,15 @@ public string SqlHistoricalDataFilter(ILoadMetadata loadMetadata, DatabaseType d switch (HistoricalFetchDuration) { case AttacherHistoricalDurations.Past24Hours: - return $" WHERE CAST({RemoteTableDateColumn} as {dateConvert}) >= {GetCorrectDateAddForDatabaseType(dbType, "DAY", "-1")}"; + return $" WHERE CAST({RemoteTableDateColumn} as {dateConvert}) > {GetCorrectDateAddForDatabaseType(dbType, "DAY", "-1")}"; case AttacherHistoricalDurations.Past7Days: - return $" WHERE CAST({RemoteTableDateColumn} as {dateConvert}) >= {GetCorrectDateAddForDatabaseType(dbType, "WEEK", "-1")}"; + return $" WHERE CAST({RemoteTableDateColumn} as {dateConvert}) > {GetCorrectDateAddForDatabaseType(dbType, "WEEK", "-1")}"; case AttacherHistoricalDurations.PastMonth: - return $" WHERE CAST({RemoteTableDateColumn} as {dateConvert}) >= {GetCorrectDateAddForDatabaseType(dbType, "MONTH", "-1")}"; + return $" WHERE CAST({RemoteTableDateColumn} as {dateConvert}) > {GetCorrectDateAddForDatabaseType(dbType, "MONTH", "-1")}"; case AttacherHistoricalDurations.PastYear: - return $" WHERE CAST({RemoteTableDateColumn} as {dateConvert}) >= {GetCorrectDateAddForDatabaseType(dbType, "YEAR", "-1")}"; + return $" WHERE CAST({RemoteTableDateColumn} as {dateConvert}) > {GetCorrectDateAddForDatabaseType(dbType, "YEAR", "-1")}"; case AttacherHistoricalDurations.SinceLastUse: - return loadMetadata.LastLoadTime is not null ? $" WHERE CAST({RemoteTableDateColumn} as {dateConvert}) >= {ConvertDateString(dbType, loadMetadata.LastLoadTime.GetValueOrDefault().ToString(RemoteTableDateFormat))}" : ""; + return loadMetadata.LastLoadTime is not null ? $" WHERE CAST({RemoteTableDateColumn} as {dateConvert}) > {ConvertDateString(dbType, loadMetadata.LastLoadTime.GetValueOrDefault().ToString(RemoteTableDateFormat))}" : ""; case AttacherHistoricalDurations.Custom: if (CustomFetchDurationStartDate == DateTime.MinValue && CustomFetchDurationEndDate != DateTime.MinValue) { @@ -133,7 +133,7 @@ public string SqlHistoricalDataFilter(ILoadMetadata loadMetadata, DatabaseType d if (DeltaReadingStartDate == DateTime.MinValue) return ""; var startDate = DeltaReadingStartDate.AddDays(-DeltaReadingLookBackDays); var endDate = DeltaReadingStartDate.AddDays(DeltaReadingLookForwardDays); - return $" WHERE CAST({RemoteTableDateColumn} as {dateConvert}) >= {ConvertDateString(dbType, startDate.ToString(RemoteTableDateFormat))} AND CAST({RemoteTableDateColumn} as {dateConvert}) <= {ConvertDateString(dbType, endDate.ToString(RemoteTableDateFormat))}"; + return $" WHERE CAST({RemoteTableDateColumn} as {dateConvert}) >= {ConvertDateString(dbType, startDate.ToString(RemoteTableDateFormat))} AND CAST({RemoteTableDateColumn} as {dateConvert}) < {ConvertDateString(dbType, endDate.ToString(RemoteTableDateFormat))}"; default: return ""; } From dbdb2b1237359e71c72815df32c10a500c0bea11 Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 8 Nov 2024 08:25:05 +0000 Subject: [PATCH 098/106] check for multi-server query --- Rdmp.UI/DataViewing/ViewSQLAndResultsWithDataGridUI.cs | 3 ++- SharedAssemblyInfo.cs | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Rdmp.UI/DataViewing/ViewSQLAndResultsWithDataGridUI.cs b/Rdmp.UI/DataViewing/ViewSQLAndResultsWithDataGridUI.cs index fb482e7411..0681d5fa83 100644 --- a/Rdmp.UI/DataViewing/ViewSQLAndResultsWithDataGridUI.cs +++ b/Rdmp.UI/DataViewing/ViewSQLAndResultsWithDataGridUI.cs @@ -167,8 +167,9 @@ private void RefreshUIFromDatabase() { try { + var dbAccessPoint = _collection.GetDataAccessPoint(); _server = DataAccessPortal - .ExpectServer(_collection.GetDataAccessPoint(), DataAccessContext.InternalDataProcessing); + .ExpectServer(dbAccessPoint, DataAccessContext.InternalDataProcessing, dbAccessPoint.Database != null); var sql = _collection.GetSql(); _originalSql = sql; diff --git a/SharedAssemblyInfo.cs b/SharedAssemblyInfo.cs index 142b75ae4f..f00dd03538 100644 --- a/SharedAssemblyInfo.cs +++ b/SharedAssemblyInfo.cs @@ -12,4 +12,4 @@ [assembly: AssemblyVersion("8.3.1")] [assembly: AssemblyFileVersion("8.3.1")] -[assembly: AssemblyInformationalVersion("8.3.1")] \ No newline at end of file +[assembly: AssemblyInformationalVersion("8.3.1-view-sql-test-r266")] \ No newline at end of file From c73f5955b76e861db61f21c01672c3fc2a3a303b Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 11 Nov 2024 15:52:44 +0000 Subject: [PATCH 099/106] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5fc793dce2..da9b4dc62c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add Ordering to Filters - Fix Delta Load off by one issue - Update Migration strategy to account for all Primary Keys when moving from staging -> live +- Fix UI issue with viewing cross-server SQL results ## [8.3.1] - 2024-10-22 From 9e461f7fbb42e9b81c8c5784889ae968ce3c2442 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 11 Nov 2024 15:53:27 +0000 Subject: [PATCH 100/106] update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da9b4dc62c..e7baeb259f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add Ordering to Filters - Fix Delta Load off by one issue - Update Migration strategy to account for all Primary Keys when moving from staging -> live -- Fix UI issue with viewing cross-server SQL results +- Fix UI issue with viewing cross-database SQL results ## [8.3.1] - 2024-10-22 From 0779ce357aca50e214c80b88ad64d256f8ac68c4 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 12 Nov 2024 15:43:09 +0000 Subject: [PATCH 101/106] Add override for RAW table date column in delta loads (#2052) * fix table attacher special column issue * add override * add extra override * fix tests * tidy up * add docs --- CHANGELOG.md | 1 + .../DataLoadEngine/RemoteAttachers.md | 1 + .../Engine/Integration/RemoteAttacherTests.cs | 22 ++++++------- .../Operations/TableInfoCloneOperation.cs | 4 --- .../Modules/Attachers/RemoteAttacher.cs | 32 ++++++++++++------- .../Attachers/RemoteDatabaseAttacher.cs | 4 +-- .../Modules/Attachers/RemoteTableAttacher.cs | 6 ++-- 7 files changed, 38 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e7baeb259f..75dd40444c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [8.4.0] - Unreleased - Add Ordering to Filters +- Add RAW Table Date Column Override for Delta Loads - Fix Delta Load off by one issue - Update Migration strategy to account for all Primary Keys when moving from staging -> live - Fix UI issue with viewing cross-database SQL results diff --git a/Documentation/DataLoadEngine/RemoteAttachers.md b/Documentation/DataLoadEngine/RemoteAttachers.md index 2026d3bb7a..31443d96bd 100644 --- a/Documentation/DataLoadEngine/RemoteAttachers.md +++ b/Documentation/DataLoadEngine/RemoteAttachers.md @@ -50,6 +50,7 @@ The full configuration options are | Culture | Optionally specify a custom date format | | Explicit Date Time Format | Optionally specify a specific datetime format | Selected Columns | Optionally select which columns you wish to pull from the remote server (defaults to "*" ) +| RawTableDateColumn | Optionally give the date column in RAW a different name than in the select statement for delta loads. This may be due to awkward joins in a custom select query ## Configuring the Remote Database Attacher The Remote Database Attacher has a number of configuration options the required fields are: diff --git a/Rdmp.Core.Tests/DataLoad/Engine/Integration/RemoteAttacherTests.cs b/Rdmp.Core.Tests/DataLoad/Engine/Integration/RemoteAttacherTests.cs index ac771182e7..733ee90aee 100644 --- a/Rdmp.Core.Tests/DataLoad/Engine/Integration/RemoteAttacherTests.cs +++ b/Rdmp.Core.Tests/DataLoad/Engine/Integration/RemoteAttacherTests.cs @@ -28,7 +28,7 @@ public void TestRemoteAttacherParameter(AttacherHistoricalDurations duration, st RemoteTableDateColumn = "date" }; var lmd = new LoadMetadata(); - Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo($" WHERE CAST(date as Date) > DATEADD({convertTime}, -1, GETDATE())")); + Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer,attacher.RemoteTableDateColumn), Is.EqualTo($" WHERE CAST(date as Date) > DATEADD({convertTime}, -1, GETDATE())")); } [Test] public void TestRemoteAttacherParameterSinceLastUse() @@ -42,7 +42,7 @@ public void TestRemoteAttacherParameterSinceLastUse() { LastLoadTime = DateTime.Now }; - Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo($" WHERE CAST(date as Date) > convert(Date,'{lmd.LastLoadTime.GetValueOrDefault():yyyy-MM-dd HH:mm:ss.fff}')")); + Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer,attacher.RemoteTableDateColumn), Is.EqualTo($" WHERE CAST(date as Date) > convert(Date,'{lmd.LastLoadTime.GetValueOrDefault():yyyy-MM-dd HH:mm:ss.fff}')")); } [Test] public void TestRemoteAttacherParameterSinceLastUse_NULL() @@ -53,7 +53,7 @@ public void TestRemoteAttacherParameterSinceLastUse_NULL() RemoteTableDateColumn = "date" }; var lmd = new LoadMetadata(); - Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo("")); + Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer, attacher.RemoteTableDateColumn), Is.EqualTo("")); } [Test] public void TestRemoteAttacherParameterCustomRange() @@ -69,7 +69,7 @@ public void TestRemoteAttacherParameterCustomRange() { LastLoadTime = DateTime.Now }; - Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo($" WHERE CAST(date as Date) >= convert(Date,'{attacher.CustomFetchDurationStartDate:yyyy-MM-dd HH:mm:ss.fff}') AND CAST(date as Date) <= convert(Date,'{attacher.CustomFetchDurationEndDate:yyyy-MM-dd HH:mm:ss.fff}')")); + Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer, attacher.RemoteTableDateColumn), Is.EqualTo($" WHERE CAST(date as Date) >= convert(Date,'{attacher.CustomFetchDurationStartDate:yyyy-MM-dd HH:mm:ss.fff}') AND CAST(date as Date) <= convert(Date,'{attacher.CustomFetchDurationEndDate:yyyy-MM-dd HH:mm:ss.fff}')")); } [Test] public void TestRemoteAttacherParameterCustomRangeNoStart() @@ -84,7 +84,7 @@ public void TestRemoteAttacherParameterCustomRangeNoStart() { LastLoadTime = DateTime.Now }; - Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo($" WHERE CAST(date as Date) <= convert(Date,'{attacher.CustomFetchDurationEndDate:yyyy-MM-dd HH:mm:ss.fff}')")); + Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer, attacher.RemoteTableDateColumn), Is.EqualTo($" WHERE CAST(date as Date) <= convert(Date,'{attacher.CustomFetchDurationEndDate:yyyy-MM-dd HH:mm:ss.fff}')")); } [Test] public void TestRemoteAttacherParameterCustomRangeNoEnd() @@ -99,7 +99,7 @@ public void TestRemoteAttacherParameterCustomRangeNoEnd() { LastLoadTime = DateTime.Now }; - Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo($" WHERE CAST(date as Date) >= convert(Date,'{attacher.CustomFetchDurationStartDate:yyyy-MM-dd HH:mm:ss.fff}')")); + Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer, attacher.RemoteTableDateColumn), Is.EqualTo($" WHERE CAST(date as Date) >= convert(Date,'{attacher.CustomFetchDurationStartDate:yyyy-MM-dd HH:mm:ss.fff}')")); } [Test] public void TestRemoteAttacherParameterCustomRangeNoDates() @@ -113,7 +113,7 @@ public void TestRemoteAttacherParameterCustomRangeNoDates() { LastLoadTime = DateTime.Now }; - Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo("")); + Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer, attacher.RemoteTableDateColumn), Is.EqualTo("")); } [Test] public void TestRemoteAttacherParameterDeltaReading() @@ -130,7 +130,7 @@ public void TestRemoteAttacherParameterDeltaReading() { LastLoadTime = DateTime.Now }; - Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo($" WHERE CAST(date as Date) >= convert(Date,'{attacher.DeltaReadingStartDate.AddDays(-attacher.DeltaReadingLookBackDays):yyyy-MM-dd HH:mm:ss.fff}') AND CAST(date as Date) < convert(Date,'{attacher.DeltaReadingStartDate.AddDays(attacher.DeltaReadingLookForwardDays):yyyy-MM-dd HH:mm:ss.fff}')")); + Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer,attacher.RemoteTableDateColumn), Is.EqualTo($" WHERE CAST(date as Date) >= convert(Date,'{attacher.DeltaReadingStartDate.AddDays(-attacher.DeltaReadingLookBackDays):yyyy-MM-dd HH:mm:ss.fff}') AND CAST(date as Date) < convert(Date,'{attacher.DeltaReadingStartDate.AddDays(attacher.DeltaReadingLookForwardDays):yyyy-MM-dd HH:mm:ss.fff}')")); } [Test] public void TestRemoteAttacherParameterDeltaReading_NoLookBack() @@ -146,7 +146,7 @@ public void TestRemoteAttacherParameterDeltaReading_NoLookBack() { LastLoadTime = DateTime.Now }; - Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo($" WHERE CAST(date as Date) >= convert(Date,'{attacher.DeltaReadingStartDate:yyyy-MM-dd HH:mm:ss.fff}') AND CAST(date as Date) < convert(Date,'{attacher.DeltaReadingStartDate.AddDays(attacher.DeltaReadingLookForwardDays):yyyy-MM-dd HH:mm:ss.fff}')")); + Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer,attacher.RemoteTableDateColumn), Is.EqualTo($" WHERE CAST(date as Date) >= convert(Date,'{attacher.DeltaReadingStartDate:yyyy-MM-dd HH:mm:ss.fff}') AND CAST(date as Date) < convert(Date,'{attacher.DeltaReadingStartDate.AddDays(attacher.DeltaReadingLookForwardDays):yyyy-MM-dd HH:mm:ss.fff}')")); } [Test] public void TestRemoteAttacherParameterDeltaReading_NoLookForward() @@ -162,7 +162,7 @@ public void TestRemoteAttacherParameterDeltaReading_NoLookForward() { LastLoadTime = DateTime.Now }; - Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo($" WHERE CAST(date as Date) >= convert(Date,'{attacher.DeltaReadingStartDate.AddDays(-attacher.DeltaReadingLookBackDays):yyyy-MM-dd HH:mm:ss.fff}') AND CAST(date as Date) < convert(Date,'{attacher.DeltaReadingStartDate:yyyy-MM-dd HH:mm:ss.fff}')")); + Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer,attacher.RemoteTableDateColumn), Is.EqualTo($" WHERE CAST(date as Date) >= convert(Date,'{attacher.DeltaReadingStartDate.AddDays(-attacher.DeltaReadingLookBackDays):yyyy-MM-dd HH:mm:ss.fff}') AND CAST(date as Date) < convert(Date,'{attacher.DeltaReadingStartDate:yyyy-MM-dd HH:mm:ss.fff}')")); } [Test] public void TestRemoteAttacherParameterDeltaReadingNoDates() @@ -173,6 +173,6 @@ public void TestRemoteAttacherParameterDeltaReadingNoDates() RemoteTableDateColumn = "date" }; var lmd = new LoadMetadata(); - Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo("")); + Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer, attacher.RemoteTableDateColumn), Is.EqualTo("")); } } diff --git a/Rdmp.Core/DataLoad/Engine/DatabaseManagement/Operations/TableInfoCloneOperation.cs b/Rdmp.Core/DataLoad/Engine/DatabaseManagement/Operations/TableInfoCloneOperation.cs index db2feda611..0e0c8fd74d 100644 --- a/Rdmp.Core/DataLoad/Engine/DatabaseManagement/Operations/TableInfoCloneOperation.cs +++ b/Rdmp.Core/DataLoad/Engine/DatabaseManagement/Operations/TableInfoCloneOperation.cs @@ -156,10 +156,6 @@ public void CloneTable(DiscoveredDatabase srcDatabaseInfo, DiscoveredDatabase de drop = true; } - //drop the data load run ID field and validFrom fields, we don't need them in STAGING or RAW, it will be hard coded in the MERGE migration with a fixed value anyway. - if (colName.Equals(SpecialFieldNames.DataLoadRunID) || colName.Equals(SpecialFieldNames.ValidFrom)) - drop = true; - var dilution = dilutionColumns.SingleOrDefault(c => c.GetRuntimeName().Equals(colName)); if (dilution != null) diff --git a/Rdmp.Core/DataLoad/Modules/Attachers/RemoteAttacher.cs b/Rdmp.Core/DataLoad/Modules/Attachers/RemoteAttacher.cs index 64a4ef0788..0a86fcc5ed 100644 --- a/Rdmp.Core/DataLoad/Modules/Attachers/RemoteAttacher.cs +++ b/Rdmp.Core/DataLoad/Modules/Attachers/RemoteAttacher.cs @@ -31,6 +31,9 @@ public RemoteAttacher() : base(true) { } [DemandsInitialization("Which column in the remote table can be used to perform time-based data selection")] public string RemoteTableDateColumn { get; set; } + [DemandsInitialization("Option name for the Date colunn table within RDMP if it differs from the remote table e.g Due to joins")] + public string RawTableDateColumn { get; set; } + private readonly string RemoteTableDateFormat = "yyyy-MM-dd HH:mm:ss.fff"; [DemandsInitialization("Earliest date when using a custom fetch duration")] @@ -93,33 +96,33 @@ private string ConvertDateString(DatabaseType dbType, string dateString) } - public string SqlHistoricalDataFilter(ILoadMetadata loadMetadata, DatabaseType dbType) + public string SqlHistoricalDataFilter(ILoadMetadata loadMetadata, DatabaseType dbType, string column) { const string dateConvert = "Date"; switch (HistoricalFetchDuration) { case AttacherHistoricalDurations.Past24Hours: - return $" WHERE CAST({RemoteTableDateColumn} as {dateConvert}) > {GetCorrectDateAddForDatabaseType(dbType, "DAY", "-1")}"; + return $" WHERE CAST({column} as {dateConvert}) > {GetCorrectDateAddForDatabaseType(dbType, "DAY", "-1")}"; case AttacherHistoricalDurations.Past7Days: - return $" WHERE CAST({RemoteTableDateColumn} as {dateConvert}) > {GetCorrectDateAddForDatabaseType(dbType, "WEEK", "-1")}"; + return $" WHERE CAST({column} as {dateConvert}) > {GetCorrectDateAddForDatabaseType(dbType, "WEEK", "-1")}"; case AttacherHistoricalDurations.PastMonth: - return $" WHERE CAST({RemoteTableDateColumn} as {dateConvert}) > {GetCorrectDateAddForDatabaseType(dbType, "MONTH", "-1")}"; + return $" WHERE CAST({column} as {dateConvert}) > {GetCorrectDateAddForDatabaseType(dbType, "MONTH", "-1")}"; case AttacherHistoricalDurations.PastYear: - return $" WHERE CAST({RemoteTableDateColumn} as {dateConvert}) > {GetCorrectDateAddForDatabaseType(dbType, "YEAR", "-1")}"; + return $" WHERE CAST({column} as {dateConvert}) > {GetCorrectDateAddForDatabaseType(dbType, "YEAR", "-1")}"; case AttacherHistoricalDurations.SinceLastUse: - return loadMetadata.LastLoadTime is not null ? $" WHERE CAST({RemoteTableDateColumn} as {dateConvert}) > {ConvertDateString(dbType, loadMetadata.LastLoadTime.GetValueOrDefault().ToString(RemoteTableDateFormat))}" : ""; + return loadMetadata.LastLoadTime is not null ? $" WHERE CAST({column} as {dateConvert}) > {ConvertDateString(dbType, loadMetadata.LastLoadTime.GetValueOrDefault().ToString(RemoteTableDateFormat))}" : ""; case AttacherHistoricalDurations.Custom: if (CustomFetchDurationStartDate == DateTime.MinValue && CustomFetchDurationEndDate != DateTime.MinValue) { //end only - return $" WHERE CAST({RemoteTableDateColumn} as {dateConvert}) <= {ConvertDateString(dbType, CustomFetchDurationEndDate.ToString(RemoteTableDateFormat))}"; + return $" WHERE CAST({column} as {dateConvert}) <= {ConvertDateString(dbType, CustomFetchDurationEndDate.ToString(RemoteTableDateFormat))}"; } if (CustomFetchDurationStartDate != DateTime.MinValue && CustomFetchDurationEndDate == DateTime.MinValue) { //start only - return $" WHERE CAST({RemoteTableDateColumn} as {dateConvert}) >= {ConvertDateString(dbType, CustomFetchDurationStartDate.ToString(RemoteTableDateFormat))}"; + return $" WHERE CAST({column} as {dateConvert}) >= {ConvertDateString(dbType, CustomFetchDurationStartDate.ToString(RemoteTableDateFormat))}"; } if (CustomFetchDurationStartDate == DateTime.MinValue && CustomFetchDurationEndDate == DateTime.MinValue) @@ -128,12 +131,12 @@ public string SqlHistoricalDataFilter(ILoadMetadata loadMetadata, DatabaseType d return ""; } - return $" WHERE CAST({RemoteTableDateColumn} as {dateConvert}) >= {ConvertDateString(dbType, CustomFetchDurationStartDate.ToString(RemoteTableDateFormat))} AND CAST({RemoteTableDateColumn} as {dateConvert}) <= {ConvertDateString(dbType, CustomFetchDurationEndDate.ToString(RemoteTableDateFormat))}"; + return $" WHERE CAST({column} as {dateConvert}) >= {ConvertDateString(dbType, CustomFetchDurationStartDate.ToString(RemoteTableDateFormat))} AND CAST({column} as {dateConvert}) <= {ConvertDateString(dbType, CustomFetchDurationEndDate.ToString(RemoteTableDateFormat))}"; case AttacherHistoricalDurations.DeltaReading: if (DeltaReadingStartDate == DateTime.MinValue) return ""; var startDate = DeltaReadingStartDate.AddDays(-DeltaReadingLookBackDays); var endDate = DeltaReadingStartDate.AddDays(DeltaReadingLookForwardDays); - return $" WHERE CAST({RemoteTableDateColumn} as {dateConvert}) >= {ConvertDateString(dbType, startDate.ToString(RemoteTableDateFormat))} AND CAST({RemoteTableDateColumn} as {dateConvert}) < {ConvertDateString(dbType, endDate.ToString(RemoteTableDateFormat))}"; + return $" WHERE CAST({column} as {dateConvert}) >= {ConvertDateString(dbType, startDate.ToString(RemoteTableDateFormat))} AND CAST({column} as {dateConvert}) < {ConvertDateString(dbType, endDate.ToString(RemoteTableDateFormat))}"; default: return ""; } @@ -173,9 +176,14 @@ private bool IsThisRemoteAttacher(IProcessTask task) return true; } - public void FindMostRecentDateInLoadedData(IQuerySyntaxHelper syntaxFrom, DatabaseType dbType, string table, IDataLoadJob job) + public void FindMostRecentDateInLoadedData(IQuerySyntaxHelper syntaxFrom, DatabaseType dbType, string table, IDataLoadJob job, bool userOverrideColumn = false) { - var maxDateSql = $"SELECT MAX({RemoteTableDateColumn}) FROM {syntaxFrom.EnsureWrapped(table)} {SqlHistoricalDataFilter(job.LoadMetadata, dbType)}"; + var column = RemoteTableDateColumn; + if (userOverrideColumn && !string.IsNullOrWhiteSpace(RawTableDateColumn)) + { + column = RawTableDateColumn; + } + var maxDateSql = $"SELECT MAX({column}) FROM {syntaxFrom.EnsureWrapped(table)} {SqlHistoricalDataFilter(job.LoadMetadata, dbType, column)}"; using var con = _dbInfo.Server.GetConnection(); using var dt = new DataTable(); diff --git a/Rdmp.Core/DataLoad/Modules/Attachers/RemoteDatabaseAttacher.cs b/Rdmp.Core/DataLoad/Modules/Attachers/RemoteDatabaseAttacher.cs index 0bc8380fed..90617a7752 100644 --- a/Rdmp.Core/DataLoad/Modules/Attachers/RemoteDatabaseAttacher.cs +++ b/Rdmp.Core/DataLoad/Modules/Attachers/RemoteDatabaseAttacher.cs @@ -104,11 +104,11 @@ public override ExitCodeType Attach(IDataLoadJob job, GracefulCancellationToken ? tableInfo.GetColumnsAtStage(LoadStage.AdjustRaw) : tableInfo.ColumnInfos; sql = - $"SELECT {string.Join(",", rawColumns.Select(c => syntaxFrom.EnsureWrapped(c.GetRuntimeName(LoadStage.AdjustRaw))))} FROM {syntaxFrom.EnsureWrapped(table)} {SqlHistoricalDataFilter(job.LoadMetadata, dbFrom.Server.DatabaseType)}"; + $"SELECT {string.Join(",", rawColumns.Select(c => syntaxFrom.EnsureWrapped(c.GetRuntimeName(LoadStage.AdjustRaw))))} FROM {syntaxFrom.EnsureWrapped(table)} {SqlHistoricalDataFilter(job.LoadMetadata, dbFrom.Server.DatabaseType,RemoteTableDateColumn)}"; } else { - sql = $"SELECT * FROM {syntaxFrom.EnsureWrapped(table)} {SqlHistoricalDataFilter(job.LoadMetadata,RemoteSource.DatabaseType)}"; + sql = $"SELECT * FROM {syntaxFrom.EnsureWrapped(table)} {SqlHistoricalDataFilter(job.LoadMetadata,RemoteSource.DatabaseType, RemoteTableDateColumn)}"; } job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"About to execute SQL:{Environment.NewLine}{sql}")); diff --git a/Rdmp.Core/DataLoad/Modules/Attachers/RemoteTableAttacher.cs b/Rdmp.Core/DataLoad/Modules/Attachers/RemoteTableAttacher.cs index 781cd5bd2d..51f231e79f 100644 --- a/Rdmp.Core/DataLoad/Modules/Attachers/RemoteTableAttacher.cs +++ b/Rdmp.Core/DataLoad/Modules/Attachers/RemoteTableAttacher.cs @@ -364,12 +364,12 @@ public override ExitCodeType Attach(IDataLoadJob job, GracefulCancellationToken string sql; if (!string.IsNullOrWhiteSpace(RemoteSelectSQL)) { - var injectedWhereClause = SqlHistoricalDataFilter(job.LoadMetadata, DatabaseType).Replace(" WHERE", ""); + var injectedWhereClause = SqlHistoricalDataFilter(job.LoadMetadata, DatabaseType, RemoteTableDateColumn).Replace(" WHERE", ""); sql = Regex.Replace(RemoteSelectSQL, "\\$RDMPDefinedWhereClause", injectedWhereClause); } else { - sql = $"Select {SelectedColumns} from {syntax.EnsureWrapped(RemoteTableName)} {SqlHistoricalDataFilter(job.LoadMetadata, DatabaseType)}"; + sql = $"Select {SelectedColumns} from {syntax.EnsureWrapped(RemoteTableName)} {SqlHistoricalDataFilter(job.LoadMetadata, DatabaseType, RemoteTableDateColumn)}"; } var scheduleMismatch = false; @@ -466,7 +466,7 @@ public override ExitCodeType Attach(IDataLoadJob job, GracefulCancellationToken if (SetDeltaReadingToLatestSeenDatePostLoad) { - FindMostRecentDateInLoadedData(rawSyntax, _dbInfo.Server.DatabaseType, rawTableName, job); + FindMostRecentDateInLoadedData(rawSyntax, _dbInfo.Server.DatabaseType, rawTableName, job,true); } From a81d859bcfd47226ce9dd975fbf393ca95e38ef7 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 19 Nov 2024 11:01:39 +0000 Subject: [PATCH 102/106] update deps --- Directory.Packages.props | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 8b927c09dc..854c027214 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -1,9 +1,9 @@ - - - - + + + + @@ -20,17 +20,17 @@ - + - + - + @@ -39,7 +39,7 @@ - + From 7bd6a333593e4f1e93edb6222cad3eca06a12490 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 19 Nov 2024 13:06:42 +0000 Subject: [PATCH 103/106] add safe ref --- Directory.Packages.props | 1 + 1 file changed, 1 insertion(+) diff --git a/Directory.Packages.props b/Directory.Packages.props index 854c027214..24e3b964b4 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -24,6 +24,7 @@ + From 5fbe4ae6b37b93fa93326e8d8711e61cb7f54ec3 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 25 Nov 2024 10:24:57 +0000 Subject: [PATCH 104/106] add links --- .../PostLoad/PureDatasetPostLoadUpdater.cs | 1 + Rdmp.Core/Datasets/PureDataset.cs | 3 + Rdmp.Core/Datasets/PureDatasetProvider.cs | 42 +----- .../PureDatasetConfigurationUI.Designer.cs | 29 ++++- .../PureDatasetConfigurationUI.cs | 37 +++++- .../PureDatasetLinkUI.Designer.cs | 68 ++++++++++ Rdmp.UI/SubComponents/PureDatasetLinkUI.cs | 45 +++++++ Rdmp.UI/SubComponents/PureDatasetLinkUI.resx | 120 ++++++++++++++++++ 8 files changed, 301 insertions(+), 44 deletions(-) create mode 100644 Rdmp.UI/SubComponents/PureDatasetLinkUI.Designer.cs create mode 100644 Rdmp.UI/SubComponents/PureDatasetLinkUI.cs create mode 100644 Rdmp.UI/SubComponents/PureDatasetLinkUI.resx diff --git a/Rdmp.Core/DataLoad/Modules/Mutilators/PostLoad/PureDatasetPostLoadUpdater.cs b/Rdmp.Core/DataLoad/Modules/Mutilators/PostLoad/PureDatasetPostLoadUpdater.cs index c76eb21f85..48bc51023f 100644 --- a/Rdmp.Core/DataLoad/Modules/Mutilators/PostLoad/PureDatasetPostLoadUpdater.cs +++ b/Rdmp.Core/DataLoad/Modules/Mutilators/PostLoad/PureDatasetPostLoadUpdater.cs @@ -119,6 +119,7 @@ public ExitCodeType Mutilate(IDataLoadJob job) _provider = new PureDatasetProvider(new ThrowImmediatelyActivator(job.RepositoryLocator), job.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("ID", Dataset.Provider_ID).First()); _pureDataset = _provider.FetchPureDataset(Dataset); var datasetUpdate = new PureDataset(); + datasetUpdate.Links = null; if (UpdateDescription) { datasetUpdate.Descriptions = GetDescriptions(); diff --git a/Rdmp.Core/Datasets/PureDataset.cs b/Rdmp.Core/Datasets/PureDataset.cs index b5a653b877..4503ea73d7 100644 --- a/Rdmp.Core/Datasets/PureDataset.cs +++ b/Rdmp.Core/Datasets/PureDataset.cs @@ -90,6 +90,9 @@ public class PureDataset : PluginDataset public TemporalCoveragePeriod? TemporalCoveragePeriod { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List Links { get; set; } + #nullable disable public PureDataset() { } diff --git a/Rdmp.Core/Datasets/PureDatasetProvider.cs b/Rdmp.Core/Datasets/PureDatasetProvider.cs index b85e2f6c81..5470f05911 100644 --- a/Rdmp.Core/Datasets/PureDatasetProvider.cs +++ b/Rdmp.Core/Datasets/PureDatasetProvider.cs @@ -69,46 +69,6 @@ public override void AddExistingDataset(string name, string url) } } - //public Curation.Data.Datasets.Dataset Create(string title, string publisherUid, List people, Visibility visibility, PureDate publicationDate) - //{ - // //this works but i'm not sure it's linking up with the person correctly - - // var pureDataset = new PureDataset(); - // pureDataset.Title = new ENGBWrapper(title); - // pureDataset.ManagingOrganization = new PureSystem(Configuration.Organisation_ID, "Organization"); - // pureDataset.Type = new URITerm("/dk/atira/pure/dataset/datasettypes/dataset/dataset", new ENGBWrapper("Dataset")); - // pureDataset.Publisher = new PureSystem(publisherUid, "Publisher"); - - // pureDataset.Persons = people; - // pureDataset.Visibility = visibility; - // pureDataset.Organizations = [new PureSystem(Configuration.Organisation_ID, "Organization")]; - // pureDataset.PublicationAvailableDate = publicationDate; - - // var serializeOptions = new JsonSerializerOptions - // { - // PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - // WriteIndented = true - // }; - - // var jsonString = JsonSerializer.Serialize(pureDataset, serializeOptions); - - // var uri = $"{Configuration.Url}/data-sets"; - // var httpContent = new StringContent(jsonString, Encoding.UTF8, "application/json"); - - // var response = Task.Run(async () => await _client.PutAsync(uri, httpContent)).Result; - // if (response.StatusCode != HttpStatusCode.Created) - // { - // throw new Exception("Error"); - // } - // var detailsString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; - // PureDataset pd = JsonConvert.DeserializeObject(detailsString); - // var dataset = new Curation.Data.Datasets.Dataset(Repository, pureDataset.Title.en_GB); - // dataset.Provider_ID = Configuration.ID; - // dataset.Url = pd.PortalURL; - // dataset.SaveToDatabase(); - // return dataset; - //} - public override void Update(string uuid, PluginDataset datasetUpdates) { var serializeOptions = new JsonSerializerOptions @@ -135,7 +95,7 @@ public override void Update(string uuid, PluginDataset datasetUpdates) } } - public PureDataset FetchPureDataset(Curation.Data.Datasets.Dataset dataset) + public PureDataset FetchPureDataset(Dataset dataset) { var uri = $"{Configuration.Url}/data-sets/{UrltoUUID(dataset.Url)}"; var response = Task.Run(async () => await _client.GetAsync(uri)).Result; diff --git a/Rdmp.UI/SubComponents/PureDatasetConfigurationUI.Designer.cs b/Rdmp.UI/SubComponents/PureDatasetConfigurationUI.Designer.cs index 604522394d..528cc50b8a 100644 --- a/Rdmp.UI/SubComponents/PureDatasetConfigurationUI.Designer.cs +++ b/Rdmp.UI/SubComponents/PureDatasetConfigurationUI.Designer.cs @@ -39,6 +39,9 @@ private void InitializeComponent() tbTemporalEnd = new System.Windows.Forms.TextBox(); label4 = new System.Windows.Forms.Label(); btnSave = new System.Windows.Forms.Button(); + panelLinks = new System.Windows.Forms.Panel(); + lblLinks = new System.Windows.Forms.Label(); + panelLinks.SuspendLayout(); SuspendLayout(); // // columnButtonRenderer1 @@ -124,7 +127,8 @@ private void InitializeComponent() // // btnSave // - btnSave.Location = new System.Drawing.Point(807, 466); + btnSave.Anchor = System.Windows.Forms.AnchorStyles.Bottom | System.Windows.Forms.AnchorStyles.Right; + btnSave.Location = new System.Drawing.Point(736, 49); btnSave.Name = "btnSave"; btnSave.Size = new System.Drawing.Size(75, 23); btnSave.TabIndex = 14; @@ -132,11 +136,29 @@ private void InitializeComponent() btnSave.UseVisualStyleBackColor = true; btnSave.Click += btnSave_Click; // + // panelLinks + // + panelLinks.Controls.Add(btnSave); + panelLinks.Location = new System.Drawing.Point(68, 471); + panelLinks.Name = "panelLinks"; + panelLinks.Size = new System.Drawing.Size(814, 84); + panelLinks.TabIndex = 15; + // + // lblLinks + // + lblLinks.AutoSize = true; + lblLinks.Location = new System.Drawing.Point(68, 453); + lblLinks.Name = "lblLinks"; + lblLinks.Size = new System.Drawing.Size(37, 15); + lblLinks.TabIndex = 16; + lblLinks.Text = "Links:"; + // // PureDatasetConfigurationUI // AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; - Controls.Add(btnSave); + Controls.Add(lblLinks); + Controls.Add(panelLinks); Controls.Add(label4); Controls.Add(tbTemporalEnd); Controls.Add(tbTemporalStart); @@ -149,6 +171,7 @@ private void InitializeComponent() Name = "PureDatasetConfigurationUI"; Size = new System.Drawing.Size(955, 648); Load += PureDatasetConfigurationUI_Load; + panelLinks.ResumeLayout(false); ResumeLayout(false); PerformLayout(); } @@ -165,5 +188,7 @@ private void InitializeComponent() private System.Windows.Forms.TextBox tbTemporalEnd; private System.Windows.Forms.Label label4; private System.Windows.Forms.Button btnSave; + private System.Windows.Forms.Panel panelLinks; + private System.Windows.Forms.Label lblLinks; } } diff --git a/Rdmp.UI/SubComponents/PureDatasetConfigurationUI.cs b/Rdmp.UI/SubComponents/PureDatasetConfigurationUI.cs index ea158dcc4b..fefa64cc2e 100644 --- a/Rdmp.UI/SubComponents/PureDatasetConfigurationUI.cs +++ b/Rdmp.UI/SubComponents/PureDatasetConfigurationUI.cs @@ -1,4 +1,5 @@ -using Rdmp.Core.Curation.Data.Datasets; +using NPOI.HSSF.Record.Aggregates.Chart; +using Rdmp.Core.Curation.Data.Datasets; using Rdmp.Core.Datasets; using Rdmp.Core.Datasets.PureItems; using Rdmp.Core.ReusableLibraryCode; @@ -25,6 +26,7 @@ public partial class PureDatasetConfigurationUI : DatsetConfigurationUI_Design, private IActivateItems _activator; private PureDatasetProvider _provider; private Dataset _dbObject; + private List links = new(); public PureDatasetConfigurationUI() { InitializeComponent(); @@ -32,6 +34,7 @@ public PureDatasetConfigurationUI() public override void SetDatabaseObject(IActivateItems activator, Dataset databaseObject) { + links = new(); _activator = activator; _dbObject = databaseObject; base.SetDatabaseObject(activator, databaseObject); @@ -48,6 +51,24 @@ public override void SetDatabaseObject(IActivateItems activator, Dataset databas tbTemporalStart.Text = _dataset.TemporalCoveragePeriod.StartDate.ToDateTime().ToString(); if (_dataset.TemporalCoveragePeriod is not null && _dataset.TemporalCoveragePeriod.EndDate is not null) tbTemporalEnd.Text = _dataset.TemporalCoveragePeriod.EndDate.ToDateTime().ToString(); + + if (_dataset.Links is not null) + { + var yOffset = 0; + foreach (var link in _dataset.Links) + { + var linkUI = new PureDatasetLinkUI(link); + links.Add(linkUI); + this.panelLinks.Controls.Add(linkUI); + linkUI.Location = new Point(linkUI.Location.X, linkUI.Location.Y + yOffset); + yOffset += 40; + } + this.panelLinks.Height += yOffset; + } + else + { + lblLinks.Visible = false; + } } public void RefreshBus_RefreshObject(object sender, RefreshObjectEventArgs e) @@ -99,6 +120,19 @@ private void btnSave_Click(object sender, EventArgs e) } if (!string.IsNullOrWhiteSpace(tbTemporalStart.Text) || !string.IsNullOrWhiteSpace(tbTemporalEnd.Text)) datasetUpdate.TemporalCoveragePeriod = _dataset.TemporalCoveragePeriod; + + datasetUpdate.Links = new(); + foreach (var link in links) + { + var original = _dataset.Links.Where(l => l.PureID == link.Link.PureID).First(); + if (original.Url == link.linkUrl && link.linkDescription == original.Description.En_GB) + { + continue; + } + var pl = new PureLink(link.Link.PureID, link.linkUrl, link.Link.Alias, new ENGBWrapper(link.linkDescription), link.Link.LinkType); + datasetUpdate.Links.Add(pl); + } + if (!datasetUpdate.Links.Any()) datasetUpdate.Links = null; _provider.Update(_dataset.UUID, datasetUpdate); SetDatabaseObject(_activator, _dbObject); _activator.Show("Successfully updated Pure dataset"); @@ -109,6 +143,7 @@ private void label4_Click(object sender, EventArgs e) { } + } [TypeDescriptionProvider( diff --git a/Rdmp.UI/SubComponents/PureDatasetLinkUI.Designer.cs b/Rdmp.UI/SubComponents/PureDatasetLinkUI.Designer.cs new file mode 100644 index 0000000000..b464a18199 --- /dev/null +++ b/Rdmp.UI/SubComponents/PureDatasetLinkUI.Designer.cs @@ -0,0 +1,68 @@ +namespace Rdmp.UI.SubComponents +{ + partial class PureDatasetLinkUI + { + /// + /// Required designer variable. + /// + private System.ComponentModel.IContainer components = null; + + /// + /// Clean up any resources being used. + /// + /// true if managed resources should be disposed; otherwise, false. + protected override void Dispose(bool disposing) + { + if (disposing && (components != null)) + { + components.Dispose(); + } + base.Dispose(disposing); + } + + #region Component Designer generated code + + /// + /// Required method for Designer support - do not modify + /// the contents of this method with the code editor. + /// + private void InitializeComponent() + { + tbLink = new System.Windows.Forms.TextBox(); + tbDescription = new System.Windows.Forms.TextBox(); + SuspendLayout(); + // + // tbLink + // + tbLink.Location = new System.Drawing.Point(3, 3); + tbLink.Name = "tbLink"; + tbLink.Size = new System.Drawing.Size(100, 23); + tbLink.TabIndex = 0; + tbLink.TextChanged += tbLink_TextChanged_1; + // + // tbDescription + // + tbDescription.Location = new System.Drawing.Point(122, 3); + tbDescription.Name = "tbDescription"; + tbDescription.Size = new System.Drawing.Size(100, 23); + tbDescription.TabIndex = 1; + tbDescription.TextChanged += tbDescription_TextChanged_1; + // + // PureDatasetLinkUI + // + AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); + AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; + Controls.Add(tbDescription); + Controls.Add(tbLink); + Name = "PureDatasetLinkUI"; + Size = new System.Drawing.Size(254, 34); + ResumeLayout(false); + PerformLayout(); + } + + #endregion + + private System.Windows.Forms.TextBox tbLink; + private System.Windows.Forms.TextBox tbDescription; + } +} diff --git a/Rdmp.UI/SubComponents/PureDatasetLinkUI.cs b/Rdmp.UI/SubComponents/PureDatasetLinkUI.cs new file mode 100644 index 0000000000..4654666436 --- /dev/null +++ b/Rdmp.UI/SubComponents/PureDatasetLinkUI.cs @@ -0,0 +1,45 @@ +using Rdmp.Core.Datasets.PureItems; +using Rdmp.UI.TestsAndSetup.ServicePropogation; +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Data; +using System.Drawing; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Windows.Forms; + +namespace Rdmp.UI.SubComponents +{ + public partial class PureDatasetLinkUI : RDMPUserControl + { + public PureLink Link { get; } + + public string linkUrl { get; set; } + public string linkDescription { get; set; } + public PureDatasetLinkUI(PureLink link) + { + InitializeComponent(); + Link = link; + tbLink.Text = link.Url; + linkUrl = link.Url; + tbDescription.Text = link.Description.En_GB; + linkDescription = link.Description.En_GB; + tbLink.Width = TextRenderer.MeasureText(link.Url, tbLink.Font).Width; + tbDescription.Width = TextRenderer.MeasureText(link.Description.En_GB, tbLink.Font).Width; + tbDescription.Location = new Point(20 + tbLink.Width, tbDescription.Location.Y); + this.Width = tbLink.Width + tbDescription.Width + 60; + } + + private void tbDescription_TextChanged_1(object sender, EventArgs e) + { + linkDescription = tbDescription.Text; + } + + private void tbLink_TextChanged_1(object sender, EventArgs e) + { + linkUrl = tbLink.Text; + } + } +} diff --git a/Rdmp.UI/SubComponents/PureDatasetLinkUI.resx b/Rdmp.UI/SubComponents/PureDatasetLinkUI.resx new file mode 100644 index 0000000000..af32865ec1 --- /dev/null +++ b/Rdmp.UI/SubComponents/PureDatasetLinkUI.resx @@ -0,0 +1,120 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + \ No newline at end of file From b73892822e5fc95bc18e060afe4253ed5d4444c4 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 25 Nov 2024 10:35:49 +0000 Subject: [PATCH 105/106] ad dmissing file --- Rdmp.Core/Datasets/PureItems/PureLink.cs | 28 ++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 Rdmp.Core/Datasets/PureItems/PureLink.cs diff --git a/Rdmp.Core/Datasets/PureItems/PureLink.cs b/Rdmp.Core/Datasets/PureItems/PureLink.cs new file mode 100644 index 0000000000..54afe50485 --- /dev/null +++ b/Rdmp.Core/Datasets/PureItems/PureLink.cs @@ -0,0 +1,28 @@ +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . +#nullable enable + +namespace Rdmp.Core.Datasets.PureItems; + +public class PureLink +{ + + public PureLink(int pureID, string url, string? alias, ENGBWrapper description, URITerm linkType) + { + PureID = pureID; + Url = url; + Alias = alias; + Description = description; + LinkType = linkType; + } + public int PureID { get; set; } + public string Url { get; set; } + + public string? Alias { get; set; } + public ENGBWrapper Description { get; set; } + public URITerm LinkType { get; set; } + +} From d61dfeac7509aaa64b8263c3090d43a40e81d47c Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 6 Jan 2025 13:38:01 +0000 Subject: [PATCH 106/106] fix bad merge --- .../Collections/ConfigurationsCollectionUI.cs | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs b/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs index d0c0b91858..c894f08133 100644 --- a/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs +++ b/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs @@ -26,12 +26,19 @@ private IAtomicCommand[] GetWhitespaceRightClickMenu() return new IAtomicCommand[] { new ExecuteCommandCreateNewDatasetUI(_activator){ - OverrideCommandName="Add New Dataset" - }, - new ExecuteCommandAddNewRegexRedactionConfigurationUI(_activator) - { - OverrideCommandName="Add New Regex Redaction Configuration" - } + OverrideCommandName="Add New Dataset", SuggestedCategory="Datasets" + }, + new ExecuteCommandImportExistingPureDatasetUI(_activator) + { + OverrideCommandName="Import Existing Pure Dataset", SuggestedCategory="Pure Datasets" + }, + new ExecuteCommandCreateNewPureConfigurationUI(_activator){ + OverrideCommandName="Create New Pure Configuration", SuggestedCategory="Pure Datasets" + }, + new ExecuteCommandAddNewRegexRedactionConfigurationUI(_activator) + { + OverrideCommandName="Add New Regex Redaction Configuration", SuggestedCategory="Regex" + } }; } @@ -44,9 +51,10 @@ public override void SetItemActivator(IActivateItems activator) CommonTreeFunctionality.WhitespaceRightClickMenuCommandsGetter = e => GetWhitespaceRightClickMenu(); Activator.RefreshBus.EstablishLifetimeSubscription(this); tlvConfigurations.AddObject(Activator.CoreChildProvider.AllDatasetsNode); + tlvConfigurations.AddObject(Activator.CoreChildProvider.AllDatasetProviderConfigurationsNode); tlvConfigurations.AddObject(Activator.CoreChildProvider.AllRegexRedactionConfigurationsNode); tlvConfigurations.Refresh(); - } + } public void RefreshBus_RefreshObject(object sender, RefreshObjectEventArgs e) {