Feature/rdmp 34 chi ingress

Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandConfirmRedactedCHI.cs

@@ -0,0 +1,19 @@
+using HICPlugin.Curation.Data;
+using Rdmp.Core.Curation.Data;
+
+namespace Rdmp.Core.CommandExecution.AtomicCommands;
+
+public class ExecuteCommandConfirmRedactedCHI : BasicCommandExecution, IAtomicCommand
+{
+    RedactedCHI _redactedCHI;
+    public ExecuteCommandConfirmRedactedCHI(IBasicActivateItems activator, [DemandsInitialization("redactionto confirm")]RedactedCHI redaction): base(activator)
+    {
+        _redactedCHI = redaction;
+    }
+
+    public override void Execute()
+    {
+        base.Execute();
+        _redactedCHI.DeleteInDatabase();
+    }
+}

Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandIdentifyCHIInCatalogue.cs

@@ -0,0 +1,120 @@
+
+using Rdmp.Core.CommandExecution;
+using Rdmp.Core.CommandExecution.AtomicCommands;
+using Rdmp.Core.Curation.Data;
+using Rdmp.Core.DataFlowPipeline;
+using Rdmp.Core.ReusableLibraryCode.DataAccess;
+using System;
+using System.Collections.Generic;
+using System.Data;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using YamlDotNet.Serialization;
+
+namespace Rdmp.Core.CommandExecution.AtomicCommands;
+
+public class ExecuteCommandIdentifyCHIInCatalogue : BasicCommandExecution, IAtomicCommand
+{
+
+    private ICatalogue _catalouge;
+    private IBasicActivateItems _activator;
+    private bool _bailOutEarly;
+    private readonly Dictionary<string, List<string>> _allowLists = new();
+
+    public ExecuteCommandIdentifyCHIInCatalogue(IBasicActivateItems activator, [DemandsInitialization("The catalogue to search")] ICatalogue catalogue, bool bailOutEarly = false, string allowListLocation = null) : base(activator)
+    {
+        _catalouge = catalogue;
+        _activator = activator;
+        _bailOutEarly = bailOutEarly;
+        if(!string.IsNullOrWhiteSpace(allowListLocation))
+        {
+            var allowListFileContent = File.ReadAllText(allowListLocation);
+            var deserializer = new DeserializerBuilder().Build();
+            var yamlObject = deserializer.Deserialize<Dictionary<string, List<string>>>(allowListFileContent);
+            foreach (var (cat, columns) in yamlObject)
+            {
+                _allowLists.Add(cat, columns);
+            }
+        }
+    }
+
+
+    public static string WrapCHIInContext(string chi, string source, int padding = 25)
+    {
+        var foundIndex = source.IndexOf(chi);
+        return $"{source[Math.Max(0, foundIndex - padding)..foundIndex]}{chi}{source[(foundIndex + chi.Length)..Math.Min(foundIndex + chi.Length + padding, source.Length)]}";
+    }
+
+
+
+    private void handleFoundCHI(string foundChi,string contextValue, string columnName)
+    {
+        if(foundChis.Rows.Count == 0)
+        {
+            //init
+            foundChis.Columns.Add("Potential CHI");
+            foundChis.Columns.Add("Context");
+            foundChis.Columns.Add("Source Column Name");
+        }
+        var shrunkContext = WrapCHIInContext(foundChi,contextValue);
+        foundChis.Rows.Add(foundChi, shrunkContext, columnName);
+    }
+    public DataTable foundChis = new();
+
+    public override void Execute()
+    {
+        base.Execute();
+        List<string> columnAllowList = new();
+        if (_allowLists.TryGetValue("RDMP_ALL", out var _extractionSpecificAllowances))
+            columnAllowList.AddRange(_extractionSpecificAllowances);
+        if (_allowLists.TryGetValue(_catalouge.Name, out var _catalogueSpecificAllowances))
+            columnAllowList.AddRange(_catalogueSpecificAllowances.ToList());
+        foreach (var item in _catalouge.CatalogueItems)
+        {
+            if (columnAllowList.Contains(item.Name)) continue;
+
+            if (_bailOutEarly && foundChis.Rows.Count > 0)
+            {
+                break;
+            }
+            var column = item.ColumnInfo.Name;
+            int idxOfLastSplit = column.LastIndexOf('.');
+            var columnName = column[(idxOfLastSplit + 1)..];
+            var server = _catalouge.GetDistinctLiveDatabaseServer(DataAccessContext.InternalDataProcessing, false);
+            var sql = $"SELECT {columnName} from {column[..idxOfLastSplit]}";
+            var dt = new DataTable();
+            dt.BeginLoadData();
+            using (var cmd = server.GetCommand(sql, server.GetConnection()))
+            {
+                using var da = server.GetDataAdapter(cmd);
+                da.Fill(dt);
+            }
+            dt.EndLoadData();
+            foreach (DataRow row in dt.Rows)
+            {
+
+                var value = row[dt.Columns[0].ColumnName].ToString();
+                var potentialCHI = CHIColumnFinder.GetPotentialCHI(value);
+                if (!string.IsNullOrWhiteSpace(potentialCHI))
+                {
+                    handleFoundCHI(potentialCHI, value, item.Name);
+                    if (_bailOutEarly)
+                    {
+                        break;
+                    }
+                }
+
+
+            }
+        }
+        Console.WriteLine($"Found {foundChis.Rows.Count} CHIs in the {_catalouge.Name} Catalogue.");
+        foreach(DataRow row in foundChis.Rows)
+        {
+            Console.WriteLine($"{row["potential CHI"]} | {row["Context"]} | {row["Source Column Name"]}");
+
+        }
+    }
+}

Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRedactCHIsFromCatalogue.cs

@@ -0,0 +1,103 @@
+using Rdmp.Core.CommandExecution.AtomicCommands;
+using Rdmp.Core.CommandExecution;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Rdmp.Core.Curation.Data;
+using YamlDotNet.Serialization;
+using System.IO;
+using Rdmp.Core.ReusableLibraryCode.DataAccess;
+using System.Data;
+using static NPOI.HSSF.Util.HSSFColor;
+using Rdmp.Core.Curation.Data.Defaults;
+using TB.ComponentModel;
+using FAnsi.Discovery.TableCreation;
+using HICPlugin.Curation.Data;
+using Rdmp.Core.DataFlowPipeline;
+
+namespace Rdmp.Core.CommandExecution.AtomicCommands;
+
+public class ExecuteCommandRedactCHIsFromCatalogue : BasicCommandExecution, IAtomicCommand
+{
+
+    private ICatalogue _catalouge;
+    private IBasicActivateItems _activator;
+    private readonly Dictionary<string, List<string>> _allowLists = new();
+    public int redactionCount = 0;
+
+    public ExecuteCommandRedactCHIsFromCatalogue(IBasicActivateItems activator, [DemandsInitialization("The catalogue to search")] ICatalogue catalogue, string allowListLocation = null) : base(activator)
+    {
+        _catalouge = catalogue;
+        _activator = activator;
+        if (!string.IsNullOrWhiteSpace(allowListLocation))
+        {
+            var allowListFileContent = File.ReadAllText(allowListLocation);
+            var deserializer = new DeserializerBuilder().Build();
+            var yamlObject = deserializer.Deserialize<Dictionary<string, List<string>>>(allowListFileContent);
+            foreach (var (cat, columns) in yamlObject)
+            {
+                _allowLists.Add(cat, columns);
+            }
+        }
+    }
+    private void handleFoundCHI(string foundChi, string table, string column, string columnValue)
+    {
+        Console.WriteLine("Found CHI!");
+        redactionCount++;
+        var rc = new RedactedCHI(_activator.RepositoryLocator.CatalogueRepository, foundChi, ExecuteCommandIdentifyCHIInCatalogue.WrapCHIInContext(foundChi,columnValue,20),$"{table}.{column}");
+        rc.SaveToDatabase();
+        var redactedValue = columnValue.Replace(foundChi, $"REDACTED_CHI_{rc.ID}");
+        //TODO can be smarted about how we wrote tothe db, can share a connection etc
+        var sql = $"UPDATE {table} SET {column}='{redactedValue}' where {column}='{columnValue}'";
+        var server = _catalouge.GetDistinctLiveDatabaseServer(DataAccessContext.InternalDataProcessing, false);
+        var conn = server.GetConnection();
+        conn.Open();
+        using (var cmd = server.GetCommand(sql, conn))
+        {
+            cmd.ExecuteNonQuery();
+            conn.Close();
+        }
+    }
+
+
+    public override void Execute()
+    {
+        base.Execute();
+        List<string> columnAllowList = new();
+        if (_allowLists.TryGetValue("RDMP_ALL", out var _extractionSpecificAllowances))
+            columnAllowList.AddRange(_extractionSpecificAllowances);
+        if (_allowLists.TryGetValue(_catalouge.Name, out var _catalogueSpecificAllowances))
+            columnAllowList.AddRange(_catalogueSpecificAllowances.ToList());
+        foreach (var item in _catalouge.CatalogueItems)
+        {
+            if (columnAllowList.Contains(item.Name)) continue;
+
+            var column = item.ColumnInfo.Name;
+            int idxOfLastSplit = column.LastIndexOf('.');
+            string table = column[..idxOfLastSplit];
+            var columnName = column[(idxOfLastSplit + 1)..];
+            var server = _catalouge.GetDistinctLiveDatabaseServer(DataAccessContext.InternalDataProcessing, false);
+            var sql = $"SELECT {columnName} from {table}";
+            var dt = new DataTable();
+            dt.BeginLoadData();
+            using (var cmd = server.GetCommand(sql, server.GetConnection()))
+            {
+                using var da = server.GetDataAdapter(cmd);
+                da.Fill(dt);
+            }
+            dt.EndLoadData();
+            foreach (DataRow row in dt.Rows)
+            {
+
+                var value = row[dt.Columns[0].ColumnName].ToString();
+                var potentialCHI = CHIColumnFinder.GetPotentialCHI(value);
+                if (!string.IsNullOrWhiteSpace(potentialCHI))
+                {
+                    handleFoundCHI(potentialCHI, table,columnName,value);
+                }
+            }
+        }
+    }
+}

Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRevertRedactedCHI.cs

@@ -0,0 +1,51 @@
+using HICPlugin.Curation.Data;
+using Microsoft.Data.SqlClient;
+using Rdmp.Core.Curation.Data;
+using Rdmp.Core.ReusableLibraryCode.DataAccess;
+using System;
+using System.Collections.Generic;
+using System.Data;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Rdmp.Core.CommandExecution.AtomicCommands;
+
+public class ExecuteCommandRevertRedactedCHI : BasicCommandExecution, IAtomicCommand
+{
+    RedactedCHI _redactedCHI;
+    IBasicActivateItems _activator;
+
+    public ExecuteCommandRevertRedactedCHI(IBasicActivateItems activator, [DemandsInitialization("Redacted CHIto Revert")] RedactedCHI redaction) : base(activator)
+    {
+        _redactedCHI = redaction;
+        _activator = activator;
+    }
+
+    public override void Execute()
+    {
+        base.Execute();
+        var splitidx = _redactedCHI.CHILocation.LastIndexOf('.');
+        var table = _redactedCHI.CHILocation[..splitidx];
+        var column = _redactedCHI.CHILocation[(splitidx + 1)..];
+        var columnInfo = _activator.RepositoryLocator.CatalogueRepository.GetAllObjects<ColumnInfo>().Where(ci => ci.Name == _redactedCHI.CHILocation).First();
+        var catalogue = columnInfo.CatalogueItems.FirstOrDefault().Catalogue;
+        var findSlq = $"select {column} from {table} where {column} like '%REDACTED_CHI_{_redactedCHI.ID}%';";
+        var existingResultsDT = new DataTable();
+        using (var con = (SqlConnection)catalogue.GetDistinctLiveDatabaseServer(DataAccessContext.InternalDataProcessing, false).GetConnection()) 
+        {
+            con.Open();
+            var da = new SqlDataAdapter(new SqlCommand(findSlq, con));
+            da.Fill(existingResultsDT);
+            if (existingResultsDT.Rows.Count > 0 && existingResultsDT.Rows[0].ItemArray.Length > 0)
+            {
+                var currentContext = existingResultsDT.Rows[0].ItemArray[0].ToString();
+                var newContext = currentContext.Replace($"REDACTED_CHI_{_redactedCHI.ID}", _redactedCHI.PotentialCHI);
+                var updateSQL = $"update {table} set {column}='{newContext}' where {column} = '{currentContext}'";
+                var updateCmd = new SqlCommand(updateSQL, con);
+                updateCmd.ExecuteNonQuery();
+            }
+            _redactedCHI.DeleteInDatabase();
+        }
+    }
+}

Rdmp.Core/Curation/Data/IRedactedCHI.cs

@@ -0,0 +1,23 @@
+using FAnsi.Naming;
+using Rdmp.Core.Curation.Data.Cohort;
+using Rdmp.Core.MapsDirectlyToDatabaseTable;
+using Rdmp.Core.Repositories;
+using Rdmp.Core.ReusableLibraryCode.Checks;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace HICPlugin.Curation.Data;
+
+public interface IRedactedCHI :IMapsDirectlyToDatabaseTable
+{
+
+    ICatalogueRepository CatalogueRepository { get; }
+
+    string PotentialCHI { get; }
+    string CHIContext{ get; }
+
+    string CHILocation { get; }
+}