Merge pull request #1844 from HicServices/release/8.1.7

Release/8.1.7

CHANGELOG.md

@@ -1,12 +1,19 @@
 
-
-
 # Changelog
 All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [8.1.7] - 2024-06-17
+
+## Changed
+
+- Add ability to update an extraction's cohort from the command line using `SetExtractionConfigurationCohort ExtractionConfiguration:{id} ExtractableCohort:{id}`
+- Fix issue with non-default named PostgreSQL Table Info not being checkable
+- Improve default timeouts on database lookups
+- Implement keepalive and liveness checks for FTP, SFTP fetches before deletion attempt
+
 ## [8.1.6] - 2024-05-27
 
 ## Changed

Rdmp.Core.Tests/CommandExecution/ExecuteCommandSetExtractionConfigurationCohortTests.cs

@@ -0,0 +1,126 @@
+// Copyright (c) The University of Dundee 2024-2024
+// This file is part of the Research Data Management Platform (RDMP).
+// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
+// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
+
+using Microsoft.Data.SqlClient;
+using NUnit.Framework;
+using Rdmp.Core.CohortCommitting.Pipeline;
+using Rdmp.Core.CommandExecution.AtomicCommands;
+using Rdmp.Core.CommandLine.Interactive;
+using Rdmp.Core.DataExport.Data;
+using Rdmp.Core.ReusableLibraryCode.Checks;
+using System.Linq;
+using Tests.Common.Scenarios;
+
+namespace Rdmp.Core.Tests.CommandExecution;
+
+public class ExecuteCommandSetExtractionConfigurationCohortTests : TestsRequiringACohort
+{
+
+    [SetUp]
+    protected override void SetUp()
+    {
+        base.SetUp();
+    }
+
+    [Test]
+    public void UpdateCohortForExtractionTest()
+    {
+        var proj = new Project(DataExportRepository, "Some Proj")
+        {
+            ProjectNumber = 999
+        };
+        proj.SaveToDatabase();
+
+        // we are replacing this imaginary cohort
+        var definition998 = new CohortDefinition(null, "CommittingNewCohorts", 1, 999, _externalCohortTable);
+        // with this one (v2)
+        var definition999 = new CohortDefinition(null, "CommittingNewCohorts", 2, 999, _externalCohortTable);
+
+        // Create a basic cohort first
+        var request1 = new CohortCreationRequest(proj, definition998, DataExportRepository, "fish");
+        request1.Check(ThrowImmediatelyCheckNotifier.Quiet);
+
+        using var con = _cohortDatabase.Server.GetManagedConnection();
+        request1.PushToServer(con);
+        request1.ImportAsExtractableCohort(true, false);
+
+        // the definition was imported and should now be a saved ExtractableCohort
+        var cohort998 = request1.CohortCreatedIfAny;
+        Assert.That(cohort998, Is.Not.Null);
+        Assert.That(cohort998.IsDeprecated, Is.False);
+
+        //create second cohort 
+        // Create a basic cohort first
+        var request2 = new CohortCreationRequest(proj, definition999, DataExportRepository, "fish");
+        request2.Check(ThrowImmediatelyCheckNotifier.Quiet);
+
+        request2.PushToServer(con);
+        request2.ImportAsExtractableCohort(true, false);
+
+        // the definition was imported and should now be a saved ExtractableCohort
+        var cohort999 = request2.CohortCreatedIfAny;
+        Assert.That(cohort999, Is.Not.Null);
+        Assert.That(cohort999.IsDeprecated, Is.False);
+
+        // legit user 1
+        var ec1 = new ExtractionConfiguration(DataExportRepository, proj)
+        {
+            IsReleased = false,
+            Cohort_ID = cohort998.ID
+        };
+        ec1.SaveToDatabase();
+        var activator = new ConsoleInputManager(RepositoryLocator, ThrowImmediatelyCheckNotifier.Quiet)
+        { DisallowInput = true };
+        var cmd = new ExecuteCommandSetExtractionConfigurationCohort(activator, ec1, cohort999);
+        Assert.DoesNotThrow(() => cmd.Execute());
+        var updatedExt = DataExportRepository.GetAllObjects<ExtractionConfiguration>().Where(ei => ei.ID == ec1.ID).ToList();
+        Assert.That(updatedExt.Count, Is.EqualTo(1));
+        Assert.That(updatedExt.First().Cohort_ID, Is.EqualTo(cohort999.ID));
+    }
+
+    [Test]
+    public void UpdateCohortForExtractionTest_BadCohort()
+    {
+        var proj = new Project(DataExportRepository, "Some Proj")
+        {
+            ProjectNumber = 999
+        };
+        proj.SaveToDatabase();
+
+        // we are replacing this imaginary cohort
+        var definition998 = new CohortDefinition(null, "CommittingNewCohorts", 1, 999, _externalCohortTable);
+        // with this one (v2)
+        var definition999 = new CohortDefinition(null, "CommittingNewCohorts", 2, 999, _externalCohortTable);
+
+        // Create a basic cohort first
+        var request1 = new CohortCreationRequest(proj, definition998, DataExportRepository, "fish");
+        request1.Check(ThrowImmediatelyCheckNotifier.Quiet);
+
+        using var con = _cohortDatabase.Server.GetManagedConnection();
+        request1.PushToServer(con);
+        request1.ImportAsExtractableCohort(true, false);
+
+        // the definition was imported and should now be a saved ExtractableCohort
+        var cohort998 = request1.CohortCreatedIfAny;
+        Assert.That(cohort998, Is.Not.Null);
+        Assert.That(cohort998.IsDeprecated, Is.False);
+
+        // legit user 1
+        var ec1 = new ExtractionConfiguration(DataExportRepository, proj)
+        {
+            IsReleased = false,
+            Cohort_ID = cohort998.ID
+        };
+        ec1.SaveToDatabase();
+        var activator = new ConsoleInputManager(RepositoryLocator, ThrowImmediatelyCheckNotifier.Quiet)
+        { DisallowInput = true };
+        var cmd = new ExecuteCommandSetExtractionConfigurationCohort(activator, ec1, new ExtractableCohort()
+        {
+            ID = -1
+        });
+        Assert.Throws<SqlException>(() => cmd.Execute());
+    }
+}

Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetExtractionConfigurationCohort.cs

@@ -0,0 +1,37 @@
+// Copyright (c) The University of Dundee 2024-2024
+// This file is part of the Research Data Management Platform (RDMP).
+// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
+// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
+
+using Rdmp.Core.DataExport.Data;
+using Rdmp.Core.Icons.IconProvision;
+using Rdmp.Core.ReusableLibraryCode.Icons.IconProvision;
+using SixLabors.ImageSharp.PixelFormats;
+using SixLabors.ImageSharp;
+
+namespace Rdmp.Core.CommandExecution.AtomicCommands;
+
+public class ExecuteCommandSetExtractionConfigurationCohort : BasicCommandExecution, IAtomicCommand
+{
+    private readonly ExtractionConfiguration _extractionConfiguration;
+    private readonly ExtractableCohort _extractableCohort;
+
+    public ExecuteCommandSetExtractionConfigurationCohort(IBasicActivateItems activator, ExtractionConfiguration extractionConfiguration, ExtractableCohort cohort) : base(activator)
+    {
+        _extractionConfiguration = extractionConfiguration;
+        _extractableCohort = cohort;
+    }
+
+
+    public override void Execute()
+    {
+        base.Execute();
+        _extractionConfiguration.Cohort_ID = _extractableCohort.ID;
+        _extractionConfiguration.SaveToDatabase();
+    }
+
+    public override Image<Rgba32> GetImage(IIconProvider iconProvider) =>
+    iconProvider.GetImage(RDMPConcept.ExtractionConfiguration, OverlayKind.None);
+
+}

Rdmp.Core/Curation/Data/Catalogue.cs

@@ -499,7 +499,7 @@ public DateTime? DatasetStartDate
     /// <inheritdoc/>
     public LoadMetadata[] LoadMetadatas()
     {
-        var loadMetadataLinkIDs = Repository.GetAllObjectsWhere<LoadMetadataCatalogueLinkage>("CatalogueID",ID).Select(l => l.LoadMetadataID);
+        var loadMetadataLinkIDs = Repository.GetAllObjectsWhere<LoadMetadataCatalogueLinkage>("CatalogueID", ID).Select(l => l.LoadMetadataID);
 
         return Repository.GetAllObjects<LoadMetadata>().Where(cat => loadMetadataLinkIDs.Contains(cat.ID)).ToArray();
     }
@@ -511,7 +511,7 @@ public LoadMetadata[] LoadMetadatas()
 
     /// <inheritdoc/>
     [NoMappingToDatabase]
-     public ExternalDatabaseServer LiveLoggingServer =>
+    public ExternalDatabaseServer LiveLoggingServer =>
        LiveLoggingServer_ID == null
                    ? null
                    : Repository.GetObjectByID<ExternalDatabaseServer>((int)LiveLoggingServer_ID);
@@ -893,8 +893,8 @@ public void Check(ICheckNotifier notifier)
 
             try
             {
-                var server = DataAccessPortal.ExpectDistinctServer(tables, accessContext, false);
-
+                var setInitialDatabase = tables.Any(static t => t.Database != null);
+                var server = DataAccessPortal.ExpectDistinctServer(tables, accessContext, setInitialDatabase);
                 using var con = server.GetConnection();
                 con.Open();
 

Rdmp.Core/DataLoad/Engine/LoadExecution/Components/Standard/MigrateRAWTableToStaging.cs

@@ -107,7 +107,7 @@ public override ExitCodeType Run(IDataLoadJob job, GracefulCancellationToken can
         //pass pre load discard
         var destination = new SqlBulkInsertDestination(destinationDatabase, destinationTableName,
             columnNamesToIgnoreForBulkInsert);
-
+        destination.Timeout = 43200; //set max copy to 12 hours
         //engine that will move data
         _pipeline = new DataFlowPipelineEngine<DataTable>(context, source, destination, job);
 

Rdmp.Core/DataLoad/Engine/Pipeline/Destinations/SqlBulkInsertDestination.cs

@@ -27,7 +27,7 @@ public class SqlBulkInsertDestination : IDataFlowDestination<DataTable>, IPipeli
     private readonly List<string> _columnNamesToIgnore;
     private string _taskBeingPerformed;
 
-    public const int Timeout = 5000;
+    public int Timeout = 5000;
 
     private IBulkCopy _copy;
     private Stopwatch _timer = new();

Rdmp.Core/DataLoad/Modules/FTP/FTPDownloader.cs

@@ -66,18 +66,18 @@ public FTPDownloader()
     [DemandsInitialization("The directory on the FTP server that you want to download files from")]
     public string? RemoteDirectory { get; set; }
 
-    [DemandsInitialization("True to set keep alive", DefaultValue = true)]
+    [DemandsInitialization("True to set keep alive",DefaultValue = true)]
     public bool KeepAlive { get; set; }
 
 
-    public void Initialize(ILoadDirectory directory, DiscoveredDatabase dbInfo)
+    public void Initialize(ILoadDirectory directory,DiscoveredDatabase dbInfo)
     {
         _directory = directory;
     }
 
-    public ExitCodeType Fetch(IDataLoadJob job, GracefulCancellationToken cancellationToken)
+    public ExitCodeType Fetch(IDataLoadJob job,GracefulCancellationToken cancellationToken)
     {
-        return DownloadFilesOnFTP(_directory ?? throw new InvalidOperationException("No output directory set"), job);
+        return DownloadFilesOnFTP(_directory ?? throw new InvalidOperationException("No output directory set"),job);
     }
 
     private FtpClient SetupFtp()
@@ -86,32 +86,35 @@ private FtpClient SetupFtp()
         var username = FTPServer.Username ?? "anonymous";
         var password = string.IsNullOrWhiteSpace(FTPServer.Password) ? "guest" : FTPServer.GetDecryptedPassword();
         var c = new FtpClient(host, username, password);
+
+        // Enable periodic NOOP keepalive operations to keep connection active until we're done
+        c.Config.Noop = true;
         c.AutoConnect();
         return c;
     }
 
-    private ExitCodeType DownloadFilesOnFTP(ILoadDirectory destination, IDataLoadEventListener listener)
+    private ExitCodeType DownloadFilesOnFTP(ILoadDirectory destination,IDataLoadEventListener listener)
     {
         var files = GetFileList().ToArray();
 
-        listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,
+        listener.OnNotify(this,new NotifyEventArgs(ProgressEventType.Information,
             $"Identified the following files on the FTP server:{string.Join(',',files)}"));
 
         var forLoadingContainedCachedFiles = false;
 
         foreach (var file in files)
         {
-            var action = GetSkipActionForFile(file, destination);
+            var action = GetSkipActionForFile(file,destination);
 
-            listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,
+            listener.OnNotify(this,new NotifyEventArgs(ProgressEventType.Information,
                 $"File {file} was evaluated as {action}"));
 
             switch (action)
             {
                 case SkipReason.DoNotSkip:
                     listener.OnNotify(this,
-                        new NotifyEventArgs(ProgressEventType.Information, $"About to download {file}"));
-                    Download(file, destination);
+                        new NotifyEventArgs(ProgressEventType.Information,$"About to download {file}"));
+                    Download(file,destination);
                     break;
                 case SkipReason.InForLoading:
                     forLoadingContainedCachedFiles = true;
@@ -120,7 +123,7 @@ private ExitCodeType DownloadFilesOnFTP(ILoadDirectory destination, IDataLoadEve
         }
 
         // it was a success - even if no files were actually retrieved... hey that's what the user said, otherwise he would have set SendLoadNotRequiredIfFileNotFound
-        if (forLoadingContainedCachedFiles || _filesRetrieved.Any() || !SendLoadNotRequiredIfFileNotFound)
+        if (forLoadingContainedCachedFiles || _filesRetrieved.Count != 0 || !SendLoadNotRequiredIfFileNotFound)
             return ExitCodeType.Success;
 
         // if no files were downloaded (and there were none skipped because they were in forLoading) and in that eventuality we have our flag set to return LoadNotRequired then do so
@@ -138,7 +141,7 @@ protected enum SkipReason
         IsImaginaryFile
     }
 
-    protected SkipReason GetSkipActionForFile(string file, ILoadDirectory destination)
+    protected SkipReason GetSkipActionForFile(string file,ILoadDirectory destination)
     {
         if (file.StartsWith(".",StringComparison.Ordinal))
             return SkipReason.IsImaginaryFile;
@@ -152,7 +155,7 @@ protected SkipReason GetSkipActionForFile(string file, ILoadDirectory destinatio
     }
 
 
-    private static bool ValidateServerCertificate(object _1, X509Certificate _2, X509Chain _3,
+    private static bool ValidateServerCertificate(object _1,X509Certificate _2,X509Chain _3,
         SslPolicyErrors _4) => true; //any cert will do! yay
 
 
@@ -161,21 +164,24 @@ protected virtual IEnumerable<string> GetFileList()
         return _connection.Value.GetNameListing().ToList().Where(_connection.Value.FileExists);
     }
 
-    protected virtual void Download(string file, ILoadDirectory destination)
+    protected virtual void Download(string file,ILoadDirectory destination)
     {
         var remotePath = !string.IsNullOrWhiteSpace(RemoteDirectory)
             ? $"{RemoteDirectory}/{file}"
             : file;
 
-        var destinationFileName = Path.Combine(destination.ForLoading.FullName, file);
-        _connection.Value.DownloadFile(destinationFileName, remotePath);
+        var destinationFileName = Path.Combine(destination.ForLoading.FullName,file);
+        _connection.Value.DownloadFile(destinationFileName,remotePath);
         _filesRetrieved.Add(remotePath);
     }
 
-    public virtual void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener)
+    public virtual void LoadCompletedSoDispose(ExitCodeType exitCode,IDataLoadEventListener postLoadEventListener)
     {
         if (exitCode != ExitCodeType.Success || !DeleteFilesOffFTPServerAfterSuccesfulDataLoad) return;
 
+        // Force a reconnection attempt if we got cut off
+        if (!_connection.Value.IsStillConnected())
+            _connection.Value.Connect(true);
         foreach (var file in _filesRetrieved) _connection.Value.DeleteFile(file);
     }
 
@@ -188,7 +194,7 @@ public void Check(ICheckNotifier notifier)
         }
         catch (Exception e)
         {
-            notifier.OnCheckPerformed(new CheckEventArgs("Failed to SetupFTP", CheckResult.Fail, e));
+            notifier.OnCheckPerformed(new CheckEventArgs("Failed to SetupFTP",CheckResult.Fail,e));
         }
     }
 }

Rdmp.Core/DataLoad/Modules/FTP/SFTPDownloader.cs

@@ -8,7 +8,6 @@
 using System.IO;
 using System.Linq;
 using System.Threading;
-using FluentFTP;
 using Rdmp.Core.Curation;
 using Rdmp.Core.Curation.Data;
 using Rdmp.Core.ReusableLibraryCode.Progress;
@@ -46,6 +45,8 @@ private SftpClient SetupSftp()
         var password = string.IsNullOrWhiteSpace(FTPServer.Password) ? "guest" : FTPServer.GetDecryptedPassword();
         var c = new SftpClient(host, username, password);
         c.Connect();
+        if (KeepAlive)
+            c.KeepAliveInterval = TimeSpan.FromMilliseconds(KeepAliveIntervalMilliseconds);
         return c;
     }
 
@@ -70,6 +71,10 @@ public override void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEven
     {
         if (exitCode != ExitCodeType.Success) return;
 
+        // Reconnect if we got cut off, for example due to idle timers
+        if (!_connection.Value.IsConnected)
+            _connection.Value.Connect();
+
         foreach (var retrievedFiles in _filesRetrieved)
             try
             {