From f100b652a57e438fbdafc1205b5a767083d3bd23 Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 23 May 2024 02:26:08 +0100 Subject: [PATCH 01/13] Bugfix/rdmp 185 table import check fail (#1836) * use server db * Update Catalogue.cs --------- Co-authored-by: James A Sutherland --- CHANGELOG.md | 6 ++++++ Rdmp.Core/Curation/Data/Catalogue.cs | 8 ++++---- SharedAssemblyInfo.cs | 6 +++--- directory.build.props | 2 +- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2906320a4e..af535cc848 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [8.1.7] - Unreleased + +## Changed + +- Fix issue with non-default named PostgreSQL Table Info not being checkable + ## [8.1.6] - Unreleased ## Changed diff --git a/Rdmp.Core/Curation/Data/Catalogue.cs b/Rdmp.Core/Curation/Data/Catalogue.cs index eca05444f2..48b71b8293 100644 --- a/Rdmp.Core/Curation/Data/Catalogue.cs +++ b/Rdmp.Core/Curation/Data/Catalogue.cs @@ -499,7 +499,7 @@ public DateTime? DatasetStartDate /// public LoadMetadata[] LoadMetadatas() { - var loadMetadataLinkIDs = Repository.GetAllObjectsWhere("CatalogueID",ID).Select(l => l.LoadMetadataID); + var loadMetadataLinkIDs = Repository.GetAllObjectsWhere("CatalogueID", ID).Select(l => l.LoadMetadataID); return Repository.GetAllObjects().Where(cat => loadMetadataLinkIDs.Contains(cat.ID)).ToArray(); } @@ -511,7 +511,7 @@ public LoadMetadata[] LoadMetadatas() /// [NoMappingToDatabase] - public ExternalDatabaseServer LiveLoggingServer => + public ExternalDatabaseServer LiveLoggingServer => LiveLoggingServer_ID == null ? null : Repository.GetObjectByID((int)LiveLoggingServer_ID); @@ -893,8 +893,8 @@ public void Check(ICheckNotifier notifier) try { - var server = DataAccessPortal.ExpectDistinctServer(tables, accessContext, false); - + var setInitialDatabase = tables.Any(static t => t.Database != null); + var server = DataAccessPortal.ExpectDistinctServer(tables, accessContext, setInitialDatabase); using var con = server.GetConnection(); con.Open(); diff --git a/SharedAssemblyInfo.cs b/SharedAssemblyInfo.cs index e7ab847e39..cac26791a7 100644 --- a/SharedAssemblyInfo.cs +++ b/SharedAssemblyInfo.cs @@ -10,6 +10,6 @@ [assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] -[assembly: AssemblyVersion("8.1.6")] -[assembly: AssemblyFileVersion("8.1.6")] -[assembly: AssemblyInformationalVersion("8.1.6-rc2")] +[assembly: AssemblyVersion("8.1.7")] +[assembly: AssemblyFileVersion("8.1.7")] +[assembly: AssemblyInformationalVersion("8.1.7")] diff --git a/directory.build.props b/directory.build.props index 5daa1d1147..0c317dcb16 100644 --- a/directory.build.props +++ b/directory.build.props @@ -1,7 +1,7 @@ 11.0 - 8.1.4 + 8.1.7 true \ No newline at end of file From 8f40a2be89a5c48c4ab03c6609b6fd31ee76b2ed Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 27 May 2024 12:02:18 +0100 Subject: [PATCH 02/13] add basic command --- ...CommandSetExtractionConfigurationCohort.cs | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetExtractionConfigurationCohort.cs diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetExtractionConfigurationCohort.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetExtractionConfigurationCohort.cs new file mode 100644 index 0000000000..2472da56d4 --- /dev/null +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetExtractionConfigurationCohort.cs @@ -0,0 +1,36 @@ +using Rdmp.Core.Curation.Data; +using Rdmp.Core.DataExport.Data; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.CommandExecution.AtomicCommands +{ + internal class ExecuteCommandSetExtractionConfigurationCohort : BasicCommandExecution + { + private ExtractionConfiguration _extractionConfiguration; + private ExtractableCohort _extractableCohort; + + public ExecuteCommandSetExtractionConfigurationCohort(IBasicActivateItems activator) : base(activator) + { + Weight = 100.1f; + } + + public ExecuteCommandSetExtractionConfigurationCohort(IBasicActivateItems activator, ExtractionConfiguration extractionConfiguration, ExtractableCohort cohort) : this(activator) + { + _extractionConfiguration = extractionConfiguration; + _extractableCohort = cohort; + } + + + public override void Execute() + { + base.Execute(); + _extractionConfiguration.Cohort_ID = _extractableCohort.ID; + _extractionConfiguration.SaveToDatabase(); + } + + } +} From 9ce3ca190623eb6543e6c674ae41ed90e77410a8 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 27 May 2024 12:09:20 +0100 Subject: [PATCH 03/13] tidy up code --- ...CommandSetExtractionConfigurationCohort.cs | 55 ++++++++++--------- Tools/rdmp/Databases.yaml | 4 +- 2 files changed, 30 insertions(+), 29 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetExtractionConfigurationCohort.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetExtractionConfigurationCohort.cs index 2472da56d4..27a3ab7b20 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetExtractionConfigurationCohort.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetExtractionConfigurationCohort.cs @@ -1,36 +1,37 @@ -using Rdmp.Core.Curation.Data; +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + using Rdmp.Core.DataExport.Data; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; +using Rdmp.Core.Icons.IconProvision; +using Rdmp.Core.ReusableLibraryCode.Icons.IconProvision; +using SixLabors.ImageSharp.PixelFormats; +using SixLabors.ImageSharp; + +namespace Rdmp.Core.CommandExecution.AtomicCommands; -namespace Rdmp.Core.CommandExecution.AtomicCommands +public class ExecuteCommandSetExtractionConfigurationCohort : BasicCommandExecution, IAtomicCommand { - internal class ExecuteCommandSetExtractionConfigurationCohort : BasicCommandExecution - { - private ExtractionConfiguration _extractionConfiguration; - private ExtractableCohort _extractableCohort; + private ExtractionConfiguration _extractionConfiguration; + private ExtractableCohort _extractableCohort; - public ExecuteCommandSetExtractionConfigurationCohort(IBasicActivateItems activator) : base(activator) - { - Weight = 100.1f; - } + public ExecuteCommandSetExtractionConfigurationCohort(IBasicActivateItems activator, ExtractionConfiguration extractionConfiguration, ExtractableCohort cohort) : base(activator) + { + _extractionConfiguration = extractionConfiguration; + _extractableCohort = cohort; + } - public ExecuteCommandSetExtractionConfigurationCohort(IBasicActivateItems activator, ExtractionConfiguration extractionConfiguration, ExtractableCohort cohort) : this(activator) - { - _extractionConfiguration = extractionConfiguration; - _extractableCohort = cohort; - } + public override void Execute() + { + base.Execute(); + _extractionConfiguration.Cohort_ID = _extractableCohort.ID; + _extractionConfiguration.SaveToDatabase(); + } - public override void Execute() - { - base.Execute(); - _extractionConfiguration.Cohort_ID = _extractableCohort.ID; - _extractionConfiguration.SaveToDatabase(); - } + public override Image GetImage(IIconProvider iconProvider) => + iconProvider.GetImage(RDMPConcept.ExtractionConfiguration, OverlayKind.None); - } } diff --git a/Tools/rdmp/Databases.yaml b/Tools/rdmp/Databases.yaml index 434fc12411..c688f62761 100644 --- a/Tools/rdmp/Databases.yaml +++ b/Tools/rdmp/Databases.yaml @@ -1,2 +1,2 @@ -CatalogueConnectionString: Server=(localdb)\MSSQLLocalDB;Database=TEST_Catalogue;Trusted_Connection=True;TrustServerCertificate=true; -DataExportConnectionString: Server=(localdb)\MSSQLLocalDB;Database=TEST_DataExport;Trusted_Connection=True;TrustServerCertificate=true; +CatalogueConnectionString: Server=(localdb)\MSSQLLocalDB;Database=RDMP_Catalogue;Trusted_Connection=True;TrustServerCertificate=true; +DataExportConnectionString: Server=(localdb)\MSSQLLocalDB;Database=RDMP_DataExport;Trusted_Connection=True;TrustServerCertificate=true; From a21ec88d7b0ba0a91bfdd74fb52a268d570c0d03 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 27 May 2024 12:10:42 +0100 Subject: [PATCH 04/13] add changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a92cb4b639..728e024e1f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [8.2.0] - Unrelased + +## Changed + +- Add ability to update an extraction's cohort from the command line using `SetExtractionConfigurationCohort ExtractionConfiguration:{id} ExtractableCohort:{id}` + ## [8.1.6] - 2024-05-27 ## Changed From ccc5c9cfe524bc88b98d6d086855ba69e32d7f11 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 27 May 2024 13:33:29 +0100 Subject: [PATCH 05/13] add tests --- ...ndSetExtractionConfigurationCohortTests.cs | 129 ++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 Rdmp.Core.Tests/CommandExecution/ExecuteCommandSetExtractionConfigurationCohortTests.cs diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandSetExtractionConfigurationCohortTests.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandSetExtractionConfigurationCohortTests.cs new file mode 100644 index 0000000000..cb763108ca --- /dev/null +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandSetExtractionConfigurationCohortTests.cs @@ -0,0 +1,129 @@ +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using Microsoft.Data.SqlClient; +using NUnit.Framework; +using Rdmp.Core.CohortCommitting.Pipeline; +using Rdmp.Core.CommandExecution; +using Rdmp.Core.CommandExecution.AtomicCommands; +using Rdmp.Core.CommandLine.Interactive; +using Rdmp.Core.DataExport.Data; +using Rdmp.Core.ReusableLibraryCode.Checks; +using System; +using System.IO; +using System.Linq; +using Tests.Common.Scenarios; + +namespace Rdmp.Core.Tests.CommandExecution; + +public class ExecuteCommandSetExtractionConfigurationCohortTests : TestsRequiringACohort +{ + + [SetUp] + protected override void SetUp() + { + base.SetUp(); + } + + [Test] + public void UpdateCohortForExtractionTest() + { + var proj = new Project(DataExportRepository, "Some Proj") + { + ProjectNumber = 999 + }; + proj.SaveToDatabase(); + + // we are replacing this imaginary cohort + var definition998 = new CohortDefinition(null, "CommittingNewCohorts", 1, 999, _externalCohortTable); + // with this one (v2) + var definition999 = new CohortDefinition(null, "CommittingNewCohorts", 2, 999, _externalCohortTable); + + // Create a basic cohort first + var request1 = new CohortCreationRequest(proj, definition998, DataExportRepository, "fish"); + request1.Check(ThrowImmediatelyCheckNotifier.Quiet); + + using var con = _cohortDatabase.Server.GetManagedConnection(); + request1.PushToServer(con); + request1.ImportAsExtractableCohort(true, false); + + // the definition was imported and should now be a saved ExtractableCohort + var cohort998 = request1.CohortCreatedIfAny; + Assert.That(cohort998, Is.Not.Null); + Assert.That(cohort998.IsDeprecated, Is.False); + + //create second cohort + // Create a basic cohort first + var request2 = new CohortCreationRequest(proj, definition999, DataExportRepository, "fish"); + request2.Check(ThrowImmediatelyCheckNotifier.Quiet); + + request2.PushToServer(con); + request2.ImportAsExtractableCohort(true, false); + + // the definition was imported and should now be a saved ExtractableCohort + var cohort999 = request2.CohortCreatedIfAny; + Assert.That(cohort999, Is.Not.Null); + Assert.That(cohort999.IsDeprecated, Is.False); + + // legit user 1 + var ec1 = new ExtractionConfiguration(DataExportRepository, proj) + { + IsReleased = false, + Cohort_ID = cohort998.ID + }; + ec1.SaveToDatabase(); + var activator = new ConsoleInputManager(RepositoryLocator, ThrowImmediatelyCheckNotifier.Quiet) + { DisallowInput = true }; + var cmd = new ExecuteCommandSetExtractionConfigurationCohort(activator, ec1, cohort999); + Assert.DoesNotThrow(() => cmd.Execute()); + var updatedExt = DataExportRepository.GetAllObjects().Where(ei => ei.ID == ec1.ID).ToList(); + Assert.That(updatedExt.Count, Is.EqualTo(1)); + Assert.That(updatedExt.First().Cohort_ID, Is.EqualTo(cohort999.ID)); + } + + [Test] + public void UpdateCohortForExtractionTest_BadCohort() + { + var proj = new Project(DataExportRepository, "Some Proj") + { + ProjectNumber = 999 + }; + proj.SaveToDatabase(); + + // we are replacing this imaginary cohort + var definition998 = new CohortDefinition(null, "CommittingNewCohorts", 1, 999, _externalCohortTable); + // with this one (v2) + var definition999 = new CohortDefinition(null, "CommittingNewCohorts", 2, 999, _externalCohortTable); + + // Create a basic cohort first + var request1 = new CohortCreationRequest(proj, definition998, DataExportRepository, "fish"); + request1.Check(ThrowImmediatelyCheckNotifier.Quiet); + + using var con = _cohortDatabase.Server.GetManagedConnection(); + request1.PushToServer(con); + request1.ImportAsExtractableCohort(true, false); + + // the definition was imported and should now be a saved ExtractableCohort + var cohort998 = request1.CohortCreatedIfAny; + Assert.That(cohort998, Is.Not.Null); + Assert.That(cohort998.IsDeprecated, Is.False); + + // legit user 1 + var ec1 = new ExtractionConfiguration(DataExportRepository, proj) + { + IsReleased = false, + Cohort_ID = cohort998.ID + }; + ec1.SaveToDatabase(); + var activator = new ConsoleInputManager(RepositoryLocator, ThrowImmediatelyCheckNotifier.Quiet) + { DisallowInput = true }; + var cmd = new ExecuteCommandSetExtractionConfigurationCohort(activator, ec1, new ExtractableCohort() + { + ID = -1 + }); + Assert.Throws(() => cmd.Execute()); + } +} From 9f973da4c6871fb23de9c751231b1339149e01a7 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 27 May 2024 13:33:54 +0100 Subject: [PATCH 06/13] change target release --- CHANGELOG.md | 2 +- Tools/rdmp/Databases.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 728e024e1f..5f71ebbae3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [8.2.0] - Unrelased +## [8.1.7] - Unrelased ## Changed diff --git a/Tools/rdmp/Databases.yaml b/Tools/rdmp/Databases.yaml index c688f62761..434fc12411 100644 --- a/Tools/rdmp/Databases.yaml +++ b/Tools/rdmp/Databases.yaml @@ -1,2 +1,2 @@ -CatalogueConnectionString: Server=(localdb)\MSSQLLocalDB;Database=RDMP_Catalogue;Trusted_Connection=True;TrustServerCertificate=true; -DataExportConnectionString: Server=(localdb)\MSSQLLocalDB;Database=RDMP_DataExport;Trusted_Connection=True;TrustServerCertificate=true; +CatalogueConnectionString: Server=(localdb)\MSSQLLocalDB;Database=TEST_Catalogue;Trusted_Connection=True;TrustServerCertificate=true; +DataExportConnectionString: Server=(localdb)\MSSQLLocalDB;Database=TEST_DataExport;Trusted_Connection=True;TrustServerCertificate=true; From 3c5bf1f00cf92e8f87ef3457231be8fd2197e68e Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 27 May 2024 13:34:47 +0100 Subject: [PATCH 07/13] tidy up imports --- .../ExecuteCommandSetExtractionConfigurationCohortTests.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandSetExtractionConfigurationCohortTests.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandSetExtractionConfigurationCohortTests.cs index cb763108ca..6634227e29 100644 --- a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandSetExtractionConfigurationCohortTests.cs +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandSetExtractionConfigurationCohortTests.cs @@ -7,13 +7,10 @@ using Microsoft.Data.SqlClient; using NUnit.Framework; using Rdmp.Core.CohortCommitting.Pipeline; -using Rdmp.Core.CommandExecution; using Rdmp.Core.CommandExecution.AtomicCommands; using Rdmp.Core.CommandLine.Interactive; using Rdmp.Core.DataExport.Data; using Rdmp.Core.ReusableLibraryCode.Checks; -using System; -using System.IO; using System.Linq; using Tests.Common.Scenarios; From 62ed28a4f365500600859698cfa93de5f25df152 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 27 May 2024 15:08:48 +0100 Subject: [PATCH 08/13] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f71ebbae3..dde772e1a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [8.1.7] - Unrelased +## [8.1.7] - Unreleased ## Changed From 2afcd8748cfc71b3ae3f3b12eed18fc6c17ed288 Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 7 Jun 2024 14:33:45 +0100 Subject: [PATCH 09/13] Bugfix/rdmp 192 connection failed (#1851) * add better async checking * ignore timeout of 0ms * tidy up * add changelog * add 12 hour timeout * use latest fansisql * update timeout * bump version --- CHANGELOG.md | 1 + Directory.Packages.props | 2 +- .../Components/Standard/MigrateRAWTableToStaging.cs | 2 +- .../Destinations/SqlBulkInsertDestination.cs | 2 +- .../SimpleControls/ServerDatabaseTableSelector.cs | 13 ++++++------- directory.build.props | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e25efa8fe..f9ce8126bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add ability to update an extraction's cohort from the command line using `SetExtractionConfigurationCohort ExtractionConfiguration:{id} ExtractableCohort:{id}` - Fix issue with non-default named PostgreSQL Table Info not being checkable +- Improve default timeouts on database lookups ## [8.1.6] - 2024-05-27 diff --git a/Directory.Packages.props b/Directory.Packages.props index 00a7797c86..802be23915 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -5,7 +5,7 @@ - + all diff --git a/Rdmp.Core/DataLoad/Engine/LoadExecution/Components/Standard/MigrateRAWTableToStaging.cs b/Rdmp.Core/DataLoad/Engine/LoadExecution/Components/Standard/MigrateRAWTableToStaging.cs index 4c55323db1..622a054c91 100644 --- a/Rdmp.Core/DataLoad/Engine/LoadExecution/Components/Standard/MigrateRAWTableToStaging.cs +++ b/Rdmp.Core/DataLoad/Engine/LoadExecution/Components/Standard/MigrateRAWTableToStaging.cs @@ -107,7 +107,7 @@ public override ExitCodeType Run(IDataLoadJob job, GracefulCancellationToken can //pass pre load discard var destination = new SqlBulkInsertDestination(destinationDatabase, destinationTableName, columnNamesToIgnoreForBulkInsert); - + destination.Timeout = 43200; //set max copy to 12 hours //engine that will move data _pipeline = new DataFlowPipelineEngine(context, source, destination, job); diff --git a/Rdmp.Core/DataLoad/Engine/Pipeline/Destinations/SqlBulkInsertDestination.cs b/Rdmp.Core/DataLoad/Engine/Pipeline/Destinations/SqlBulkInsertDestination.cs index 2f115af40e..4fed6bc89f 100644 --- a/Rdmp.Core/DataLoad/Engine/Pipeline/Destinations/SqlBulkInsertDestination.cs +++ b/Rdmp.Core/DataLoad/Engine/Pipeline/Destinations/SqlBulkInsertDestination.cs @@ -27,7 +27,7 @@ public class SqlBulkInsertDestination : IDataFlowDestination, IPipeli private readonly List _columnNamesToIgnore; private string _taskBeingPerformed; - public const int Timeout = 5000; + public int Timeout = 5000; private IBulkCopy _copy; private Stopwatch _timer = new(); diff --git a/Rdmp.UI/SimpleControls/ServerDatabaseTableSelector.cs b/Rdmp.UI/SimpleControls/ServerDatabaseTableSelector.cs index 150217a752..b3295490d6 100644 --- a/Rdmp.UI/SimpleControls/ServerDatabaseTableSelector.cs +++ b/Rdmp.UI/SimpleControls/ServerDatabaseTableSelector.cs @@ -114,7 +114,7 @@ public ServerDatabaseTableSelector() #region Async Stuff - private void UpdateTablesListAsync(object sender, DoWorkEventArgs e) + private async void UpdateTablesListAsync(object sender, DoWorkEventArgs e) { var builder = (DbConnectionStringBuilder)((object[])e.Argument)[0]; if (!string.IsNullOrWhiteSpace(Timeout) && int.TryParse(Timeout, out var _timeout)) @@ -132,9 +132,9 @@ private void UpdateTablesListAsync(object sender, DoWorkEventArgs e) var syntaxHelper = discoveredDatabase.Server.GetQuerySyntaxHelper(); try { - using var con = discoveredDatabase.Server.GetConnection(); - var openTask = con.OpenAsync(_workerRefreshTablesToken.Token); - openTask.Wait(_workerRefreshTablesToken.Token); + await using var con = discoveredDatabase.Server.GetConnection(); + await con.OpenAsync(_workerRefreshTablesToken.Token); + //openTask.Wait(_workerRefreshTablesToken.Token); var result = new List(); @@ -177,7 +177,7 @@ private void UpdateTablesAsyncCompleted(object sender, RunWorkerCompletedEventAr private void UpdateDatabaseListAsync(object sender, DoWorkEventArgs e) { var builder = (DbConnectionStringBuilder)((object[])e.Argument)[0]; - if (!string.IsNullOrWhiteSpace(Timeout) && int.TryParse(Timeout, out var _timeout)) + if (!string.IsNullOrWhiteSpace(Timeout) && int.TryParse(Timeout, out var _timeout) && _timeout > 0) { builder["Timeout"] = _timeout; } @@ -190,8 +190,7 @@ private void UpdateDatabaseListAsync(object sender, DoWorkEventArgs e) _workerRefreshDatabasesToken = new CancellationTokenSource(); try { - _listDatabasesAsyncResult = _helper.ListDatabasesAsync(builder, _workerRefreshDatabasesToken.Token) - .ToBlockingEnumerable(_workerRefreshDatabasesToken.Token).ToArray(); + _listDatabasesAsyncResult = _helper.ListDatabasesAsync(builder, _workerRefreshDatabasesToken.Token).ToBlockingEnumerable(_workerRefreshDatabasesToken.Token).ToArray(); } catch (OperationCanceledException) { diff --git a/directory.build.props b/directory.build.props index 0c317dcb16..e9fe2ce210 100644 --- a/directory.build.props +++ b/directory.build.props @@ -4,4 +4,4 @@ 8.1.7 true - \ No newline at end of file + From b15793d4c038a44fd96b75281fc7370054078a8a Mon Sep 17 00:00:00 2001 From: James A Sutherland Date: Mon, 17 Jun 2024 02:49:50 -0500 Subject: [PATCH 10/13] FTP, SFTP timeouts and liveness checks (#1856) * Bump HIC.FAnsiSql from 3.2.3 to 3.2.4 (#1850) Bumps [HIC.FAnsiSql](https://github.com/HicServices/FAnsiSql) from 3.2.3 to 3.2.4. - [Release notes](https://github.com/HicServices/FAnsiSql/releases) - [Changelog](https://github.com/HicServices/FAnsiSql/blob/main/CHANGELOG.md) - [Commits](https://github.com/HicServices/FAnsiSql/compare/v3.2.3...v3.2.4) --- updated-dependencies: - dependency-name: HIC.FAnsiSql dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump HIC.FAnsiSql from 3.2.4 to 3.2.5 (#1852) Bumps [HIC.FAnsiSql](https://github.com/HicServices/FAnsiSql) from 3.2.4 to 3.2.5. - [Release notes](https://github.com/HicServices/FAnsiSql/releases) - [Changelog](https://github.com/HicServices/FAnsiSql/blob/main/CHANGELOG.md) - [Commits](https://github.com/HicServices/FAnsiSql/compare/v3.2.4...v3.2.5) --- updated-dependencies: - dependency-name: HIC.FAnsiSql dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump Autoupdater.NET.Official from 1.8.6 to 1.9.0 Bumps [Autoupdater.NET.Official](https://github.com/ravibpatel/AutoUpdater.NET) from 1.8.6 to 1.9.0. - [Release notes](https://github.com/ravibpatel/AutoUpdater.NET/releases) - [Commits](https://github.com/ravibpatel/AutoUpdater.NET/compare/v1.8.6...v1.9.0) --- updated-dependencies: - dependency-name: Autoupdater.NET.Official dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * FTP, SFTP timeouts and liveness checks --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: James Friel --- CHANGELOG.md | 2 +- Directory.Packages.props | 2 +- .../DataLoad/Modules/FTP/FTPDownloader.cs | 42 +++++++++++-------- .../DataLoad/Modules/FTP/SFTPDownloader.cs | 7 +++- 4 files changed, 32 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9ce8126bd..4ddfc6f452 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add ability to update an extraction's cohort from the command line using `SetExtractionConfigurationCohort ExtractionConfiguration:{id} ExtractableCohort:{id}` - Fix issue with non-default named PostgreSQL Table Info not being checkable - Improve default timeouts on database lookups - +- Implement keepalive and liveness checks for FTP, SFTP fetches before deletion attempt ## [8.1.6] - 2024-05-27 diff --git a/Directory.Packages.props b/Directory.Packages.props index 802be23915..202b6dbad1 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -22,7 +22,7 @@ - + diff --git a/Rdmp.Core/DataLoad/Modules/FTP/FTPDownloader.cs b/Rdmp.Core/DataLoad/Modules/FTP/FTPDownloader.cs index c90631e536..1f9246a7b4 100644 --- a/Rdmp.Core/DataLoad/Modules/FTP/FTPDownloader.cs +++ b/Rdmp.Core/DataLoad/Modules/FTP/FTPDownloader.cs @@ -66,18 +66,18 @@ public FTPDownloader() [DemandsInitialization("The directory on the FTP server that you want to download files from")] public string? RemoteDirectory { get; set; } - [DemandsInitialization("True to set keep alive", DefaultValue = true)] + [DemandsInitialization("True to set keep alive",DefaultValue = true)] public bool KeepAlive { get; set; } - public void Initialize(ILoadDirectory directory, DiscoveredDatabase dbInfo) + public void Initialize(ILoadDirectory directory,DiscoveredDatabase dbInfo) { _directory = directory; } - public ExitCodeType Fetch(IDataLoadJob job, GracefulCancellationToken cancellationToken) + public ExitCodeType Fetch(IDataLoadJob job,GracefulCancellationToken cancellationToken) { - return DownloadFilesOnFTP(_directory ?? throw new InvalidOperationException("No output directory set"), job); + return DownloadFilesOnFTP(_directory ?? throw new InvalidOperationException("No output directory set"),job); } private FtpClient SetupFtp() @@ -86,32 +86,35 @@ private FtpClient SetupFtp() var username = FTPServer.Username ?? "anonymous"; var password = string.IsNullOrWhiteSpace(FTPServer.Password) ? "guest" : FTPServer.GetDecryptedPassword(); var c = new FtpClient(host, username, password); + + // Enable periodic NOOP keepalive operations to keep connection active until we're done + c.Config.Noop = true; c.AutoConnect(); return c; } - private ExitCodeType DownloadFilesOnFTP(ILoadDirectory destination, IDataLoadEventListener listener) + private ExitCodeType DownloadFilesOnFTP(ILoadDirectory destination,IDataLoadEventListener listener) { var files = GetFileList().ToArray(); - listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, + listener.OnNotify(this,new NotifyEventArgs(ProgressEventType.Information, $"Identified the following files on the FTP server:{string.Join(',',files)}")); var forLoadingContainedCachedFiles = false; foreach (var file in files) { - var action = GetSkipActionForFile(file, destination); + var action = GetSkipActionForFile(file,destination); - listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, + listener.OnNotify(this,new NotifyEventArgs(ProgressEventType.Information, $"File {file} was evaluated as {action}")); switch (action) { case SkipReason.DoNotSkip: listener.OnNotify(this, - new NotifyEventArgs(ProgressEventType.Information, $"About to download {file}")); - Download(file, destination); + new NotifyEventArgs(ProgressEventType.Information,$"About to download {file}")); + Download(file,destination); break; case SkipReason.InForLoading: forLoadingContainedCachedFiles = true; @@ -120,7 +123,7 @@ private ExitCodeType DownloadFilesOnFTP(ILoadDirectory destination, IDataLoadEve } // it was a success - even if no files were actually retrieved... hey that's what the user said, otherwise he would have set SendLoadNotRequiredIfFileNotFound - if (forLoadingContainedCachedFiles || _filesRetrieved.Any() || !SendLoadNotRequiredIfFileNotFound) + if (forLoadingContainedCachedFiles || _filesRetrieved.Count != 0 || !SendLoadNotRequiredIfFileNotFound) return ExitCodeType.Success; // if no files were downloaded (and there were none skipped because they were in forLoading) and in that eventuality we have our flag set to return LoadNotRequired then do so @@ -138,7 +141,7 @@ protected enum SkipReason IsImaginaryFile } - protected SkipReason GetSkipActionForFile(string file, ILoadDirectory destination) + protected SkipReason GetSkipActionForFile(string file,ILoadDirectory destination) { if (file.StartsWith(".",StringComparison.Ordinal)) return SkipReason.IsImaginaryFile; @@ -152,7 +155,7 @@ protected SkipReason GetSkipActionForFile(string file, ILoadDirectory destinatio } - private static bool ValidateServerCertificate(object _1, X509Certificate _2, X509Chain _3, + private static bool ValidateServerCertificate(object _1,X509Certificate _2,X509Chain _3, SslPolicyErrors _4) => true; //any cert will do! yay @@ -161,21 +164,24 @@ protected virtual IEnumerable GetFileList() return _connection.Value.GetNameListing().ToList().Where(_connection.Value.FileExists); } - protected virtual void Download(string file, ILoadDirectory destination) + protected virtual void Download(string file,ILoadDirectory destination) { var remotePath = !string.IsNullOrWhiteSpace(RemoteDirectory) ? $"{RemoteDirectory}/{file}" : file; - var destinationFileName = Path.Combine(destination.ForLoading.FullName, file); - _connection.Value.DownloadFile(destinationFileName, remotePath); + var destinationFileName = Path.Combine(destination.ForLoading.FullName,file); + _connection.Value.DownloadFile(destinationFileName,remotePath); _filesRetrieved.Add(remotePath); } - public virtual void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener) + public virtual void LoadCompletedSoDispose(ExitCodeType exitCode,IDataLoadEventListener postLoadEventListener) { if (exitCode != ExitCodeType.Success || !DeleteFilesOffFTPServerAfterSuccesfulDataLoad) return; + // Force a reconnection attempt if we got cut off + if (!_connection.Value.IsStillConnected()) + _connection.Value.Connect(true); foreach (var file in _filesRetrieved) _connection.Value.DeleteFile(file); } @@ -188,7 +194,7 @@ public void Check(ICheckNotifier notifier) } catch (Exception e) { - notifier.OnCheckPerformed(new CheckEventArgs("Failed to SetupFTP", CheckResult.Fail, e)); + notifier.OnCheckPerformed(new CheckEventArgs("Failed to SetupFTP",CheckResult.Fail,e)); } } } \ No newline at end of file diff --git a/Rdmp.Core/DataLoad/Modules/FTP/SFTPDownloader.cs b/Rdmp.Core/DataLoad/Modules/FTP/SFTPDownloader.cs index 8c489cf276..a33972c609 100644 --- a/Rdmp.Core/DataLoad/Modules/FTP/SFTPDownloader.cs +++ b/Rdmp.Core/DataLoad/Modules/FTP/SFTPDownloader.cs @@ -8,7 +8,6 @@ using System.IO; using System.Linq; using System.Threading; -using FluentFTP; using Rdmp.Core.Curation; using Rdmp.Core.Curation.Data; using Rdmp.Core.ReusableLibraryCode.Progress; @@ -46,6 +45,8 @@ private SftpClient SetupSftp() var password = string.IsNullOrWhiteSpace(FTPServer.Password) ? "guest" : FTPServer.GetDecryptedPassword(); var c = new SftpClient(host, username, password); c.Connect(); + if (KeepAlive) + c.KeepAliveInterval = TimeSpan.FromMilliseconds(KeepAliveIntervalMilliseconds); return c; } @@ -70,6 +71,10 @@ public override void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEven { if (exitCode != ExitCodeType.Success) return; + // Reconnect if we got cut off, for example due to idle timers + if (!_connection.Value.IsConnected) + _connection.Value.Connect(); + foreach (var retrievedFiles in _filesRetrieved) try { From 5ff49bc34b85b3c3641ef4b75e3c25b1955a6a49 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 17 Jun 2024 10:04:32 +0100 Subject: [PATCH 11/13] update changelog --- CHANGELOG.md | 4 +--- .../ExecuteCommandSetExtractionConfigurationCohort.cs | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ddfc6f452..3136a1cb20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,11 @@ - - # Changelog All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [8.1.7] - Unreleased +## [8.1.7] - 2024-06-17 ## Changed diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetExtractionConfigurationCohort.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetExtractionConfigurationCohort.cs index 27a3ab7b20..f8da43f79c 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetExtractionConfigurationCohort.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetExtractionConfigurationCohort.cs @@ -14,8 +14,8 @@ namespace Rdmp.Core.CommandExecution.AtomicCommands; public class ExecuteCommandSetExtractionConfigurationCohort : BasicCommandExecution, IAtomicCommand { - private ExtractionConfiguration _extractionConfiguration; - private ExtractableCohort _extractableCohort; + private readonly ExtractionConfiguration _extractionConfiguration; + private readonly ExtractableCohort _extractableCohort; public ExecuteCommandSetExtractionConfigurationCohort(IBasicActivateItems activator, ExtractionConfiguration extractionConfiguration, ExtractableCohort cohort) : base(activator) { From 7bec477affd1e1262cc6a0c1b0e450ec8c263687 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 17 Jun 2024 10:41:05 +0100 Subject: [PATCH 12/13] tidy up --- Rdmp.UI/SimpleControls/ServerDatabaseTableSelector.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Rdmp.UI/SimpleControls/ServerDatabaseTableSelector.cs b/Rdmp.UI/SimpleControls/ServerDatabaseTableSelector.cs index b3295490d6..1edc000f97 100644 --- a/Rdmp.UI/SimpleControls/ServerDatabaseTableSelector.cs +++ b/Rdmp.UI/SimpleControls/ServerDatabaseTableSelector.cs @@ -114,7 +114,7 @@ public ServerDatabaseTableSelector() #region Async Stuff - private async void UpdateTablesListAsync(object sender, DoWorkEventArgs e) + private void UpdateTablesListAsync(object sender, DoWorkEventArgs e) { var builder = (DbConnectionStringBuilder)((object[])e.Argument)[0]; if (!string.IsNullOrWhiteSpace(Timeout) && int.TryParse(Timeout, out var _timeout)) @@ -132,9 +132,9 @@ private async void UpdateTablesListAsync(object sender, DoWorkEventArgs e) var syntaxHelper = discoveredDatabase.Server.GetQuerySyntaxHelper(); try { - await using var con = discoveredDatabase.Server.GetConnection(); - await con.OpenAsync(_workerRefreshTablesToken.Token); - //openTask.Wait(_workerRefreshTablesToken.Token); + using var con = discoveredDatabase.Server.GetConnection(); + var openTask = con.OpenAsync(_workerRefreshTablesToken.Token); + openTask.Wait(_workerRefreshTablesToken.Token); var result = new List(); From 70dcb48e4b6831e7a583c8d38ad70e568b1ac2fd Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 17 Jun 2024 10:41:48 +0100 Subject: [PATCH 13/13] update updater --- rdmp-client.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rdmp-client.xml b/rdmp-client.xml index c0a16d535d..40a3c6da56 100644 --- a/rdmp-client.xml +++ b/rdmp-client.xml @@ -1,7 +1,7 @@ - 8.1.6.0 - https://github.com/HicServices/RDMP/releases/download/v8.1.6/rdmp-8.1.6-client.zip + 8.1.7.0 + https://github.com/HicServices/RDMP/releases/download/v8.1.7/rdmp-8.1.7-client.zip https://github.com/HicServices/RDMP/blob/main/CHANGELOG.md#7 true \ No newline at end of file