From d1c80e0c11979f900aaa44b0327060d764cb5b9a Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 4 Feb 2025 09:58:04 +0000 Subject: [PATCH 1/8] interim --- .../Pipeline/Sources/ExecuteDatasetExtractionSource.cs | 2 +- Rdmp.Core/QueryBuilding/ExtractionQueryBuilder.cs | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecuteDatasetExtractionSource.cs b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecuteDatasetExtractionSource.cs index d6e602bb4b..903507f540 100644 --- a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecuteDatasetExtractionSource.cs +++ b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecuteDatasetExtractionSource.cs @@ -190,7 +190,7 @@ public virtual DataTable GetChunk(IDataLoadEventListener listener, GracefulCance if (Request.DatasetBundle.DataSet.DisableExtraction) throw new Exception( $"Cannot extract {Request.DatasetBundle.DataSet} because DisableExtraction is set to true"); - + var x = GetCommandSQL(listener); _hostedSource = new DbDataCommandDataFlowSource(GetCommandSQL(listener), $"ExecuteDatasetExtraction {Request.DatasetBundle.DataSet}", Request.GetDistinctLiveDatabaseServer().Builder, diff --git a/Rdmp.Core/QueryBuilding/ExtractionQueryBuilder.cs b/Rdmp.Core/QueryBuilding/ExtractionQueryBuilder.cs index 80cb333d23..8158ad4d63 100644 --- a/Rdmp.Core/QueryBuilding/ExtractionQueryBuilder.cs +++ b/Rdmp.Core/QueryBuilding/ExtractionQueryBuilder.cs @@ -8,6 +8,7 @@ using System.Collections.Generic; using System.Linq; using FAnsi.Discovery.QuerySyntax; +using Rdmp.Core.CohortCommitting; using Rdmp.Core.DataExport; using Rdmp.Core.DataExport.Data; using Rdmp.Core.DataExport.DataExtraction.Commands; @@ -120,7 +121,8 @@ public QueryBuilder GetSQLCommandForFullExtractionSet(ExtractDatasetCommand requ var externalCohortTable = _repository.GetObjectByID(request.ExtractableCohort.ExternalCohortTable_ID); - + //todo here is where we want to create a temp table + CreateCohortTempTable(externalCohortTable); if (request.ExtractableCohort != null) { //the JOIN with the cohort table: @@ -141,6 +143,8 @@ public QueryBuilder GetSQLCommandForFullExtractionSet(ExtractDatasetCommand requ return queryBuilder; } + private void CreateCohortTempTable(ExternalCohortTable externalCohortTable) { } + private void HandleBatching(ExtractDatasetCommand request, QueryBuilder queryBuilder, IQuerySyntaxHelper syntaxHelper) { From c36ccacb8fca791fe817db61bd31b4c52ee49292 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 4 Feb 2025 13:53:54 +0000 Subject: [PATCH 2/8] add cohort temp table --- .../Sources/ExecuteDatasetExtractionSource.cs | 47 +++++++++++++++++-- .../Sources/DbDataCommandDataFlowSource.cs | 28 ++++++++--- .../QueryBuilding/ExtractionQueryBuilder.cs | 35 +++++++++++++- 3 files changed, 98 insertions(+), 12 deletions(-) diff --git a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecuteDatasetExtractionSource.cs b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecuteDatasetExtractionSource.cs index 903507f540..6fbd750342 100644 --- a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecuteDatasetExtractionSource.cs +++ b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecuteDatasetExtractionSource.cs @@ -7,10 +7,12 @@ using System; using System.Collections.Generic; using System.Data; +using System.Data.Common; using System.Diagnostics; using System.Linq; using System.Threading.Tasks; using FAnsi; +using FAnsi.Discovery; using FAnsi.Discovery.QuerySyntax; using Rdmp.Core.Curation.Data; using Rdmp.Core.DataExport.Data; @@ -99,13 +101,18 @@ public class ExecuteDatasetExtractionSource : IPluginDataFlowSource, private DbDataCommandDataFlowSource _hostedSource; + private IExternalCohortTable _externalCohortTable; + private string _whereSQL; + private DbConnection _con; + private string _uuid; protected virtual void Initialize(ExtractDatasetCommand request) { Request = request; if (request == ExtractDatasetCommand.EmptyCommand) return; - + _externalCohortTable = request.ExtractableCohort.ExternalCohortTable; + _whereSQL = request.ExtractableCohort.WhereSQL(); _timeSpentValidating = new Stopwatch(); _timeSpentCalculatingDISTINCT = new Stopwatch(); _timeSpentBuckettingDates = new Stopwatch(); @@ -156,6 +163,32 @@ private void Initialize(ExtractGlobalsCommand request) private RowPeeker _peeker = new(); + private static Random random = new Random(); + + private static string RandomString(int length) + { + const string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + return new string(Enumerable.Repeat(chars, length) + .Select(s => s[random.Next(s.Length)]).ToArray()); + } + + private void CreateCohortTempTable(DbConnection con) + { + var db = _externalCohortTable.Discover(); + _uuid = $"#{RandomString(24)}"; + var sql = $""" + SELECT * + INTO {_uuid} + FROM( + SELECT * FROM {_externalCohortTable.TableName} + WHERE {_whereSQL} + ) as cohortTempTable + + """; + using var cmd = db.Server.GetCommand(sql, con); + cmd.ExecuteNonQuery(); + } + public virtual DataTable GetChunk(IDataLoadEventListener listener, GracefulCancellationToken cancellationToken) { // we are in the Global Commands case, let's return an empty DataTable (not null) @@ -185,15 +218,18 @@ public virtual DataTable GetChunk(IDataLoadEventListener listener, GracefulCance if (_hostedSource == null) { + _con = DatabaseCommandHelper.GetConnection(Request.GetDistinctLiveDatabaseServer().Builder); + _con.Open(); + CreateCohortTempTable(_con); StartAudit(Request.QueryBuilder.SQL); if (Request.DatasetBundle.DataSet.DisableExtraction) throw new Exception( $"Cannot extract {Request.DatasetBundle.DataSet} because DisableExtraction is set to true"); - var x = GetCommandSQL(listener); + _hostedSource = new DbDataCommandDataFlowSource(GetCommandSQL(listener), $"ExecuteDatasetExtraction {Request.DatasetBundle.DataSet}", - Request.GetDistinctLiveDatabaseServer().Builder, + _con, ExecutionTimeout) { // If we are running in batches then always allow empty extractions @@ -451,7 +487,10 @@ private string GetCommandSQL(IDataLoadEventListener listener) listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"/*Decided on extraction SQL:*/{Environment.NewLine}{sql}")); - + if(_uuid is not null) + { + sql = sql.Replace(_externalCohortTable.TableName, _uuid); + } return sql; } diff --git a/Rdmp.Core/DataLoad/Engine/Pipeline/Sources/DbDataCommandDataFlowSource.cs b/Rdmp.Core/DataLoad/Engine/Pipeline/Sources/DbDataCommandDataFlowSource.cs index 67213043cc..440b848446 100644 --- a/Rdmp.Core/DataLoad/Engine/Pipeline/Sources/DbDataCommandDataFlowSource.cs +++ b/Rdmp.Core/DataLoad/Engine/Pipeline/Sources/DbDataCommandDataFlowSource.cs @@ -49,6 +49,17 @@ public DbDataCommandDataFlowSource(string sql, string taskBeingPerformed, DbConn BatchSize = 10000; } + public DbDataCommandDataFlowSource(string sql, string taskBeingPerformed, DbConnection con, + int timeout) + { + Sql = sql; + _taskBeingPerformed = taskBeingPerformed; + _con = con; + _timeout = timeout; + + BatchSize = 10000; + } + private int _numberOfColumns; private bool _firstChunk = true; @@ -59,8 +70,11 @@ public DataTable GetChunk(IDataLoadEventListener job, GracefulCancellationToken { if (_reader == null) { - _con = DatabaseCommandHelper.GetConnection(_builder); - _con.Open(); + _con = _con ==null?DatabaseCommandHelper.GetConnection(_builder):_con; + if(_con != null && _con.State == ConnectionState.Closed) +{ + _con.Open(); + } job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"Running SQL:{Environment.NewLine}{Sql}")); @@ -209,11 +223,13 @@ private void CloseReader(IDataLoadEventListener listener) public DataTable TryGetPreview() { var chunk = new DataTable(); - using var con = DatabaseCommandHelper.GetConnection(_builder); - con.Open(); - using var da = DatabaseCommandHelper.GetDataAdapter(DatabaseCommandHelper.GetCommand(Sql, con)); + _con = _con == null ? DatabaseCommandHelper.GetConnection(_builder) : _con; + if (_con != null && _con.State == ConnectionState.Closed) + { + _con.Open(); + } + using var da = DatabaseCommandHelper.GetDataAdapter(DatabaseCommandHelper.GetCommand(Sql, _con)); var read = da.Fill(0, 100, chunk); - return read == 0 ? null : chunk; } } \ No newline at end of file diff --git a/Rdmp.Core/QueryBuilding/ExtractionQueryBuilder.cs b/Rdmp.Core/QueryBuilding/ExtractionQueryBuilder.cs index 8158ad4d63..4916df7f13 100644 --- a/Rdmp.Core/QueryBuilding/ExtractionQueryBuilder.cs +++ b/Rdmp.Core/QueryBuilding/ExtractionQueryBuilder.cs @@ -6,8 +6,11 @@ using System; using System.Collections.Generic; +using System.Data.Common; using System.Linq; +using FAnsi.Discovery; using FAnsi.Discovery.QuerySyntax; +using NPOI.SS.Formula.Functions; using Rdmp.Core.CohortCommitting; using Rdmp.Core.DataExport; using Rdmp.Core.DataExport.Data; @@ -122,7 +125,11 @@ public QueryBuilder GetSQLCommandForFullExtractionSet(ExtractDatasetCommand requ var externalCohortTable = _repository.GetObjectByID(request.ExtractableCohort.ExternalCohortTable_ID); //todo here is where we want to create a temp table - CreateCohortTempTable(externalCohortTable); + + //var db = externalCohortTable.Discover(); + //var con = db.Server.GetConnection(); + //con.Open(); + //var tempTable = CreateCohortTempTable(externalCohortTable, request.ExtractableCohort.WhereSQL(),con.db); if (request.ExtractableCohort != null) { //the JOIN with the cohort table: @@ -143,7 +150,31 @@ public QueryBuilder GetSQLCommandForFullExtractionSet(ExtractDatasetCommand requ return queryBuilder; } - private void CreateCohortTempTable(ExternalCohortTable externalCohortTable) { } + //private static Random random = new Random(); + + //private static string RandomString(int length) + //{ + // const string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + // return new string(Enumerable.Repeat(chars, length) + // .Select(s => s[random.Next(s.Length)]).ToArray()); + //} + + //private string CreateCohortTempTable(ExternalCohortTable externalCohortTable, string whereSQL,DbConnection con, DiscoveredDatabase db) + //{ + // var uuid = $"#{RandomString(24)}"; + // var sql = $""" + // SELECT * + // INTO {uuid} + // FROM( + // SELECT * FROM {externalCohortTable.TableName} + // WHERE {whereSQL} + // ) as cohortTempTable + + // """; + // using var cmd = db.Server.GetCommand(sql, con); + // cmd.ExecuteNonQuery(); + // return uuid; + //} private void HandleBatching(ExtractDatasetCommand request, QueryBuilder queryBuilder, IQuerySyntaxHelper syntaxHelper) From dccd3dbc46bbc36176eb5650cbca9a379cc336bf Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 4 Feb 2025 16:15:03 +0000 Subject: [PATCH 3/8] tidy up code --- .../Sources/ExecuteDatasetExtractionSource.cs | 39 +++++++++++++------ .../QueryBuilding/ExtractionQueryBuilder.cs | 31 --------------- 2 files changed, 27 insertions(+), 43 deletions(-) diff --git a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecuteDatasetExtractionSource.cs b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecuteDatasetExtractionSource.cs index 6fbd750342..890857015a 100644 --- a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecuteDatasetExtractionSource.cs +++ b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecuteDatasetExtractionSource.cs @@ -90,6 +90,9 @@ public class ExecuteDatasetExtractionSource : IPluginDataFlowSource, "Exclusion list. A collection of Catalogues which will never be considered for HASH JOIN even when UseHashJoins is enabled. Being on this list takes precedence for a Catalogue even if it is on UseHashJoinsForCatalogues.")] public Catalogue[] DoNotUseHashJoinsForCatalogues { get; set; } + [DemandsInitialization("When performing an extracton, copy the cohort into a temporary table to improve extraction speed", defaultValue: false)] + public bool UseTempTablesWhenExtractingCohort { get; set; } + /// /// This is a dictionary containing all the CatalogueItems used in the query, the underlying datatype in the origin database and the @@ -172,7 +175,7 @@ private static string RandomString(int length) .Select(s => s[random.Next(s.Length)]).ToArray()); } - private void CreateCohortTempTable(DbConnection con) + private void CreateCohortTempTable(DbConnection con, IDataLoadEventListener listener) { var db = _externalCohortTable.Discover(); _uuid = $"#{RandomString(24)}"; @@ -185,8 +188,13 @@ private void CreateCohortTempTable(DbConnection con) ) as cohortTempTable """; + listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"About to copy the cohort into a temporary table using the SQL: {sql}")); + using var cmd = db.Server.GetCommand(sql, con); + cmd.CommandTimeout = ExecutionTimeout; cmd.ExecuteNonQuery(); + listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"Cohort successfully copied to temporary table")); + } public virtual DataTable GetChunk(IDataLoadEventListener listener, GracefulCancellationToken cancellationToken) @@ -218,24 +226,31 @@ public virtual DataTable GetChunk(IDataLoadEventListener listener, GracefulCance if (_hostedSource == null) { - _con = DatabaseCommandHelper.GetConnection(Request.GetDistinctLiveDatabaseServer().Builder); - _con.Open(); - CreateCohortTempTable(_con); - StartAudit(Request.QueryBuilder.SQL); + if (UseTempTablesWhenExtractingCohort) + { + _con = DatabaseCommandHelper.GetConnection(Request.GetDistinctLiveDatabaseServer().Builder); + _con.Open(); + CreateCohortTempTable(_con, listener); + } + var cmdSql = GetCommandSQL(listener); + StartAudit(cmdSql); if (Request.DatasetBundle.DataSet.DisableExtraction) throw new Exception( $"Cannot extract {Request.DatasetBundle.DataSet} because DisableExtraction is set to true"); - _hostedSource = new DbDataCommandDataFlowSource(GetCommandSQL(listener), + _hostedSource = UseTempTablesWhenExtractingCohort ? new DbDataCommandDataFlowSource(cmdSql, $"ExecuteDatasetExtraction {Request.DatasetBundle.DataSet}", _con, + ExecutionTimeout) : new DbDataCommandDataFlowSource(cmdSql, + $"ExecuteDatasetExtraction {Request.DatasetBundle.DataSet}", + Request.GetDistinctLiveDatabaseServer().Builder, ExecutionTimeout) - { - // If we are running in batches then always allow empty extractions - AllowEmptyResultSets = AllowEmptyExtractions || Request.IsBatchResume, - BatchSize = BatchSize - }; + { + // If we are running in batches then always allow empty extractions + AllowEmptyResultSets = AllowEmptyExtractions || Request.IsBatchResume, + BatchSize = BatchSize + }; } DataTable chunk = null; @@ -487,7 +502,7 @@ private string GetCommandSQL(IDataLoadEventListener listener) listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"/*Decided on extraction SQL:*/{Environment.NewLine}{sql}")); - if(_uuid is not null) + if (UseTempTablesWhenExtractingCohort && _uuid is not null) { sql = sql.Replace(_externalCohortTable.TableName, _uuid); } diff --git a/Rdmp.Core/QueryBuilding/ExtractionQueryBuilder.cs b/Rdmp.Core/QueryBuilding/ExtractionQueryBuilder.cs index 4916df7f13..a95d066bfa 100644 --- a/Rdmp.Core/QueryBuilding/ExtractionQueryBuilder.cs +++ b/Rdmp.Core/QueryBuilding/ExtractionQueryBuilder.cs @@ -124,12 +124,7 @@ public QueryBuilder GetSQLCommandForFullExtractionSet(ExtractDatasetCommand requ var externalCohortTable = _repository.GetObjectByID(request.ExtractableCohort.ExternalCohortTable_ID); - //todo here is where we want to create a temp table - //var db = externalCohortTable.Discover(); - //var con = db.Server.GetConnection(); - //con.Open(); - //var tempTable = CreateCohortTempTable(externalCohortTable, request.ExtractableCohort.WhereSQL(),con.db); if (request.ExtractableCohort != null) { //the JOIN with the cohort table: @@ -150,32 +145,6 @@ public QueryBuilder GetSQLCommandForFullExtractionSet(ExtractDatasetCommand requ return queryBuilder; } - //private static Random random = new Random(); - - //private static string RandomString(int length) - //{ - // const string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; - // return new string(Enumerable.Repeat(chars, length) - // .Select(s => s[random.Next(s.Length)]).ToArray()); - //} - - //private string CreateCohortTempTable(ExternalCohortTable externalCohortTable, string whereSQL,DbConnection con, DiscoveredDatabase db) - //{ - // var uuid = $"#{RandomString(24)}"; - // var sql = $""" - // SELECT * - // INTO {uuid} - // FROM( - // SELECT * FROM {externalCohortTable.TableName} - // WHERE {whereSQL} - // ) as cohortTempTable - - // """; - // using var cmd = db.Server.GetCommand(sql, con); - // cmd.ExecuteNonQuery(); - // return uuid; - //} - private void HandleBatching(ExtractDatasetCommand request, QueryBuilder queryBuilder, IQuerySyntaxHelper syntaxHelper) { From 4761d779cc17dacef060ed72b1042d095aab63dd Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 4 Feb 2025 16:38:00 +0000 Subject: [PATCH 4/8] handle all types of db --- .../Sources/ExecuteDatasetExtractionSource.cs | 49 ++++++++++++++----- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecuteDatasetExtractionSource.cs b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecuteDatasetExtractionSource.cs index 890857015a..a1385bd0cd 100644 --- a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecuteDatasetExtractionSource.cs +++ b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecuteDatasetExtractionSource.cs @@ -12,8 +12,8 @@ using System.Linq; using System.Threading.Tasks; using FAnsi; -using FAnsi.Discovery; using FAnsi.Discovery.QuerySyntax; +using MySqlConnector; using Rdmp.Core.Curation.Data; using Rdmp.Core.DataExport.Data; using Rdmp.Core.DataExport.DataExtraction.Commands; @@ -177,17 +177,44 @@ private static string RandomString(int length) private void CreateCohortTempTable(DbConnection con, IDataLoadEventListener listener) { - var db = _externalCohortTable.Discover(); _uuid = $"#{RandomString(24)}"; - var sql = $""" - SELECT * - INTO {_uuid} - FROM( - SELECT * FROM {_externalCohortTable.TableName} - WHERE {_whereSQL} - ) as cohortTempTable - - """; + var sql = ""; + var db = _externalCohortTable.Discover(); + switch (db.Server.DatabaseType) + { + case DatabaseType.MicrosoftSQLServer: + sql = $""" + SELECT * + INTO {_uuid} + FROM( + SELECT * FROM {_externalCohortTable.TableName} + WHERE {_whereSQL} + ) as cohortTempTable + """; + break; + case DatabaseType.MySql: + sql = $""" + CREATE TEMPORARY TABLE {_uuid} ENGINE=MEMORY + as (SELECT * FROM {_externalCohortTable.TableName} WHERE {_whereSQL}) + """; + break; + case DatabaseType.Oracle: + sql= $""" + CREATE TEMPORARY TABLE {_uuid} SELECT * FROM {_externalCohortTable.TableName} WHERE {_whereSQL} + """; + break; + case DatabaseType.PostgreSql: + sql= $""" + CREATE TEMP TABLE {_uuid} AS + SELECT * FROM {_externalCohortTable.TableName} WHERE {_whereSQL} + """; + break; + default: + listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, $"Unable to create temporary table for cohort. Original cohort table will be used")); + return; + + + } listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"About to copy the cohort into a temporary table using the SQL: {sql}")); using var cmd = db.Server.GetCommand(sql, con); From c1f4a3cadfd29349ff39436eff43dbf1ff8dcd56 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 4 Feb 2025 16:41:51 +0000 Subject: [PATCH 5/8] tidy up --- .../Pipeline/Sources/ExecuteDatasetExtractionSource.cs | 3 +-- Rdmp.Core/QueryBuilding/ExtractionQueryBuilder.cs | 4 ---- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecuteDatasetExtractionSource.cs b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecuteDatasetExtractionSource.cs index a1385bd0cd..6243a76c7a 100644 --- a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecuteDatasetExtractionSource.cs +++ b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecuteDatasetExtractionSource.cs @@ -1,4 +1,4 @@ -// Copyright (c) The University of Dundee 2018-2019 +// Copyright (c) The University of Dundee 2018-2025 // This file is part of the Research Data Management Platform (RDMP). // RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. @@ -13,7 +13,6 @@ using System.Threading.Tasks; using FAnsi; using FAnsi.Discovery.QuerySyntax; -using MySqlConnector; using Rdmp.Core.Curation.Data; using Rdmp.Core.DataExport.Data; using Rdmp.Core.DataExport.DataExtraction.Commands; diff --git a/Rdmp.Core/QueryBuilding/ExtractionQueryBuilder.cs b/Rdmp.Core/QueryBuilding/ExtractionQueryBuilder.cs index a95d066bfa..80cb333d23 100644 --- a/Rdmp.Core/QueryBuilding/ExtractionQueryBuilder.cs +++ b/Rdmp.Core/QueryBuilding/ExtractionQueryBuilder.cs @@ -6,12 +6,8 @@ using System; using System.Collections.Generic; -using System.Data.Common; using System.Linq; -using FAnsi.Discovery; using FAnsi.Discovery.QuerySyntax; -using NPOI.SS.Formula.Functions; -using Rdmp.Core.CohortCommitting; using Rdmp.Core.DataExport; using Rdmp.Core.DataExport.Data; using Rdmp.Core.DataExport.DataExtraction.Commands; From f0ff75e3c9708e9af0a79a55fd734ce582755200 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 5 Feb 2025 09:50:13 +0000 Subject: [PATCH 6/8] tidy up --- .../Pipeline/Sources/ExecuteDatasetExtractionSource.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecuteDatasetExtractionSource.cs b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecuteDatasetExtractionSource.cs index 6243a76c7a..91edc06736 100644 --- a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecuteDatasetExtractionSource.cs +++ b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecuteDatasetExtractionSource.cs @@ -165,7 +165,7 @@ private void Initialize(ExtractGlobalsCommand request) private RowPeeker _peeker = new(); - private static Random random = new Random(); + private static readonly Random random = new Random(); private static string RandomString(int length) { From 7a73b88bc40c27510ed215aa9f77c38ff88df917 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 5 Feb 2025 14:21:16 +0000 Subject: [PATCH 7/8] add changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5cc94376cd..70f315472a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fix application restart not closing all windows - Improve cohort deprecation override test - Add Filters for CatalogueItems to Dashboard graphs +- Add ability to use cohort temp table during extractions ## [8.4.2] - 2024-12-18 From f92552531191439fec876b80d04b43c5e2a824c3 Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 6 Feb 2025 07:53:03 +0000 Subject: [PATCH 8/8] add error handling --- .../Pipeline/Sources/ExecuteDatasetExtractionSource.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecuteDatasetExtractionSource.cs b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecuteDatasetExtractionSource.cs index 91edc06736..3daced40fa 100644 --- a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecuteDatasetExtractionSource.cs +++ b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecuteDatasetExtractionSource.cs @@ -218,7 +218,14 @@ CREATE TEMP TABLE {_uuid} AS using var cmd = db.Server.GetCommand(sql, con); cmd.CommandTimeout = ExecutionTimeout; - cmd.ExecuteNonQuery(); + try + { + cmd.ExecuteNonQuery(); + } + catch (Exception ex) { + listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, $"Unable to create temporary table for cohort. Original cohort table will be used",ex)); + _uuid = null; + } listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"Cohort successfully copied to temporary table")); }