Skip to content

Commit

Permalink
interim
Browse files Browse the repository at this point in the history
  • Loading branch information
@JFriel
JFriel committed Jan 6, 2025
1 parent a109bd3 commit cd05458
Show file tree
Hide file tree
Showing 2 changed files with 78 additions and 231 deletions.
277 changes: 69 additions & 208 deletions Rdmp.Core/Curation/Data/Overview/OverviewModel.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
using Rdmp.Core.Curation.Data.DataLoad;
using Rdmp.Core.DataExport.Data;
using Rdmp.Core.DataLoad.Triggers;
using Rdmp.Core.DataQualityEngine.Data;
using Rdmp.Core.DataViewing;
using Rdmp.Core.Logging;
using Rdmp.Core.MapsDirectlyToDatabaseTable;
Expand All @@ -32,257 +33,117 @@ public class OverviewModel

private readonly ICatalogue _catalogue;
private readonly IBasicActivateItems _activator;
private CatalogueOverview _catalogueOverview;
//private CatalogueOverview _catalogueOverview;
private Evaluation _evaluation;
private DataTable _dqeTable = new();
//private DateTime? _extractionDate;
private DateTime? _dataLoadDate;

public OverviewModel(IBasicActivateItems activator, ICatalogue catalogue)
{
_activator = activator;
_catalogue = catalogue;
_catalogueOverview = activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere<CatalogueOverview>("Catalogue_ID", catalogue.ID).FirstOrDefault();
_dqeTable = GetCountsByDatePeriod();
_dataLoadDate = GetDataLoadDate();
}

public void Generate(int dateColumn_ID)
public DataTable GetTableData()
{
var recordCount = 1;// GetRecordCount();
var peopleCount = 1;// GetPeopleCount();
if (_catalogueOverview is null)
{
_catalogueOverview = new CatalogueOverview(_activator.RepositoryLocator.CatalogueRepository, _catalogue.ID, recordCount, peopleCount, dateColumn_ID);
}
//_catalogueOverview.LastDataLoad = GetDataLoadDate();
//_catalogueOverview.LastExtractionTime = GetExtractionTime();
//_catalogueOverview.DateColumn_ID = dateColumn_ID;
//var dates = GetDates();
//_catalogueOverview.StartDate = dates?.Item1;
//_catalogueOverview.EndDate = dates?.Item2;
_catalogueOverview.SaveToDatabase();

GetCountsByDate();
}

private int GetRecordCount()
{
var column = _catalogue.CatalogueItems.Where(ci => ci.ExtractionInformation.IsExtractionIdentifier).FirstOrDefault();
if (column is null)
{
column = _catalogue.CatalogueItems.FirstOrDefault();
}
if (column is null) return 0;
var discoveredColumn = column.ColumnInfo.Discover(DataAccessContext.InternalDataProcessing);
return discoveredColumn.Table.GetRowCount();
}

private int GetPeopleCount()
{
var column = _catalogue.CatalogueItems.Where(ci => ci.ExtractionInformation.IsExtractionIdentifier).FirstOrDefault();
if (column is null) return 0;
var discoveredColumn = column.ColumnInfo.Discover(DataAccessContext.InternalDataProcessing);
var qb = new QueryBuilder("DISTINCT", null, null);
var memRepo = new MemoryRepository();
qb.AddColumn(new ColumnInfoToIColumn(memRepo, column.ColumnInfo));
var server = discoveredColumn.Table.Database.Server;
using var con = server.GetConnection();
con.Open();
using var cmd = server.GetCommand($"SELECT count(*) from ({qb.SQL})as dt", con);
return int.Parse(cmd.ExecuteScalar().ToString());
return _dqeTable;
}

private DateTime? GetDataLoadDate()
{
var column = _catalogue.CatalogueItems.Where(c => c.Name == SpecialFieldNames.ValidFrom).FirstOrDefault();
if (column is null) return null;
var discoveredColumn = column.ColumnInfo.Discover(DataAccessContext.InternalDataProcessing);
var qb = new QueryBuilder("DISTINCT", null, null);
var memRepo = new MemoryRepository();
qb.AddColumn(new ColumnInfoToIColumn(memRepo, column.ColumnInfo));
var server = discoveredColumn.Table.Database.Server;
using var con = server.GetConnection();
con.Open();
using var cmd = server.GetCommand(qb.SQL, con);
using var da = server.GetDataAdapter(cmd);
var dt = new DataTable();
da.Fill(dt);
var rows = dt.AsEnumerable().Where(r => !string.IsNullOrEmpty(r[0].ToString()));
if (!rows.Any()) return null;
return rows.Select(r => DateTime.Parse(r[0].ToString())).Max();
}

private DateTime? GetExtractionTime()
{
var datasets = _activator.RepositoryLocator.DataExportRepository.GetAllObjectsWhere<ExtractableDataSet>("Catalogue_ID", _catalogue.ID).Select(d => d.ID);
var results = _activator.RepositoryLocator.DataExportRepository.GetAllObjects<CumulativeExtractionResults>().Where(result => datasets.Contains(result.ExtractableDataSet_ID)).ToList();
if (!results.Any()) return null;
return results.Select(r => r.DateOfExtraction).Max();

}


private Tuple<DateTime?, DateTime?> GetDates()
{
DateTime? min = null;
DateTime? max = null;
var column = _catalogue.CatalogueItems.Where(c => c.ID == _catalogueOverview.DateColumn_ID).FirstOrDefault();
if (column == null) return new Tuple<DateTime?, DateTime?>(min, max); ;
var discoveredColumn = column.ColumnInfo.Discover(DataAccessContext.InternalDataProcessing);
var server = discoveredColumn.Table.Database.Server;
if (server.DatabaseType == FAnsi.DatabaseType.MicrosoftSQLServer)
{
var qb = new QueryBuilder(null, null, null);
var memRepo = new MemoryRepository();
qb.AddColumn(new ColumnInfoToIColumn(memRepo, column.ColumnInfo));
using var con = server.GetConnection();
con.Open();
using var cmd = server.GetCommand($"SELECT max({column.Name}) from {discoveredColumn.Table.GetRuntimeName()}", con);
max = DateTime.Parse((cmd.ExecuteScalar().ToString()));
using var cmd2 = server.GetCommand($"SELECT min({column.Name}) from {discoveredColumn.Table.GetRuntimeName()}", con);
min = DateTime.Parse((cmd2.ExecuteScalar().ToString()));
}
else
{
var qb = new QueryBuilder("DISTINCT", null, null);
var memRepo = new MemoryRepository();
qb.AddColumn(new ColumnInfoToIColumn(memRepo, column.ColumnInfo));
using var con = server.GetConnection();
con.Open();
using var cmd = server.GetCommand(qb.SQL, con);
cmd.CommandTimeout = 300;
using var da = server.GetDataAdapter(cmd);
var dt = new DataTable();
da.Fill(dt);
var rows = dt.AsEnumerable().Where(r => !string.IsNullOrEmpty(r[0].ToString()));
if (!rows.Any()) return new Tuple<DateTime?, DateTime?>(min, max);
var dateTimeRows = rows.Select(r => DateTime.Parse(r[0].ToString()));
max = dateTimeRows.Max();
min = dateTimeRows.Min();
}
return new Tuple<DateTime?, DateTime?>(min, max);
}

private void GetCountsByDate()
{
var column = _catalogue.CatalogueItems.Where(c => c.ID == _catalogueOverview.DateColumn_ID).FirstOrDefault();
var dateString = "yyyy-MM";
var discoveredColumn = column.ColumnInfo.Discover(DataAccessContext.InternalDataProcessing);
var server = discoveredColumn.Table.Database.Server;
using var con = server.GetConnection();
con.Open();
var counts = new DataTable();
if (server.DatabaseType == FAnsi.DatabaseType.MicrosoftSQLServer)
{

var sql = @$"
SELECT
FORMAT({column.Name},'{dateString}'), count(FORMAT({column.Name},'{dateString}'))
FROM
{discoveredColumn.Table.GetRuntimeName()}
WHERE
{column.Name} IS NOT NULL
Group by FORMAT({column.Name},'{dateString}')
ORDER BY FORMAT({column.Name},'{dateString}')";
var cmd = server.GetCommand(sql, con);
using var da = server.GetDataAdapter(cmd);
counts.BeginLoadData();
da.Fill(counts);
counts.EndLoadData();
con.Dispose();
}
else
{
//nonsql v slow
var repo = new MemoryCatalogueRepository();
var qb = new QueryBuilder(null, null);
qb.AddColumn(new ColumnInfoToIColumn(repo, column.ColumnInfo));
qb.AddCustomLine($"{column.ColumnInfo.Name} IS NOT NULL", FAnsi.Discovery.QuerySyntax.QueryComponent.WHERE);
var cmd = server.GetCommand(qb.SQL, con);
using var da = server.GetDataAdapter(cmd);
var dt = new DataTable();
dt.BeginLoadData();
da.Fill(dt);
dt.EndLoadData();
con.Dispose();
Dictionary<string, int> _counts = [];
foreach (var key in dt.AsEnumerable().Select(row => DateTime.Parse(row[0].ToString()).ToString(dateString)))
{
if (_counts.TryGetValue(key, out var count))
{
_counts[key]++;
}
else
{
_counts[key] = 1;
}
}
foreach (var item in _counts)
{
DataRow dr = counts.NewRow();
dr[0] = item.Key;
dr[1] = item.Value;
}
}
if (_activator.RepositoryLocator.CatalogueRepository.GetType() == typeof(CatalogueRepository))
{
//fast
((CatalogueRepository)_activator.RepositoryLocator.CatalogueRepository).Delete($"DELETE CatalogueOverviewDataPoint WHERE CatalogueOverview_ID = {_catalogueOverview.ID}", null, false);
}
else
{
//very slow
foreach (CatalogueOverviewDataPoint dp in _activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere<CatalogueOverviewDataPoint>("CatalogueOverview_ID", _catalogueOverview.ID).ToList())
int dataLoadID = _evaluation.RowStates.Max(rs => rs.DataLoadRunID);
if (dataLoadID > 0) {
//can use the hic_validFrom in the underlying catalogue table to get the data load date
var pk = _catalogue.CatalogueItems.Select(c => c.ColumnInfo).Where(c => c.IsPrimaryKey).FirstOrDefault();
if (pk is not null)
{
dp.DeleteInDatabase();
var table = pk.TableInfo.Discover(DataAccessContext.InternalDataProcessing);
table.

Check failure on line 64 in Rdmp.Core/Curation/Data/Overview/OverviewModel.cs

View workflow job for this annotation

GitHub Actions / Bundle Build 

Identifier expected

Check failure on line 64 in Rdmp.Core/Curation/Data/Overview/OverviewModel.cs

View workflow job for this annotation

GitHub Actions / Bundle Build 

; expected

Check failure on line 64 in Rdmp.Core/Curation/Data/Overview/OverviewModel.cs

View workflow job for this annotation

GitHub Actions / Bundle Build 

Identifier expected

Check failure on line 64 in Rdmp.Core/Curation/Data/Overview/OverviewModel.cs

View workflow job for this annotation

GitHub Actions / Bundle Build 

; expected

Check failure on line 64 in Rdmp.Core/Curation/Data/Overview/OverviewModel.cs

View workflow job for this annotation

GitHub Actions / Bundle Build 

Identifier expected

Check failure on line 64 in Rdmp.Core/Curation/Data/Overview/OverviewModel.cs

View workflow job for this annotation

GitHub Actions / Bundle Build 

; expected
}
}

foreach (var item in counts.AsEnumerable())
{
var dp = new CatalogueOverviewDataPoint(_activator.RepositoryLocator.CatalogueRepository, _catalogueOverview.ID, DateTime.Parse(item[0].ToString()), (int)item[1]);
dp.SaveToDatabase();
}
return null;
}

//private int GetRecordCount()
//{
// var column = _catalogue.CatalogueItems.Where(ci => ci.ExtractionInformation.IsExtractionIdentifier).FirstOrDefault();
// if (column is null)
// {
// column = _catalogue.CatalogueItems.FirstOrDefault();
// }
// if (column is null) return 0;
// var discoveredColumn = column.ColumnInfo.Discover(DataAccessContext.InternalDataProcessing);
// return discoveredColumn.Table.GetRowCount();
//}

//private int GetPeopleCount()
//{
// var column = _catalogue.CatalogueItems.Where(ci => ci.ExtractionInformation.IsExtractionIdentifier).FirstOrDefault();
// if (column is null) return 0;
// var discoveredColumn = column.ColumnInfo.Discover(DataAccessContext.InternalDataProcessing);
// var qb = new QueryBuilder("DISTINCT", null, null);
// var memRepo = new MemoryRepository();
// qb.AddColumn(new ColumnInfoToIColumn(memRepo, column.ColumnInfo));
// var server = discoveredColumn.Table.Database.Server;
// using var con = server.GetConnection();
// con.Open();
// using var cmd = server.GetCommand($"SELECT count(*) from ({qb.SQL})as dt", con);
// return int.Parse(cmd.ExecuteScalar().ToString());
//}

public int GetNumberOfRecords()
{
return _catalogueOverview != null ? _catalogueOverview.NumberOfRecords : 0;
return _dqeTable.AsEnumerable()
.Sum(x => int.Parse(x["# Records"].ToString()));
}

public int GetNumberOfPeople()
{
return _catalogueOverview != null ? _catalogueOverview.NumberOfPeople : 0;
return 0;// _catalogueOverview != null ? _catalogueOverview.NumberOfPeople : 0;
}

public string GetLatestExtraction()
{
return _catalogueOverview?.LastExtractionTime != null ? ((DateTime)_catalogueOverview.LastExtractionTime).ToString("dd/MM/yyyy") : null;
return null;// _extractionDate != null ? ((DateTime)_extractionDate).ToString("MM-yyyy") : null;
}

public string GetLatestDataLoad()
{
return _catalogueOverview?.LastDataLoad != null ? ((DateTime)_catalogueOverview.LastDataLoad).ToString("dd/MM/yyyy") : null;
return _dataLoadDate!= null ? ((DateTime)_dataLoadDate).ToString("MM-yyyy") : null;
}

public Tuple<DateTime?, DateTime?> GetStartEndDates()
{
return new Tuple<DateTime?, DateTime?>(_catalogueOverview?.StartDate, _catalogueOverview?.EndDate);
var start = DateTime.Parse(_dqeTable.AsEnumerable().First()["YearMonth"].ToString());
var end= DateTime.Parse(_dqeTable.AsEnumerable().Last()["YearMonth"].ToString());
return new Tuple<DateTime?, DateTime?>(start,end);
}

public int? GetDateColumn()
{
return _catalogueOverview != null ? _catalogueOverview.DateColumn_ID : null;
}
public DataTable GetCountsByDatePeriod()
private DataTable GetCountsByDatePeriod()
{

var dt = new DataTable();
dt.Columns.Add("YearMonth");
dt.Columns.Add("# Records");
if (_catalogueOverview is null) return dt;
var counts = _activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere<CatalogueOverviewDataPoint>("CatalogueOverview_ID", _catalogueOverview.ID).ToList().OrderBy(c => c.Date);
foreach (var item in counts)
var repo = new DQERepository(_catalogue.CatalogueRepository);
_evaluation = repo.GetAllObjectsWhere<Evaluation>("CatalogueID", _catalogue.ID).LastOrDefault();
if (_evaluation != null)
{
dt.Rows.Add([item.Date.ToString("yyyy-MM"), item.Count]);
dt = PeriodicityState.GetPeriodicityForDataTableForEvaluation(_evaluation, "ALL", true);
dt.Columns.Add("# Records");
foreach (DataRow row in dt.Rows)
{
row["# Records"] = int.Parse(row["Correct"].ToString()) + int.Parse(row["Wrong"].ToString()) + int.Parse(row["Missing"].ToString()) + int.Parse(row["InvalidatesRow"].ToString());
}
dt.Columns.Remove("Year");
dt.Columns.Remove("Month");
dt.Columns.Remove("Correct");
dt.Columns.Remove("Wrong");
dt.Columns.Remove("Missing");
dt.Columns.Remove("InvalidatesRow");
}

return dt;
}

Expand Down
32 changes: 9 additions & 23 deletions Rdmp.UI/Overview/ViewCatalogueOverviewUI.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,26 +63,12 @@ private void UpdateCatalogueData()
}
lblPeople.Text = $"{_overview.GetNumberOfPeople()} People";

areaChartUI.GenerateChart(_overview.GetCountsByDatePeriod(), $"Records per month");
areaChartUI.GenerateChart(_overview.GetTableData(), $"Records per month");

var syntaxHelper = _catalogue.GetDistinctLiveDatabaseServer(DataAccessContext.InternalDataProcessing, false)?.GetQuerySyntaxHelper();
var dateTypeString = syntaxHelper.TypeTranslater.GetSQLDBTypeForCSharpType(new TypeGuesser.DatabaseTypeRequest(typeof(DateTime)));
var dateColumns = _catalogue.CatalogueItems.Where(ci => ci.ColumnInfo.Data_type == dateTypeString).ToList();

if (cbTimeColumns.Items.Count == 0)
{
var x = dateColumns.ToList().ToArray();
cbTimeColumns.Items.AddRange(x);
}
if (_overview.GetDateColumn() != null)
{
cbTimeColumns.SelectedIndex = dateColumns.IndexOf(dateColumns.Where(ci => ci.ID == _overview.GetDateColumn()).FirstOrDefault());
}
else
{
cbTimeColumns.SelectedIndex = 0;
}

}

private int OnTabChange(int tabIndex)
Expand All @@ -107,14 +93,14 @@ private void btnSettings_Click(object sender, EventArgs e)

private void btnGenerate_Click(object sender, EventArgs e)
{
var dateColumnID = ((CatalogueItem)cbTimeColumns.SelectedItem).ID;
Task.Run(() =>
{
_overview.Generate(dateColumnID);
}).ContinueWith((task) =>
{
UpdateCatalogueData();
}, TaskScheduler.FromCurrentSynchronizationContext());
//var dateColumnID = ((CatalogueItem)cbTimeColumns.SelectedItem).ID;
//Task.Run(() =>
//{
// _overview.Generate(dateColumnID);
//}).ContinueWith((task) =>
//{
// UpdateCatalogueData();
//}, TaskScheduler.FromCurrentSynchronizationContext());
}

}
Expand Down

0 comments on commit cd05458

Please sign in to comment.