From 0dcd99a36be4fa6d9c2bcecc7707b379470465d5 Mon Sep 17 00:00:00 2001 From: "Martin Hinshelwood nkdAgility.com" Date: Wed, 17 Jul 2024 11:32:51 +0100 Subject: [PATCH 1/6] feat(WorkItemCloneCommand.cs): replace List with CashedWorkItems to include queryDatetime refactor(WorkItemCloneCommand.cs): optimize cache loading and testing logic to improve performance feat(WorkItem.cs): add CashedWorkItems class to store workitems and queryDatetime for better cache management --- .../Commands/WorkItemCloneCommand.cs | 41 ++++++++++--------- .../DataContracts/WorkItem.cs | 5 +++ 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/AzureDevOps.WorkItemClone.ConsoleUI/Commands/WorkItemCloneCommand.cs b/AzureDevOps.WorkItemClone.ConsoleUI/Commands/WorkItemCloneCommand.cs index 2e9b650..5a55e82 100644 --- a/AzureDevOps.WorkItemClone.ConsoleUI/Commands/WorkItemCloneCommand.cs +++ b/AzureDevOps.WorkItemClone.ConsoleUI/Commands/WorkItemCloneCommand.cs @@ -9,6 +9,7 @@ using Microsoft.VisualStudio.Services.CircuitBreaker; using Newtonsoft.Json.Linq; using System.Diagnostics.Eventing.Reader; +using AzureDevOps.WorkItemClone.Repositories; namespace AzureDevOps.WorkItemClone.ConsoleUI.Commands { @@ -86,7 +87,7 @@ await AnsiConsole.Progress() } task1.MaxValue = 1; - List templateWorkItems = null; + CashedWorkItems templateWorkItems = null; @@ -95,12 +96,11 @@ await AnsiConsole.Progress() if (System.IO.File.Exists(cacheTemplateWorkItemsFile)) { - - var changedDate = System.IO.File.GetLastWriteTime(cacheTemplateWorkItemsFile).AddDays(1).Date; - //Test Cache - QueryResults fakeItemsFromTemplateQuery; - fakeItemsFromTemplateQuery = await templateApi.GetWiqlQueryResults("Select [System.Id] From WorkItems Where [System.TeamProject] = '@project' AND [System.Parent] = @id AND [System.ChangedDate] > '@changeddate' order by [System.CreatedDate] desc", new Dictionary() { { "@id", config.templateParentId.ToString() }, { "@changeddate", changedDate.ToString("yyyy-MM-dd") } }); - if (fakeItemsFromTemplateQuery.workItems.Length == 0) + // load Cache + templateWorkItems = JsonConvert.DeserializeObject(System.IO.File.ReadAllText(cacheTemplateWorkItemsFile)); + //Test Cache date + QueryResults changedWorkItems = await templateApi.GetWiqlQueryResults("Select [System.Id] From WorkItems Where [System.TeamProject] = '@project' AND [System.Parent] = @id AND [System.ChangedDate] > '@changeddate' order by [System.CreatedDate] desc", new Dictionary() { { "@id", config.templateParentId.ToString() }, { "@changeddate", templateWorkItems.queryDatetime.AddDays(-1).ToString("yyyy-MM-dd") } }); + if (changedWorkItems.workItems.Length == 0) { AnsiConsole.WriteLine($"Stage 1: Checked template for changes. None Detected. Loading Cache"); @@ -111,10 +111,12 @@ await AnsiConsole.Progress() await Task.Delay(250); task1.StopTask(); ////////////////////// - templateWorkItems = JsonConvert.DeserializeObject>(System.IO.File.ReadAllText(cacheTemplateWorkItemsFile)); - task2.Increment(templateWorkItems.Count); + task2.Increment(templateWorkItems.workitems.Count()); task2.Description = task2.Description + " (cache)"; - AnsiConsole.WriteLine($"Stage 2: Loaded {templateWorkItems.Count()} work items from cache."); + AnsiConsole.WriteLine($"Stage 2: Loaded {templateWorkItems.workitems.Count()} work items from cache."); + } else + { + templateWorkItems = null; } } @@ -126,23 +128,24 @@ await AnsiConsole.Progress() task1.StartTask(); //AnsiConsole.WriteLine("Stage 1: Executing items from Query"); - QueryResults fakeItemsFromTemplateQuery; - fakeItemsFromTemplateQuery = await templateApi.GetWiqlQueryResults("Select [System.Id] From WorkItems Where [System.TeamProject] = '@project' AND [System.Parent] = @id order by [System.CreatedDate] desc", new Dictionary() { { "@id", config.templateParentId.ToString() } }); - AnsiConsole.WriteLine($"Stage 1: Query returned {fakeItemsFromTemplateQuery.workItems.Count()} items id's from the template."); + QueryResults templateWorkItemLight; + DateTime queryDatetime = DateTime.Now; + templateWorkItemLight = await templateApi.GetWiqlQueryResults("Select [System.Id] From WorkItems Where [System.TeamProject] = '@project' AND [System.Parent] = @id order by [System.CreatedDate] desc", new Dictionary() { { "@id", config.templateParentId.ToString() } }); + AnsiConsole.WriteLine($"Stage 1: Query returned {templateWorkItemLight.workItems.Count()} items id's from the template."); task1.Increment(1); task1.StopTask(); // -------------------------------------------------------------- // Task 2: getting work items and their full data - task2.MaxValue = fakeItemsFromTemplateQuery.workItems.Count(); + task2.MaxValue = templateWorkItemLight.workItems.Count(); task2.StartTask(); await Task.Delay(250); //AnsiConsole.WriteLine($"Stage 2: Starting process of {task2.MaxValue} work items to get their full data "); - templateWorkItems = new List(); + templateWorkItems.workitems = new List(); //AnsiConsole.WriteLine($"Stage 2: Loading {fakeItemsFromTemplateQuery.workItems.Count()} work items from template."); - await foreach (var workItem in templateApi.GetWorkItemsFullAsync(fakeItemsFromTemplateQuery.workItems)) + await foreach (var workItem in templateApi.GetWorkItemsFullAsync(templateWorkItemLight.workItems)) { //AnsiConsole.WriteLine($"Stage 2: Processing {workItem.id}:`{workItem.fields.SystemTitle}`"); - templateWorkItems.Add(workItem); + templateWorkItems.workitems.Add(workItem); task2.Increment(1); } System.IO.File.WriteAllText(cacheTemplateWorkItemsFile, JsonConvert.SerializeObject(templateWorkItems, Formatting.Indented)); @@ -206,7 +209,7 @@ await AnsiConsole.Progress() await Task.Delay(250); //AnsiConsole.WriteLine($"Stage 4: First Pass generation of Work Items to build will merge the provided json work items with the data from the template."); buildItems = new List(); - await foreach (WorkItemToBuild witb in generateWorkItemsToBuildList(inputWorkItems, templateWorkItems, projectItem, config.targetProject)) + await foreach (WorkItemToBuild witb in generateWorkItemsToBuildList(inputWorkItems, templateWorkItems.workitems, projectItem, config.targetProject)) { // AnsiConsole.WriteLine($"Stage 4: processing {witb.guid}"); buildItems.Add(witb); @@ -221,7 +224,7 @@ await AnsiConsole.Progress() //AnsiConsole.WriteLine($"Stage 5: Second Pass generate relations."); task5.StartTask(); await Task.Delay(250); - await foreach (WorkItemToBuild witb in generateWorkItemsToBuildRelations(buildItems, templateWorkItems)) + await foreach (WorkItemToBuild witb in generateWorkItemsToBuildRelations(buildItems, templateWorkItems.workitems)) { //AnsiConsole.WriteLine($"Stage 5: processing {witb.guid} for output of {witb.relations.Count-1} relations"); task5.Increment(1); diff --git a/AzureDevOps.WorkItemClone/DataContracts/WorkItem.cs b/AzureDevOps.WorkItemClone/DataContracts/WorkItem.cs index 27a56a0..1049567 100644 --- a/AzureDevOps.WorkItemClone/DataContracts/WorkItem.cs +++ b/AzureDevOps.WorkItemClone/DataContracts/WorkItem.cs @@ -7,7 +7,12 @@ namespace AzureDevOps.WorkItemClone.DataContracts { + public class CashedWorkItems + { + public List workitems { get; set; } + public DateTime queryDatetime { get; set; } + } public class WorkItemFull { From 2e89eeb431be2a455562fdc03a0e5e1af64c679f Mon Sep 17 00:00:00 2001 From: "Martin Hinshelwood nkdAgility.com" Date: Wed, 17 Jul 2024 11:33:50 +0100 Subject: [PATCH 2/6] feat(AzureDevOps.WorkItemClone.csproj): add Repositories folder to the project structure for better organization of repository related classes --- AzureDevOps.WorkItemClone/AzureDevOps.WorkItemClone.csproj | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/AzureDevOps.WorkItemClone/AzureDevOps.WorkItemClone.csproj b/AzureDevOps.WorkItemClone/AzureDevOps.WorkItemClone.csproj index 0d9c711..8ea0d4f 100644 --- a/AzureDevOps.WorkItemClone/AzureDevOps.WorkItemClone.csproj +++ b/AzureDevOps.WorkItemClone/AzureDevOps.WorkItemClone.csproj @@ -14,4 +14,8 @@ + + + + From c003a57e9749dd135b67c6c7ba43ef8ee41f6f36 Mon Sep 17 00:00:00 2001 From: "Martin Hinshelwood nkdAgility.com" Date: Wed, 17 Jul 2024 11:59:25 +0100 Subject: [PATCH 3/6] refactor(WorkItemCloneCommand.cs): add exception handling for cache loading to prevent crashes feat(WorkItemCloneCommand.cs): add cache stale check to ensure data freshness fix(WorkItemCloneCommand.cs): correct task count display off-by-one error in Stage 6 description --- .../Commands/WorkItemCloneCommand.cs | 59 +++++++++++-------- 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/AzureDevOps.WorkItemClone.ConsoleUI/Commands/WorkItemCloneCommand.cs b/AzureDevOps.WorkItemClone.ConsoleUI/Commands/WorkItemCloneCommand.cs index 5a55e82..02d09f0 100644 --- a/AzureDevOps.WorkItemClone.ConsoleUI/Commands/WorkItemCloneCommand.cs +++ b/AzureDevOps.WorkItemClone.ConsoleUI/Commands/WorkItemCloneCommand.cs @@ -9,7 +9,6 @@ using Microsoft.VisualStudio.Services.CircuitBreaker; using Newtonsoft.Json.Linq; using System.Diagnostics.Eventing.Reader; -using AzureDevOps.WorkItemClone.Repositories; namespace AzureDevOps.WorkItemClone.ConsoleUI.Commands { @@ -97,27 +96,40 @@ await AnsiConsole.Progress() if (System.IO.File.Exists(cacheTemplateWorkItemsFile)) { // load Cache - templateWorkItems = JsonConvert.DeserializeObject(System.IO.File.ReadAllText(cacheTemplateWorkItemsFile)); - //Test Cache date - QueryResults changedWorkItems = await templateApi.GetWiqlQueryResults("Select [System.Id] From WorkItems Where [System.TeamProject] = '@project' AND [System.Parent] = @id AND [System.ChangedDate] > '@changeddate' order by [System.CreatedDate] desc", new Dictionary() { { "@id", config.templateParentId.ToString() }, { "@changeddate", templateWorkItems.queryDatetime.AddDays(-1).ToString("yyyy-MM-dd") } }); - if (changedWorkItems.workItems.Length == 0) + try { - AnsiConsole.WriteLine($"Stage 1: Checked template for changes. None Detected. Loading Cache"); - - // Load from Cache - - task1.Increment(1); - task1.Description = task1.Description + " (cache)"; - await Task.Delay(250); - task1.StopTask(); - ////////////////////// - task2.Increment(templateWorkItems.workitems.Count()); - task2.Description = task2.Description + " (cache)"; - AnsiConsole.WriteLine($"Stage 2: Loaded {templateWorkItems.workitems.Count()} work items from cache."); - } else + templateWorkItems = JsonConvert.DeserializeObject(System.IO.File.ReadAllText(cacheTemplateWorkItemsFile)); + } + catch (Exception ex) { - templateWorkItems = null; + // failed to load + AnsiConsole.WriteLine($"Cache is moldy, reloading.."); } + if (templateWorkItems != null) + { + //Test Cache date + QueryResults changedWorkItems = await templateApi.GetWiqlQueryResults("Select [System.Id] From WorkItems Where [System.TeamProject] = '@project' AND [System.Parent] = @id AND [System.ChangedDate] > '@changeddate' order by [System.CreatedDate] desc", new Dictionary() { { "@id", config.templateParentId.ToString() }, { "@changeddate", templateWorkItems.queryDatetime.AddDays(-1).ToString("yyyy-MM-dd") } }); + if (changedWorkItems.workItems.Length == 0) + { + AnsiConsole.WriteLine($"Stage 1: Checked template for changes. None Detected. Loading Cache"); + + // Load from Cache + + task1.Increment(1); + task1.Description = task1.Description + " (cache)"; + await Task.Delay(250); + task1.StopTask(); + ////////////////////// + task2.Increment(templateWorkItems.workitems.Count()); + task2.Description = task2.Description + " (cache)"; + AnsiConsole.WriteLine($"Stage 2: Loaded {templateWorkItems.workitems.Count()} work items from cache."); + } + else + { + AnsiConsole.WriteLine($"Cache is stale, reloading.."); + templateWorkItems = null; + } + } } if (templateWorkItems == null) @@ -126,10 +138,9 @@ await AnsiConsole.Progress() // -------------------------------------------------------------- // Task 1: query for template work items task1.StartTask(); - + templateWorkItems = new CashedWorkItems() { workitems = new List(), queryDatetime = DateTime.Now }; //AnsiConsole.WriteLine("Stage 1: Executing items from Query"); QueryResults templateWorkItemLight; - DateTime queryDatetime = DateTime.Now; templateWorkItemLight = await templateApi.GetWiqlQueryResults("Select [System.Id] From WorkItems Where [System.TeamProject] = '@project' AND [System.Parent] = @id order by [System.CreatedDate] desc", new Dictionary() { { "@id", config.templateParentId.ToString() } }); AnsiConsole.WriteLine($"Stage 1: Query returned {templateWorkItemLight.workItems.Count()} items id's from the template."); task1.Increment(1); @@ -138,9 +149,7 @@ await AnsiConsole.Progress() // Task 2: getting work items and their full data task2.MaxValue = templateWorkItemLight.workItems.Count(); task2.StartTask(); - await Task.Delay(250); - //AnsiConsole.WriteLine($"Stage 2: Starting process of {task2.MaxValue} work items to get their full data "); - templateWorkItems.workitems = new List(); + await Task.Delay(250); //AnsiConsole.WriteLine($"Stage 2: Loading {fakeItemsFromTemplateQuery.workItems.Count()} work items from template."); await foreach (var workItem in templateApi.GetWorkItemsFullAsync(templateWorkItemLight.workItems)) { @@ -255,7 +264,7 @@ await AnsiConsole.Progress() //AnsiConsole.WriteLine($"Stage 6: Processing {witb.guid} for output of {witb.relations.Count - 1} relations"); task6.Increment(1); taskCount++; - task6.Description = $"[bold]Stage 6[/]: Create Work Items ({taskCount}/{buildItems.Count()} c:{result.created}, s:{result.skipped}, f:{result.failed})"; + task6.Description = $"[bold]Stage 6[/]: Create Work Items ({taskCount-1}/{buildItems.Count()} c:{result.created}, s:{result.skipped}, f:{result.failed})"; switch (result.status) { case "created": From 0e02d48658719a1aa736ad8e2c7e8090caf68e9c Mon Sep 17 00:00:00 2001 From: "Martin Hinshelwood nkdAgility.com" Date: Wed, 17 Jul 2024 12:37:46 +0100 Subject: [PATCH 4/6] feat(WorkItemCloneCommand.cs): remove unnecessary whitespace for cleaner code refactor(AzureDevOps.WorkItemClone.csproj): remove unused Repositories folder reference style(WorkItem.cs): remove unnecessary whitespace for cleaner code feat(WorkItemRepo.cs): add new file to handle work item repository operations, improving code organization and separation of concerns --- .../Commands/WorkItemCloneCommand.cs | 1 + .../AzureDevOps.WorkItemClone.csproj | 4 - .../DataContracts/WorkItem.cs | 1 - .../Repositories/WorkItemRepo.cs | 116 ++++++++++++++++++ 4 files changed, 117 insertions(+), 5 deletions(-) create mode 100644 AzureDevOps.WorkItemClone/Repositories/WorkItemRepo.cs diff --git a/AzureDevOps.WorkItemClone.ConsoleUI/Commands/WorkItemCloneCommand.cs b/AzureDevOps.WorkItemClone.ConsoleUI/Commands/WorkItemCloneCommand.cs index 02d09f0..f8fc734 100644 --- a/AzureDevOps.WorkItemClone.ConsoleUI/Commands/WorkItemCloneCommand.cs +++ b/AzureDevOps.WorkItemClone.ConsoleUI/Commands/WorkItemCloneCommand.cs @@ -93,6 +93,7 @@ await AnsiConsole.Progress() task1.StartTask(); task2.StartTask(); + if (System.IO.File.Exists(cacheTemplateWorkItemsFile)) { // load Cache diff --git a/AzureDevOps.WorkItemClone/AzureDevOps.WorkItemClone.csproj b/AzureDevOps.WorkItemClone/AzureDevOps.WorkItemClone.csproj index 8ea0d4f..0d9c711 100644 --- a/AzureDevOps.WorkItemClone/AzureDevOps.WorkItemClone.csproj +++ b/AzureDevOps.WorkItemClone/AzureDevOps.WorkItemClone.csproj @@ -14,8 +14,4 @@ - - - - diff --git a/AzureDevOps.WorkItemClone/DataContracts/WorkItem.cs b/AzureDevOps.WorkItemClone/DataContracts/WorkItem.cs index 1049567..84509c3 100644 --- a/AzureDevOps.WorkItemClone/DataContracts/WorkItem.cs +++ b/AzureDevOps.WorkItemClone/DataContracts/WorkItem.cs @@ -9,7 +9,6 @@ namespace AzureDevOps.WorkItemClone.DataContracts { public class CashedWorkItems { - public List workitems { get; set; } public DateTime queryDatetime { get; set; } } diff --git a/AzureDevOps.WorkItemClone/Repositories/WorkItemRepo.cs b/AzureDevOps.WorkItemClone/Repositories/WorkItemRepo.cs new file mode 100644 index 0000000..ef2598c --- /dev/null +++ b/AzureDevOps.WorkItemClone/Repositories/WorkItemRepo.cs @@ -0,0 +1,116 @@ +using AzureDevOps.WorkItemClone.DataContracts; +using Newtonsoft.Json; +using Spectre.Console; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace AzureDevOps.WorkItemClone.Repositories +{ + public interface IWorkItemRepository + { + IAsyncEnumerable<(int total, int processed)> GetWorkItemsFullAsync(); + //Task GetWorkItemByIdAsync(int id); + //Task> GetAllWorkItemAsync(); + //Task AddWorkItemAsync(WorkItemFull wif); + //Task UpdateWorkItemAsync(WorkItemFull wif); + //Task DeleteWorkItemAsync(int id); + } + public interface IPersistantCache + { + Task SaveToCache(); + Task LoadFromCache(); + } + + public class WorkItemRepository : IWorkItemRepository + { + public string OrganisationName { get; private set; } + public string ProjectName { get; private set; } + private string AccesToken { get; set; } + public string ParentId { get; private set; } + + private AzureDevOpsApi _context; + private string cacheWorkItemsFile; + public CashedWorkItems WorkItems { get { return cachedWorkItems; } } + + CashedWorkItems cachedWorkItems = null; + + public WorkItemRepository(string cachePath, string organisationName, string projectName, string accessToken, string parentId) + { + if (string.IsNullOrEmpty(organisationName)) + { + throw new ArgumentNullException(nameof(organisationName)); + } + this.OrganisationName = organisationName; + if (string.IsNullOrEmpty(projectName)) + { + throw new ArgumentNullException(nameof(projectName)); + } + this.ProjectName = projectName; + if (string.IsNullOrEmpty(accessToken)) + { + throw new ArgumentNullException(nameof(accessToken)); + } + this.AccesToken = accessToken; + if (string.IsNullOrEmpty(ParentId)) + { + throw new ArgumentNullException(nameof(parentId)); + } + this.ParentId = parentId; + _context = new AzureDevOpsApi(organisationName, projectName, accessToken); + cacheWorkItemsFile = $"{cachePath}\\cache-{organisationName}-{projectName}-{ParentId}.json"; + } + + + public async IAsyncEnumerable<(int total, int processed)> GetWorkItemsFullAsync() + { + if (System.IO.File.Exists(cacheWorkItemsFile)) + { + // load Cache + try + { + cachedWorkItems = JsonConvert.DeserializeObject(System.IO.File.ReadAllText(cacheWorkItemsFile)); + } + catch (Exception ex) + { + // failed to load:: do nothing we will refresh the cache. + } + if (cachedWorkItems != null) + { + //Test Cache date + QueryResults? changedWorkItems = await _context.GetWiqlQueryResults("Select [System.Id] From WorkItems Where [System.TeamProject] = '@project' AND [System.Parent] = @id AND [System.ChangedDate] > '@changeddate' order by [System.CreatedDate] desc", new Dictionary() { { "@id", ParentId }, { "@changeddate", cachedWorkItems.queryDatetime.AddDays(-1).ToString("yyyy-MM-dd") } }); + if (changedWorkItems?.workItems.Length == 0) + { + yield return (cachedWorkItems.workitems.Count(), cachedWorkItems.workitems.Count()); + } + else + { + cachedWorkItems = null; + } + } + } + if (cachedWorkItems == null) + { + cachedWorkItems = new CashedWorkItems() { queryDatetime = DateTime.Now, workitems = new List() }; + QueryResults? templateWorkItemLight; + templateWorkItemLight = await _context.GetWiqlQueryResults("Select [System.Id] From WorkItems Where [System.TeamProject] = '@project' AND [System.Parent] = @id order by [System.CreatedDate] desc", new Dictionary() { { "@id", ParentId.ToString() } }); + int count = 1; + foreach (var item in templateWorkItemLight?.workItems) + { + WorkItemFull result = await _context.GetWorkItem((int)item.id); + if (result != null) + { + cachedWorkItems.workitems.Add(result); + } + yield return (cachedWorkItems.workitems.Count(), count); + count++; + } + System.IO.File.WriteAllText(cacheWorkItemsFile, JsonConvert.SerializeObject(cachedWorkItems, Formatting.Indented)); + } + + } + + } +} From b96f263982d1fcb727c10b4972b65c52bb3ee08f Mon Sep 17 00:00:00 2001 From: "Martin Hinshelwood nkdAgility.com" Date: Wed, 17 Jul 2024 12:59:44 +0100 Subject: [PATCH 5/6] refactor(WorkItemCloneCommand.cs): merge two tasks into one for loading template items feat(WorkItemCloneCommand.cs): replace direct API calls with repository pattern for better code organization and separation of concerns refactor(WorkItemRepo.cs): update WorkItemRepository to include loading source in GetWorkItemsFullAsync method fix(WorkItemRepo.cs): change parentId type from string to int for correct type usage refactor(WorkItemRepo.cs): update GetWorkItemsFullAsync method to return loading source along with total and processed count --- .../Commands/WorkItemCloneCommand.cs | 90 +++---------------- .../Repositories/WorkItemRepo.cs | 29 +++--- 2 files changed, 26 insertions(+), 93 deletions(-) diff --git a/AzureDevOps.WorkItemClone.ConsoleUI/Commands/WorkItemCloneCommand.cs b/AzureDevOps.WorkItemClone.ConsoleUI/Commands/WorkItemCloneCommand.cs index f8fc734..bf4a8e2 100644 --- a/AzureDevOps.WorkItemClone.ConsoleUI/Commands/WorkItemCloneCommand.cs +++ b/AzureDevOps.WorkItemClone.ConsoleUI/Commands/WorkItemCloneCommand.cs @@ -9,6 +9,7 @@ using Microsoft.VisualStudio.Services.CircuitBreaker; using Newtonsoft.Json.Linq; using System.Diagnostics.Eventing.Reader; +using AzureDevOps.WorkItemClone.Repositories; namespace AzureDevOps.WorkItemClone.ConsoleUI.Commands { @@ -70,8 +71,7 @@ await AnsiConsole.Progress() .StartAsync(async ctx => { // Define tasks - var task1 = ctx.AddTask("[bold]Stage 1[/]: Get Template Items", false); - var task2 = ctx.AddTask("[bold]Stage 2[/]: Load Template Items", false); + var task1 = ctx.AddTask("[bold]Stage 1+2[/]: Load Template Items", false); var task3 = ctx.AddTask("[bold]Stage 3[/]: Get Target Project", false); var task4 = ctx.AddTask("[bold]Stage 4[/]: Create Output Plan", false); var task5 = ctx.AddTask("[bold]Stage 5[/]: Create Output Plan Relations ", false); @@ -85,85 +85,21 @@ await AnsiConsole.Progress() System.IO.File.Delete(cacheTemplateWorkItemsFile); } - task1.MaxValue = 1; - CashedWorkItems templateWorkItems = null; - - - task1.StartTask(); - task2.StartTask(); - - - if (System.IO.File.Exists(cacheTemplateWorkItemsFile)) + IWorkItemRepository templateWor = new WorkItemRepository(config.CachePath, config.templateOrganization, config.templateProject, config.templateAccessToken, (int)config.templateParentId); + await foreach (var result in templateWor.GetWorkItemsFullAsync()) { - // load Cache - try + //AnsiConsole.WriteLine($"Stage 2: Processing {workItem.id}:`{workItem.fields.SystemTitle}`"); + task1.MaxValue = result.total; + if (result.total == result.processed) { - templateWorkItems = JsonConvert.DeserializeObject(System.IO.File.ReadAllText(cacheTemplateWorkItemsFile)); + task1.Increment(result.processed); + await Task.Delay(250); } - catch (Exception ex) - { - // failed to load - AnsiConsole.WriteLine($"Cache is moldy, reloading.."); - } - if (templateWorkItems != null) - { - //Test Cache date - QueryResults changedWorkItems = await templateApi.GetWiqlQueryResults("Select [System.Id] From WorkItems Where [System.TeamProject] = '@project' AND [System.Parent] = @id AND [System.ChangedDate] > '@changeddate' order by [System.CreatedDate] desc", new Dictionary() { { "@id", config.templateParentId.ToString() }, { "@changeddate", templateWorkItems.queryDatetime.AddDays(-1).ToString("yyyy-MM-dd") } }); - if (changedWorkItems.workItems.Length == 0) - { - AnsiConsole.WriteLine($"Stage 1: Checked template for changes. None Detected. Loading Cache"); - - // Load from Cache - - task1.Increment(1); - task1.Description = task1.Description + " (cache)"; - await Task.Delay(250); - task1.StopTask(); - ////////////////////// - task2.Increment(templateWorkItems.workitems.Count()); - task2.Description = task2.Description + " (cache)"; - AnsiConsole.WriteLine($"Stage 2: Loaded {templateWorkItems.workitems.Count()} work items from cache."); - } - else - { - AnsiConsole.WriteLine($"Cache is stale, reloading.."); - templateWorkItems = null; - } - } - } - - if (templateWorkItems == null) - { - // Get From Server - // -------------------------------------------------------------- - // Task 1: query for template work items - task1.StartTask(); - templateWorkItems = new CashedWorkItems() { workitems = new List(), queryDatetime = DateTime.Now }; - //AnsiConsole.WriteLine("Stage 1: Executing items from Query"); - QueryResults templateWorkItemLight; - templateWorkItemLight = await templateApi.GetWiqlQueryResults("Select [System.Id] From WorkItems Where [System.TeamProject] = '@project' AND [System.Parent] = @id order by [System.CreatedDate] desc", new Dictionary() { { "@id", config.templateParentId.ToString() } }); - AnsiConsole.WriteLine($"Stage 1: Query returned {templateWorkItemLight.workItems.Count()} items id's from the template."); + task1.Description = $"[bold]Stage 1[/]: Load Template Items ({result.loadingFrom})"; task1.Increment(1); - task1.StopTask(); - // -------------------------------------------------------------- - // Task 2: getting work items and their full data - task2.MaxValue = templateWorkItemLight.workItems.Count(); - task2.StartTask(); - await Task.Delay(250); - //AnsiConsole.WriteLine($"Stage 2: Loading {fakeItemsFromTemplateQuery.workItems.Count()} work items from template."); - await foreach (var workItem in templateApi.GetWorkItemsFullAsync(templateWorkItemLight.workItems)) - { - //AnsiConsole.WriteLine($"Stage 2: Processing {workItem.id}:`{workItem.fields.SystemTitle}`"); - templateWorkItems.workitems.Add(workItem); - task2.Increment(1); - } - System.IO.File.WriteAllText(cacheTemplateWorkItemsFile, JsonConvert.SerializeObject(templateWorkItems, Formatting.Indented)); - //AnsiConsole.WriteLine($"Stage 2: All {task2.MaxValue} work items loaded"); - await Task.Delay(250); - task2.StopTask(); } - await Task.Delay(250); + task1.StopTask(); // -------------------------------------------------------------- string targetProjectRunFile = $"{runCache}\\targetProject.json"; @@ -219,7 +155,7 @@ await AnsiConsole.Progress() await Task.Delay(250); //AnsiConsole.WriteLine($"Stage 4: First Pass generation of Work Items to build will merge the provided json work items with the data from the template."); buildItems = new List(); - await foreach (WorkItemToBuild witb in generateWorkItemsToBuildList(inputWorkItems, templateWorkItems.workitems, projectItem, config.targetProject)) + await foreach (WorkItemToBuild witb in generateWorkItemsToBuildList(inputWorkItems, templateWor.Data.workitems, projectItem, config.targetProject)) { // AnsiConsole.WriteLine($"Stage 4: processing {witb.guid}"); buildItems.Add(witb); @@ -234,7 +170,7 @@ await AnsiConsole.Progress() //AnsiConsole.WriteLine($"Stage 5: Second Pass generate relations."); task5.StartTask(); await Task.Delay(250); - await foreach (WorkItemToBuild witb in generateWorkItemsToBuildRelations(buildItems, templateWorkItems.workitems)) + await foreach (WorkItemToBuild witb in generateWorkItemsToBuildRelations(buildItems, templateWor.Data.workitems)) { //AnsiConsole.WriteLine($"Stage 5: processing {witb.guid} for output of {witb.relations.Count-1} relations"); task5.Increment(1); diff --git a/AzureDevOps.WorkItemClone/Repositories/WorkItemRepo.cs b/AzureDevOps.WorkItemClone/Repositories/WorkItemRepo.cs index ef2598c..3647954 100644 --- a/AzureDevOps.WorkItemClone/Repositories/WorkItemRepo.cs +++ b/AzureDevOps.WorkItemClone/Repositories/WorkItemRepo.cs @@ -11,12 +11,8 @@ namespace AzureDevOps.WorkItemClone.Repositories { public interface IWorkItemRepository { - IAsyncEnumerable<(int total, int processed)> GetWorkItemsFullAsync(); - //Task GetWorkItemByIdAsync(int id); - //Task> GetAllWorkItemAsync(); - //Task AddWorkItemAsync(WorkItemFull wif); - //Task UpdateWorkItemAsync(WorkItemFull wif); - //Task DeleteWorkItemAsync(int id); + CashedWorkItems Data {get;} + IAsyncEnumerable<(int total, int processed, string loadingFrom)> GetWorkItemsFullAsync(); } public interface IPersistantCache { @@ -29,15 +25,15 @@ public class WorkItemRepository : IWorkItemRepository public string OrganisationName { get; private set; } public string ProjectName { get; private set; } private string AccesToken { get; set; } - public string ParentId { get; private set; } + public int ParentId { get; private set; } private AzureDevOpsApi _context; private string cacheWorkItemsFile; - public CashedWorkItems WorkItems { get { return cachedWorkItems; } } + public CashedWorkItems Data { get { return cachedWorkItems; } } CashedWorkItems cachedWorkItems = null; - public WorkItemRepository(string cachePath, string organisationName, string projectName, string accessToken, string parentId) + public WorkItemRepository(string cachePath, string organisationName, string projectName, string accessToken, int parentId) { if (string.IsNullOrEmpty(organisationName)) { @@ -54,17 +50,17 @@ public WorkItemRepository(string cachePath, string organisationName, string proj throw new ArgumentNullException(nameof(accessToken)); } this.AccesToken = accessToken; - if (string.IsNullOrEmpty(ParentId)) + if (parentId == 0) { throw new ArgumentNullException(nameof(parentId)); } this.ParentId = parentId; - _context = new AzureDevOpsApi(organisationName, projectName, accessToken); + _context = new AzureDevOpsApi(accessToken, organisationName, projectName); cacheWorkItemsFile = $"{cachePath}\\cache-{organisationName}-{projectName}-{ParentId}.json"; } - public async IAsyncEnumerable<(int total, int processed)> GetWorkItemsFullAsync() + public async IAsyncEnumerable<(int total, int processed, string loadingFrom)> GetWorkItemsFullAsync() { if (System.IO.File.Exists(cacheWorkItemsFile)) { @@ -80,10 +76,10 @@ public WorkItemRepository(string cachePath, string organisationName, string proj if (cachedWorkItems != null) { //Test Cache date - QueryResults? changedWorkItems = await _context.GetWiqlQueryResults("Select [System.Id] From WorkItems Where [System.TeamProject] = '@project' AND [System.Parent] = @id AND [System.ChangedDate] > '@changeddate' order by [System.CreatedDate] desc", new Dictionary() { { "@id", ParentId }, { "@changeddate", cachedWorkItems.queryDatetime.AddDays(-1).ToString("yyyy-MM-dd") } }); + QueryResults? changedWorkItems = await _context.GetWiqlQueryResults("Select [System.Id] From WorkItems Where [System.TeamProject] = '@project' AND [System.Parent] = @id AND [System.ChangedDate] > '@changeddate' order by [System.CreatedDate] desc", new Dictionary() { { "@id", ParentId.ToString() }, { "@changeddate", cachedWorkItems.queryDatetime.AddDays(-1).ToString("yyyy-MM-dd") } }); if (changedWorkItems?.workItems.Length == 0) { - yield return (cachedWorkItems.workitems.Count(), cachedWorkItems.workitems.Count()); + yield return (cachedWorkItems.workitems.Count(), cachedWorkItems.workitems.Count(), "cache"); } else { @@ -93,9 +89,10 @@ public WorkItemRepository(string cachePath, string organisationName, string proj } if (cachedWorkItems == null) { - cachedWorkItems = new CashedWorkItems() { queryDatetime = DateTime.Now, workitems = new List() }; + QueryResults? templateWorkItemLight; templateWorkItemLight = await _context.GetWiqlQueryResults("Select [System.Id] From WorkItems Where [System.TeamProject] = '@project' AND [System.Parent] = @id order by [System.CreatedDate] desc", new Dictionary() { { "@id", ParentId.ToString() } }); + cachedWorkItems = new CashedWorkItems() { queryDatetime = templateWorkItemLight.asOf, workitems = new List() }; int count = 1; foreach (var item in templateWorkItemLight?.workItems) { @@ -104,7 +101,7 @@ public WorkItemRepository(string cachePath, string organisationName, string proj { cachedWorkItems.workitems.Add(result); } - yield return (cachedWorkItems.workitems.Count(), count); + yield return (templateWorkItemLight.workItems.Count(), count, "server"); count++; } System.IO.File.WriteAllText(cacheWorkItemsFile, JsonConvert.SerializeObject(cachedWorkItems, Formatting.Indented)); From 114dff557ebe48f8ccc8b6fbc11ef61825695169 Mon Sep 17 00:00:00 2001 From: "Martin Hinshelwood nkdAgility.com" Date: Wed, 17 Jul 2024 13:00:26 +0100 Subject: [PATCH 6/6] refactor(WorkItemCloneCommand.cs): remove unused AzureDevOpsApi templateApi instance to improve code readability and performance --- .../Commands/WorkItemCloneCommand.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/AzureDevOps.WorkItemClone.ConsoleUI/Commands/WorkItemCloneCommand.cs b/AzureDevOps.WorkItemClone.ConsoleUI/Commands/WorkItemCloneCommand.cs index bf4a8e2..c173b29 100644 --- a/AzureDevOps.WorkItemClone.ConsoleUI/Commands/WorkItemCloneCommand.cs +++ b/AzureDevOps.WorkItemClone.ConsoleUI/Commands/WorkItemCloneCommand.cs @@ -31,8 +31,6 @@ public override async Task ExecuteAsync(CommandContext context, WorkItemClo AnsiConsole.MarkupLine($"[red]Run: [/] {config.RunName}"); string runCache = $"{config.CachePath}\\{config.RunName}"; DirectoryInfo outputPathInfo = CreateOutputPath(runCache); - - AzureDevOpsApi templateApi = CreateAzureDevOpsConnection(config.templateAccessToken, config.templateOrganization, config.templateProject); AzureDevOpsApi targetApi = CreateAzureDevOpsConnection(config.targetAccessToken, config.targetOrganization, config.targetProject); JArray inputWorkItems = DeserializeWorkItemList(config);