Skip to content

Commit 097cafb

Browse files
MrHinshMrHinsh
authored
Topic/53 cache not reloading correctly after multiple changes to template work items (#54)
Focus on fixing the cache issue. 1. Resolve #53 2. Bonus Resolve #50
2 parents 8a172cb + 114dff5 commit 097cafb

3 files changed

Lines changed: 131 additions & 67 deletions

File tree

‎AzureDevOps.WorkItemClone.ConsoleUI/Commands/WorkItemCloneCommand.cs‎

Lines changed: 13 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
using Microsoft.VisualStudio.Services.CircuitBreaker;
1010
using Newtonsoft.Json.Linq;
1111
using System.Diagnostics.Eventing.Reader;
12+
using AzureDevOps.WorkItemClone.Repositories;
1213

1314
namespace AzureDevOps.WorkItemClone.ConsoleUI.Commands
1415
{
@@ -30,8 +31,6 @@ public override async Task<int> ExecuteAsync(CommandContext context, WorkItemClo
3031
AnsiConsole.MarkupLine($"[red]Run: [/] {config.RunName}");
3132
string runCache = $"{config.CachePath}\\{config.RunName}";
3233
DirectoryInfo outputPathInfo = CreateOutputPath(runCache);
33-
34-
AzureDevOpsApi templateApi = CreateAzureDevOpsConnection(config.templateAccessToken, config.templateOrganization, config.templateProject);
3534
AzureDevOpsApi targetApi = CreateAzureDevOpsConnection(config.targetAccessToken, config.targetOrganization, config.targetProject);
3635

3736
JArray inputWorkItems = DeserializeWorkItemList(config);
@@ -70,8 +69,7 @@ await AnsiConsole.Progress()
7069
.StartAsync(async ctx =>
7170
{
7271
// Define tasks
73-
var task1 = ctx.AddTask("[bold]Stage 1[/]: Get Template Items", false);
74-
var task2 = ctx.AddTask("[bold]Stage 2[/]: Load Template Items", false);
72+
var task1 = ctx.AddTask("[bold]Stage 1+2[/]: Load Template Items", false);
7573
var task3 = ctx.AddTask("[bold]Stage 3[/]: Get Target Project", false);
7674
var task4 = ctx.AddTask("[bold]Stage 4[/]: Create Output Plan", false);
7775
var task5 = ctx.AddTask("[bold]Stage 5[/]: Create Output Plan Relations ", false);
@@ -85,72 +83,21 @@ await AnsiConsole.Progress()
8583
System.IO.File.Delete(cacheTemplateWorkItemsFile);
8684
}
8785

88-
task1.MaxValue = 1;
89-
List<WorkItemFull> templateWorkItems = null;
90-
91-
92-
9386
task1.StartTask();
94-
task2.StartTask();
95-
96-
if (System.IO.File.Exists(cacheTemplateWorkItemsFile))
87+
IWorkItemRepository templateWor = new WorkItemRepository(config.CachePath, config.templateOrganization, config.templateProject, config.templateAccessToken, (int)config.templateParentId);
88+
await foreach (var result in templateWor.GetWorkItemsFullAsync())
9789
{
98-
99-
var changedDate = System.IO.File.GetLastWriteTime(cacheTemplateWorkItemsFile).AddDays(1).Date;
100-
//Test Cache
101-
QueryResults fakeItemsFromTemplateQuery;
102-
fakeItemsFromTemplateQuery = await templateApi.GetWiqlQueryResults("Select [System.Id] From WorkItems Where [System.TeamProject] = '@project' AND [System.Parent] = @id AND [System.ChangedDate] > '@changeddate' order by [System.CreatedDate] desc", new Dictionary<string, string>() { { "@id", config.templateParentId.ToString() }, { "@changeddate", changedDate.ToString("yyyy-MM-dd") } });
103-
if (fakeItemsFromTemplateQuery.workItems.Length == 0)
90+
//AnsiConsole.WriteLine($"Stage 2: Processing {workItem.id}:`{workItem.fields.SystemTitle}`");
91+
task1.MaxValue = result.total;
92+
if (result.total == result.processed)
10493
{
105-
AnsiConsole.WriteLine($"Stage 1: Checked template for changes. None Detected. Loading Cache");
106-
107-
// Load from Cache
108-
109-
task1.Increment(1);
110-
task1.Description = task1.Description + " (cache)";
94+
task1.Increment(result.processed);
11195
await Task.Delay(250);
112-
task1.StopTask();
113-
//////////////////////
114-
templateWorkItems = JsonConvert.DeserializeObject<List<WorkItemFull>>(System.IO.File.ReadAllText(cacheTemplateWorkItemsFile));
115-
task2.Increment(templateWorkItems.Count);
116-
task2.Description = task2.Description + " (cache)";
117-
AnsiConsole.WriteLine($"Stage 2: Loaded {templateWorkItems.Count()} work items from cache.");
11896
}
119-
}
120-
121-
if (templateWorkItems == null)
122-
{
123-
// Get From Server
124-
// --------------------------------------------------------------
125-
// Task 1: query for template work items
126-
task1.StartTask();
127-
128-
//AnsiConsole.WriteLine("Stage 1: Executing items from Query");
129-
QueryResults fakeItemsFromTemplateQuery;
130-
fakeItemsFromTemplateQuery = await templateApi.GetWiqlQueryResults("Select [System.Id] From WorkItems Where [System.TeamProject] = '@project' AND [System.Parent] = @id order by [System.CreatedDate] desc", new Dictionary<string, string>() { { "@id", config.templateParentId.ToString() } });
131-
AnsiConsole.WriteLine($"Stage 1: Query returned {fakeItemsFromTemplateQuery.workItems.Count()} items id's from the template.");
97+
task1.Description = $"[bold]Stage 1[/]: Load Template Items ({result.loadingFrom})";
13298
task1.Increment(1);
133-
task1.StopTask();
134-
// --------------------------------------------------------------
135-
// Task 2: getting work items and their full data
136-
task2.MaxValue = fakeItemsFromTemplateQuery.workItems.Count();
137-
task2.StartTask();
138-
await Task.Delay(250);
139-
//AnsiConsole.WriteLine($"Stage 2: Starting process of {task2.MaxValue} work items to get their full data ");
140-
templateWorkItems = new List<WorkItemFull>();
141-
//AnsiConsole.WriteLine($"Stage 2: Loading {fakeItemsFromTemplateQuery.workItems.Count()} work items from template.");
142-
await foreach (var workItem in templateApi.GetWorkItemsFullAsync(fakeItemsFromTemplateQuery.workItems))
143-
{
144-
//AnsiConsole.WriteLine($"Stage 2: Processing {workItem.id}:`{workItem.fields.SystemTitle}`");
145-
templateWorkItems.Add(workItem);
146-
task2.Increment(1);
147-
}
148-
System.IO.File.WriteAllText(cacheTemplateWorkItemsFile, JsonConvert.SerializeObject(templateWorkItems, Formatting.Indented));
149-
//AnsiConsole.WriteLine($"Stage 2: All {task2.MaxValue} work items loaded");
150-
await Task.Delay(250);
151-
task2.StopTask();
15299
}
153-
await Task.Delay(250);
100+
task1.StopTask();
154101

155102
// --------------------------------------------------------------
156103
string targetProjectRunFile = $"{runCache}\\targetProject.json";
@@ -206,7 +153,7 @@ await AnsiConsole.Progress()
206153
await Task.Delay(250);
207154
//AnsiConsole.WriteLine($"Stage 4: First Pass generation of Work Items to build will merge the provided json work items with the data from the template.");
208155
buildItems = new List<WorkItemToBuild>();
209-
await foreach (WorkItemToBuild witb in generateWorkItemsToBuildList(inputWorkItems, templateWorkItems, projectItem, config.targetProject))
156+
await foreach (WorkItemToBuild witb in generateWorkItemsToBuildList(inputWorkItems, templateWor.Data.workitems, projectItem, config.targetProject))
210157
{
211158
// AnsiConsole.WriteLine($"Stage 4: processing {witb.guid}");
212159
buildItems.Add(witb);
@@ -221,7 +168,7 @@ await AnsiConsole.Progress()
221168
//AnsiConsole.WriteLine($"Stage 5: Second Pass generate relations.");
222169
task5.StartTask();
223170
await Task.Delay(250);
224-
await foreach (WorkItemToBuild witb in generateWorkItemsToBuildRelations(buildItems, templateWorkItems))
171+
await foreach (WorkItemToBuild witb in generateWorkItemsToBuildRelations(buildItems, templateWor.Data.workitems))
225172
{
226173
//AnsiConsole.WriteLine($"Stage 5: processing {witb.guid} for output of {witb.relations.Count-1} relations");
227174
task5.Increment(1);
@@ -252,7 +199,7 @@ await AnsiConsole.Progress()
252199
//AnsiConsole.WriteLine($"Stage 6: Processing {witb.guid} for output of {witb.relations.Count - 1} relations");
253200
task6.Increment(1);
254201
taskCount++;
255-
task6.Description = $"[bold]Stage 6[/]: Create Work Items ({taskCount}/{buildItems.Count()} c:{result.created}, s:{result.skipped}, f:{result.failed})";
202+
task6.Description = $"[bold]Stage 6[/]: Create Work Items ({taskCount-1}/{buildItems.Count()} c:{result.created}, s:{result.skipped}, f:{result.failed})";
256203
switch (result.status)
257204
{
258205
case "created":

‎AzureDevOps.WorkItemClone/DataContracts/WorkItem.cs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,11 @@
77

88
namespace AzureDevOps.WorkItemClone.DataContracts
99
{
10-
10+
public class CashedWorkItems
11+
{
12+
public List<WorkItemFull> workitems { get; set; }
13+
public DateTime queryDatetime { get; set; }
14+
}
1115

1216
public class WorkItemFull
1317
{

‎AzureDevOps.WorkItemClone/Repositories/WorkItemRepo.cs‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
using AzureDevOps.WorkItemClone.DataContracts;
2+
using Newtonsoft.Json;
3+
using Spectre.Console;
4+
using System;
5+
using System.Collections.Generic;
6+
using System.Linq;
7+
using System.Text;
8+
using System.Threading.Tasks;
9+
10+
namespace AzureDevOps.WorkItemClone.Repositories
11+
{
12+
public interface IWorkItemRepository
13+
{
14+
CashedWorkItems Data {get;}
15+
IAsyncEnumerable<(int total, int processed, string loadingFrom)> GetWorkItemsFullAsync();
16+
}
17+
public interface IPersistantCache
18+
{
19+
Task SaveToCache();
20+
Task LoadFromCache();
21+
}
22+
23+
public class WorkItemRepository : IWorkItemRepository
24+
{
25+
public string OrganisationName { get; private set; }
26+
public string ProjectName { get; private set; }
27+
private string AccesToken { get; set; }
28+
public int ParentId { get; private set; }
29+
30+
private AzureDevOpsApi _context;
31+
private string cacheWorkItemsFile;
32+
public CashedWorkItems Data { get { return cachedWorkItems; } }
33+
34+
CashedWorkItems cachedWorkItems = null;
35+
36+
public WorkItemRepository(string cachePath, string organisationName, string projectName, string accessToken, int parentId)
37+
{
38+
if (string.IsNullOrEmpty(organisationName))
39+
{
40+
throw new ArgumentNullException(nameof(organisationName));
41+
}
42+
this.OrganisationName = organisationName;
43+
if (string.IsNullOrEmpty(projectName))
44+
{
45+
throw new ArgumentNullException(nameof(projectName));
46+
}
47+
this.ProjectName = projectName;
48+
if (string.IsNullOrEmpty(accessToken))
49+
{
50+
throw new ArgumentNullException(nameof(accessToken));
51+
}
52+
this.AccesToken = accessToken;
53+
if (parentId == 0)
54+
{
55+
throw new ArgumentNullException(nameof(parentId));
56+
}
57+
this.ParentId = parentId;
58+
_context = new AzureDevOpsApi(accessToken, organisationName, projectName);
59+
cacheWorkItemsFile = $"{cachePath}\\cache-{organisationName}-{projectName}-{ParentId}.json";
60+
}
61+
62+
63+
public async IAsyncEnumerable<(int total, int processed, string loadingFrom)> GetWorkItemsFullAsync()
64+
{
65+
if (System.IO.File.Exists(cacheWorkItemsFile))
66+
{
67+
// load Cache
68+
try
69+
{
70+
cachedWorkItems = JsonConvert.DeserializeObject<CashedWorkItems>(System.IO.File.ReadAllText(cacheWorkItemsFile));
71+
}
72+
catch (Exception ex)
73+
{
74+
// failed to load:: do nothing we will refresh the cache.
75+
}
76+
if (cachedWorkItems != null)
77+
{
78+
//Test Cache date
79+
QueryResults? changedWorkItems = await _context.GetWiqlQueryResults("Select [System.Id] From WorkItems Where [System.TeamProject] = '@project' AND [System.Parent] = @id AND [System.ChangedDate] > '@changeddate' order by [System.CreatedDate] desc", new Dictionary<string, string>() { { "@id", ParentId.ToString() }, { "@changeddate", cachedWorkItems.queryDatetime.AddDays(-1).ToString("yyyy-MM-dd") } });
80+
if (changedWorkItems?.workItems.Length == 0)
81+
{
82+
yield return (cachedWorkItems.workitems.Count(), cachedWorkItems.workitems.Count(), "cache");
83+
}
84+
else
85+
{
86+
cachedWorkItems = null;
87+
}
88+
}
89+
}
90+
if (cachedWorkItems == null)
91+
{
92+
93+
QueryResults? templateWorkItemLight;
94+
templateWorkItemLight = await _context.GetWiqlQueryResults("Select [System.Id] From WorkItems Where [System.TeamProject] = '@project' AND [System.Parent] = @id order by [System.CreatedDate] desc", new Dictionary<string, string>() { { "@id", ParentId.ToString() } });
95+
cachedWorkItems = new CashedWorkItems() { queryDatetime = templateWorkItemLight.asOf, workitems = new List<WorkItemFull>() };
96+
int count = 1;
97+
foreach (var item in templateWorkItemLight?.workItems)
98+
{
99+
WorkItemFull result = await _context.GetWorkItem((int)item.id);
100+
if (result != null)
101+
{
102+
cachedWorkItems.workitems.Add(result);
103+
}
104+
yield return (templateWorkItemLight.workItems.Count(), count, "server");
105+
count++;
106+
}
107+
System.IO.File.WriteAllText(cacheWorkItemsFile, JsonConvert.SerializeObject(cachedWorkItems, Formatting.Indented));
108+
}
109+
110+
}
111+
112+
}
113+
}

0 commit comments

Comments
 (0)