Skip to content

Commit a9937d1

Browse files
directhexdirecthex
authored
Add lots of debug information to stage 1 uploader (#164)
* Add lots of debug information to stage 1 uploader * debuggier stage 1 downloader as well
1 parent 286612e commit a9937d1

File tree

2 files changed

+75
-8
lines changed

2 files changed

+75
-8
lines changed

‎src/Microsoft.SourceIndexer.Tasks/DownloadStage1Index.cs‎

Lines changed: 37 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ private void ExecuteCore()
5555
}
5656

5757
DefaultAzureCredential credential;
58+
DefaultAzureCredentialOptions credentialoptions;
5859

5960
if (string.IsNullOrEmpty(ClientId) && !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("ARM_CLIENT_ID")))
6061
{
@@ -64,21 +65,46 @@ private void ExecuteCore()
6465

6566
if (string.IsNullOrEmpty(ClientId))
6667
{
67-
credential = new DefaultAzureCredential();
68+
credentialoptions = new DefaultAzureCredentialOptions
69+
{
70+
Diagnostics =
71+
{
72+
LoggedHeaderNames = { "x-ms-request-id" },
73+
LoggedQueryParameters = { "api-version" },
74+
IsAccountIdentifierLoggingEnabled = true
75+
}
76+
};
6877
Log.LogMessage($"Trying to use managed identity without default identity");
6978
}
7079
else
7180
{
72-
credential = new DefaultAzureCredential(new DefaultAzureCredentialOptions { ManagedIdentityClientId = ClientId });
81+
credentialoptions = new DefaultAzureCredentialOptions
82+
{
83+
Diagnostics =
84+
{
85+
LoggedHeaderNames = { "x-ms-request-id" },
86+
LoggedQueryParameters = { "api-version" },
87+
IsAccountIdentifierLoggingEnabled = true
88+
},
89+
ManagedIdentityClientId = clientId };
7390
Log.LogMessage($"Trying to use managed identity with client id: {ClientId}");
7491
}
7592

93+
credential = new DefaultAzureCredential(credentialoptions);
94+
7695
BlobServiceClient blobServiceClient = new(
7796
new Uri(StorageAccount),
7897
credential);
7998

8099
var containerClient = blobServiceClient.GetBlobContainerClient(BlobContainer);
81-
Pageable<BlobItem> blobs = containerClient.GetBlobs(prefix: RepoName + "/");
100+
try
101+
{
102+
Pageable<BlobItem> blobs = containerClient.GetBlobs(prefix: RepoName + "/");
103+
}
104+
catch (AuthenticationFailedException e)
105+
{
106+
Fatal($"*** BLOB ENUMERATION FAILED: {e.Message}");
107+
}
82108
BlobItem newest = blobs.OrderByDescending(b => b.Name).FirstOrDefault();
83109
if (newest == null)
84110
{
@@ -89,7 +115,14 @@ private void ExecuteCore()
89115
BlobClient blobClient = containerClient.GetBlobClient(newest.Name);
90116
var loggableUrl = new UriBuilder(blobClient.Uri) {Fragment = "", Query = ""};
91117
Log.LogMessage($"Extracting {loggableUrl} to {OutputDirectory}");
92-
using Stream fileStream = blobClient.OpenRead();
118+
try
119+
{
120+
using Stream fileStream = blobClient.OpenRead();
121+
}
122+
catch (AuthenticationFailedException e)
123+
{
124+
Fatal($"*** STREAM READ FAILED: {e.Message}");
125+
}
93126
using var input = new GZipInputStream(fileStream);
94127
using var archive = TarArchive.CreateInputTarArchive(input, Encoding.UTF8);
95128
archive.ExtractContents(OutputDirectory, true); // would like this to be false, but SharpZipLib has a bug in 1.3.3

‎src/UploadIndexStage1/Program.cs‎

Lines changed: 38 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
using System.Linq;
66
using System.Text;
77
using System.Threading.Tasks;
8+
using Azure.Core;
89
using Azure.Identity;
910
using Azure.Storage.Blobs;
1011
using Azure.Storage.Blobs.Models;
@@ -65,6 +66,7 @@ static async Task Main(string[] args)
6566
}
6667

6768
DefaultAzureCredential credential;
69+
DefaultAzureCredentialOptions credentialoptions;
6870

6971
if (string.IsNullOrEmpty(clientId) && !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("ARM_CLIENT_ID")))
7072
{
@@ -74,15 +76,33 @@ static async Task Main(string[] args)
7476

7577
if (string.IsNullOrEmpty(clientId))
7678
{
77-
credential = new DefaultAzureCredential();
79+
credentialoptions = new DefaultAzureCredentialOptions
80+
{
81+
Diagnostics =
82+
{
83+
LoggedHeaderNames = { "x-ms-request-id" },
84+
LoggedQueryParameters = { "api-version" },
85+
IsAccountIdentifierLoggingEnabled = true
86+
}
87+
};
7888
System.Console.WriteLine("Trying to use managed identity without default identity");
7989
}
8090
else
8191
{
82-
credential = new DefaultAzureCredential(new DefaultAzureCredentialOptions { ManagedIdentityClientId = clientId });
92+
credentialoptions = new DefaultAzureCredentialOptions
93+
{
94+
Diagnostics =
95+
{
96+
LoggedHeaderNames = { "x-ms-request-id" },
97+
LoggedQueryParameters = { "api-version" },
98+
IsAccountIdentifierLoggingEnabled = true
99+
},
100+
ManagedIdentityClientId = clientId };
83101
System.Console.WriteLine("Trying to use managed identity with client id: " + clientId);
84102
}
85103

104+
credential = new DefaultAzureCredential(credentialoptions);
105+
86106
BlobServiceClient blobServiceClient = new(
87107
new Uri(storageAccount),
88108
credential);
@@ -127,7 +147,14 @@ void AddFolder(string path)
127147
}
128148

129149
outputFileStream.Position = 0;
130-
await newBlobClient.UploadAsync(outputFileStream);
150+
try
151+
{
152+
await newBlobClient.UploadAsync(outputFileStream);
153+
}
154+
catch (AuthenticationFailedException e)
155+
{
156+
Fatal($"*** UPLOAD FAILED: {e.Message}");
157+
}
131158
}
132159

133160
Console.WriteLine("Cleaning up old blobs");
@@ -136,7 +163,14 @@ void AddFolder(string path)
136163
foreach (BlobItem d in toDelete)
137164
{
138165
Console.WriteLine($"Deleting blob {d.Name}");
139-
await containerClient.DeleteBlobAsync(d.Name);
166+
try
167+
{
168+
await containerClient.DeleteBlobAsync(d.Name);
169+
}
170+
catch (AuthenticationFailedException e)
171+
{
172+
Fatal($"*** CONTAINER \"{d.Name}\" CLEANUP FAILED: {e.Message}");
173+
}
140174
}
141175
Console.WriteLine("Finished.");
142176
}

0 commit comments

Comments
 (0)