Logging and Progress Update

Added per job file logs
fixed up the progress bar system to be cleaner

Author: curin

sources/SilkTouch/SilkTouch/Clang/ClangScraper.cs

@@ -22,6 +22,7 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using Silk.NET.SilkTouch.Caching;
+using Silk.NET.SilkTouch.Logging;
 using Silk.NET.SilkTouch.Mods;
 using Silk.NET.SilkTouch.Sources;
 using Silk.NET.SilkTouch.Utility;
@@ -41,12 +42,14 @@ namespace Silk.NET.SilkTouch.Clang;
 /// <param name="inputResolver">The input resolver to use.</param>
 /// <param name="cacheProvider">The cache provider into which ClangSharp outputs are cached.</param>
 /// <param name="responseFileMods">The mods that modify response files before they are fed to ClangSharp.</param>
+/// <param name="progressService">the progress service to use</param>
 [ModConfiguration<Configuration>]
 public sealed class ClangScraper(
     ResponseFileHandler rspHandler,
     IOptionsSnapshot<ClangScraper.Configuration> config,
     ILogger<ClangScraper> logger,
     IInputResolver inputResolver,
+    IProgressService progressService,
     ICacheProvider? cacheProvider = null,
     IEnumerable<IJobDependency<IResponseFileMod>>? responseFileMods = null
 ) : IMod
@@ -549,7 +552,7 @@ public async Task ExecuteAsync(IModContext ctx, CancellationToken ct = default)
         );
 
         // Resolve any foreign paths referenced in the response files
-        await inputResolver.ResolveInPlace(rsps);
+        await inputResolver.ResolveInPlace(rsps, progressService);
 
         // Should we completely skip running ClangSharp (e.g. we can't get Windows SDK bindings on macOS)
         var skip = (cfg.SkipScrapeIf?.Any(applicableSkipIfs.Contains)).GetValueOrDefault();

sources/SilkTouch/SilkTouch/Clang/ResponseFileHandler.cs

@@ -11,6 +11,7 @@
 using ClangSharp;
 using Microsoft.Extensions.FileSystemGlobbing;
 using Microsoft.Extensions.Logging;
+using Silk.NET.SilkTouch.Logging;
 using Silk.NET.SilkTouch.Utility;
 using static ClangSharp.Interop.CXTranslationUnit_Flags;
 
@@ -20,7 +21,7 @@ namespace Silk.NET.SilkTouch.Clang;
 /// Reads a response file (rsp) containing ClangSharpPInvokeGenerator command line arguments.
 /// </summary>
 [SuppressMessage("ReSharper", "InconsistentNaming")]
-public class ResponseFileHandler(ILogger<ResponseFileHandler> logger)
+public class ResponseFileHandler(ILogger<ResponseFileHandler> logger, IProgressService progressService)
 {
     // Begin verbatim ClangSharp code
     private static readonly string[] s_additionalOptionAliases = ["--additional", "-a"];
@@ -1356,7 +1357,6 @@ out Dictionary<string, string> withPackings
         {
             foreach (var error in errorList)
             {
-                ProgressBarUtility.Hide(LogLevel.Information);
                 logger.LogError($"Error in args for '{files.FirstOrDefault()}': {error}");
             }
         }
@@ -1497,8 +1497,7 @@ IReadOnlyList<string> globs
         IEnumerable<string> rsps = Glob(globs);
         int index = 0;
         int count = rsps.Count();
-        ProgressBarUtility.SetPercentage(0);
-        ProgressBarUtility.Show(LogLevel.Information);
+        progressService.SetTask("Reading ResponseFiles");
         foreach (var rsp in rsps)
         {
             index++;
@@ -1508,14 +1507,12 @@ IReadOnlyList<string> globs
                 ?? throw new InvalidOperationException("Couldn't get directory name of path");
             var read = ReadResponseFile(RspRelativeTo(dir, rsp).ToArray(), dir, rsp);
 
-            ProgressBarUtility.SetPercentage(index / (float)count);
+            progressService.SetProgress(index / (float)count);
             yield return read with
             {
                 FileDirectory = dir,
             };
         }
-
-        ProgressBarUtility.Hide(LogLevel.Information);
     }
 
     private IEnumerable<string> RspRelativeTo(string directory, string fullPath)

sources/SilkTouch/SilkTouch/JobContext.cs

@@ -0,0 +1,34 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Silk.NET.SilkTouch
+{
+    /// <summary>
+    /// Information on the current Job
+    /// </summary>
+    public class JobContext
+    {
+        private readonly AsyncLocal<string?> _key = new AsyncLocal<string?>();
+
+        /// <summary>
+        /// The key for the relevant job
+        /// </summary>
+        public string? JobKey
+        {
+            get => _key.Value;
+            set => _key.Value = value;
+        }
+
+        /// <summary>
+        /// A dictionary of all current open log file writers
+        /// </summary>
+        public readonly ConcurrentDictionary<string, StreamWriter> LogWriters = new ConcurrentDictionary<string, StreamWriter>();
+    }
+}

sources/SilkTouch/SilkTouch/Logging/ConsoleRenderer.cs

@@ -0,0 +1,87 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Silk.NET.SilkTouch.Logging
+{
+    /// <summary>
+    /// A console renderer which handles progress bar writing
+    /// </summary>
+    public class ConsoleRenderer
+    {
+        private readonly IProgressService _progressService;
+        private readonly TextWriter _originalOut;
+        private readonly StringWriter _consoleOutput;
+        private readonly Timer _timer;
+        private bool _isRunning = true;
+        private int _progressBarCount;
+        private readonly object _lockObject = new object();
+
+        /// <summary>
+        /// Creates and instance of the <see cref="ConsoleRenderer"/>
+        /// </summary>
+        /// <param name="progressService">the progress service</param>
+        public ConsoleRenderer(IProgressService progressService)
+        {
+            _progressService = progressService;
+            _originalOut = Console.Out;
+            _consoleOutput = new StringWriter();
+            Console.SetOut(_consoleOutput); // Redirect console output
+
+            _timer = new Timer(Render, null, 0, 500);
+        }
+
+        /// <summary>
+        /// Stops the console renderer processing and restores console to original state
+        /// </summary>
+        public void Stop()
+        {
+            _isRunning = false;
+            _timer.Dispose();
+            Console.SetOut(_originalOut); // Restore original output
+        }
+
+        private void Render(object? state)
+        {
+            if (!_isRunning && _consoleOutput.GetStringBuilder().Length > 0)
+                return;
+
+            lock (_lockObject)
+            {
+                // Clear the progress bars by moving the cursor and overwriting with spaces
+                if (_progressBarCount > 0)
+                {
+                    int currentTop = Console.CursorTop;
+                    Console.SetCursorPosition(0, currentTop - _progressBarCount);
+                    for (int i = 0; i < _progressBarCount; i++)
+                    {
+                        _originalOut.Write(new string(' ', Console.WindowWidth));
+                    }
+                    Console.SetCursorPosition(0, currentTop - _progressBarCount);
+                }
+
+                var progressDictionary = _progressService.GetAllProgress();
+                _progressBarCount = progressDictionary.Count();
+
+                // Write all console output captured since the last render
+                var output = _consoleOutput.GetStringBuilder().ToString();
+                if (!string.IsNullOrWhiteSpace(output))
+                {
+                    _originalOut.Write(output);
+                    _consoleOutput.GetStringBuilder().Clear();
+                }
+
+                foreach (var kvp in progressDictionary)
+                {
+                    string task = string.IsNullOrWhiteSpace(kvp.Value.Item1) ? string.Empty : $" - {kvp.Value.Item1}";
+                    _originalOut.WriteLine($"{kvp.Key}{task}: ({new string('|', (int)(kvp.Value.Item2 * 20))}{new string('-', 20 - (int)(kvp.Value.Item2 * 20))}) ({(kvp.Value.Item2 * 100):F2}%)");
+                }
+            }
+        }
+    }
+}

sources/SilkTouch/SilkTouch/Logging/IProgressService.cs

@@ -0,0 +1,44 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Silk.NET.SilkTouch.Logging
+{
+    /// <summary>
+    /// An interface for handling current task progress in a job
+    /// </summary>
+    public interface IProgressService
+    {
+        /// <summary>
+        /// Returns an enumerable of all progress values with job as the key, and then the task and progress in a tuple
+        /// </summary>
+        /// <returns></returns>
+        IEnumerable<KeyValuePair<string, (string, float)>> GetAllProgress();
+        /// <summary>
+        /// Get the current task and progress for this job
+        /// </summary>
+        /// <returns>current task and progress for this job</returns>
+        (string, float) GetCurrentTaskAndProgress();
+        /// <summary>
+        /// Set the name of the task for this job
+        /// Resets the progress to 0
+        /// </summary>
+        /// <param name="task">name of the current task</param>
+        void SetTask(string task);
+        /// <summary>
+        /// Sets the progress of the current job's task
+        /// Values are expected between 0 and 1
+        /// </summary>
+        /// <param name="progress">current progress value</param>
+        void SetProgress(float progress);
+        /// <summary>
+        /// Remove the current job from the progress service
+        /// </summary>
+        void RemoveProgress();
+    }
+}

sources/SilkTouch/SilkTouch/Logging/ProgressService.cs

@@ -0,0 +1,64 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Newtonsoft.Json.Linq;
+
+namespace Silk.NET.SilkTouch.Logging
+{
+    /// <summary>
+    /// Default Implementation for ProgressService
+    /// </summary>
+    public class ProgressService : IProgressService
+    {
+        private ConcurrentDictionary<string, (string, float)> Progress = new ConcurrentDictionary<string, (string, float)>();
+        private JobContext _jobContext;
+
+        /// <summary>
+        /// creates an instance of <see cref="ProgressService"/>
+        /// </summary>
+        /// <param name="context">the current job context</param>
+        public ProgressService(JobContext context)
+        {
+            _jobContext = context;
+        }
+
+        /// <inheritdoc/>
+        public IEnumerable<KeyValuePair<string, (string, float)>> GetAllProgress() => Progress;
+        /// <inheritdoc/>
+        public (string, float) GetCurrentTaskAndProgress() => Progress.TryGetValue(_jobContext.JobKey ?? string.Empty, out var value) ? value : (string.Empty, 0);
+        /// <inheritdoc/>
+        public void RemoveProgress() => Progress.TryRemove(_jobContext.JobKey ?? string.Empty, out _);
+        /// <inheritdoc/>
+        public void SetProgress(float progress)
+        {
+            if (_jobContext.JobKey is null)
+            {
+                return;
+            }
+
+            if (!Progress.TryGetValue(_jobContext.JobKey, out var value))
+            {
+                value = (string.Empty, 0);
+            }
+
+            Progress[_jobContext.JobKey] = (value.Item1, progress);
+        }
+        /// <inheritdoc/>
+        public void SetTask(string task)
+        {
+            if (_jobContext.JobKey is null)
+            {
+                return;
+            }
+
+            Progress[_jobContext.JobKey] = (task, 0);
+        }
+    }
+}