Some optimizations for component detection (#1384)

* Optimize component detection scanning

Author: narasamdya

docs/schema/manifest.schema.json

@@ -118,7 +118,8 @@
               "Spdx",
               "Vcpkg",
               "DockerReference",
-              "DotNet"
+              "DotNet",
+              "Conan"
             ]
           }
         }
@@ -345,7 +346,8 @@
             "Spdx",
             "Vcpkg",
             "DockerReference",
-            "DotNet"
+            "DotNet",
+            "Conan"
           ]
         },
         "id": {

src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs

@@ -12,11 +12,12 @@
 
 namespace Microsoft.ComponentDetection.Common.DependencyGraph;
 
+using System.Collections.ObjectModel;
 using Microsoft.Extensions.Logging;
 
 public class ComponentRecorder : IComponentRecorder
 {
-    private readonly ConcurrentBag<SingleFileComponentRecorder> singleFileRecorders = [];
+    private readonly ConcurrentDictionary<string, SingleFileComponentRecorder> singleFileRecorders = [];
 
     private readonly bool enableManualTrackingOfExplicitReferences;
 
@@ -30,7 +31,7 @@ public ComponentRecorder(ILogger logger = null, bool enableManualTrackingOfExpli
 
     public TypedComponent GetComponent(string componentId)
     {
-        return this.singleFileRecorders.Select(x => x.GetComponent(componentId)?.Component).FirstOrDefault(x => x != null);
+        return this.singleFileRecorders.Values.Select(x => x.GetComponent(componentId)?.Component).FirstOrDefault(x => x != null);
     }
 
     public IEnumerable<DetectedComponent> GetDetectedComponents()
@@ -41,25 +42,21 @@ public IEnumerable<DetectedComponent> GetDetectedComponents()
             return [];
         }
 
-        detectedComponents = this.singleFileRecorders
-            .Select(singleFileRecorder => singleFileRecorder.GetDetectedComponents().Values)
-            .SelectMany(x => x)
+        detectedComponents = this.singleFileRecorders.Values
+            .SelectMany(singleFileRecorder => singleFileRecorder.GetDetectedComponents().Values)
             .GroupBy(x => x.Component.Id)
             .Select(grouping =>
             {
                 // We pick a winner here -- any stateful props could get lost at this point. Only stateful prop still outstanding is ContainerDetails.
                 var winningDetectedComponent = grouping.First();
-                foreach (var component in grouping)
+                foreach (var component in grouping.Skip(1))
                 {
-                    foreach (var containerDetailId in component.ContainerDetailIds)
-                    {
-                        winningDetectedComponent.ContainerDetailIds.Add(containerDetailId);
-                    }
+                    winningDetectedComponent.ContainerDetailIds.UnionWith(component.ContainerDetailIds);
                 }
 
                 return winningDetectedComponent;
             })
-            .ToImmutableList();
+            .ToArray();
 
         return detectedComponents;
     }
@@ -71,11 +68,10 @@ public IEnumerable<string> GetSkippedComponents()
             return [];
         }
 
-        return this.singleFileRecorders
-            .Select(x => x.GetSkippedComponents().Keys)
-            .SelectMany(x => x)
+        return this.singleFileRecorders.Values
+            .SelectMany(x => x.GetSkippedComponents().Keys)
             .Distinct()
-            .ToImmutableList();
+            .ToArray();
     }
 
     public ISingleFileComponentRecorder CreateSingleFileComponentRecorder(string location)
@@ -85,25 +81,20 @@ public ISingleFileComponentRecorder CreateSingleFileComponentRecorder(string loc
             throw new ArgumentNullException(nameof(location));
         }
 
-        var matching = this.singleFileRecorders.FirstOrDefault(x => x.ManifestFileLocation == location);
-        if (matching == null)
-        {
-            matching = new SingleFileComponentRecorder(location, this, this.enableManualTrackingOfExplicitReferences, this.logger);
-            this.singleFileRecorders.Add(matching);
-        }
-
-        return matching;
+        return this.singleFileRecorders.GetOrAdd(location, loc => new SingleFileComponentRecorder(loc, this, this.enableManualTrackingOfExplicitReferences, this.logger));
     }
 
     public IReadOnlyDictionary<string, IDependencyGraph> GetDependencyGraphsByLocation()
     {
-        return this.singleFileRecorders.Where(x => x.DependencyGraph.HasComponents())
-            .ToImmutableDictionary(x => x.ManifestFileLocation, x => x.DependencyGraph as IDependencyGraph);
+        return new ReadOnlyDictionary<string, IDependencyGraph>(
+            this.singleFileRecorders.Values
+                .Where(x => x.DependencyGraph.HasComponents())
+                .ToDictionary(x => x.ManifestFileLocation, x => x.DependencyGraph as IDependencyGraph));
     }
 
     internal DependencyGraph GetDependencyGraphForLocation(string location)
     {
-        return this.singleFileRecorders.Single(x => x.ManifestFileLocation == location).DependencyGraph;
+        return this.singleFileRecorders[location].DependencyGraph;
     }
 
     public sealed class SingleFileComponentRecorder : ISingleFileComponentRecorder
@@ -183,16 +174,15 @@ public void RegisterUsage(
 #endif
 
             var componentId = detectedComponent.Component.Id;
-            DetectedComponent storedComponent = null;
-            lock (this.registerUsageLock)
-            {
-                storedComponent = this.detectedComponentsInternal.GetOrAdd(componentId, detectedComponent);
+            var storedComponent = this.detectedComponentsInternal.GetOrAdd(componentId, detectedComponent);
 
-                if (!string.IsNullOrWhiteSpace(targetFramework))
-                {
-                    storedComponent.TargetFrameworks.Add(targetFramework.Trim());
-                }
+            if (!string.IsNullOrWhiteSpace(targetFramework))
+            {
+                storedComponent.TargetFrameworks.Add(targetFramework.Trim());
+            }
 
+            lock (this.registerUsageLock)
+            {
                 this.AddComponentToGraph(this.ManifestFileLocation, detectedComponent, isExplicitReferencedDependency, parentComponentId, isDevelopmentDependency, dependencyScope);
             }
         }

src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs

@@ -97,7 +97,7 @@ public void AddAdditionalRelatedFile(string additionalRelatedFile)
 
     public HashSet<string> GetAdditionalRelatedFiles()
     {
-        return this.AdditionalRelatedFiles.Keys.ToImmutableHashSet().ToHashSet();
+        return this.AdditionalRelatedFiles.Keys.ToHashSet();
     }
 
     public bool HasComponents()

src/Microsoft.ComponentDetection.Contracts/FileComponentDetector.cs

@@ -1,8 +1,10 @@
 namespace Microsoft.ComponentDetection.Contracts;
 
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Reactive.Linq;
 using System.Threading;
 using System.Threading.Tasks;
@@ -56,7 +58,7 @@ public abstract class FileComponentDetector : IComponentDetector
 
     public virtual bool NeedsAutomaticRootDependencyCalculation { get; protected set; }
 
-    protected Dictionary<string, string> Telemetry { get; set; } = [];
+    protected ConcurrentDictionary<string, string> Telemetry { get; set; } = [];
 
     /// <summary>
     /// List of any any additional properties as key-value pairs that we would like to capture for the detector.
@@ -135,7 +137,7 @@ private async Task<IndividualDetectorScanResult> ProcessAsync(
         return new IndividualDetectorScanResult
         {
             ResultCode = ProcessingResultCode.Success,
-            AdditionalTelemetryDetails = this.Telemetry,
+            AdditionalTelemetryDetails = this.Telemetry.ToDictionary(kvp => kvp.Key, kvp => kvp.Value),
         };
     }
 

src/Microsoft.ComponentDetection.Contracts/TypedComponent/CargoComponent.cs

@@ -36,7 +36,7 @@ public CargoComponent(string name, string version, string author = null, string
 
     public override ComponentType Type => ComponentType.Cargo;
 
-    public override string Id => $"{this.Name} {this.Version} - {this.Type}";
-
     public override PackageURL PackageUrl => new PackageURL("cargo", string.Empty, this.Name, this.Version, null, string.Empty);
+
+    protected override string ComputeId() => $"{this.Name} {this.Version} - {this.Type}";
 }

src/Microsoft.ComponentDetection.Contracts/TypedComponent/ConanComponent.cs

@@ -29,7 +29,7 @@ public ConanComponent(string name, string version, string previous, string packa
 
     public override ComponentType Type => ComponentType.Conan;
 
-    public override string Id => $"{this.Name} {this.Version} - {this.Type}";
-
     public override PackageURL PackageUrl => new PackageURL("conan", string.Empty, this.Name, this.Version, null, string.Empty);
+
+    protected override string ComputeId() => $"{this.Name} {this.Version} - {this.Type}";
 }

src/Microsoft.ComponentDetection.Contracts/TypedComponent/CondaComponent.cs

@@ -37,5 +37,5 @@ private CondaComponent()
 
     public override ComponentType Type => ComponentType.Conda;
 
-    public override string Id => $"{this.Name} {this.Version} {this.Build} {this.Channel} {this.Subdir} {this.Namespace} {this.Url} {this.MD5} - {this.Type}";
+    protected override string ComputeId() => $"{this.Name} {this.Version} {this.Build} {this.Channel} {this.Subdir} {this.Namespace} {this.Url} {this.MD5} - {this.Type}";
 }

src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerImageComponent.cs

@@ -22,5 +22,5 @@ public DockerImageComponent(string hash, string name = null, string tag = null)
 
     public override ComponentType Type => ComponentType.DockerImage;
 
-    public override string Id => $"{this.Name} {this.Tag} {this.Digest}";
+    protected override string ComputeId() => $"{this.Name} {this.Tag} {this.Digest}";
 }

src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerReferenceComponent.cs

@@ -36,5 +36,5 @@ public DockerReference FullReference
         }
     }
 
-    public override string Id => $"{this.Repository} {this.Tag} {this.Digest}";
+    protected override string ComputeId() => $"{this.Repository} {this.Tag} {this.Digest}";
 }

src/Microsoft.ComponentDetection.Contracts/TypedComponent/DotNetComponent.cs

@@ -43,5 +43,6 @@ public DotNetComponent(string sdkVersion, string targetFramework = null, string
     /// <summary>
     /// Provides an id like `{SdkVersion} - {TargetFramework} - {ProjectType} - dotnet` where unspecified values are represented as 'unknown'.
     /// </summary>
-    public override string Id => $"{this.SdkVersion} {this.TargetFramework} {this.ProjectType} - {this.Type}";
+    /// <returns>Id of the component.</returns>
+    protected override string ComputeId() => $"{this.SdkVersion} {this.TargetFramework} {this.ProjectType} - {this.Type}";
 }