Plotting Limits Handling Improvements

Algorithm/QCAlgorithm.Plotting.cs

@@ -340,32 +340,22 @@ private bool TryGetChartSeries<T>(string chartName, string seriesName, out T ser
             {
                 if (reservedSeriesNames.Count == 0)
                 {
-                    throw new Exception($"Algorithm.Plot(): '{chartName}' is a reserved chart name.");
+                    throw new ArgumentException($"'{chartName}' is a reserved chart name.");
                 }
                 if (reservedSeriesNames.Contains(seriesName))
                 {
-                    throw new Exception($"Algorithm.Plot(): '{seriesName}' is a reserved series name for chart '{chartName}'.");
+                    throw new ArgumentException($"'{seriesName}' is a reserved series name for chart '{chartName}'.");
                 }
             }
 
-            // If we don't have the chart, create it
-            _charts.TryAdd(chartName, new Chart(chartName));
+            if(!_charts.TryGetValue(chartName, out var chart))
+            {
+                // If we don't have the chart, create it
+                _charts[chartName] = chart = new Chart(chartName);
+            }
 
-            var chart = _charts[chartName];
             if (!chart.Series.TryGetValue(seriesName, out var chartSeries))
             {
-                //Number of series in total, excluding reserved charts
-                var seriesCount = _charts.Select(x => x.Value)
-                    .Aggregate(0, (i, c) => ReservedChartSeriesNames.TryGetValue(c.Name, out reservedSeriesNames)
-                    ? i + c.Series.Values.Count(s => reservedSeriesNames.Count > 0 && !reservedSeriesNames.Contains(s.Name))
-                    : i + c.Series.Count);
-
-                if (seriesCount > 10)
-                {
-                    Error("Exceeded maximum series count: Each backtest can have up to 10 series in total.");
-                    return false;
-                }
-
                 chartSeries = new T() { Name = seriesName };
                 chart.AddSeries(chartSeries);
             }
@@ -549,22 +539,20 @@ public void SetSummaryStatistic(string name, decimal value)
         /// <returns>List of chart updates since the last request</returns>
         /// <remarks>GetChartUpdates returns the latest updates since previous request.</remarks>
         [DocumentationAttribute(Charting)]
-        public List<Chart> GetChartUpdates(bool clearChartData = false)
+        public IEnumerable<Chart> GetChartUpdates(bool clearChartData = false)
         {
-            var updates = _charts.Select(x => x.Value).Select(chart => chart.GetUpdates()).ToList();
-
-            if (clearChartData)
+            foreach (var chart in _charts.Values)
             {
-                // we can clear this data out after getting updates to prevent unnecessary memory usage
-                foreach (var chart in _charts)
+                yield return chart.GetUpdates();
+                if (clearChartData)
                 {
-                    foreach (var series in chart.Value.Series)
+                    // we can clear this data out after getting updates to prevent unnecessary memory usage
+                    foreach (var series in chart.Series)
                     {
                         series.Value.Purge();
                     }
                 }
             }
-            return updates;
         }
     }
 }

AlgorithmFactory/Python/Wrappers/AlgorithmPythonWrapper.cs

@@ -574,7 +574,7 @@ public void OnEndOfTimeStep()
         /// </summary>
         /// <param name="clearChartData"></param>
         /// <returns>List of Chart Updates</returns>
-        public List<Chart> GetChartUpdates(bool clearChartData = false) => _baseAlgorithm.GetChartUpdates(clearChartData);
+        public IEnumerable<Chart> GetChartUpdates(bool clearChartData = false) => _baseAlgorithm.GetChartUpdates(clearChartData);
 
         /// <summary>
         /// Gets whether or not this algorithm has been locked and fully initialized

Common/Interfaces/IAlgorithm.cs

@@ -642,7 +642,7 @@ InsightManager Insights
         /// </summary>
         /// <param name="clearChartData"></param>
         /// <returns>List of Chart Updates</returns>
-        List<Chart> GetChartUpdates(bool clearChartData = false);
+        IEnumerable<Chart> GetChartUpdates(bool clearChartData = false);
 
         /// <summary>
         /// Set a required SecurityType-symbol and resolution for algorithm

Common/Packets/Controls.cs

@@ -103,6 +103,12 @@ public class Controls
         [JsonProperty(PropertyName = "iMaximumDataPointsPerChartSeries")]
         public int MaximumDataPointsPerChartSeries;
 
+        /// <summary>
+        /// Limits the amount of chart series. Applies only for backtesting
+        /// </summary>
+        [JsonProperty(PropertyName = "iMaximumChartSeries")]
+        public int MaximumChartSeries;
+
         /// <summary>
         /// The amount seconds used for timeout limits
         /// </summary>
@@ -162,6 +168,7 @@ public Controls()
             RemainingLogAllowance = 10000;
             MaximumRuntimeMinutes = 60 * 24 * 100; // 100 days default
             BacktestingMaxInsights = 10000;
+            MaximumChartSeries = 10;
             MaximumDataPointsPerChartSeries = 4000;
             SecondTimeOut = 300;
             StorageLimit = 10737418240;

Engine/Results/BacktestingResultHandler.cs

@@ -44,6 +44,8 @@ public class BacktestingResultHandler : BaseResultsHandler, IResultHandler
         private string _errorMessage;
         private int _daysProcessedFrontier;
         private readonly HashSet<string> _chartSeriesExceededDataPoints;
+        private readonly HashSet<string> _chartSeriesCount;
+        private bool _chartSeriesCountExceededError;
 
         private BacktestProgressMonitor _progressMonitor;
 
@@ -70,7 +72,8 @@ public BacktestingResultHandler()
             ResamplePeriod = TimeSpan.FromMinutes(4);
             NotificationPeriod = TimeSpan.FromSeconds(2);
 
-            _chartSeriesExceededDataPoints = new HashSet<string>();
+            _chartSeriesExceededDataPoints = new();
+            _chartSeriesCount = new();
 
             // Delay uploading first packet
             _nextS3Update = StartTime.AddSeconds(5);
@@ -570,7 +573,7 @@ protected override void SampleCapacity(DateTime time)
         /// Add a range of samples from the users algorithms to the end of our current list.
         /// </summary>
         /// <param name="updates">Chart updates since the last request.</param>
-        protected void SampleRange(List<Chart> updates)
+        protected void SampleRange(IEnumerable<Chart> updates)
         {
             lock (ChartLock)
             {
@@ -587,6 +590,22 @@ protected void SampleRange(List<Chart> updates)
                     //Add these samples to this chart.
                     foreach (var series in update.Series.Values)
                     {
+                        // let's assert we are within series count limit
+                        if (_chartSeriesCount.Count < _job.Controls.MaximumChartSeries)
+                        {
+                            _chartSeriesCount.Add(series.Name);
+                        }
+                        else if (!_chartSeriesCount.Contains(series.Name))
+                        {
+                            // above the limit and this is a new series
+                            if(!_chartSeriesCountExceededError)
+                            {
+                                _chartSeriesCountExceededError = true;
+                                DebugMessage($"Exceeded maximum chart series count, new series will be ignored. Limit is currently set at {_job.Controls.MaximumChartSeries}");
+                            }
+                            continue;
+                        }
+
                         if (series.Values.Count > 0)
                         {
                             var thisSeries = chart.TryAddAndGetSeries(series.Name, series, forceAddNew: false);

Engine/Results/LiveTradingResultHandler.cs

@@ -653,7 +653,7 @@ protected override void Sample(string chartName, string seriesName, int seriesIn
         /// </summary>
         /// <param name="updates">Chart updates since the last request.</param>
         /// <seealso cref="Sample(string,string,int,SeriesType,ISeriesPoint,string)"/>
-        protected void SampleRange(List<Chart> updates)
+        protected void SampleRange(IEnumerable<Chart> updates)
         {
             Log.Debug("LiveTradingResultHandler.SampleRange(): Begin sampling");
             lock (ChartLock)

Launcher/config.json

@@ -55,7 +55,8 @@
   "maximum-warmup-history-days-look-back": 5,
 
   // limits the amount of data points per chart series. Applies only for backtesting
-  "maximum-data-points-per-chart-series": 4000,
+  "maximum-data-points-per-chart-series": 1000000,
+  "maximum-chart-series": 30,
 
   // if one uses true in following token, market hours will remain open all hours and all days.
   // if one uses false will make lean operate only during regular market hours.

Queues/JobQueue.cs

@@ -153,7 +153,8 @@ public AlgorithmNodePacket NextJob(out string location)
                 SecondLimit = Config.GetInt("symbol-second-limit", 10000),
                 TickLimit = Config.GetInt("symbol-tick-limit", 10000),
                 RamAllocation = int.MaxValue,
-                MaximumDataPointsPerChartSeries = Config.GetInt("maximum-data-points-per-chart-series", 4000),
+                MaximumDataPointsPerChartSeries = Config.GetInt("maximum-data-points-per-chart-series", 1000000),
+                MaximumChartSeries = Config.GetInt("maximum-chart-series", 30),
                 StorageLimit = Config.GetValue("storage-limit", 10737418240L),
                 StorageFileCount = Config.GetInt("storage-file-count", 10000),
                 StoragePermissions = (FileAccess)Config.GetInt("storage-permissions", (int)FileAccess.ReadWrite)

Tests/Algorithm/AlgorithmPlottingTests.cs

@@ -29,7 +29,7 @@
 
 namespace QuantConnect.Tests.Algorithm
 {
-    [TestFixture]
+    [TestFixture, Parallelizable(ParallelScope.Fixtures)]
     public class AlgorithmPlottingTests
     {
         private Symbol _spy;
@@ -105,7 +105,7 @@ public void TestGetChartUpdatesWhileAdding()
             {
                 for (var i = 0; i < 1000; i++)
                 {
-                    _algorithm.GetChartUpdates(true);
+                    _algorithm.GetChartUpdates(true).ToList();
                     Thread.Sleep(1);
                 }
             });
@@ -168,7 +168,7 @@ def Update(self, input):
                 customIndicator.Update(input);
                 customIndicator.Current.Value = customIndicator.Value;
                 Assert.DoesNotThrow(() => _algorithm.Plot("PlotTest", customIndicator));
-                var charts = _algorithm.GetChartUpdates();
+                var charts = _algorithm.GetChartUpdates().ToList();
                 Assert.IsTrue(charts.Where(x => x.Name == "PlotTest").Any());
                 Assert.AreEqual(10, charts.First().Series["custom"].GetValues<ChartPoint>().First().y);
             }
@@ -194,7 +194,7 @@ def Update(self, input):
 
                 var customIndicator = module.GetAttr("CustomIndicator").Invoke();
                 Assert.DoesNotThrow(() => _algorithm.Plot("PlotTest", customIndicator));
-                var charts = _algorithm.GetChartUpdates();
+                var charts = _algorithm.GetChartUpdates().ToList();
                 Assert.IsFalse(charts.Where(x => x.Name == "PlotTest").Any());
                 Assert.IsTrue(charts.Where(x => x.Name == "Strategy Equity").Any());
                 Assert.AreEqual(10, charts.First().Series["PlotTest"].GetValues<ChartPoint>().First().y);
@@ -213,7 +213,7 @@ public void PlotIndicatorPlotsBaseIndicator()
             sma1.Update(new DateTime(2022, 11, 15), 1);
             sma2.Update(new DateTime(2022, 11, 15), 2);
 
-            var charts = _algorithm.GetChartUpdates();
+            var charts = _algorithm.GetChartUpdates().ToList();
             Assert.IsTrue(charts.Where(x => x.Name == "PlotTest").Any());
 
             var chart = charts.First();