Feat: split history request (#10)

* fix: several deprecate href

* feat: use max available date by subs

* feat: request parameters const names

* feat: Execute History Request Parallel

* refactor: synchronize request in API class

* feat: develop StopWatch Wrapper

* remove: RestClient huge Timeout value

* fix: GenerateDateRangesWithInterval extension

* refactor: ExecuteRequestParallelAsync
test:feat: GetHistoryRequestWithLongRange

* refactor: ExecuteRequestParallelAsync

* refactor: ExecuteRequestParallelAsync if scope

* Revert "feat: use max available date by subs"

This reverts commit a1819dc.

* refactor: use Parallel.ForEachAsync instead of  semaphore and task to get history data

* remove: NullDisposable

* feat: use max available date by subs

* feat: request retry if failed in ApiClient

* fix: GenerateDateRangesWithInterval
refactor: ExecuteRequest

* feat: filter of HistoryRequest with ExtendedMarketHours
test:feat: amount of bars with different resolution

* fix: use Utc time for tick requests
remove: extra logs

* feat: validate history request on null

* remove: not used lib

Author: Romazes

QuantConnect.ThetaData.Tests/TestHelpers.cs

@@ -174,11 +174,11 @@ public static void AssertTradeBars(IEnumerable<TradeBar> tradeBars, Symbol symbo
         }
 
         public static HistoryRequest CreateHistoryRequest(Symbol symbol, Resolution resolution, TickType tickType, DateTime startDateTime, DateTime endDateTime,
-            SecurityExchangeHours exchangeHours = null, DateTimeZone dataTimeZone = null)
+            SecurityExchangeHours exchangeHours = null, DateTimeZone dataTimeZone = null, bool includeExtendedMarketHours = true)
         {
             if (exchangeHours == null)
             {
-                exchangeHours = SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork);
+                exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType);
             }
 
             if (dataTimeZone == null)
@@ -188,15 +188,15 @@ public static HistoryRequest CreateHistoryRequest(Symbol symbol, Resolution reso
 
             var dataType = LeanData.GetDataType(resolution, tickType);
             return new HistoryRequest(
-                startDateTime,
-                endDateTime,
+                startDateTime.ConvertToUtc(exchangeHours.TimeZone),
+                endDateTime.ConvertToUtc(exchangeHours.TimeZone),
                 dataType,
                 symbol,
                 resolution,
                 exchangeHours,
                 dataTimeZone,
-                null,
-                true,
+                resolution,
+                includeExtendedMarketHours,
                 false,
                 DataNormalizationMode.Raw,
                 tickType

QuantConnect.ThetaData.Tests/ThetaDataAdditionalTests.cs

@@ -0,0 +1,87 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using NUnit.Framework;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.DataSource.ThetaData.Tests;
+
+[TestFixture]
+public class ThetaDataAdditionalTests
+{
+    [Test]
+    public void GenerateDateRangesWithNinetyDaysInterval()
+    {
+        var intervalDays = 90;
+        var startDate = new DateTime(2020, 07, 18);
+        var endDate = new DateTime(2021, 01, 14);
+
+        var expectedRanges = new List<(DateTime startDate, DateTime endDate)>
+        {
+            (new DateTime(2020, 07, 18), new DateTime(2020, 10, 16)),
+            (new DateTime(2020, 10, 17), new DateTime(2021, 01, 14)),
+        };
+
+        var actualRanges = new List<(DateTime startDate, DateTime endDate)>(ThetaDataExtensions.GenerateDateRangesWithInterval(startDate, endDate, intervalDays));
+
+        Assert.AreEqual(expectedRanges.Count, actualRanges.Count, "The number of ranges should match.");
+
+        for (int i = 0; i < expectedRanges.Count; i++)
+        {
+            Assert.AreEqual(expectedRanges[i].startDate, actualRanges[i].startDate, $"Start date mismatch at index {i}");
+            Assert.AreEqual(expectedRanges[i].endDate, actualRanges[i].endDate, $"End date mismatch at index {i}");
+        }
+    }
+
+    [Test]
+    public void GenerateDateRangesWithOneDayInterval()
+    {
+        var intervalDays = 1;
+
+        var startDate = new DateTime(2024, 07, 26);
+        var endDate = new DateTime(2024, 07, 30);
+
+        var expectedRanges = new List<(DateTime startDate, DateTime endDate)>
+        {
+            (new DateTime(2024, 07, 26), new DateTime(2024, 07, 27)),
+            (new DateTime(2024, 07, 28), new DateTime(2024, 07, 29)),
+            (new DateTime(2024, 07, 30), new DateTime(2024, 07, 30))
+        };
+
+        var actualRanges = new List<(DateTime startDate, DateTime endDate)>(ThetaDataExtensions.GenerateDateRangesWithInterval(startDate, endDate, intervalDays));
+
+        Assert.AreEqual(expectedRanges.Count, actualRanges.Count, "The number of ranges should match.");
+
+        for (int i = 0; i < expectedRanges.Count; i++)
+        {
+            Assert.AreEqual(expectedRanges[i].startDate, actualRanges[i].startDate, $"Start date mismatch at index {i}");
+            Assert.AreEqual(expectedRanges[i].endDate, actualRanges[i].endDate, $"End date mismatch at index {i}");
+        }
+    }
+
+    [Test]
+    public void GenerateDateRangesWithInterval_ShouldHandleSameStartEndDate()
+    {
+        DateTime startDate = new DateTime(2025, 2, 1);
+        DateTime endDate = new DateTime(2025, 2, 1);
+
+        var ranges = new List<(DateTime startDate, DateTime endDate)>(
+            ThetaDataExtensions.GenerateDateRangesWithInterval(startDate, endDate, 1)
+        );
+
+        Assert.AreEqual(1, ranges.Count, "There should be no date ranges generated.");
+    }
+}

QuantConnect.ThetaData.Tests/ThetaDataHistoryProviderTests..cs

@@ -16,6 +16,11 @@
 using System;
 using System.Linq;
 using NUnit.Framework;
+using QuantConnect.Data;
+using System.Diagnostics;
+using QuantConnect.Logging;
+using Microsoft.CodeAnalysis;
+using System.Collections.Generic;
 
 namespace QuantConnect.Lean.DataSource.ThetaData.Tests
 {
@@ -114,5 +119,109 @@ public void GetHistoryTickTradeValidateOnDistinctData(string ticker, Resolution
 
             Assert.That(history.Count, Is.EqualTo(distinctHistory.Count));
         }
+
+        [TestCase("SPY", SecurityType.Equity, Resolution.Hour, "1998/01/02", "2025/02/16", new[] { TickType.Quote, TickType.Trade })]
+        [TestCase("SPY", SecurityType.Equity, Resolution.Daily, "1998/01/02", "2025/02/16", new[] { TickType.Quote, TickType.Trade })]
+        [TestCase("SPY", SecurityType.Equity, Resolution.Minute, "2025/01/02", "2025/02/16", new[] { TickType.Quote, TickType.Trade })]
+        [TestCase("AAPL", SecurityType.Equity, Resolution.Hour, "1998/01/02", "2025/02/16", new[] { TickType.Quote, TickType.Trade })]
+        public void GetHistoryRequestWithLongRange(string ticker, SecurityType securityType, Resolution resolution, DateTime startDate, DateTime endDate, TickType[] tickTypes)
+        {
+            var symbol = TestHelpers.CreateSymbol(ticker, securityType);
+
+            var historyRequests = new List<HistoryRequest>();
+            foreach (var tickType in tickTypes)
+            {
+                historyRequests.Add(TestHelpers.CreateHistoryRequest(symbol, resolution, tickType, startDate, endDate));
+            }
+
+            foreach (var historyRequest in historyRequests)
+            {
+                var stopwatch = Stopwatch.StartNew();
+                var history = _thetaDataProvider.GetHistory(historyRequest).ToList();
+                stopwatch.Stop();
+
+                Assert.IsNotEmpty(history);
+
+                var firstDate = history.First().Time;
+                var lastDate = history.Last().Time;
+
+                Log.Trace($"[{nameof(ThetaDataHistoryProviderTests)}] Execution completed in {stopwatch.Elapsed.TotalMinutes:F2} min | " +
+                          $"Symbol: {historyRequest.Symbol}, Resolution: {resolution}, TickType: {historyRequest.TickType}, Count: {history.Count}, " +
+                          $"First Date: {firstDate:yyyy-MM-dd HH:mm:ss}, Last Date: {lastDate:yyyy-MM-dd HH:mm:ss}");
+
+                // Ensure historical data is returned in chronological order
+                for (var i = 1; i < history.Count; i++)
+                {
+                    if (history[i].Time < history[i - 1].Time)
+                        Assert.Fail("Historical data is not in chronological order.");
+                }
+            }
+        }
+
+        [TestCase("AAPL", SecurityType.Equity, Resolution.Minute, "2025/02/19", "2025/02/20", new[] { TickType.Quote, TickType.Trade })]
+        [TestCase("AAPL", SecurityType.Equity, Resolution.Minute, "2025/02/18", "2025/02/20", new[] { TickType.Quote, TickType.Trade })]
+        [TestCase("AAPL", SecurityType.Equity, Resolution.Minute, "2025/02/15", "2025/02/20", new[] { TickType.Quote, TickType.Trade })]
+        [TestCase("AAPL", SecurityType.Equity, Resolution.Minute, "2025/02/10", "2025/02/20", new[] { TickType.Quote, TickType.Trade })]
+        [TestCase("AAPL", SecurityType.Equity, Resolution.Hour, "2025/02/19", "2025/02/20", new[] { TickType.Quote, TickType.Trade })]
+        [TestCase("AAPL", SecurityType.Equity, Resolution.Hour, "2025/02/18", "2025/02/20", new[] { TickType.Quote, TickType.Trade })]
+        [TestCase("AAPL", SecurityType.Equity, Resolution.Hour, "2025/02/10", "2025/02/20", new[] { TickType.Quote, TickType.Trade })]
+        [TestCase("AAPL", SecurityType.Equity, Resolution.Hour, "2025/02/01", "2025/02/20", new[] { TickType.Quote, TickType.Trade })]
+        [TestCase("AAPL", SecurityType.Equity, Resolution.Hour, "2025/01/01", "2025/02/20", new[] { TickType.Quote, TickType.Trade })]
+        [TestCase("AAPL", SecurityType.Equity, Resolution.Daily, "2025/02/19", "2025/02/20", new[] { TickType.Quote, TickType.Trade })]
+        [TestCase("AAPL", SecurityType.Equity, Resolution.Daily, "2025/02/18", "2025/02/20", new[] { TickType.Quote, TickType.Trade })]
+        [TestCase("AAPL", SecurityType.Equity, Resolution.Daily, "2025/02/15", "2025/02/20", new[] { TickType.Quote, TickType.Trade })]
+        [TestCase("AAPL", SecurityType.Equity, Resolution.Daily, "2025/02/10", "2025/02/20", new[] { TickType.Quote, TickType.Trade })]
+        [TestCase("AAPL", SecurityType.Equity, Resolution.Daily, "2025/01/01", "2025/02/20", new[] { TickType.Quote, TickType.Trade })]
+        public void GetHistoryRequestWithCalculateAmountReturnsData(string ticker, SecurityType securityType, Resolution resolution, DateTime startDate, DateTime endDate, TickType[] tickTypes)
+        {
+            var symbol = TestHelpers.CreateSymbol(ticker, securityType);
+
+            var historyRequests = new List<HistoryRequest>();
+            foreach (var tickType in tickTypes)
+            {
+                historyRequests.Add(TestHelpers.CreateHistoryRequest(symbol, resolution, tickType, startDate, endDate, includeExtendedMarketHours: false));
+            }
+
+            foreach (var historyRequest in historyRequests)
+            {
+                var history = _thetaDataProvider.GetHistory(historyRequest).ToList();
+                //Log.Trace(string.Join("\n", history.Select(x => new { Time = x.Time, EndTime = x.EndTime, Data = x })));
+
+                int expectedAmount = CalculateExpectedHistoryAmount(historyRequest);
+
+                Assert.AreEqual(expectedAmount, history.Count, "History data count does not match expected amount.");
+            }
+        }
+
+        private int CalculateExpectedHistoryAmount(HistoryRequest request)
+        {
+            var endTime = request.EndTimeUtc.ConvertFromUtc(request.DataTimeZone);
+            var currentDate = request.StartTimeUtc.ConvertFromUtc(request.DataTimeZone);
+            int totalDataPoints = 0;
+
+            while (currentDate < endTime)
+            {
+                if (request.ExchangeHours.IsDateOpen(currentDate, request.IncludeExtendedMarketHours))
+                {
+                    int dataPointsPerDay = GetDataPointsPerDay(request.Resolution);
+                    totalDataPoints += dataPointsPerDay;
+                }
+
+                currentDate = currentDate.AddDays(1);
+            }
+
+            return totalDataPoints;
+        }
+
+        private int GetDataPointsPerDay(Resolution resolution)
+        {
+            return resolution switch
+            {
+                Resolution.Minute => 390,  // 720 minutes from 9:30 AM to 4:00 PM (Trading Hours)
+                Resolution.Hour => 7,
+                Resolution.Daily => 1,     // 1 bar per day
+                _ => throw new ArgumentOutOfRangeException(nameof(resolution), "Unsupported resolution")
+            };
+        }
     }
 }

QuantConnect.ThetaData/Models/Interfaces/ISubscriptionPlan.cs

@@ -20,8 +20,16 @@ namespace QuantConnect.Lean.DataSource.ThetaData.Models.Interfaces;
 /// <summary>
 /// The <c>ISubscriptionPlan</c> interface defines the base structure for different price plans offered by ThetaData for users.
 /// For detailed documentation on ThetaData subscription plans, refer to the following links:
-/// <see href="https://www.thetadata.net/subscribe" /> 
-/// <see href="https://http-docs.thetadata.us/docs/theta-data-rest-api-v2/1floxgrco3si8-us-options#historical-endpoint-access" />
+/// <list type="bullet">
+///    <item>
+///        <term>https://www.thetadata.net/subscribe</term>
+///        <description>Institutional Data Retail Pricing</description>
+///    </item>
+///    <item>
+///        <term>https://http-docs.thetadata.us/Articles/Getting-Started/Subscriptions.html#options-data</term>
+///        <description>Initial Access Date Based on Subscription Plan</description>
+///    </item>
+///</list>
 /// </summary>
 public interface ISubscriptionPlan
 {

QuantConnect.ThetaData/Models/Rest/RequestParameters.cs

@@ -0,0 +1,40 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+namespace QuantConnect.Lean.DataSource.ThetaData.Models.Rest;
+
+/// <summary>
+/// Contains constant values for various request parameters used in API queries.
+/// </summary>
+public static class RequestParameters
+{
+    /// <summary>
+    /// Represents the time interval in milliseconds since midnight Eastern Time (ET).
+    /// Example values:
+    /// - 09:30:00 ET = 34_200_000 ms
+    /// - 16:00:00 ET = 57_600_000 ms
+    /// </summary>
+    public const string IntervalInMilliseconds = "ivl";
+
+    /// <summary>
+    /// Represents the start date for a query or request.
+    /// </summary>
+    public const string StartDate = "start_date";
+
+    /// <summary>
+    /// Represents the end date for a query or request.
+    /// </summary>
+    public const string EndDate = "end_date";
+}

QuantConnect.ThetaData/Models/Wrappers/StopwatchWrapper.cs

@@ -0,0 +1,67 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System.Diagnostics;
+using QuantConnect.Logging;
+using QuantConnect.Lean.DataSource.ThetaData.Models.Common;
+
+namespace QuantConnect.Lean.DataSource.ThetaData.Models.Wrappers;
+
+/// <summary>
+/// A utility class that conditionally starts a stopwatch for measuring execution time 
+/// when debugging is enabled. Implements <see cref="IDisposable"/> to ensure 
+/// automatic logging upon completion.
+/// </summary>
+public class StopwatchWrapper : IDisposable
+{
+    private readonly Stopwatch? _stopwatch;
+    private readonly string _message;
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="StopwatchWrapper"/> class
+    /// and starts a stopwatch to measure execution time.
+    /// </summary>
+    /// <param name="message">A descriptive message to include in the log output.</param>
+    private StopwatchWrapper(string message)
+    {
+        _message = message;
+        _stopwatch = Stopwatch.StartNew();
+    }
+
+    /// <summary>
+    /// Starts a stopwatch if debugging is enabled and returns an appropriate disposable instance.
+    /// </summary>
+    /// <param name="message">A descriptive message to include in the log output.</param>
+    /// <returns>
+    /// A <see cref="StopwatchWrapper"/> instance if debugging is enabled, 
+    /// otherwise a no-op <see cref="NullDisposable"/> instance.
+    /// </returns>
+    public static IDisposable? StartIfEnabled(string message)
+    {
+        return Log.DebuggingEnabled ? new StopwatchWrapper(message) : null;
+    }
+
+    /// <summary>
+    /// Stops the stopwatch and logs the elapsed time if debugging is enabled.
+    /// </summary>
+    public void Dispose()
+    {
+        if (_stopwatch != null)
+        {
+            _stopwatch.Stop();
+            Log.Debug($"{_message} completed in {_stopwatch.ElapsedMilliseconds} ms");
+        }
+    }
+}