From 679866db5639bdbeb2df7adca57fe2675af5d7a0 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Wed, 8 Jan 2025 09:16:16 -0400 Subject: [PATCH] Remove ZipEntryName other classes and unused code Removed ZipEntryName class and references across various files. Removed DataQueueFuturesChainUniverseDataCollectionEnumerator and DataQueueOptionChainUniverseDataCollectionEnumerator classes. Removed OptionChainUniverseSubscriptionEnumeratorFactory class. Removed unused code for handling OptionChainUniverse and FuturesChainUniverse in FileSystemDataFeed.cs and LiveTradingDataFeed.cs. Removed several test files related to enumerator factories and universe data collection. --- Common/Data/Auxiliary/ZipEntryName.cs | 98 --------- Common/Extensions.cs | 2 +- Common/Util/LeanData.cs | 15 +- ...esChainUniverseDataCollectionEnumerator.cs | 135 ------------ ...onChainUniverseDataCollectionEnumerator.cs | 164 -------------- .../BaseDataSubscriptionEnumeratorFactory.cs | 87 -------- ...inUniverseSubscriptionEnumeratorFactory.cs | 100 --------- Engine/DataFeeds/FileSystemDataFeed.cs | 18 -- Engine/DataFeeds/LiveTradingDataFeed.cs | 34 --- Engine/HistoricalData/FakeHistoryProvider.cs | 28 --- .../Common/Securities/SecurityServiceTests.cs | 35 +-- ...inUniverseDataCollectionEnumeratorTests.cs | 153 ------------- ...inUniverseDataCollectionEnumeratorTests.cs | 198 ----------------- ...eDataSubscriptionEnumeratorFactoryTests.cs | 76 ------- ...verseSubscriptionEnumeratorFactoryTests.cs | 202 ------------------ Tests/Engine/DataFeeds/TimeSliceTests.cs | 5 +- .../ZipEntryNameSubscriptionFactoryTests.cs | 20 +- Tests/ToolBox/LeanDataReaderTests.cs | 17 +- 18 files changed, 59 insertions(+), 1328 deletions(-) delete mode 100644 Common/Data/Auxiliary/ZipEntryName.cs delete mode 100644 Engine/DataFeeds/Enumerators/DataQueueFuturesChainUniverseDataCollectionEnumerator.cs delete mode 100644 Engine/DataFeeds/Enumerators/DataQueueOptionChainUniverseDataCollectionEnumerator.cs delete mode 100644 Engine/DataFeeds/Enumerators/Factories/BaseDataSubscriptionEnumeratorFactory.cs delete mode 100644 Engine/DataFeeds/Enumerators/Factories/OptionChainUniverseSubscriptionEnumeratorFactory.cs delete mode 100644 Tests/Engine/DataFeeds/Enumerators/DataQueueFuturesChainUniverseDataCollectionEnumeratorTests.cs delete mode 100644 Tests/Engine/DataFeeds/Enumerators/DataQueueOptionChainUniverseDataCollectionEnumeratorTests.cs delete mode 100644 Tests/Engine/DataFeeds/Enumerators/Factories/BaseDataSubscriptionEnumeratorFactoryTests.cs delete mode 100644 Tests/Engine/DataFeeds/Enumerators/Factories/OptionChainUniverseSubscriptionEnumeratorFactoryTests.cs diff --git a/Common/Data/Auxiliary/ZipEntryName.cs b/Common/Data/Auxiliary/ZipEntryName.cs deleted file mode 100644 index cd2abbff1e75..000000000000 --- a/Common/Data/Auxiliary/ZipEntryName.cs +++ /dev/null @@ -1,98 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -using System; -using QuantConnect.Util; - -namespace QuantConnect.Data.Auxiliary -{ - /// - /// Defines a data type that just produces data points from the zip entry names in a zip file - /// - public class ZipEntryName : BaseData - { - private DateTime _endTime; - - /// - /// Initializes a new instance of the class - /// - public ZipEntryName() - { - DataType = MarketDataType.Auxiliary; - } - - /// - /// Gets or sets the end time of this data - /// - public override DateTime EndTime - { - get - { - if (_endTime == default) - { - // to be user friendly let's return Time if not set, like BaseData does - return Time; - } - return _endTime; - } - set - { - _endTime = value; - } - } - - /// - /// Indicates whether this contains data that should be stored in the security cache - /// - /// Whether this contains data that should be stored in the security cache - public override bool ShouldCacheToSecurity() - { - return false; - } - - /// - /// Reader converts each line of the data source into BaseData objects. Each data type creates its own factory method, and returns a new instance of the object - /// each time it is called. The returned object is assumed to be time stamped in the config.ExchangeTimeZone. - /// - /// Subscription data config setup object - /// Line of the source document - /// Date of the requested data - /// true if we're in live mode, false for backtesting mode - /// Instance of the T:BaseData object generated by this line of the CSV - public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode) - { - var symbol = LeanData.ReadSymbolFromZipEntry(config.Symbol, config.Resolution, line); - return new ZipEntryName {Time = date, Symbol = symbol}; - } - - /// - /// Return the URL string source of the file. This will be converted to a stream - /// - /// Configuration object - /// Date of this source file - /// true if we're in live mode, false for backtesting mode - /// String URL of source file. - public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode) - { - if (isLiveMode) - { - return new SubscriptionDataSource(string.Empty, SubscriptionTransportMedium.LocalFile); - } - - var source = LeanData.GenerateZipFilePath(Globals.DataFolder, config.Symbol, date, config.Resolution, config.TickType); - return new SubscriptionDataSource(source, SubscriptionTransportMedium.LocalFile, FileFormat.ZipEntryName); - } - } -} diff --git a/Common/Extensions.cs b/Common/Extensions.cs index 5e176bddb3a9..8335a927d895 100644 --- a/Common/Extensions.cs +++ b/Common/Extensions.cs @@ -4114,7 +4114,7 @@ public static bool ShouldEmitData(this SubscriptionDataConfig config, BaseData d var expectedType = type.IsAssignableTo(config.Type); // Check our config type first to be lazy about using data.GetType() unless required - var configTypeFilter = (config.Type == typeof(TradeBar) || config.Type == typeof(ZipEntryName) || + var configTypeFilter = (config.Type == typeof(TradeBar) || config.Type.IsAssignableTo(typeof(BaseChainUniverseData)) || config.Type == typeof(Tick) && config.TickType == TickType.Trade || config.IsCustomData); if (!configTypeFilter) diff --git a/Common/Util/LeanData.cs b/Common/Util/LeanData.cs index 12daf548965a..186d204281ca 100644 --- a/Common/Util/LeanData.cs +++ b/Common/Util/LeanData.cs @@ -42,7 +42,7 @@ public static class LeanData private static readonly HashSet _strictDailyEndTimesDataTypes = new() { // the underlying could yield auxiliary data which we don't want to change - typeof(TradeBar), typeof(QuoteBar), typeof(ZipEntryName), typeof(BaseDataCollection), typeof(OpenInterest) + typeof(TradeBar), typeof(QuoteBar), typeof(BaseDataCollection), typeof(OpenInterest) }; /// @@ -1030,7 +1030,7 @@ public static TickType GetCommonTickTypeForCommonDataTypes(Type type, SecurityTy { return TickType.OpenInterest; } - if (type == typeof(ZipEntryName)) + if (type.IsAssignableTo(typeof(BaseChainUniverseData))) { return TickType.Quote; } @@ -1516,17 +1516,8 @@ public static bool SetStrictEndTimes(IBaseData baseData, SecurityExchangeHours e return false; } - var isZipEntryName = dataType == typeof(ZipEntryName); - if (isZipEntryName && baseData.Time.Hour == 0) - { - // zip entry names are emitted point in time for a date, see BaseDataSubscriptionEnumeratorFactory. When setting the strict end times - // we will move it to the previous day daily times, because daily market data on disk end time is midnight next day, so here we add 1 day - baseData.Time += Time.OneDay; - baseData.EndTime += Time.OneDay; - } - var dailyCalendar = GetDailyCalendar(baseData.EndTime, exchange, extendedMarketHours: false); - if (!isZipEntryName && dailyCalendar.End < baseData.Time) + if (dailyCalendar.End < baseData.Time) { // this data point we were given is probably from extended market hours which we don't support for daily backtesting data return false; diff --git a/Engine/DataFeeds/Enumerators/DataQueueFuturesChainUniverseDataCollectionEnumerator.cs b/Engine/DataFeeds/Enumerators/DataQueueFuturesChainUniverseDataCollectionEnumerator.cs deleted file mode 100644 index 60aca0cdf44a..000000000000 --- a/Engine/DataFeeds/Enumerators/DataQueueFuturesChainUniverseDataCollectionEnumerator.cs +++ /dev/null @@ -1,135 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ - -using System; -using System.Collections.Generic; -using QuantConnect.Data; -using QuantConnect.Data.UniverseSelection; -using System.Collections; -using System.Linq; -using QuantConnect.Interfaces; -using QuantConnect.Logging; - -namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators -{ - /// - /// Enumerates live futures symbol universe data into instances - /// - public class DataQueueFuturesChainUniverseDataCollectionEnumerator : IEnumerator - { - private readonly SubscriptionRequest _subscriptionRequest; - private readonly IDataQueueUniverseProvider _universeProvider; - private readonly ITimeProvider _timeProvider; - - private bool _needNewCurrent; - private DateTime _lastEmitTime; - - /// - /// Initializes a new instance of the class. - /// - /// The subscription request to be used - /// Symbol universe provider of the data queue - /// The time provider to be used - public DataQueueFuturesChainUniverseDataCollectionEnumerator( - SubscriptionRequest subscriptionRequest, - IDataQueueUniverseProvider universeProvider, - ITimeProvider timeProvider) - { - _subscriptionRequest = subscriptionRequest; - _universeProvider = universeProvider; - _timeProvider = timeProvider; - - _needNewCurrent = true; - } - - /// - /// Returns current futures chain enumerator position - /// - public BaseDataCollection Current { get; private set; } - - /// - /// Returns current futures chain enumerator position - /// - object IEnumerator.Current => Current; - - /// - /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. - /// - public void Dispose() - { - } - - /// - /// Advances the enumerator to the next element of the collection. - /// - /// - /// true if the enumerator was successfully advanced to the next element; false if the enumerator has passed the end of the collection. - /// - public bool MoveNext() - { - if (!_needNewCurrent) - { - // refresh on date change (in exchange time zone) - _needNewCurrent = _timeProvider.GetUtcNow().ConvertFromUtc(_subscriptionRequest.Configuration.ExchangeTimeZone).Date != _lastEmitTime.Date; - } - - if (_needNewCurrent) - { - if (!_universeProvider.CanPerformSelection()) - { - Current = null; - return true; - } - - var localTime = _timeProvider.GetUtcNow() - .ConvertFromUtc(_subscriptionRequest.Configuration.ExchangeTimeZone) - .RoundDown(_subscriptionRequest.Configuration.Increment); - - // loading the list of futures contracts and converting them into zip entries - var symbols = _universeProvider.LookupSymbols(_subscriptionRequest.Security.Symbol, false); - var universeData = symbols.Select(x => new FutureUniverse { Time = localTime, Symbol = x } as BaseData).ToList(); - var current = new BaseDataCollection - { - Symbol = _subscriptionRequest.Security.Symbol, - Data = universeData, - Time = localTime, - EndTime = localTime - }; - - _lastEmitTime = localTime; - - Log.Trace($"DataQueueFuturesChainUniverseDataCollectionEnumerator({current.Symbol}): Emitting data point: {current.EndTime}. Count: {current.Data.Count}"); - - Current = current; - _needNewCurrent = false; - } - else - { - Current = null; - } - - return true; - } - - /// - /// Sets the enumerator to its initial position, which is before the first element in the collection. - /// - public void Reset() - { - _needNewCurrent = true; - } - } -} diff --git a/Engine/DataFeeds/Enumerators/DataQueueOptionChainUniverseDataCollectionEnumerator.cs b/Engine/DataFeeds/Enumerators/DataQueueOptionChainUniverseDataCollectionEnumerator.cs deleted file mode 100644 index 31b940e13858..000000000000 --- a/Engine/DataFeeds/Enumerators/DataQueueOptionChainUniverseDataCollectionEnumerator.cs +++ /dev/null @@ -1,164 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ - -using System; -using System.Collections.Generic; -using System.Linq; -using QuantConnect.Data; -using QuantConnect.Data.UniverseSelection; -using QuantConnect.Interfaces; -using QuantConnect.Data.Auxiliary; -using System.Collections; -using QuantConnect.Logging; - -namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators -{ - /// - /// Enumerates live options symbol universe data into instances - /// - public class DataQueueOptionChainUniverseDataCollectionEnumerator : IEnumerator - { - private readonly SubscriptionRequest _subscriptionRequest; - private readonly IDataQueueUniverseProvider _universeProvider; - private readonly ITimeProvider _timeProvider; - private bool _needNewCurrent; - private DateTime _lastEmitTime; - private BaseDataCollection _currentData; - - /// - /// Gets the enumerator for the underlying asset - /// - public IEnumerator Underlying { get; } - - /// - /// Initializes a new instance of the class. - /// - /// The subscription request to be used - /// Underlying enumerator - /// Symbol universe provider of the data queue - /// The time provider to be used - public DataQueueOptionChainUniverseDataCollectionEnumerator( - SubscriptionRequest subscriptionRequest, - IEnumerator underlying, - IDataQueueUniverseProvider universeProvider, - ITimeProvider timeProvider) - { - _subscriptionRequest = subscriptionRequest; - Underlying = underlying; - _universeProvider = universeProvider; - _timeProvider = timeProvider; - - _needNewCurrent = true; - } - - /// - /// Returns current option chain enumerator position - /// - public BaseDataCollection Current { get; private set; } - - /// - /// Returns current option chain enumerator position - /// - object IEnumerator.Current => Current; - - /// - /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. - /// - public void Dispose() - { - Underlying.Dispose(); - } - - /// - /// Advances the enumerator to the next element of the collection. - /// - /// - /// true if the enumerator was successfully advanced to the next element; false if the enumerator has passed the end of the collection. - /// - public bool MoveNext() - { - Underlying.MoveNext(); - - if (Underlying.Current == null) - { - Current = null; - return true; - } - - if (!_needNewCurrent) - { - // refresh on date change (in exchange time zone) - _needNewCurrent = _timeProvider.GetUtcNow().ConvertFromUtc(_subscriptionRequest.Configuration.ExchangeTimeZone).Date != _lastEmitTime.Date; - } - - if (_needNewCurrent) - { - if (!_universeProvider.CanPerformSelection()) - { - Current = null; - return true; - } - - var localTime = _timeProvider.GetUtcNow() - .ConvertFromUtc(_subscriptionRequest.Configuration.ExchangeTimeZone) - .RoundDown(_subscriptionRequest.Configuration.Increment); - - // loading the list of futures contracts and converting them into zip entries - var symbols = _universeProvider.LookupSymbols(_subscriptionRequest.Security.Symbol, false); - var zipEntries = symbols.Select(x => new ZipEntryName { Time = localTime, Symbol = x } as BaseData).ToList(); - _currentData = new BaseDataCollection - { - Symbol = _subscriptionRequest.Security.Symbol, - Underlying = Underlying.Current, - Data = zipEntries, - Time = localTime, - EndTime = localTime - }; - - Log.Trace($"DataQueueOptionChainUniverseDataCollectionEnumerator({_currentData.Symbol}): Emitting data point: {_currentData.EndTime}. " + - $"Count: {_currentData.Data.Count}. Underlying: {_currentData.Underlying} Underlying.EndTime: {_currentData.Underlying.EndTime}"); - - _lastEmitTime = localTime; - - Current = _currentData; - _needNewCurrent = false; - } - else - { - if (Current == null) - { - Current = _currentData; - } - - Current = (BaseDataCollection)Current.Clone(fillForward: true); - Current.Underlying = Underlying.Current; - Current.Time = Underlying.Current.EndTime; - Current.EndTime = Underlying.Current.EndTime; - } - - return true; - } - - /// - /// Sets the enumerator to its initial position, which is before the first element in the collection. - /// - public void Reset() - { - Underlying.Reset(); - _needNewCurrent = true; - } - } -} diff --git a/Engine/DataFeeds/Enumerators/Factories/BaseDataSubscriptionEnumeratorFactory.cs b/Engine/DataFeeds/Enumerators/Factories/BaseDataSubscriptionEnumeratorFactory.cs deleted file mode 100644 index 02a0223cdab1..000000000000 --- a/Engine/DataFeeds/Enumerators/Factories/BaseDataSubscriptionEnumeratorFactory.cs +++ /dev/null @@ -1,87 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ - -using System; -using QuantConnect.Data; -using QuantConnect.Interfaces; -using System.Collections.Generic; -using QuantConnect.Data.Auxiliary; -using QuantConnect.Data.UniverseSelection; - -namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories -{ - /// - /// Provides a default implementation of that uses - /// factory methods for reading sources - /// - public class BaseDataSubscriptionEnumeratorFactory : ISubscriptionEnumeratorFactory - { - private readonly IOptionChainProvider _optionChainProvider; - private readonly IFutureChainProvider _futureChainProvider; - private readonly Func> _tradableDaysProvider; - - /// - /// Initializes a new instance of the class - /// - /// The option chain provider - /// The future chain provider - public BaseDataSubscriptionEnumeratorFactory(IOptionChainProvider optionChainProvider, IFutureChainProvider futureChainProvider) - { - _futureChainProvider = futureChainProvider; - _optionChainProvider = optionChainProvider; - _tradableDaysProvider = (request => request.TradableDaysInDataTimeZone); - } - - /// - /// Creates an enumerator to read the specified request - /// - /// The subscription request to be read - /// Provider used to get data when it is not present on disk - /// An enumerator reading the subscription request - public IEnumerator CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider) - { - // We decide to use the ZipDataCacheProvider instead of the SingleEntryDataCacheProvider here - // for resiliency and as a fix for an issue preventing us from reading non-equity options data. - // It has the added benefit of caching any zip files that we request from the filesystem, and reading - // files contained within the zip file, which the SingleEntryDataCacheProvider does not support. - var sourceFactory = request.Configuration.GetBaseDataInstance(); - foreach (var date in _tradableDaysProvider(request)) - { - IEnumerable symbols; - if (request.Configuration.SecurityType.IsOption()) - { - symbols = _optionChainProvider.GetOptionContractList(request.Configuration.Symbol, date); - } - else if (request.Configuration.SecurityType == SecurityType.Future) - { - symbols = _futureChainProvider.GetFutureContractList(request.Configuration.Symbol, date); - } - else - { - throw new NotImplementedException($"{request.Configuration.SecurityType} is not supported"); - } - - // we are going to use these symbols to create a collection that for options will also have the underlying that will be emitted in exchange time zone - // note the merging of the data will happen based on their end time so time zones are important to respect - var exchangeTimeZoneDate = date.ConvertTo(request.Configuration.DataTimeZone, request.ExchangeHours.TimeZone); - foreach (var symbol in symbols) - { - yield return new ZipEntryName { Symbol = symbol, Time = exchangeTimeZoneDate }; - } - } - } - } -} diff --git a/Engine/DataFeeds/Enumerators/Factories/OptionChainUniverseSubscriptionEnumeratorFactory.cs b/Engine/DataFeeds/Enumerators/Factories/OptionChainUniverseSubscriptionEnumeratorFactory.cs deleted file mode 100644 index 9b21dddadf37..000000000000 --- a/Engine/DataFeeds/Enumerators/Factories/OptionChainUniverseSubscriptionEnumeratorFactory.cs +++ /dev/null @@ -1,100 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ - -using System; -using System.Collections.Generic; -using System.Linq; -using QuantConnect.Data; -using QuantConnect.Data.Market; -using QuantConnect.Data.UniverseSelection; -using QuantConnect.Interfaces; - -namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories -{ - /// - /// Provides an implementation of for the - /// - public class OptionChainUniverseSubscriptionEnumeratorFactory : ISubscriptionEnumeratorFactory - { - private readonly Func> _enumeratorConfigurator; - private readonly bool _isLiveMode; - - private readonly IDataQueueUniverseProvider _symbolUniverse; - private readonly ITimeProvider _timeProvider; - - /// - /// Initializes a new instance of the class - /// - /// Function used to configure the sub-enumerators before sync (fill-forward/filter/ect...) - /// Symbol universe provider of the data queue - /// The time provider instance used to determine when bars are completed and can be emitted - public OptionChainUniverseSubscriptionEnumeratorFactory(Func> enumeratorConfigurator, - IDataQueueUniverseProvider symbolUniverse, ITimeProvider timeProvider) - { - _isLiveMode = true; - _symbolUniverse = symbolUniverse; - _timeProvider = timeProvider; - _enumeratorConfigurator = enumeratorConfigurator; - } - - /// - /// Creates an enumerator to read the specified request - /// - /// The subscription request to be read - /// Provider used to get data when it is not present on disk - /// An enumerator reading the subscription request - public IEnumerator CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider) - { - if (_isLiveMode) - { - // configuring the enumerator - var subscriptionConfiguration = GetSubscriptionConfigurations(request).First(); - var subscriptionRequest = new SubscriptionRequest(request, configuration: subscriptionConfiguration); - var configuredEnumerator = _enumeratorConfigurator(subscriptionRequest); - - return new DataQueueOptionChainUniverseDataCollectionEnumerator(subscriptionRequest, configuredEnumerator, _symbolUniverse, _timeProvider); - } - else - { - throw new InvalidOperationException($"Backtesting is expected to be using {nameof(BaseDataSubscriptionEnumeratorFactory)}"); - } - } - - private IEnumerable GetSubscriptionConfigurations(SubscriptionRequest request) - { - // canonical also needs underlying price data - var config = request.Configuration; - var underlying = config.Symbol.Underlying; - - // Making sure data is non-tick - var resolution = config.Resolution == Resolution.Tick ? Resolution.Second : config.Resolution; - - var configurations = new List - { - // add underlying trade data - new SubscriptionDataConfig(config, resolution: resolution, fillForward: true, symbol: underlying, objectType: typeof (TradeBar), tickType: TickType.Trade), - }; - - if (!_isLiveMode) - { - // rewrite the primary to be fill forward - configurations.Add(new SubscriptionDataConfig(config, resolution: resolution, fillForward: true)); - } - - return configurations; - } - } -} diff --git a/Engine/DataFeeds/FileSystemDataFeed.cs b/Engine/DataFeeds/FileSystemDataFeed.cs index 5cbe1ac1781d..d107fa3bd1f9 100644 --- a/Engine/DataFeeds/FileSystemDataFeed.cs +++ b/Engine/DataFeeds/FileSystemDataFeed.cs @@ -201,24 +201,6 @@ protected IEnumerator CreateUniverseEnumerator(SubscriptionRequest req { factory = new BaseDataCollectionSubscriptionEnumeratorFactory(_algorithm.ObjectStore); } - else if (request.Configuration.Type == typeof(ZipEntryName)) - { - // TODO: subscription should already come in correctly built - var resolution = request.Configuration.Resolution == Resolution.Tick ? Resolution.Second : request.Configuration.Resolution; - - // TODO: subscription should already come in as fill forward true - request = new SubscriptionRequest(request, configuration: new SubscriptionDataConfig(request.Configuration, fillForward: true, resolution: resolution)); - - var result = new BaseDataSubscriptionEnumeratorFactory(_algorithm.OptionChainProvider, _algorithm.FutureChainProvider) - .CreateEnumerator(request, _dataProvider); - - if (LeanData.UseDailyStrictEndTimes(_algorithm.Settings, request, request.Configuration.Symbol, request.Configuration.Increment)) - { - result = new StrictDailyEndTimesEnumerator(result, request.ExchangeHours, request.StartTimeLocal); - } - result = ConfigureEnumerator(request, true, result, fillForwardResolution); - return TryAppendUnderlyingEnumerator(request, result, createUnderlyingEnumerator, fillForwardResolution); - } // define our data enumerator var enumerator = factory.CreateEnumerator(request, _dataProvider); diff --git a/Engine/DataFeeds/LiveTradingDataFeed.cs b/Engine/DataFeeds/LiveTradingDataFeed.cs index 1d3c86076604..8383e93d6eb1 100644 --- a/Engine/DataFeeds/LiveTradingDataFeed.cs +++ b/Engine/DataFeeds/LiveTradingDataFeed.cs @@ -369,40 +369,6 @@ request.Universe is OptionChainUniverse || // advance time if before 23pm or after 5am and not on Saturdays time => time.Hour < 23 && time.Hour > 5 && time.DayOfWeek != DayOfWeek.Saturday); } - else if (request.Universe is OptionChainUniverse) - { - Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating option chain universe: " + config.Symbol.ID); - - Func> configure = (subRequest) => - { - // Pass the security exchange hours explicitly to avoid using the ones in the request, since - // those could be different. e.g. when requests are created for open interest data the exchange - // hours are set to always open to avoid OI data being filtered out due to the exchange being closed. - var useDailyStrictEndTimes = LeanData.UseDailyStrictEndTimes(_algorithm.Settings, request, request.Configuration.Symbol, - request.Configuration.Increment, request.Security.Exchange.Hours); - var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(subRequest.Configuration); - var input = Subscribe(subRequest.Configuration, (sender, args) => subscription?.OnNewDataAvailable(), (_) => false); - return new LiveFillForwardEnumerator(_frontierTimeProvider, input, subRequest.Security.Exchange, fillForwardResolution, - subRequest.Configuration.ExtendedMarketHours, localEndTime, subRequest.Configuration.Resolution, - subRequest.Configuration.DataTimeZone, useDailyStrictEndTimes, request.Configuration.Type); - }; - - var symbolUniverse = GetUniverseProvider(request.Configuration.SecurityType); - - var enumeratorFactory = new OptionChainUniverseSubscriptionEnumeratorFactory(configure, symbolUniverse, _timeProvider); - enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider); - - enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider); - } - else if (request.Universe is FuturesChainUniverse) - { - Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating futures chain universe: " + config.Symbol.ID); - - var symbolUniverse = GetUniverseProvider(SecurityType.Future); - - enumerator = new DataQueueFuturesChainUniverseDataCollectionEnumerator(request, symbolUniverse, _timeProvider); - enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider); - } else { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ID); diff --git a/Engine/HistoricalData/FakeHistoryProvider.cs b/Engine/HistoricalData/FakeHistoryProvider.cs index 8717a5b4d543..d772276bb531 100644 --- a/Engine/HistoricalData/FakeHistoryProvider.cs +++ b/Engine/HistoricalData/FakeHistoryProvider.cs @@ -91,34 +91,6 @@ public override IEnumerable GetHistory(IEnumerable reques Period = single.Resolution.ToTimeSpan() }; } - else if (single.DataType == typeof(ZipEntryName)) - { - if (single.Symbol.SecurityType == SecurityType.Future) - { - data = new ZipEntryName - { - Symbol = Symbol.CreateFuture(single.Symbol.ID.Symbol, single.Symbol.ID.Market, currentLocalTime.AddDays(20)), - Time = currentLocalTime - }; - } - else if (single.Symbol.SecurityType.IsOption()) - { - data = new ZipEntryName - { - Symbol = Symbol.CreateOption(single.Symbol.Underlying.ID.Symbol, - single.Symbol.ID.Market, - single.Symbol.Underlying.SecurityType.DefaultOptionStyle(), - default(OptionRight), - 0m, - currentLocalTime.AddDays(20)), - Time = currentLocalTime - }; - } - else - { - yield break; - } - } else { yield break; diff --git a/Tests/Common/Securities/SecurityServiceTests.cs b/Tests/Common/Securities/SecurityServiceTests.cs index 484dcb06669d..8b17a27f4452 100644 --- a/Tests/Common/Securities/SecurityServiceTests.cs +++ b/Tests/Common/Securities/SecurityServiceTests.cs @@ -22,6 +22,7 @@ using QuantConnect.Data; using QuantConnect.Data.Auxiliary; using QuantConnect.Data.Market; +using QuantConnect.Data.UniverseSelection; using QuantConnect.Interfaces; using QuantConnect.Securities; using QuantConnect.Tests.Engine.DataFeeds; @@ -66,11 +67,11 @@ public void CanCreate_CanonicalOption_WithCorrectSubscriptions() { var optionSymbol = Symbol.Create("GOOG", SecurityType.Option, Market.USA); - var configs = _subscriptionManager.SubscriptionDataConfigService.Add(typeof(ZipEntryName), optionSymbol, Resolution.Minute, false, false, false); + var configs = _subscriptionManager.SubscriptionDataConfigService.Add(typeof(OptionUniverse), optionSymbol, Resolution.Minute, false, false, false); var option = _securityService.CreateSecurity(optionSymbol, configs, 1.0m, false); Assert.AreEqual(option.Subscriptions.Count(), 1); - Assert.AreEqual(option.Subscriptions.First().Type, typeof(ZipEntryName)); + Assert.AreEqual(option.Subscriptions.First().Type, typeof(OptionUniverse)); Assert.AreEqual(option.Subscriptions.First().TickType, TickType.Quote); } @@ -191,19 +192,19 @@ public void CannotCreateSecurityWhenBaseCurrencyNotFound(string ticker, Security var configs = _subscriptionManager.SubscriptionDataConfigService.Add(symbol, Resolution.Second, false, false, false, false, false, subscriptionTypes); Assert.Throws(() => _securityService.CreateSecurity(symbol, configs, 1.0m, false)); } - + [Test] public void CreatesEquityOptionWithContractMultiplierEqualsToContractUnitOfTrade() { var underlying = Symbol.Create("TWX", SecurityType.Equity, Market.USA); var equityOption = Symbol.CreateOption( - underlying, - Market.USA, - OptionStyle.American, - OptionRight.Call, + underlying, + Market.USA, + OptionStyle.American, + OptionRight.Call, 320m, new DateTime(2020, 12, 18)); - + var subscriptionTypes = new List> { new Tuple(typeof(TradeBar), TickType.Trade), @@ -213,7 +214,7 @@ public void CreatesEquityOptionWithContractMultiplierEqualsToContractUnitOfTrade var configs = _subscriptionManager.SubscriptionDataConfigService.Add(equityOption, Resolution.Minute, true, false, false, false, false, subscriptionTypes); var equityOptionSecurity = (QuantConnect.Securities.Option.Option)_securityService.CreateSecurity(equityOption, configs, 1.0m); - + Assert.AreEqual(100, equityOptionSecurity.ContractMultiplier); Assert.AreEqual(100,equityOptionSecurity.ContractUnitOfTrade); } @@ -227,13 +228,13 @@ public void CreatesFutureOptionWithContractMultiplierEqualsToFutureContractMulti new DateTime(2020, 12, 18)); var futureOption = Symbol.CreateOption( - underlying, - Market.CME, - OptionStyle.American, - OptionRight.Call, + underlying, + Market.CME, + OptionStyle.American, + OptionRight.Call, 3250m, new DateTime(2020, 12, 18)); - + var subscriptionTypes = new List> { new Tuple(typeof(TradeBar), TickType.Trade), @@ -243,7 +244,7 @@ public void CreatesFutureOptionWithContractMultiplierEqualsToFutureContractMulti var configs = _subscriptionManager.SubscriptionDataConfigService.Add(futureOption, Resolution.Minute, true, false, false, false, false, subscriptionTypes); var futureOptionSecurity = (QuantConnect.Securities.Option.Option)_securityService.CreateSecurity(futureOption, configs, 1.0m); - + Assert.AreEqual(50, futureOptionSecurity.ContractMultiplier); Assert.AreEqual(1, futureOptionSecurity.ContractUnitOfTrade); } @@ -262,11 +263,11 @@ public void AddPrimaryExchangeToSecurityObject() SymbolPropertiesDatabase.FromDataFolder(), algorithm, new RegisteredSecurityDataTypesProvider(), - new SecurityCacheProvider(algorithm.Portfolio), + new SecurityCacheProvider(algorithm.Portfolio), mockedPrimaryExchangeProvider.Object); var configs = _subscriptionManager.SubscriptionDataConfigService.Add(typeof(TradeBar), equitySymbol, Resolution.Second, false, false, false); - + // Act var equity = securityService.CreateSecurity(equitySymbol, configs, 1.0m, false); diff --git a/Tests/Engine/DataFeeds/Enumerators/DataQueueFuturesChainUniverseDataCollectionEnumeratorTests.cs b/Tests/Engine/DataFeeds/Enumerators/DataQueueFuturesChainUniverseDataCollectionEnumeratorTests.cs deleted file mode 100644 index 01aa0f9d0610..000000000000 --- a/Tests/Engine/DataFeeds/Enumerators/DataQueueFuturesChainUniverseDataCollectionEnumeratorTests.cs +++ /dev/null @@ -1,153 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ - -using System; -using NUnit.Framework; -using QuantConnect.Data; -using QuantConnect.Interfaces; -using QuantConnect.Securities; -using System.Collections.Generic; -using QuantConnect.Data.Auxiliary; -using QuantConnect.Lean.Engine.DataFeeds; -using QuantConnect.Data.UniverseSelection; -using QuantConnect.Lean.Engine.DataFeeds.Enumerators; - -namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators -{ - [TestFixture, Parallelizable(ParallelScope.All)] - public class DataQueueFuturesChainUniverseDataCollectionEnumeratorTests - { - [TestCase(Resolution.Tick, "20181017 05:00", 5)] - [TestCase(Resolution.Second, "20181017 05:00", 5)] - [TestCase(Resolution.Minute, "20181017 05:00", 5)] - [TestCase(Resolution.Hour, "20181017 05:00", 5)] - [TestCase(Resolution.Daily, "20181017 05:00", 5)] - - [TestCase(Resolution.Tick, "20181017 10:00", 10)] - [TestCase(Resolution.Second, "20181017 10:00", 10)] - [TestCase(Resolution.Minute, "20181017 10:00", 10)] - [TestCase(Resolution.Hour, "20181017 10:00", 10)] - [TestCase(Resolution.Daily, "20181017 10:00", 10)] - public void RefreshesFutureChainUniverseOnDateChange(Resolution resolution, string dateTime, int expectedStartHour) - { - var startTime = Time.ParseDate(dateTime); - Assert.AreEqual(expectedStartHour, startTime.Hour); - var timeProvider = new ManualTimeProvider(startTime); - - var symbolUniverse = new TestDataQueueUniverseProvider(timeProvider); - - var canonicalSymbol = Symbol.Create(Futures.Indices.VIX, SecurityType.Future, Market.CFE, "/VX"); - - var request = GetRequest(canonicalSymbol, startTime, resolution); - var enumerator = new DataQueueFuturesChainUniverseDataCollectionEnumerator(request, symbolUniverse, timeProvider); - - Assert.IsTrue(enumerator.MoveNext()); - Assert.IsNotNull(enumerator.Current); - Assert.AreEqual(1, symbolUniverse.TotalLookupCalls); - var data = enumerator.Current; - Assert.IsNotNull(data); - Assert.AreEqual(1, data.Data.Count); - - timeProvider.Advance(Time.OneSecond); - - Assert.IsTrue(enumerator.MoveNext()); - Assert.IsNull(enumerator.Current); - Assert.AreEqual(1, symbolUniverse.TotalLookupCalls); - - timeProvider.Advance(Time.OneMinute); - - Assert.IsTrue(enumerator.MoveNext()); - Assert.IsNull(enumerator.Current); - Assert.AreEqual(1, symbolUniverse.TotalLookupCalls); - - timeProvider.Advance(Time.OneDay); - - Assert.IsTrue(enumerator.MoveNext()); - Assert.IsNotNull(enumerator.Current); - Assert.AreEqual(2, symbolUniverse.TotalLookupCalls); - data = enumerator.Current; - Assert.IsNotNull(data); - Assert.AreEqual(2, data.Data.Count); - - timeProvider.Advance(Time.OneMinute); - - Assert.IsTrue(enumerator.MoveNext()); - Assert.IsNull(enumerator.Current); - Assert.AreEqual(2, symbolUniverse.TotalLookupCalls); - - enumerator.Dispose(); - request.Universe.Dispose(); - } - - private static SubscriptionRequest GetRequest(Symbol canonicalSymbol, DateTime startTime, Resolution resolution) - { - var entry = MarketHoursDatabase.FromDataFolder().GetEntry(canonicalSymbol.ID.Market, canonicalSymbol, canonicalSymbol.SecurityType); - var config = new SubscriptionDataConfig( - typeof(ZipEntryName), - canonicalSymbol, - resolution, - entry.DataTimeZone, - entry.ExchangeHours.TimeZone, - true, - false, - false, - false, - TickType.Quote, - false, - DataNormalizationMode.Raw - ); - - var algo = new AlgorithmStub(); - var future = algo.AddFuture(canonicalSymbol.Value); - - var universeSettings = new UniverseSettings(resolution, 0, true, false, TimeSpan.Zero); -#pragma warning disable CA2000 - var universe = new FuturesChainUniverse(future, universeSettings); -#pragma warning restore CA2000 - return new SubscriptionRequest(true, universe, future, config, startTime, Time.EndOfTime); - } - - private class TestDataQueueUniverseProvider : IDataQueueUniverseProvider - { - private readonly Symbol[] _symbolList1 = - { - Symbol.CreateFuture(Futures.Indices.VIX, Market.CFE, new DateTime(2018, 10, 31)) - }; - private readonly Symbol[] _symbolList2 = - { - Symbol.CreateFuture(Futures.Indices.VIX, Market.CFE, new DateTime(2018, 10, 31)), - Symbol.CreateFuture(Futures.Indices.VIX, Market.CFE, new DateTime(2018, 11, 30)), - }; - - private readonly ITimeProvider _timeProvider; - - public int TotalLookupCalls { get; set; } - - public TestDataQueueUniverseProvider(ITimeProvider timeProvider) - { - _timeProvider = timeProvider; - } - - public IEnumerable LookupSymbols(Symbol symbol, bool includeExpired, string securityCurrency = null) - { - TotalLookupCalls++; - return _timeProvider.GetUtcNow().Date.Day >= 18 ? _symbolList2 : _symbolList1; - } - - public bool CanPerformSelection() => true; - } - } -} diff --git a/Tests/Engine/DataFeeds/Enumerators/DataQueueOptionChainUniverseDataCollectionEnumeratorTests.cs b/Tests/Engine/DataFeeds/Enumerators/DataQueueOptionChainUniverseDataCollectionEnumeratorTests.cs deleted file mode 100644 index 0402a7ec8a25..000000000000 --- a/Tests/Engine/DataFeeds/Enumerators/DataQueueOptionChainUniverseDataCollectionEnumeratorTests.cs +++ /dev/null @@ -1,198 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ - -using System; -using System.Linq; -using NUnit.Framework; -using QuantConnect.Data; -using QuantConnect.Interfaces; -using QuantConnect.Securities; -using QuantConnect.Data.Market; -using System.Collections.Generic; -using QuantConnect.Data.Auxiliary; -using QuantConnect.Securities.Option; -using QuantConnect.Lean.Engine.DataFeeds; -using QuantConnect.Data.UniverseSelection; -using QuantConnect.Lean.Engine.DataFeeds.Enumerators; - -namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators -{ - [TestFixture, Parallelizable(ParallelScope.All)] - public class DataQueueOptionChainUniverseDataCollectionEnumeratorTests - { - [TestCase(Resolution.Tick)] - [TestCase(Resolution.Second)] - [TestCase(Resolution.Minute)] - [TestCase(Resolution.Hour)] - [TestCase(Resolution.Daily)] - public void NullUnderlying(Resolution resolution) - { - var startTime = new DateTime(2018, 10, 17, 5, 0, 0); - var timeProvider = new ManualTimeProvider(startTime); - - var symbolUniverse = new TestDataQueueUniverseProvider(timeProvider); - - var canonicalSymbol = Symbols.SPY_Option_Chain; - var request = GetRequest(canonicalSymbol, startTime, resolution); - using var underlying = new EnqueueableEnumerator(); - using var enumerator = new DataQueueOptionChainUniverseDataCollectionEnumerator(request, underlying, symbolUniverse, timeProvider); - - Assert.IsTrue(enumerator.MoveNext()); - Assert.IsNull(enumerator.Current); - - request.Universe.Dispose(); - } - - [TestCase(Resolution.Tick, "20181017 05:00", 5)] - [TestCase(Resolution.Second, "20181017 05:00", 5)] - [TestCase(Resolution.Minute, "20181017 05:00", 5)] - [TestCase(Resolution.Hour, "20181017 05:00", 5)] - [TestCase(Resolution.Daily, "20181017 05:00", 5)] - - [TestCase(Resolution.Tick, "20181017 10:00", 10)] - [TestCase(Resolution.Second, "20181017 10:00", 10)] - [TestCase(Resolution.Minute, "20181017 10:00", 10)] - [TestCase(Resolution.Hour, "20181017 10:00", 10)] - [TestCase(Resolution.Daily, "20181017 10:00", 10)] - public void RefreshesUniverseChainOnDateChange(Resolution resolution, string dateTime, int expectedStartHour) - { - var startTime = Time.ParseDate(dateTime); - Assert.AreEqual(expectedStartHour, startTime.Hour); - var timeProvider = new ManualTimeProvider(startTime); - - var symbolUniverse = new TestDataQueueUniverseProvider(timeProvider); - - var canonicalSymbol = Symbols.SPY_Option_Chain; - var request = GetRequest(canonicalSymbol, startTime, resolution); - var underlyingSymbol = (request.Security as Option).Underlying.Symbol; - using var underlying = new EnqueueableEnumerator(); - using var enumerator = new DataQueueOptionChainUniverseDataCollectionEnumerator(request, underlying, symbolUniverse, timeProvider); - - underlying.Enqueue(new Tick(timeProvider.GetUtcNow().ConvertFromUtc(request.Security.Exchange.TimeZone), underlyingSymbol, 9, 10)); - Assert.IsTrue(enumerator.MoveNext()); - Assert.IsNotNull(enumerator.Current); - Assert.AreEqual(1, symbolUniverse.TotalLookupCalls); - var data = enumerator.Current; - Assert.IsNotNull(data); - Assert.AreEqual(1, data.Data.Count); - Assert.IsFalse(enumerator.Current.IsFillForward); - - timeProvider.Advance(Time.OneSecond); - underlying.Enqueue(new Tick(timeProvider.GetUtcNow().ConvertFromUtc(request.Security.Exchange.TimeZone), underlyingSymbol, 9, 10)); - - Assert.IsTrue(enumerator.MoveNext()); - Assert.IsNotNull(enumerator.Current); - Assert.AreEqual(1, symbolUniverse.TotalLookupCalls); - // fill forwarding the chain selection, but time should advance - Assert.AreEqual(data.Data.Single().Symbol, enumerator.Current.Data.Single().Symbol); - Assert.IsTrue(enumerator.Current.EndTime > data.EndTime); - Assert.IsTrue(enumerator.Current.IsFillForward); - data = enumerator.Current; - - timeProvider.Advance(Time.OneMinute); - underlying.Enqueue(new Tick(timeProvider.GetUtcNow().ConvertFromUtc(request.Security.Exchange.TimeZone), underlyingSymbol, 9, 10)); - - Assert.IsTrue(enumerator.MoveNext()); - Assert.IsNotNull(enumerator.Current); - Assert.AreEqual(1, symbolUniverse.TotalLookupCalls); - // fill forwarding the chain selection, but time should advance - Assert.AreEqual(data.Data.Single().Symbol, enumerator.Current.Data.Single().Symbol); - Assert.IsTrue(enumerator.Current.EndTime > data.EndTime); - Assert.IsTrue(enumerator.Current.IsFillForward); - data = enumerator.Current; - - timeProvider.Advance(Time.OneDay); - underlying.Enqueue(new Tick(timeProvider.GetUtcNow().ConvertFromUtc(request.Security.Exchange.TimeZone), underlyingSymbol, 9, 10)); - - Assert.IsTrue(enumerator.MoveNext()); - Assert.IsNotNull(enumerator.Current); - // selection should of have changed - Assert.AreNotEqual(data.Data.Single().Symbol, enumerator.Current.Data.Single().Symbol); - Assert.IsTrue(enumerator.Current.EndTime > data.EndTime); - Assert.IsFalse(enumerator.Current.IsFillForward); - data = enumerator.Current; - Assert.AreEqual(2, symbolUniverse.TotalLookupCalls); - data = enumerator.Current; - Assert.IsNotNull(data); - Assert.AreEqual(1, data.Data.Count); - - timeProvider.Advance(Time.OneMinute); - underlying.Enqueue(new Tick(timeProvider.GetUtcNow().ConvertFromUtc(request.Security.Exchange.TimeZone), underlyingSymbol, 9, 10)); - - Assert.IsTrue(enumerator.MoveNext()); - Assert.AreEqual(2, symbolUniverse.TotalLookupCalls); - // fill forwarding the chain selection, but time should advance - Assert.AreEqual(data.Data.Single().Symbol, enumerator.Current.Data.Single().Symbol); - Assert.IsTrue(enumerator.Current.EndTime > data.EndTime); - Assert.IsTrue(enumerator.Current.IsFillForward); - - enumerator.Dispose(); - request.Universe.Dispose(); - } - - private static SubscriptionRequest GetRequest(Symbol canonicalSymbol, DateTime startTime, Resolution resolution) - { - var entry = MarketHoursDatabase.FromDataFolder().GetEntry(canonicalSymbol.ID.Market, canonicalSymbol, canonicalSymbol.SecurityType); - var config = new SubscriptionDataConfig( - typeof(ZipEntryName), - canonicalSymbol, - resolution, - entry.DataTimeZone, - entry.ExchangeHours.TimeZone, - true, - false, - false, - false, - TickType.Quote, - false, - DataNormalizationMode.Raw - ); - - var algo = new AlgorithmStub(); - var underlying = algo.AddEquity("SPY"); - var option = algo.AddOption(underlying.Symbol); - option.Underlying = underlying; - - var universeSettings = new UniverseSettings(resolution, 0, true, false, TimeSpan.Zero); -# pragma warning disable CA2000 - var universe = new OptionChainUniverse(option, universeSettings); -#pragma warning restore CA2000 - return new SubscriptionRequest(true, universe, option, config, startTime, Time.EndOfTime); - } - - private class TestDataQueueUniverseProvider : IDataQueueUniverseProvider - { - private readonly Symbol[] _symbolList1 = { Symbols.SPY_C_192_Feb19_2016 }; - private readonly Symbol[] _symbolList2 = { Symbols.SPY_P_192_Feb19_2016 }; - private readonly ITimeProvider _timeProvider; - - public int TotalLookupCalls { get; set; } - - public TestDataQueueUniverseProvider(ITimeProvider timeProvider) - { - _timeProvider = timeProvider; - } - - public IEnumerable LookupSymbols(Symbol symbol, bool includeExpired, string securityCurrency = null) - { - TotalLookupCalls++; - return _timeProvider.GetUtcNow().Date.Day >= 18 ? _symbolList2 : _symbolList1; - } - - public bool CanPerformSelection() => true; - } - } -} diff --git a/Tests/Engine/DataFeeds/Enumerators/Factories/BaseDataSubscriptionEnumeratorFactoryTests.cs b/Tests/Engine/DataFeeds/Enumerators/Factories/BaseDataSubscriptionEnumeratorFactoryTests.cs deleted file mode 100644 index 95b22aa94a29..000000000000 --- a/Tests/Engine/DataFeeds/Enumerators/Factories/BaseDataSubscriptionEnumeratorFactoryTests.cs +++ /dev/null @@ -1,76 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ - -using System; -using NUnit.Framework; -using QuantConnect.Data; -using QuantConnect.Data.Market; -using QuantConnect.Data.UniverseSelection; -using QuantConnect.Lean.Engine.DataFeeds; -using QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories; -using QuantConnect.Logging; -using QuantConnect.Securities; - -namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators.Factories -{ - [TestFixture] - public class BaseDataSubscriptionEnumeratorFactoryTests - { - // This test reports higher memory usage when ran with Travis, so we exclude it for now - [Test, Category("TravisExclude")] - public void DoesNotLeakMemory() - { - var symbol = Symbols.SPY_Option_Chain; - var config = new SubscriptionDataConfig(typeof(TradeBar), symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false, false, TickType.Trade, false); - var security = new Security( - SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork), - config, - new Cash(Currencies.USD, 0, 1), - SymbolProperties.GetDefault(Currencies.USD), - ErrorCurrencyConverter.Instance, - RegisteredSecurityDataTypesProvider.Null, - new SecurityCache() - ); - - var fileProvider = TestGlobals.DataProvider; - var factory = new BaseDataSubscriptionEnumeratorFactory(new BacktestingOptionChainProvider(TestGlobals.MapFileProvider), new BacktestingFutureChainProvider()); - - GC.Collect(); - var ramUsageBeforeLoop = OS.TotalPhysicalMemoryUsed; - - var date = new DateTime(1998, 1, 1); - - const int iterations = 1000; - for (var i = 0; i < iterations; i++) - { - var request = new SubscriptionRequest(false, null, security, config, date, date); - using (var enumerator = factory.CreateEnumerator(request, fileProvider)) - { - enumerator.MoveNext(); - } - date = date.AddDays(1); - } - - GC.Collect(); - var ramUsageAfterLoop = OS.TotalPhysicalMemoryUsed; - - Log.Trace($"RAM usage - before: {ramUsageBeforeLoop} MB, after: {ramUsageAfterLoop} MB"); - - Assert.IsTrue(ramUsageAfterLoop - ramUsageBeforeLoop < 10); - } - - } -} diff --git a/Tests/Engine/DataFeeds/Enumerators/Factories/OptionChainUniverseSubscriptionEnumeratorFactoryTests.cs b/Tests/Engine/DataFeeds/Enumerators/Factories/OptionChainUniverseSubscriptionEnumeratorFactoryTests.cs deleted file mode 100644 index c994e55dd676..000000000000 --- a/Tests/Engine/DataFeeds/Enumerators/Factories/OptionChainUniverseSubscriptionEnumeratorFactoryTests.cs +++ /dev/null @@ -1,202 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ - -using System; -using System.Collections.Generic; -using NUnit.Framework; -using QuantConnect.Data; -using QuantConnect.Data.Auxiliary; -using QuantConnect.Data.Market; -using QuantConnect.Data.UniverseSelection; -using QuantConnect.Interfaces; -using QuantConnect.Lean.Engine.DataFeeds; -using QuantConnect.Lean.Engine.DataFeeds.Enumerators; -using QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories; -using QuantConnect.Securities; -using QuantConnect.Securities.Option; -using QuantConnect.Util; - -namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators.Factories -{ - [TestFixture] - public class OptionChainUniverseSubscriptionEnumeratorFactoryTests - { - [Test] - public void RefreshesOptionChainUniverseOnDateChange() - { - var startTime = new DateTime(2018, 10, 19, 10, 0, 0); - var timeProvider = new ManualTimeProvider(startTime); - - var canonicalSymbol = Symbol.Create("SPY", SecurityType.Option, Market.USA, "?SPY"); - - var quoteCurrency = new Cash(Currencies.USD, 0, 1); - var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.USA, canonicalSymbol, SecurityType.Option); - var config = new SubscriptionDataConfig( - typeof(ZipEntryName), - canonicalSymbol, - Resolution.Minute, - TimeZones.Utc, - TimeZones.NewYork, - true, - false, - false, - false, - TickType.Quote, - false, - DataNormalizationMode.Raw - ); - - var option = new Option( - canonicalSymbol, - exchangeHours, - quoteCurrency, - new OptionSymbolProperties(SymbolProperties.GetDefault(Currencies.USD)), - ErrorCurrencyConverter.Instance, - RegisteredSecurityDataTypesProvider.Null, - new SecurityCache(), - null - ); - - var fillForwardResolution = Ref.CreateReadOnly(() => Resolution.Minute.ToTimeSpan()); - var symbolUniverse = new TestDataQueueUniverseProvider(timeProvider); - EnqueueableEnumerator underlyingEnumerator = null; - Func> underlyingEnumeratorFunc = - (req) => - { - underlyingEnumerator = new EnqueueableEnumerator(); - return new LiveFillForwardEnumerator( - timeProvider, - underlyingEnumerator, - option.Exchange, - fillForwardResolution, - false, - Time.EndOfTime, - Resolution.Minute, - TimeZones.Utc, false); - }; - var factory = new OptionChainUniverseSubscriptionEnumeratorFactory(underlyingEnumeratorFunc, symbolUniverse, timeProvider); - - var universeSettings = new UniverseSettings(Resolution.Minute, 0, true, false, TimeSpan.Zero); - using var universe = new OptionChainUniverse(option, universeSettings); - var request = new SubscriptionRequest(true, universe, option, config, startTime, Time.EndOfTime); - var enumerator = (DataQueueOptionChainUniverseDataCollectionEnumerator) factory.CreateEnumerator(request, TestGlobals.DataProvider); - - // 2018-10-19 10:00 AM UTC - underlyingEnumerator.Enqueue(new Tick { Symbol = Symbols.SPY, Value = 280m }); - - // 2018-10-19 10:01 AM UTC - timeProvider.Advance(Time.OneMinute); - - underlyingEnumerator.Enqueue(new Tick { Symbol = Symbols.SPY, Value = 280m }); - - Assert.IsTrue(enumerator.MoveNext()); - Assert.IsNotNull(enumerator.Current); - Assert.AreEqual(1, symbolUniverse.TotalLookupCalls); - var data = enumerator.Current; - Assert.IsNotNull(data); - Assert.AreEqual(1, data.Data.Count); - Assert.IsNotNull(data.Underlying); - - // 2018-10-19 10:02 AM UTC - timeProvider.Advance(Time.OneMinute); - - underlyingEnumerator.Enqueue(new Tick { Symbol = Symbols.SPY, Value = 280m }); - - Assert.IsTrue(enumerator.MoveNext()); - Assert.IsNotNull(enumerator.Current); - Assert.AreEqual(1, symbolUniverse.TotalLookupCalls); - data = enumerator.Current; - Assert.IsNotNull(data); - Assert.AreEqual(1, data.Data.Count); - Assert.IsNotNull(data.Underlying); - - // 2018-10-19 10:03 AM UTC - timeProvider.Advance(Time.OneMinute); - - underlyingEnumerator.Enqueue(new Tick { Symbol = Symbols.SPY, Value = 280m }); - - Assert.IsTrue(enumerator.MoveNext()); - Assert.IsNotNull(enumerator.Current); - Assert.AreEqual(1, symbolUniverse.TotalLookupCalls); - data = enumerator.Current; - Assert.IsNotNull(data); - Assert.AreEqual(1, data.Data.Count); - Assert.IsNotNull(data.Underlying); - - // 2018-10-20 10:03 AM UTC - timeProvider.Advance(Time.OneDay); - - underlyingEnumerator.Enqueue(new Tick { Symbol = Symbols.SPY, Value = 280m }); - - Assert.IsTrue(enumerator.MoveNext()); - Assert.IsNotNull(enumerator.Current); - Assert.AreEqual(2, symbolUniverse.TotalLookupCalls); - data = enumerator.Current; - Assert.IsNotNull(data); - Assert.AreEqual(2, data.Data.Count); - Assert.IsNotNull(data.Underlying); - - // 2018-10-20 10:04 AM UTC - timeProvider.Advance(Time.OneMinute); - - underlyingEnumerator.Enqueue(new Tick { Symbol = Symbols.SPY, Value = 280m }); - - Assert.IsTrue(enumerator.MoveNext()); - Assert.IsNotNull(enumerator.Current); - Assert.AreEqual(2, symbolUniverse.TotalLookupCalls); - data = enumerator.Current; - Assert.IsNotNull(data); - Assert.AreEqual(2, data.Data.Count); - Assert.IsNotNull(data.Underlying); - - enumerator.Dispose(); - } - - public class TestDataQueueUniverseProvider : IDataQueueUniverseProvider - { - private readonly Symbol[] _symbolList1 = - { - Symbol.CreateOption("SPY", Market.USA, OptionStyle.American, OptionRight.Call, 280m, new DateTime(2018, 10, 19)) - }; - private readonly Symbol[] _symbolList2 = - { - Symbol.CreateOption("SPY", Market.USA, OptionStyle.American, OptionRight.Call, 280m, new DateTime(2018, 10, 19)), - Symbol.CreateOption("SPY", Market.USA, OptionStyle.American, OptionRight.Put, 280m, new DateTime(2018, 10, 19)) - }; - - private readonly ITimeProvider _timeProvider; - - public int TotalLookupCalls { get; set; } - - public TestDataQueueUniverseProvider(ITimeProvider timeProvider) - { - _timeProvider = timeProvider; - } - - public IEnumerable LookupSymbols(Symbol symbol, bool includeExpired, string securityCurrency = null) - { - TotalLookupCalls++; - - return _timeProvider.GetUtcNow().Date.Day >= 20 ? _symbolList2 : _symbolList1; - } - - public bool CanPerformSelection() - { - return true; - } - } - } -} diff --git a/Tests/Engine/DataFeeds/TimeSliceTests.cs b/Tests/Engine/DataFeeds/TimeSliceTests.cs index 9d867b5621e9..95ef83abccbf 100644 --- a/Tests/Engine/DataFeeds/TimeSliceTests.cs +++ b/Tests/Engine/DataFeeds/TimeSliceTests.cs @@ -219,7 +219,8 @@ public void SuspiciousTicksAreNotAddedToConsolidatorUpdateData() private IEnumerable GetSlices(Symbol symbol, int initialVolume) { - var subscriptionDataConfig = new SubscriptionDataConfig(typeof(ZipEntryName), symbol, Resolution.Second, TimeZones.Utc, TimeZones.Utc, true, true, false); + var dataType = symbol.SecurityType.IsOption() ? typeof(OptionUniverse) : typeof(FutureUniverse); + var subscriptionDataConfig = new SubscriptionDataConfig(dataType, symbol, Resolution.Second, TimeZones.Utc, TimeZones.Utc, true, true, false); var security = GetSecurity(subscriptionDataConfig); var refTime = DateTime.UtcNow; @@ -271,7 +272,7 @@ private Security GetSecurity(SubscriptionDataConfig config) new OptionSymbolProperties(SymbolProperties.GetDefault(Currencies.USD)), ErrorCurrencyConverter.Instance, RegisteredSecurityDataTypesProvider.Null); - var underlyingConfig = new SubscriptionDataConfig(typeof(ZipEntryName), config.Symbol.Underlying, Resolution.Second, + var underlyingConfig = new SubscriptionDataConfig(typeof(TradeBar), config.Symbol.Underlying, Resolution.Second, TimeZones.Utc, TimeZones.Utc, true, true, false); var equity = new Equity( SecurityExchangeHours.AlwaysOpen(TimeZones.Utc), diff --git a/Tests/Engine/DataFeeds/ZipEntryNameSubscriptionFactoryTests.cs b/Tests/Engine/DataFeeds/ZipEntryNameSubscriptionFactoryTests.cs index f1eb0a2dfcdf..60cb66e1b04b 100644 --- a/Tests/Engine/DataFeeds/ZipEntryNameSubscriptionFactoryTests.cs +++ b/Tests/Engine/DataFeeds/ZipEntryNameSubscriptionFactoryTests.cs @@ -20,6 +20,7 @@ using QuantConnect.Data; using QuantConnect.Data.Auxiliary; using QuantConnect.Lean.Engine.DataFeeds; +using QuantConnect.Util; namespace QuantConnect.Tests.Engine.DataFeeds { @@ -31,7 +32,7 @@ public void ReadsZipEntryNames() { var time = new DateTime(2016, 03, 03, 12, 48, 15); var source = Path.Combine("TestData", "20151224_quote_american.zip"); - var config = new SubscriptionDataConfig(typeof (ZipEntryName), Symbol.Create("XLRE", SecurityType.Option, Market.USA), Resolution.Tick, + var config = new SubscriptionDataConfig(typeof (CustomData), Symbol.Create("XLRE", SecurityType.Option, Market.USA), Resolution.Tick, TimeZones.NewYork, TimeZones.NewYork, false, false, false); using var cacheProvider = new ZipDataCacheProvider(TestGlobals.DataProvider); var factory = new ZipEntryNameSubscriptionDataSourceReader(cacheProvider, config, time, false); @@ -46,7 +47,22 @@ public void ReadsZipEntryNames() // we only really care about the symbols CollectionAssert.AreEqual(expected, actual.Select(x => x.Symbol)); - Assert.IsTrue(actual.All(x => x is ZipEntryName)); + Assert.IsTrue(actual.All(x => x is CustomData)); + } + + private class CustomData : BaseData + { + public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode) + { + var symbol = LeanData.ReadSymbolFromZipEntry(config.Symbol, config.Resolution, line); + return new CustomData { Time = date, Symbol = symbol }; + } + + public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode) + { + var source = LeanData.GenerateZipFilePath(Globals.DataFolder, config.Symbol, date, config.Resolution, config.TickType); + return new SubscriptionDataSource(source, SubscriptionTransportMedium.LocalFile, FileFormat.ZipEntryName); + } } } } diff --git a/Tests/ToolBox/LeanDataReaderTests.cs b/Tests/ToolBox/LeanDataReaderTests.cs index 80c975b3221d..c5b34b9a3710 100644 --- a/Tests/ToolBox/LeanDataReaderTests.cs +++ b/Tests/ToolBox/LeanDataReaderTests.cs @@ -177,7 +177,7 @@ private List LoadFutureChain(Symbol baseFuture, DateTime date, TickType var filePath = LeanData.GenerateZipFilePath(_dataDirectory, baseFuture, date, res, tickType); //load future chain first - var config = new SubscriptionDataConfig(typeof(ZipEntryName), baseFuture, res, + var config = new SubscriptionDataConfig(typeof(ZipEntryNameData), baseFuture, res, TimeZones.NewYork, TimeZones.NewYork, false, false, false, false, tickType); var factory = new ZipEntryNameSubscriptionDataSourceReader(TestGlobals.DataCacheProvider, config, date, false); @@ -499,5 +499,20 @@ public static string GenerateFilepathForTesting(string dataDirectory, string sec return filepath; } + private class ZipEntryNameData : BaseData + { + public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode) + { + var symbol = LeanData.ReadSymbolFromZipEntry(config.Symbol, config.Resolution, line); + return new ZipEntryNameData { Time = date, Symbol = symbol }; + } + + public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode) + { + var source = LeanData.GenerateZipFilePath(Globals.DataFolder, config.Symbol, date, config.Resolution, config.TickType); + return new SubscriptionDataSource(source, SubscriptionTransportMedium.LocalFile, FileFormat.ZipEntryName); + } + } + } }