From 8149c291fca8c993c2c100f20dbeb559633979c7 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Fri, 31 Jan 2025 15:00:14 -0400 Subject: [PATCH] Dynamic market hours for time and date rules (#8541) * Add unit test reproducing the bug * Improve unit test * Apply market hours database updates to instance Make sure entries are updated instead of overridden so that consumers holding the mhdb can pick up the changes * Improved unit tests * Update data folder MHDB entries instead of overriding instance * Optionally reset custom mhdb entries on reset * Simplify mhdb reset in live trading Cleanup and minor fixes * Ensure symbol properties database updates are picked up by consumers Update SPDB entries instead of overriding them, just like for the MHDB so that consumers pick up updates after refresh * Fix for symbol properties update thread safety * Minor improvements and cleanup * Mhdb and spdb update logic simplification * Move "force exchange always open" logic to mhdb * Simplify symbol properties holder and updates * Refactor security databases to use a common base class Introduce BaseSecurityDatabase to encapsulate common functionality for MarketHoursDatabase and SymbolPropertiesDatabase. * Cleanup --- Common/Securities/BaseSecurityDatabase.cs | 206 +++++++++ Common/Securities/MarketHoursDatabase.cs | 157 +++---- .../Option/OptionSymbolProperties.cs | 42 +- Common/Securities/SymbolProperties.cs | 157 ++++--- Common/Securities/SymbolPropertiesDatabase.cs | 81 +--- Engine/RealTime/LiveTradingRealTimeHandler.cs | 102 +---- .../Securities/MarketHoursDatabaseTests.cs | 14 +- .../SymbolPropertiesDatabaseTests.cs | 4 +- .../LiveTradingRealTimeHandlerTests.cs | 431 +++++++++++++++--- Tests/QuantConnect.Tests.csproj | 15 + .../market-hours/market-hours-database.json | 49 ++ .../market-hours/market-hours-database.json | 100 ++++ .../market-hours/market-hours-database.json | 99 ++++ .../symbol-properties-database.csv | 2 + .../symbol-properties-database.csv | 2 + 15 files changed, 1051 insertions(+), 410 deletions(-) create mode 100644 Common/Securities/BaseSecurityDatabase.cs create mode 100644 Tests/TestData/dynamic-market-hours/modified-close/market-hours/market-hours-database.json create mode 100644 Tests/TestData/dynamic-market-hours/modified-holidays/market-hours/market-hours-database.json create mode 100644 Tests/TestData/dynamic-market-hours/original/market-hours/market-hours-database.json create mode 100644 Tests/TestData/dynamic-symbol-properties/modified/symbol-properties/symbol-properties-database.csv create mode 100644 Tests/TestData/dynamic-symbol-properties/original/symbol-properties/symbol-properties-database.csv diff --git a/Common/Securities/BaseSecurityDatabase.cs b/Common/Securities/BaseSecurityDatabase.cs new file mode 100644 index 000000000000..044afc0b7bd4 --- /dev/null +++ b/Common/Securities/BaseSecurityDatabase.cs @@ -0,0 +1,206 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using QuantConnect.Util; + +namespace QuantConnect.Securities +{ + /// + /// Base class for security databases, including market hours and symbol properties. + /// + public abstract class BaseSecurityDatabase + where T : BaseSecurityDatabase + { + /// + /// The database instance loaded from the data folder + /// + protected static T DataFolderDatabase { get; set; } + + /// + /// Lock object for the data folder database + /// + protected static readonly object DataFolderDatabaseLock = new object(); + + /// + /// The database entries + /// + protected Dictionary Entries { get; set; } + + /// + /// Custom entries set by the user. + /// + protected HashSet CustomEntries { get; } + + // _loadFromFromDataFolder and _updateEntry are used to load the database from + // the data folder and update an entry respectively. + // These are not abstract or virtual methods because they might be static methods. + private readonly Func _loadFromFromDataFolder; + private readonly Action _updateEntry; + + /// + /// Initializes a new instance of the class + /// + /// The full listing of exchange hours by key + /// Method to load the database form the data folder + /// Method to update a database entry + protected BaseSecurityDatabase(Dictionary entries, + Func fromDataFolder, Action updateEntry) + { + Entries = entries; + CustomEntries = new(); + _loadFromFromDataFolder = fromDataFolder; + _updateEntry = updateEntry; + } + + /// + /// Resets the database, forcing a reload when reused. + /// Called in tests where multiple algorithms are run sequentially, + /// and we need to guarantee that every test starts with the same environment. + /// +#pragma warning disable CA1000 // Do not declare static members on generic types + public static void Reset() +#pragma warning restore CA1000 // Do not declare static members on generic types + { + lock (DataFolderDatabaseLock) + { + DataFolderDatabase = null; + } + } + + /// + /// Reload entries dictionary from file and merge them with previous custom ones + /// + internal void UpdateDataFolderDatabase() + { + lock (DataFolderDatabaseLock) + { + Reset(); + var newDatabase = _loadFromFromDataFolder(); + Merge(newDatabase, resetCustomEntries: false); + // Make sure we keep this as the data folder database + DataFolderDatabase = (T)this; + } + } + + /// + /// Updates the entries dictionary with the new entries from the specified database + /// + internal virtual void Merge(T newDatabase, bool resetCustomEntries) + { + var newEntries = new List>(); + + foreach (var newEntry in newDatabase.Entries) + { + if (Entries.TryGetValue(newEntry.Key, out var entry)) + { + if (resetCustomEntries || !CustomEntries.Contains(newEntry.Key)) + { + _updateEntry(entry, newEntry.Value); + } + } + else + { + newEntries.Add(KeyValuePair.Create(newEntry.Key, newEntry.Value)); + } + } + + Entries = Entries + .Where(kvp => (!resetCustomEntries && CustomEntries.Contains(kvp.Key)) || newDatabase.Entries.ContainsKey(kvp.Key)) + .Concat(newEntries) + .ToDictionary(); + + if (resetCustomEntries) + { + CustomEntries.Clear(); + } + } + + /// + /// Determines if the database contains the specified key + /// + /// The key to search for + /// True if an entry is found, otherwise false + protected bool ContainsKey(SecurityDatabaseKey key) + { + return Entries.ContainsKey(key); + } + + /// + /// Check whether an entry exists for the specified market/symbol/security-type + /// + /// The market the exchange resides in, i.e, 'usa', 'fxcm', ect... + /// The particular symbol being traded + /// The security type of the symbol + public bool ContainsKey(string market, string symbol, SecurityType securityType) + { + return ContainsKey(new SecurityDatabaseKey(market, symbol, securityType)); + } + + /// + /// Check whether an entry exists for the specified market/symbol/security-type + /// + /// The market the exchange resides in, i.e, 'usa', 'fxcm', ect... + /// The particular symbol being traded (Symbol class) + /// The security type of the symbol + public bool ContainsKey(string market, Symbol symbol, SecurityType securityType) + { + return ContainsKey( + market, + GetDatabaseSymbolKey(symbol), + securityType); + } + + /// + /// Gets the correct string symbol to use as a database key + /// + /// The symbol + /// The symbol string used in the database ke +#pragma warning disable CA1000 // Do not declare static members on generic types + public static string GetDatabaseSymbolKey(Symbol symbol) +#pragma warning restore CA1000 // Do not declare static members on generic types + { + string stringSymbol; + if (symbol == null) + { + stringSymbol = string.Empty; + } + else + { + switch (symbol.ID.SecurityType) + { + case SecurityType.Option: + stringSymbol = symbol.HasUnderlying ? symbol.Underlying.Value : string.Empty; + break; + case SecurityType.IndexOption: + case SecurityType.FutureOption: + stringSymbol = symbol.HasUnderlying ? symbol.ID.Symbol : string.Empty; + break; + case SecurityType.Base: + case SecurityType.Future: + stringSymbol = symbol.ID.Symbol; + break; + default: + stringSymbol = symbol.Value; + break; + } + } + + return stringSymbol; + } + } +} diff --git a/Common/Securities/MarketHoursDatabase.cs b/Common/Securities/MarketHoursDatabase.cs index e446c0e788e2..af0116d256e1 100644 --- a/Common/Securities/MarketHoursDatabase.cs +++ b/Common/Securities/MarketHoursDatabase.cs @@ -19,6 +19,7 @@ using System.Linq; using Newtonsoft.Json; using NodaTime; +using QuantConnect.Configuration; using QuantConnect.Data; using QuantConnect.Logging; using QuantConnect.Securities.Future; @@ -30,19 +31,16 @@ namespace QuantConnect.Securities /// Provides access to exchange hours and raw data times zones in various markets /// [JsonConverter(typeof(MarketHoursDatabaseJsonConverter))] - public class MarketHoursDatabase + public class MarketHoursDatabase : BaseSecurityDatabase { - private static MarketHoursDatabase _dataFolderMarketHoursDatabase; - private static MarketHoursDatabase _alwaysOpenMarketHoursDatabase; - private static readonly object DataFolderMarketHoursDatabaseLock = new object(); + private readonly bool _forceExchangeAlwaysOpen = Config.GetBool("force-exchange-always-open"); - private Dictionary _entries; - private readonly Dictionary _customEntries = new(); + private static MarketHoursDatabase _alwaysOpenMarketHoursDatabase; /// /// Gets all the exchange hours held by this provider /// - public List> ExchangeHoursListing => _entries.ToList(); + public List> ExchangeHoursListing => Entries.ToList(); /// /// Gets a that always returns @@ -60,13 +58,21 @@ public static MarketHoursDatabase AlwaysOpen } } + /// + /// Initializes a new instance of the class + /// + private MarketHoursDatabase() + : this(new()) + { + } + /// /// Initializes a new instance of the class /// /// The full listing of exchange hours by key public MarketHoursDatabase(Dictionary exchangeHours) + : base(exchangeHours, FromDataFolder, (entry, other) => entry.Update(other)) { - _entries = exchangeHours; } /// @@ -104,33 +110,6 @@ public DateTimeZone GetDataTimeZone(string market, Symbol symbol, SecurityType s return GetEntry(market, GetDatabaseSymbolKey(symbol), securityType).DataTimeZone; } - /// - /// Resets the market hours database, forcing a reload when reused. - /// Called in tests where multiple algorithms are run sequentially, - /// and we need to guarantee that every test starts with the same environment. - /// - public static void Reset() - { - lock (DataFolderMarketHoursDatabaseLock) - { - _dataFolderMarketHoursDatabase = null; - } - } - - /// - /// Reload entries dictionary from MHDB file and merge them with previous custom ones - /// - internal void ReloadEntries() - { - lock (DataFolderMarketHoursDatabaseLock) - { - Reset(); - var fileEntries = FromDataFolder()._entries.Where(x => !_customEntries.ContainsKey(x.Key)); - var newEntries = fileEntries.Concat(_customEntries).ToDictionary(); - _entries = newEntries; - } - } - /// /// Gets the instance of the class produced by reading in the market hours /// data found in /Data/market-hours/ @@ -138,20 +117,19 @@ internal void ReloadEntries() /// A class that represents the data in the market-hours folder public static MarketHoursDatabase FromDataFolder() { - var result = _dataFolderMarketHoursDatabase; - if (result == null) + if (DataFolderDatabase == null) { - lock (DataFolderMarketHoursDatabaseLock) + lock (DataFolderDatabaseLock) { - if (_dataFolderMarketHoursDatabase == null) + if (DataFolderDatabase == null) { var path = Path.Combine(Globals.GetDataFolderPath("market-hours"), "market-hours-database.json"); - _dataFolderMarketHoursDatabase = FromFile(path); + DataFolderDatabase = FromFile(path); } - result = _dataFolderMarketHoursDatabase; } } - return result; + + return DataFolderDatabase; } /// @@ -181,10 +159,10 @@ public virtual Entry SetEntry(string market, string symbol, SecurityType securit dataTimeZone = dataTimeZone ?? exchangeHours.TimeZone; var key = new SecurityDatabaseKey(market, symbol, securityType); var entry = new Entry(dataTimeZone, exchangeHours); - lock (DataFolderMarketHoursDatabaseLock) + lock (DataFolderDatabaseLock) { - _entries[key] = entry; - _customEntries[key] = entry; + Entries[key] = entry; + CustomEntries.Add(key); } return entry; } @@ -221,7 +199,7 @@ public virtual Entry GetEntry(string market, string symbol, SecurityType securit if (!TryGetEntry(market, symbol, securityType, out entry)) { var key = new SecurityDatabaseKey(market, symbol, securityType); - Log.Error($"MarketHoursDatabase.GetExchangeHours(): {Messages.MarketHoursDatabase.ExchangeHoursNotFound(key, _entries.Keys)}"); + Log.Error($"MarketHoursDatabase.GetExchangeHours(): {Messages.MarketHoursDatabase.ExchangeHoursNotFound(key, Entries.Keys)}"); if (securityType == SecurityType.Future && market == Market.USA) { @@ -262,18 +240,23 @@ public bool TryGetEntry(string market, Symbol symbol, SecurityType securityType, /// The security type of the symbol /// The entry found if any /// True if the entry was present, else false - public bool TryGetEntry(string market, string symbol, SecurityType securityType, out Entry entry) + public virtual bool TryGetEntry(string market, string symbol, SecurityType securityType, out Entry entry) { + if (_forceExchangeAlwaysOpen) + { + return AlwaysOpen.TryGetEntry(market, symbol, securityType, out entry); + } + var symbolKey = new SecurityDatabaseKey(market, symbol, securityType); - return _entries.TryGetValue(symbolKey, out entry) + return Entries.TryGetValue(symbolKey, out entry) // now check with null symbol key - || _entries.TryGetValue(symbolKey.CreateCommonKey(), out entry) + || Entries.TryGetValue(symbolKey.CreateCommonKey(), out entry) // if FOP check for future || securityType == SecurityType.FutureOption && TryGetEntry(market, FuturesOptionsSymbolMappings.MapFromOption(symbol), SecurityType.Future, out entry) // if custom data type check for type specific entry || (securityType == SecurityType.Base && SecurityIdentifier.TryGetCustomDataType(symbol, out var customType) - && _entries.TryGetValue(new SecurityDatabaseKey(market, $"TYPE.{customType}", securityType), out entry)); + && Entries.TryGetValue(new SecurityDatabaseKey(market, $"TYPE.{customType}", securityType), out entry)); } /// @@ -288,52 +271,6 @@ public virtual Entry GetEntry(string market, Symbol symbol, SecurityType securit return GetEntry(market, GetDatabaseSymbolKey(symbol), securityType); } - /// - /// Gets the correct string symbol to use as a database key - /// - /// The symbol - /// The symbol string used in the database ke - public static string GetDatabaseSymbolKey(Symbol symbol) - { - string stringSymbol; - if (symbol == null) - { - stringSymbol = string.Empty; - } - else - { - switch (symbol.ID.SecurityType) - { - case SecurityType.Option: - stringSymbol = symbol.HasUnderlying ? symbol.Underlying.Value : string.Empty; - break; - case SecurityType.IndexOption: - case SecurityType.FutureOption: - stringSymbol = symbol.HasUnderlying ? symbol.ID.Symbol : string.Empty; - break; - case SecurityType.Base: - case SecurityType.Future: - stringSymbol = symbol.ID.Symbol; - break; - default: - stringSymbol = symbol.Value; - break; - } - } - - return stringSymbol; - } - - /// - /// Determines if the database contains the specified key - /// - /// The key to search for - /// True if an entry is found, otherwise false - protected bool ContainsKey(SecurityDatabaseKey key) - { - return _entries.ContainsKey(key); - } - /// /// Represents a single entry in the /// @@ -342,7 +279,7 @@ public class Entry /// /// Gets the raw data time zone for this entry /// - public DateTimeZone DataTimeZone { get; init; } + public DateTimeZone DataTimeZone { get; private set; } /// /// Gets the exchange hours for this entry /// @@ -357,18 +294,32 @@ public Entry(DateTimeZone dataTimeZone, SecurityExchangeHours exchangeHours) DataTimeZone = dataTimeZone; ExchangeHours = exchangeHours; } + + internal void Update(Entry other) + { + DataTimeZone = other.DataTimeZone; + ExchangeHours.Update(other.ExchangeHours); + } } class AlwaysOpenMarketHoursDatabaseImpl : MarketHoursDatabase { - public override Entry GetEntry(string market, string symbol, SecurityType securityType) + public override bool TryGetEntry(string market, string symbol, SecurityType securityType, out Entry entry) { - var key = new SecurityDatabaseKey(market, symbol, securityType); - var tz = ContainsKey(key) - ? base.GetEntry(market, symbol, securityType).ExchangeHours.TimeZone - : DateTimeZone.Utc; + DateTimeZone dataTimeZone; + DateTimeZone exchangeTimeZone; + if (TryGetEntry(market, symbol, securityType, out entry)) + { + dataTimeZone = entry.DataTimeZone; + exchangeTimeZone = entry.ExchangeHours.TimeZone; + } + else + { + dataTimeZone = exchangeTimeZone = TimeZones.Utc; + } - return new Entry(tz, SecurityExchangeHours.AlwaysOpen(tz)); + entry = new Entry(dataTimeZone, SecurityExchangeHours.AlwaysOpen(exchangeTimeZone)); + return true; } public AlwaysOpenMarketHoursDatabaseImpl() diff --git a/Common/Securities/Option/OptionSymbolProperties.cs b/Common/Securities/Option/OptionSymbolProperties.cs index c0435260f483..c01176a9f63a 100644 --- a/Common/Securities/Option/OptionSymbolProperties.cs +++ b/Common/Securities/Option/OptionSymbolProperties.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -20,6 +20,21 @@ namespace QuantConnect.Securities.Option /// public class OptionSymbolProperties : SymbolProperties { + /// + /// The contract multiplier for the security. + /// + /// + /// If manually set by a consumer, this value will be used instead of the + /// and also allows to make + /// sure it is not overridden when the symbol properties database gets updated. + /// + private decimal? _contractMultiplier; + + /// + /// The contract multiplier for the security + /// + public override decimal ContractMultiplier => _contractMultiplier ?? base.ContractMultiplier; + /// /// When the holder of an equity option exercises one contract, or when the writer of an equity option is assigned /// an exercise notice on one contract, this unit of trade, usually 100 shares of the underlying security, changes hands. @@ -29,38 +44,19 @@ public int ContractUnitOfTrade get; protected set; } - /// - /// Overridable minimum price variation, required for index options contracts with - /// variable sized quoted prices depending on the premium of the option. - /// - public override decimal MinimumPriceVariation - { - get; - protected set; - } - /// /// Creates an instance of the class /// public OptionSymbolProperties(string description, string quoteCurrency, decimal contractMultiplier, decimal pipSize, decimal lotSize) - : base(description, quoteCurrency, contractMultiplier, pipSize, lotSize, string.Empty) + : this(new SymbolProperties(description, quoteCurrency, contractMultiplier, pipSize, lotSize, string.Empty)) { - ContractUnitOfTrade = (int)contractMultiplier; } /// /// Creates an instance of the class from class /// public OptionSymbolProperties(SymbolProperties properties) - : base(properties.Description, - properties.QuoteCurrency, - properties.ContractMultiplier, - properties.MinimumPriceVariation, - properties.LotSize, - properties.MarketTicker, - properties.MinimumOrderSize, - properties.PriceMagnifier, - properties.StrikeMultiplier) + : base(properties) { ContractUnitOfTrade = (int)properties.ContractMultiplier; } @@ -72,7 +68,7 @@ internal void SetContractUnitOfTrade(int unitOfTrade) internal void SetContractMultiplier(decimal multiplier) { - ContractMultiplier = multiplier; + _contractMultiplier = multiplier; } } } diff --git a/Common/Securities/SymbolProperties.cs b/Common/Securities/SymbolProperties.cs index e5f98f5fd3d4..9c3db7e38b65 100644 --- a/Common/Securities/SymbolProperties.cs +++ b/Common/Securities/SymbolProperties.cs @@ -22,55 +22,48 @@ namespace QuantConnect.Securities /// public class SymbolProperties { + /// + /// DTO used to hold the properties of the symbol + /// + /// + /// A DTO is used to handle updates to the symbol properties. Since some properties are decimals, + /// which is not thread-safe, we get around it by creating a new instance of the DTO and assigning to this property + /// + private SymbolPropertiesHolder _properties; + /// /// The description of the security /// - public string Description - { - get; - } + public string Description => _properties.Description; /// /// The quote currency of the security /// - public string QuoteCurrency - { - get; - } + public string QuoteCurrency => _properties.QuoteCurrency; /// /// The contract multiplier for the security /// - public decimal ContractMultiplier + public virtual decimal ContractMultiplier { - get; - protected set; + get => _properties.ContractMultiplier; + internal set => _properties.ContractMultiplier = value; } /// /// The minimum price variation (tick size) for the security /// - public virtual decimal MinimumPriceVariation - { - get; - protected set; - } + public virtual decimal MinimumPriceVariation => _properties.MinimumPriceVariation; /// /// The lot size (lot size of the order) for the security /// - public decimal LotSize - { - get; - } + public decimal LotSize => _properties.LotSize; /// /// The market ticker /// - public string MarketTicker - { - get; - } + public string MarketTicker => _properties.MarketTicker; /// /// The minimum order size allowed @@ -78,20 +71,14 @@ public string MarketTicker /// i.e For BTC/USD the minimum order size allowed with Coinbase is 0.0001 BTC /// while on Binance the minimum order size allowed is 10 USD /// - public decimal? MinimumOrderSize - { - get; - } + public decimal? MinimumOrderSize => _properties.MinimumOrderSize; /// /// Allows normalizing live asset prices to US Dollars for Lean consumption. In some exchanges, /// for some securities, data is expressed in cents like for example for corn futures ('ZC'). /// /// Default value is 1 but for some futures in cents it's 100 - public decimal PriceMagnifier - { - get; - } + public decimal PriceMagnifier => _properties.PriceMagnifier; /// /// Scale factor for option's strike price. For some options, such as NQX, the strike price @@ -99,41 +86,25 @@ public decimal PriceMagnifier /// that it can be used in comparation with the underlying such as /// in /// - public decimal StrikeMultiplier + public decimal StrikeMultiplier => _properties.StrikeMultiplier; + + /// + /// Creates an instance of the class + /// + protected SymbolProperties(SymbolProperties properties) { - get; + _properties = properties._properties; } /// /// Creates an instance of the class /// - public SymbolProperties(string description, string quoteCurrency, decimal contractMultiplier, decimal minimumPriceVariation, decimal lotSize, string marketTicker, decimal? minimumOrderSize = null, decimal priceMagnifier = 1, decimal strikeMultiplier = 1) + public SymbolProperties(string description, string quoteCurrency, decimal contractMultiplier, + decimal minimumPriceVariation, decimal lotSize, string marketTicker, + decimal? minimumOrderSize = null, decimal priceMagnifier = 1, decimal strikeMultiplier = 1) { - Description = description; - QuoteCurrency = quoteCurrency; - ContractMultiplier = contractMultiplier; - MinimumPriceVariation = minimumPriceVariation; - LotSize = lotSize; - - if (LotSize <= 0) - { - throw new ArgumentException(Messages.SymbolProperties.InvalidLotSize); - } - - MarketTicker = marketTicker; - MinimumOrderSize = minimumOrderSize; - - PriceMagnifier = priceMagnifier; - if (PriceMagnifier <= 0) - { - throw new ArgumentException(Messages.SymbolProperties.InvalidPriceMagnifier); - } - - StrikeMultiplier = strikeMultiplier; - if (strikeMultiplier <= 0) - { - throw new ArgumentException(Messages.SymbolProperties.InvalidStrikeMultiplier); - } + _properties = new SymbolPropertiesHolder(description, quoteCurrency, contractMultiplier, + minimumPriceVariation, lotSize, marketTicker, minimumOrderSize, priceMagnifier, strikeMultiplier); } /// @@ -153,5 +124,71 @@ public static SymbolProperties GetDefault(string quoteCurrency) { return new SymbolProperties(string.Empty, quoteCurrency.LazyToUpper(), 1, 0.01m, 1, string.Empty); } + + /// + /// Updates the symbol properties with the values from the specified + /// + /// The symbol properties to take values from + internal virtual void Update(SymbolProperties other) + { + _properties = other._properties; + } + + /// + /// DTO used to hold the properties of the symbol + /// + private class SymbolPropertiesHolder + { + public string Description { get; } + + public string QuoteCurrency { get; } + + public decimal ContractMultiplier { get; set; } + + public decimal MinimumPriceVariation { get; } + + public decimal LotSize { get; } + + public string MarketTicker { get; } + + public decimal? MinimumOrderSize { get; } + + public decimal PriceMagnifier { get; } + + public decimal StrikeMultiplier { get; } + + + /// + /// Creates an instance of the class + /// + public SymbolPropertiesHolder(string description, string quoteCurrency, decimal contractMultiplier, decimal minimumPriceVariation, decimal lotSize, string marketTicker, decimal? minimumOrderSize, decimal priceMagnifier, decimal strikeMultiplier) + { + Description = description; + QuoteCurrency = quoteCurrency; + ContractMultiplier = contractMultiplier; + MinimumPriceVariation = minimumPriceVariation; + LotSize = lotSize; + + if (LotSize <= 0) + { + throw new ArgumentException(Messages.SymbolProperties.InvalidLotSize); + } + + MarketTicker = marketTicker; + MinimumOrderSize = minimumOrderSize; + + PriceMagnifier = priceMagnifier; + if (PriceMagnifier <= 0) + { + throw new ArgumentException(Messages.SymbolProperties.InvalidPriceMagnifier); + } + + StrikeMultiplier = strikeMultiplier; + if (strikeMultiplier <= 0) + { + throw new ArgumentException(Messages.SymbolProperties.InvalidStrikeMultiplier); + } + } + } } } diff --git a/Common/Securities/SymbolPropertiesDatabase.cs b/Common/Securities/SymbolPropertiesDatabase.cs index 23d971b8faba..3af6c9dc2ab4 100644 --- a/Common/Securities/SymbolPropertiesDatabase.cs +++ b/Common/Securities/SymbolPropertiesDatabase.cs @@ -24,13 +24,8 @@ namespace QuantConnect.Securities /// /// Provides access to specific properties for various symbols /// - public class SymbolPropertiesDatabase + public class SymbolPropertiesDatabase : BaseSecurityDatabase { - private static SymbolPropertiesDatabase _dataFolderSymbolPropertiesDatabase; - private static readonly object DataFolderSymbolPropertiesDatabaseLock = new object(); - - private Dictionary _entries; - private readonly Dictionary _customEntries; private IReadOnlyDictionary _keyBySecurityType; /// @@ -38,6 +33,7 @@ public class SymbolPropertiesDatabase /// /// File to read from protected SymbolPropertiesDatabase(string file) + : base(null, FromDataFolder, (entry, newEntry) => entry.Update(newEntry)) { var allEntries = new Dictionary(); var entriesBySecurityType = new Dictionary(); @@ -58,37 +54,10 @@ protected SymbolPropertiesDatabase(string file) allEntries[keyValuePair.Key] = keyValuePair.Value; } - _entries = allEntries; - _customEntries = new(); + Entries = allEntries; _keyBySecurityType = entriesBySecurityType; } - /// - /// Check whether symbol properties exists for the specified market/symbol/security-type - /// - /// The market the exchange resides in, i.e, 'usa', 'fxcm', ect... - /// The particular symbol being traded - /// The security type of the symbol - public bool ContainsKey(string market, string symbol, SecurityType securityType) - { - var key = new SecurityDatabaseKey(market, symbol, securityType); - return _entries.ContainsKey(key); - } - - /// - /// Check whether symbol properties exists for the specified market/symbol/security-type - /// - /// The market the exchange resides in, i.e, 'usa', 'fxcm', ect... - /// The particular symbol being traded (Symbol class) - /// The security type of the symbol - public bool ContainsKey(string market, Symbol symbol, SecurityType securityType) - { - return ContainsKey( - market, - MarketHoursDatabase.GetDatabaseSymbolKey(symbol), - securityType); - } - /// /// Tries to get the market for the provided symbol/security type /// @@ -125,7 +94,7 @@ public SymbolProperties GetSymbolProperties(string market, Symbol symbol, Securi var lookupTicker = MarketHoursDatabase.GetDatabaseSymbolKey(symbol); var key = new SecurityDatabaseKey(market, lookupTicker, securityType); - if (!_entries.TryGetValue(key, out symbolProperties)) + if (!Entries.TryGetValue(key, out symbolProperties)) { if (symbol != null && symbol.SecurityType == SecurityType.FutureOption) { @@ -134,14 +103,14 @@ public SymbolProperties GetSymbolProperties(string market, Symbol symbol, Securi lookupTicker = MarketHoursDatabase.GetDatabaseSymbolKey(symbol.Underlying); key = new SecurityDatabaseKey(market, lookupTicker, symbol.Underlying.SecurityType); - if (_entries.TryGetValue(key, out symbolProperties)) + if (Entries.TryGetValue(key, out symbolProperties)) { return symbolProperties; } } // now check with null symbol key - if (!_entries.TryGetValue(new SecurityDatabaseKey(market, null, securityType), out symbolProperties)) + if (!Entries.TryGetValue(new SecurityDatabaseKey(market, null, securityType), out symbolProperties)) { // no properties found, return object with default property values return SymbolProperties.GetDefault(defaultQuoteCurrency); @@ -159,7 +128,7 @@ public SymbolProperties GetSymbolProperties(string market, Symbol symbol, Securi /// An IEnumerable of symbol properties matching the specified market/security-type public IEnumerable> GetSymbolPropertiesList(string market, SecurityType securityType) { - foreach (var entry in _entries) + foreach (var entry in Entries) { var key = entry.Key; var symbolProperties = entry.Value; @@ -178,7 +147,7 @@ public IEnumerable> GetSymbo /// An IEnumerable of symbol properties matching the specified market public IEnumerable> GetSymbolPropertiesList(string market) { - foreach (var entry in _entries) + foreach (var entry in Entries) { var key = entry.Key; var symbolProperties = entry.Value; @@ -201,10 +170,10 @@ public IEnumerable> GetSymbo public bool SetEntry(string market, string symbol, SecurityType securityType, SymbolProperties properties) { var key = new SecurityDatabaseKey(market, symbol, securityType); - lock (DataFolderSymbolPropertiesDatabaseLock) + lock (DataFolderDatabaseLock) { - _entries[key] = properties; - _customEntries[key] = properties; + Entries[key] = properties; + CustomEntries.Add(key); } return true; } @@ -216,14 +185,18 @@ public bool SetEntry(string market, string symbol, SecurityType securityType, Sy /// A class that represents the data in the symbol-properties folder public static SymbolPropertiesDatabase FromDataFolder() { - lock (DataFolderSymbolPropertiesDatabaseLock) + if (DataFolderDatabase == null) { - if (_dataFolderSymbolPropertiesDatabase == null) + lock (DataFolderDatabaseLock) { - _dataFolderSymbolPropertiesDatabase = new SymbolPropertiesDatabase(Path.Combine(Globals.GetDataFolderPath("symbol-properties"), "symbol-properties-database.csv")); + if (DataFolderDatabase == null) + { + var path = Path.Combine(Globals.GetDataFolderPath("symbol-properties"), "symbol-properties-database.csv"); + DataFolderDatabase = new SymbolPropertiesDatabase(path); + } } } - return _dataFolderSymbolPropertiesDatabase; + return DataFolderDatabase; } /// @@ -291,20 +264,10 @@ private static bool HasValidValue(string[] array, uint position) return array.Length > position && !string.IsNullOrEmpty(array[position]); } - /// - /// Reload entries dictionary from SPDB file and merge them with previous custom ones - /// - internal void ReloadEntries() + internal override void Merge(SymbolPropertiesDatabase newDatabase, bool resetCustomEntries) { - lock (DataFolderSymbolPropertiesDatabaseLock) - { - _dataFolderSymbolPropertiesDatabase = null; - var newInstance = FromDataFolder(); - var fileEntries = newInstance._entries.Where(x => !_customEntries.ContainsKey(x.Key)); - var newEntries = fileEntries.Concat(_customEntries).ToDictionary(); - _entries = newEntries; - _keyBySecurityType = newInstance._keyBySecurityType.ToReadOnlyDictionary(); - } + base.Merge(newDatabase, resetCustomEntries); + _keyBySecurityType = newDatabase._keyBySecurityType.ToReadOnlyDictionary(); } } } diff --git a/Engine/RealTime/LiveTradingRealTimeHandler.cs b/Engine/RealTime/LiveTradingRealTimeHandler.cs index 684beca83e2c..b5af0a85d2d5 100644 --- a/Engine/RealTime/LiveTradingRealTimeHandler.cs +++ b/Engine/RealTime/LiveTradingRealTimeHandler.cs @@ -23,7 +23,6 @@ using QuantConnect.Interfaces; using QuantConnect.Scheduling; using QuantConnect.Securities; -using QuantConnect.Configuration; using QuantConnect.Lean.Engine.Results; namespace QuantConnect.Lean.Engine.RealTime @@ -35,7 +34,6 @@ public class LiveTradingRealTimeHandler : BacktestingRealTimeHandler { private Thread _realTimeThread; private CancellationTokenSource _cancellationTokenSource = new(); - private readonly bool _forceExchangeAlwaysOpen = Config.GetBool("force-exchange-always-open"); /// /// Gets the current market hours database instance @@ -70,17 +68,13 @@ public override void Setup(IAlgorithm algorithm, AlgorithmNodePacket job, IResul var utcNow = TimeProvider.GetUtcNow(); var todayInAlgorithmTimeZone = utcNow.ConvertFromUtc(Algorithm.TimeZone).Date; - // refresh the market hours and symbol properties for today explicitly - RefreshMarketHours(todayInAlgorithmTimeZone); - RefreshSymbolProperties(); - // set up an scheduled event to refresh market hours and symbol properties every certain period of time var times = Time.DateTimeRange(utcNow.Date, Time.EndOfTime, Algorithm.Settings.DatabasesRefreshPeriod).Where(date => date > utcNow); Add(new ScheduledEvent("RefreshMarketHoursAndSymbolProperties", times, (name, triggerTime) => { - RefreshMarketHours(triggerTime.ConvertFromUtc(Algorithm.TimeZone).Date); - RefreshSymbolProperties(); + ResetMarketHoursDatabase(); + ResetSymbolPropertiesDatabase(); })); } @@ -129,59 +123,6 @@ private void Run() Log.Trace("LiveTradingRealTimeHandler.Run(): Exiting thread... Exit triggered: " + _cancellationTokenSource.IsCancellationRequested); } - /// - /// Refresh the market hours for each security in the given date - /// - /// Each time this method is called, the MarketHoursDatabase is reset - protected virtual void RefreshMarketHours(DateTime date) - { - date = date.Date; - ResetMarketHoursDatabase(); - - // update market hours for each security - foreach (var kvp in Algorithm.Securities) - { - var security = kvp.Value; - UpdateMarketHours(security); - - var localMarketHours = security.Exchange.Hours.GetMarketHours(date); - - // All future and option contracts sharing the same canonical symbol, share the same market - // hours too. Thus, in order to reduce logs, we log the market hours using the canonical - // symbol. See the optional parameter "overrideMessageFloodProtection" in Log.Trace() - // method for further information - var symbol = security.Symbol.HasCanonical() ? security.Symbol.Canonical : security.Symbol; - Log.Trace($"LiveTradingRealTimeHandler.RefreshMarketHoursToday({security.Type}): Market hours set: Symbol: {symbol} {localMarketHours} ({security.Exchange.Hours.TimeZone})"); - } - } - - /// - /// Refresh the symbol properties for each security - /// - /// - /// - Each time this method is called, the SymbolPropertiesDatabase is reset - /// - Made protected virtual for testing purposes - /// - protected virtual void RefreshSymbolProperties() - { - ResetSymbolPropertiesDatabase(); - - // update market hours for each security - foreach (var kvp in Algorithm.Securities) - { - var security = kvp.Value; - UpdateSymbolProperties(security); - - // All future and option contracts sharing the same canonical symbol, share the same symbol - // properties too. Thus, in order to reduce logs, we log the symbol properties using the - // canonical symbol. See the optional parameter "overrideMessageFloodProtection" in - // Log.Trace() method for further information - var symbol = security.Symbol.HasCanonical() ? security.Symbol.Canonical : security.Symbol; - Log.Trace($"LiveTradingRealTimeHandler.RefreshSymbolPropertiesToday(): Symbol properties set: " + - $"Symbol: {symbol} {security.SymbolProperties}"); - } - } - /// /// Set the current time. If the date changes re-start the realtime event setup routines. /// @@ -225,37 +166,6 @@ public override void Exit() base.Exit(); } - /// - /// Updates the market hours for the specified security. - /// - /// - /// - This is done after a MHDB refresh - /// - Made protected virtual for testing purposes - /// - protected virtual void UpdateMarketHours(Security security) - { - var hours = _forceExchangeAlwaysOpen - ? SecurityExchangeHours.AlwaysOpen(security.Exchange.TimeZone) - : MarketHoursDatabase.GetExchangeHours(security.Symbol.ID.Market, security.Symbol, security.Symbol.ID.SecurityType); - - // Use Update method to avoid replacing the reference - security.Exchange.Hours.Update(hours); - } - - /// - /// Updates the symbol properties for the specified security. - /// - /// - /// - This is done after a SPDB refresh - /// - Made protected virtual for testing purposes - /// - protected virtual void UpdateSymbolProperties(Security security) - { - var symbolProperties = SymbolPropertiesDatabase.GetSymbolProperties(security.Symbol.ID.Market, security.Symbol, - security.Symbol.ID.SecurityType, security.QuoteCurrency.Symbol); - security.UpdateSymbolProperties(symbolProperties); - } - /// /// Resets the market hours database, forcing a reload when reused. /// Called in tests where multiple algorithms are run sequentially, @@ -263,15 +173,17 @@ protected virtual void UpdateSymbolProperties(Security security) /// protected virtual void ResetMarketHoursDatabase() { - MarketHoursDatabase.ReloadEntries(); + MarketHoursDatabase.UpdateDataFolderDatabase(); + Log.Trace("LiveTradingRealTimeHandler.ResetMarketHoursDatabase(): Updated market hours database."); } /// /// Resets the symbol properties database, forcing a reload when reused. /// - private void ResetSymbolPropertiesDatabase() + protected virtual void ResetSymbolPropertiesDatabase() { - SymbolPropertiesDatabase.ReloadEntries(); + SymbolPropertiesDatabase.UpdateDataFolderDatabase(); + Log.Trace("LiveTradingRealTimeHandler.ResetSymbolPropertiesDatabase(): Updated symbol properties database."); } } } diff --git a/Tests/Common/Securities/MarketHoursDatabaseTests.cs b/Tests/Common/Securities/MarketHoursDatabaseTests.cs index a3eb557b1021..d703df5d37eb 100644 --- a/Tests/Common/Securities/MarketHoursDatabaseTests.cs +++ b/Tests/Common/Securities/MarketHoursDatabaseTests.cs @@ -26,6 +26,18 @@ namespace QuantConnect.Tests.Common.Securities [TestFixture] public class MarketHoursDatabaseTests { + [SetUp] + public void Setup() + { + MarketHoursDatabase.Reset(); + } + + [TearDown] + public void TearDown() + { + MarketHoursDatabase.Reset(); + } + [Test] public void InitializesFromFile() { @@ -395,7 +407,7 @@ public void CustomEntriesAreNotLostWhenReset(string ticker, SecurityType securit MarketHoursDatabase.Entry returnedEntry; Assert.IsTrue(database.TryGetEntry(Market.USA, ticker, securityType, out returnedEntry)); Assert.AreEqual(returnedEntry, entry); - Assert.DoesNotThrow(() => database.ReloadEntries()); + Assert.DoesNotThrow(() => database.UpdateDataFolderDatabase()); Assert.IsTrue(database.TryGetEntry(Market.USA, ticker, securityType, out returnedEntry)); Assert.AreEqual(returnedEntry, entry); } diff --git a/Tests/Common/Securities/SymbolPropertiesDatabaseTests.cs b/Tests/Common/Securities/SymbolPropertiesDatabaseTests.cs index f44f751c0402..73d8fb34a9f3 100644 --- a/Tests/Common/Securities/SymbolPropertiesDatabaseTests.cs +++ b/Tests/Common/Securities/SymbolPropertiesDatabaseTests.cs @@ -162,7 +162,7 @@ public void CustomEntriesAreKeptAfterARefresh() Assert.AreSame(properties, fetchedProperties); // Refresh the database - database.ReloadEntries(); + database.UpdateDataFolderDatabase(); // Fetch the custom entry again to make sure it was not overridden fetchedProperties = database.GetSymbolProperties(Market.USA, symbol, SecurityType.Base, "USD"); @@ -185,7 +185,7 @@ public void CanQueryMarketAfterRefresh() Globals.Reset(); // Refresh the database - database.ReloadEntries(); + database.UpdateDataFolderDatabase(); // Get market again result = database.TryGetMarket("AU200AUD", SecurityType.Cfd, out market); diff --git a/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs b/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs index 5427a6054000..d9ea09272a46 100644 --- a/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs +++ b/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs @@ -34,16 +34,29 @@ using System.Reflection; using QuantConnect.Lean.Engine.HistoricalData; using QuantConnect.Lean.Engine.DataFeeds; -using QuantConnect.Util; using QuantConnect.Securities.Option; using QuantConnect.Securities.IndexOption; +using QuantConnect.Configuration; +using NodaTime; namespace QuantConnect.Tests.Engine.RealTime { [TestFixture] - [Parallelizable(ParallelScope.Children)] + [NonParallelizable] public class LiveTradingRealTimeHandlerTests { + [SetUp] + public void SetUp() + { + MarketHoursDatabase.Reset(); + } + + [TearDown] + public void TearDown() + { + MarketHoursDatabase.Reset(); + } + [Test] public void ThreadSafety() { @@ -61,8 +74,8 @@ public void ThreadSafety() realTimeHandler.SetTime(DateTime.UtcNow); // wait for the internal thread to start WaitUntilActive(realTimeHandler); - using var scheduledEvent = new ScheduledEvent("1", new []{ Time.EndOfTime }, (_, _) => { }); - using var scheduledEvent2 = new ScheduledEvent("2", new []{ Time.EndOfTime }, (_, _) => { }); + using var scheduledEvent = new ScheduledEvent("1", new[] { Time.EndOfTime }, (_, _) => { }); + using var scheduledEvent2 = new ScheduledEvent("2", new[] { Time.EndOfTime }, (_, _) => { }); Assert.DoesNotThrow(() => { for (var i = 0; i < 100000; i++) @@ -95,7 +108,7 @@ public void RefreshesMarketHoursCorrectly(SecurityExchangeHours securityExchange var time = new DateTime(2023, 5, 30).Date; var entry = new MarketHoursDatabase.Entry(TimeZones.NewYork, securityExchangeHours); var key = new SecurityDatabaseKey(Market.USA, null, SecurityType.Equity); - var mhdb = new MarketHoursDatabase(new Dictionary() { { key, entry} }); + var mhdb = new MarketHoursDatabase(new Dictionary() { { key, entry } }); realTimeHandler.SetMarketHoursDatabase(mhdb); realTimeHandler.TestRefreshMarketHoursToday(security, time, expectedSegment); } @@ -108,8 +121,8 @@ public void ResetMarketHoursCorrectly() algorithm.SetCash(100000); algorithm.SetStartDate(2023, 5, 30); algorithm.SetEndDate(2023, 5, 30); + MarketHoursDatabase.FromDataFolder().SetEntry(Market.USA, null, SecurityType.Equity, SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork)); var security = algorithm.AddEquity("SPY"); - security.Exchange = new SecurityExchange(SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork)); var symbol = security.Symbol; algorithm.SetFinishedWarmingUp(); @@ -187,13 +200,6 @@ public void RefreshesSymbolProperties(string refreshPeriodStr) // wait for the internal thread to start WaitUntilActive(realTimeHandler); - Assert.IsTrue(realTimeHandler.SpdbRefreshed.IsSet); - Assert.IsTrue(realTimeHandler.SecuritySymbolPropertiesUpdated.IsSet); - - realTimeHandler.SpdbRefreshed.Reset(); - realTimeHandler.SecuritySymbolPropertiesUpdated.Reset(); - - var events = new[] { realTimeHandler.SpdbRefreshed.WaitHandle, realTimeHandler.SecuritySymbolPropertiesUpdated.WaitHandle }; for (var i = 0; i < 10; i++) { timeProvider.Advance(step); @@ -201,13 +207,12 @@ public void RefreshesSymbolProperties(string refreshPeriodStr) // We only advanced half the time, so we should not have refreshed yet if (i % 2 == 0) { - Assert.IsFalse(WaitHandle.WaitAll(events, 5000)); + Assert.IsFalse(realTimeHandler.SpdbRefreshed.Wait(5000)); } else { - Assert.IsTrue(WaitHandle.WaitAll(events, 5000)); + Assert.IsTrue(realTimeHandler.SpdbRefreshed.Wait(5000)); realTimeHandler.SpdbRefreshed.Reset(); - realTimeHandler.SecuritySymbolPropertiesUpdated.Reset(); } } } @@ -250,22 +255,15 @@ public void SecuritySymbolPropertiesTypeIsRespectedAfterRefresh(SecurityType sec // wait for the internal thread to start WaitUntilActive(realTimeHandler); - Assert.IsTrue(realTimeHandler.SpdbRefreshed.IsSet); - Assert.IsTrue(realTimeHandler.SecuritySymbolPropertiesUpdated.IsSet); - - realTimeHandler.SpdbRefreshed.Reset(); - realTimeHandler.SecuritySymbolPropertiesUpdated.Reset(); - var previousSymbolProperties = security.SymbolProperties; // Refresh the spdb timeProvider.Advance(refreshPeriod); Assert.IsTrue(realTimeHandler.SpdbRefreshed.Wait(5000)); - Assert.IsTrue(realTimeHandler.SecuritySymbolPropertiesUpdated.Wait(5000)); // Access the symbol properties again - // The instance must have been changed - Assert.AreNotSame(security.SymbolProperties, previousSymbolProperties); + // The instance must have not been changed + Assert.AreSame(security.SymbolProperties, previousSymbolProperties); Assert.IsInstanceOf(expectedSymbolPropertiesType, security.SymbolProperties); } @@ -317,9 +315,8 @@ public bool TryRequestAdditionalTime(int minutes) } } - public class TestLiveTradingRealTimeHandler: LiveTradingRealTimeHandler + public class TestLiveTradingRealTimeHandler : LiveTradingRealTimeHandler { - private static AutoResetEvent OnSecurityUpdated = new AutoResetEvent(false); private MarketHoursDatabase newMarketHoursDatabase; public void SetMarketHoursDatabase(MarketHoursDatabase marketHoursDatabase) { @@ -329,7 +326,7 @@ protected override void ResetMarketHoursDatabase() { if (newMarketHoursDatabase != null) { - MarketHoursDatabase = newMarketHoursDatabase; + MarketHoursDatabase.Merge(newMarketHoursDatabase, resetCustomEntries: false); } else { @@ -339,18 +336,10 @@ protected override void ResetMarketHoursDatabase() public void TestRefreshMarketHoursToday(Security security, DateTime time, MarketHoursSegment expectedSegment) { - OnSecurityUpdated.Reset(); - RefreshMarketHours(time); - OnSecurityUpdated.WaitOne(); + ResetMarketHoursDatabase(); AssertMarketHours(security, time, expectedSegment); } - protected override void UpdateMarketHours(Security security) - { - base.UpdateMarketHours(security); - OnSecurityUpdated.Set(); - } - public void AssertMarketHours(Security security, DateTime time, MarketHoursSegment expectedSegment) { var marketHours = security.Exchange.Hours.GetMarketHours(time); @@ -385,7 +374,7 @@ public void AddRefreshHoursScheduledEvent() using var scheduledEvent = new ScheduledEvent("RefreshHours", new[] { new DateTime(2023, 6, 29) }, (name, triggerTime) => { // refresh market hours from api every day - RefreshMarketHours((new DateTime(2023, 5, 30)).Date); + ResetMarketHoursDatabase(); }); Add(scheduledEvent); OnSecurityUpdated.Reset(); @@ -395,61 +384,37 @@ public void AddRefreshHoursScheduledEvent() Exit(); } - protected override void UpdateMarketHours(Security security) - { - base.UpdateMarketHours(security); - OnSecurityUpdated.Set(); - } - protected override void ResetMarketHoursDatabase() { var entry = new MarketHoursDatabase.Entry(TimeZones.NewYork, ExchangeHoursDataClass.CreateExchangeHoursWithHolidays()); var key = new SecurityDatabaseKey(Market.USA, null, SecurityType.Equity); var mhdb = new MarketHoursDatabase(new Dictionary() { { key, entry } }); - MarketHoursDatabase = mhdb; + MarketHoursDatabase.Merge(mhdb, resetCustomEntries: true); + OnSecurityUpdated.Set(); } } private class SPDBTestLiveTradingRealTimeHandler : LiveTradingRealTimeHandler, IDisposable { private bool _disposed; - private int _securitiesUpdated; public ManualTimeProvider PublicTimeProvider = new ManualTimeProvider(); protected override ITimeProvider TimeProvider { get { return PublicTimeProvider; } } public ManualResetEventSlim SpdbRefreshed = new ManualResetEventSlim(false); - public ManualResetEventSlim SecuritySymbolPropertiesUpdated = new ManualResetEventSlim(false); - protected override void RefreshSymbolProperties() + protected override void ResetSymbolPropertiesDatabase() { - if (_disposed) return; - - base.RefreshSymbolProperties(); + base.ResetSymbolPropertiesDatabase(); SpdbRefreshed.Set(); } - protected override void UpdateSymbolProperties(Security security) - { - if (_disposed) return; - - base.UpdateSymbolProperties(security); - Algorithm.Log($"{Algorithm.Securities.Count}"); - - if (++_securitiesUpdated == Algorithm.Securities.Count) - { - SecuritySymbolPropertiesUpdated.Set(); - _securitiesUpdated = 0; - } - } - public void Dispose() { if (_disposed) return; Exit(); SpdbRefreshed.Dispose(); - SecuritySymbolPropertiesUpdated.Dispose(); _disposed = true; } } @@ -468,7 +433,7 @@ public static IEnumerable TestCases { get { - yield return new TestCaseData(CreateExchangeHoursWithEarlyCloseAndLateOpen(), new MarketHoursSegment(MarketHoursState.Market,new TimeSpan(10, 0, 0), new TimeSpan(13, 0, 0))); + yield return new TestCaseData(CreateExchangeHoursWithEarlyCloseAndLateOpen(), new MarketHoursSegment(MarketHoursState.Market, new TimeSpan(10, 0, 0), new TimeSpan(13, 0, 0))); yield return new TestCaseData(CreateExchangeHoursWithEarlyClose(), new MarketHoursSegment(MarketHoursState.Market, new TimeSpan(9, 30, 0), new TimeSpan(13, 0, 0))); yield return new TestCaseData(CreateExchangeHoursWithLateOpen(), new MarketHoursSegment(MarketHoursState.Market, new TimeSpan(10, 0, 0), new TimeSpan(16, 0, 0))); yield return new TestCaseData(CreateExchangeHoursWithHolidays(), null); @@ -521,5 +486,337 @@ public static SecurityExchangeHours CreateExchangeHoursWithHolidays() } } + + [TestFixture] + [NonParallelizable] + public class DateTimeRulesPickUpMarketHoursUpdates + { + private string _originalCacheDataFolder; + + [SetUp] + public void SetUp() + { + _originalCacheDataFolder = Config.Get("cache-location"); + Config.Set("cache-location", "TestData/dynamic-market-hours/original"); + Globals.Reset(); + MarketHoursDatabase.Reset(); + } + + [TearDown] + public void TearDown() + { + Config.Set("cache-location", _originalCacheDataFolder); + Globals.Reset(); + MarketHoursDatabase.Reset(); + } + + private static IEnumerable TestCases() + { + // For this test case, market close will be updated from 4pm to 1pm. + // So we will schedule an event to be fired on market close + var expectedEventsFireTimesBeforeUpdate = new List + { + new(2024, 12, 02, 16, 0, 0), + new(2024, 12, 03, 16, 0, 0), + new(2024, 12, 04, 16, 0, 0), + new(2024, 12, 05, 16, 0, 0), + new(2024, 12, 06, 16, 0, 0), + new(2024, 12, 09, 16, 0, 0), + new(2024, 12, 10, 16, 0, 0) + }; + var expectedEventsFireTimesAfterUpdate = new List + { + // Move next will already happen, so this first event will still be fired on the old market close time + new(2024, 12, 11, 16, 0, 0), + new(2024, 12, 12, 13, 0, 0), + new(2024, 12, 13, 13, 0, 0), + new(2024, 12, 16, 13, 0, 0), + new(2024, 12, 17, 13, 0, 0), + new(2024, 12, 18, 13, 0, 0) + }; + var updatedMhdbFile = "TestData/dynamic-market-hours/modified-close"; + + foreach (var withAddedSecurity in new[] { true, false }) + { + yield return new TestCaseData(updatedMhdbFile, + expectedEventsFireTimesBeforeUpdate, + expectedEventsFireTimesAfterUpdate, + false, + withAddedSecurity); + } + + // For this test case a holiday will be added, so we will schedule an event to be fired every day at noon. + expectedEventsFireTimesBeforeUpdate = new List + { + new(2024, 12, 02, 12, 0, 0), + new(2024, 12, 03, 12, 0, 0), + new(2024, 12, 04, 12, 0, 0), + new(2024, 12, 05, 12, 0, 0), + new(2024, 12, 06, 12, 0, 0), + new(2024, 12, 09, 12, 0, 0), + new(2024, 12, 10, 12, 0, 0) + }; + expectedEventsFireTimesAfterUpdate = new List + { + new(2024, 12, 11, 12, 0, 0), + new(2024, 12, 12, 12, 0, 0), + // 13th is a holiday, and 14th and 15th are weekend days + new(2024, 12, 16, 12, 0, 0), + new(2024, 12, 17, 12, 0, 0), + new(2024, 12, 18, 12, 0, 0) + }; + updatedMhdbFile = "TestData/dynamic-market-hours/modified-holidays"; + + foreach (var withAddedSecurity in new[] { true, false }) + { + yield return new TestCaseData(updatedMhdbFile, + expectedEventsFireTimesBeforeUpdate, + expectedEventsFireTimesAfterUpdate, + true, + withAddedSecurity); + } + } + + [TestCaseSource(nameof(TestCases))] + public void EventsAreFiredOnUpdatedRules(string updatedMhdbFile, + List expectedEventsFireTimesBeforeUpdate, + List expectedEventsFireTimesAfterUpdate, + bool updatedHolidays, + bool addedSecurity) + { + var algorithm = new AlgorithmStub(); + algorithm.SetStartDate(2024, 12, 02); + + // "Disable" mhdb automatic refresh to avoid interference with the test + algorithm.Settings.DatabasesRefreshPeriod = TimeSpan.FromDays(30); + algorithm.SetFinishedWarmingUp(); + + var symbol = addedSecurity ? algorithm.AddEquity("SPY").Symbol : Symbols.SPY; + + var realTimeHandler = new TestRealTimeHandler(); + realTimeHandler.PublicTimeProvider.SetCurrentTimeUtc(algorithm.StartDate.ConvertToUtc(algorithm.TimeZone)); + realTimeHandler.Setup(algorithm, + new AlgorithmNodePacket(PacketType.AlgorithmNode), + new BacktestingResultHandler(), + null, + new TestTimeLimitManager()); + + algorithm.Schedule.SetEventSchedule(realTimeHandler); + + // Start the real time handler thread + realTimeHandler.SetTime(realTimeHandler.PublicTimeProvider.GetUtcNow()); + + WaitUntilActive(realTimeHandler); + + try + { + var mhdb = MarketHoursDatabase.FromDataFolder(); + var marketHoursEntry = mhdb.GetEntry(symbol.ID.Market, symbol, symbol.SecurityType); + var exchangeTimeZone = marketHoursEntry.ExchangeHours.TimeZone; + + // Schedule an event every day at market close + var firedEventTimes = new List(); + using var fireEvent = new ManualResetEventSlim(); + + if (updatedHolidays) + { + algorithm.Schedule.On(algorithm.DateRules.EveryDay(symbol), algorithm.TimeRules.Noon, () => + { + firedEventTimes.Add(realTimeHandler.PublicTimeProvider.GetUtcNow().ConvertFromUtc(algorithm.TimeZone)); + fireEvent.Set(); + }); + } + else + { + algorithm.Schedule.On(algorithm.DateRules.EveryDay(symbol), algorithm.TimeRules.BeforeMarketClose(symbol, 0), () => + { + firedEventTimes.Add(realTimeHandler.PublicTimeProvider.GetUtcNow().ConvertFromUtc(exchangeTimeZone)); + fireEvent.Set(); + }); + } + + // Events should be fired every week day at 16:00 (market close) + + AssertScheduledEvents(realTimeHandler, exchangeTimeZone, fireEvent, firedEventTimes, expectedEventsFireTimesBeforeUpdate); + + Config.Set("cache-location", updatedMhdbFile); + Globals.Reset(); + realTimeHandler.ResetMarketHoursPublic(); + + firedEventTimes.Clear(); + + AssertScheduledEvents(realTimeHandler, exchangeTimeZone, fireEvent, firedEventTimes, expectedEventsFireTimesAfterUpdate); + + // Just a final check: directly check for the market hours update in the data base + marketHoursEntry = mhdb.GetEntry(symbol.ID.Market, symbol, symbol.SecurityType); + if (updatedHolidays) + { + CollectionAssert.Contains(marketHoursEntry.ExchangeHours.Holidays, new DateTime(2024, 12, 13)); + } + else + { + foreach (var hours in marketHoursEntry.ExchangeHours.MarketHours.Values.Where(x => x.DayOfWeek != DayOfWeek.Saturday && x.DayOfWeek != DayOfWeek.Sunday)) + { + Assert.AreEqual(1, hours.Segments.Count); + Assert.AreEqual(new TimeSpan(13, 0, 0), hours.Segments[0].End); + } + } + } + finally + { + realTimeHandler.Exit(); + } + } + + private static void AssertScheduledEvents(TestRealTimeHandler realTimeHandler, DateTimeZone timeZone, + ManualResetEventSlim fireEvent, List firedEventTimes, List expectedEventsFireTimes) + { + while (firedEventTimes.Count < expectedEventsFireTimes.Count) + { + var currentEventsCount = firedEventTimes.Count; + var utcNow = realTimeHandler.PublicTimeProvider.GetUtcNow(); + var nextTimeUtc = utcNow.AddMinutes(60); + + realTimeHandler.PublicTimeProvider.SetCurrentTimeUtc(nextTimeUtc); + + if (currentEventsCount < expectedEventsFireTimes.Count && + nextTimeUtc.ConvertFromUtc(timeZone) >= expectedEventsFireTimes[currentEventsCount]) + { + Assert.IsTrue(fireEvent.Wait(1000)); + fireEvent.Reset(); + + Assert.AreEqual(currentEventsCount + 1, firedEventTimes.Count); + Assert.AreEqual(expectedEventsFireTimes[currentEventsCount], firedEventTimes.Last()); + } + } + } + + private class TestRealTimeHandler : LiveTradingRealTimeHandler + { + public ManualTimeProvider PublicTimeProvider { get; set; } = new ManualTimeProvider(); + + protected override ITimeProvider TimeProvider => PublicTimeProvider; + + public void ResetMarketHoursPublic() + { + ResetMarketHoursDatabase(); + } + } + } + + [TestFixture] + [NonParallelizable] + public class SymbolPropertiesAreUpdated + { + private string _originalCacheDataFolder; + + [SetUp] + public void SetUp() + { + _originalCacheDataFolder = Config.Get("cache-location"); + Config.Set("cache-location", "TestData/dynamic-symbol-properties/original"); + Globals.Reset(); + SymbolPropertiesDatabase.Reset(); + } + + [TearDown] + public void TearDown() + { + Config.Set("cache-location", _originalCacheDataFolder); + Globals.Reset(); + SymbolPropertiesDatabase.Reset(); + } + + [Test] + public void SecurityGetsSymbolPropertiesUpdates() + { + var algorithm = new AlgorithmStub(); + algorithm.SetStartDate(2024, 12, 02); + + // "Disable" automatic refresh to avoid interference with the test + algorithm.Settings.DatabasesRefreshPeriod = TimeSpan.FromDays(30); + algorithm.SetFinishedWarmingUp(); + + var security = algorithm.AddEquity("SPY"); + var symbol = security.Symbol; + + var realTimeHandler = new TestRealTimeHandler(); + realTimeHandler.PublicTimeProvider.SetCurrentTimeUtc(algorithm.StartDate.ConvertToUtc(algorithm.TimeZone)); + realTimeHandler.Setup(algorithm, + new AlgorithmNodePacket(PacketType.AlgorithmNode), + new BacktestingResultHandler(), + null, + new TestTimeLimitManager()); + + algorithm.Schedule.SetEventSchedule(realTimeHandler); + + // Start the real time handler thread + realTimeHandler.SetTime(realTimeHandler.PublicTimeProvider.GetUtcNow()); + WaitUntilActive(realTimeHandler); + + try + { + var spdb = SymbolPropertiesDatabase.FromDataFolder(); + var entry = spdb.GetSymbolProperties(Market.USA, symbol, symbol.SecurityType, "USD"); + var securityEntry = security.SymbolProperties; + + Assert.AreEqual(entry.Description, securityEntry.Description); + Assert.AreEqual(entry.QuoteCurrency, securityEntry.QuoteCurrency); + Assert.AreEqual(entry.ContractMultiplier, securityEntry.ContractMultiplier); + Assert.AreEqual(entry.MinimumPriceVariation, securityEntry.MinimumPriceVariation); + Assert.AreEqual(entry.LotSize, securityEntry.LotSize); + Assert.AreEqual(entry.MarketTicker, securityEntry.MarketTicker); + Assert.AreEqual(entry.MinimumOrderSize, securityEntry.MinimumOrderSize); + Assert.AreEqual(entry.PriceMagnifier, securityEntry.PriceMagnifier); + Assert.AreEqual(entry.StrikeMultiplier, securityEntry.StrikeMultiplier); + + // Back up entry + entry = new SymbolProperties(entry.Description, entry.QuoteCurrency, entry.ContractMultiplier, entry.MinimumPriceVariation, entry.LotSize, entry.MarketTicker, entry.MinimumOrderSize, entry.PriceMagnifier, entry.StrikeMultiplier); + + Config.Set("cache-location", "TestData/dynamic-symbol-properties/modified"); + Globals.Reset(); + realTimeHandler.ResetSymbolPropertiesDatabasePublic(); + + var newEntry = spdb.GetSymbolProperties(Market.USA, symbol, symbol.SecurityType, "USD"); + + Assert.AreEqual(newEntry.Description, securityEntry.Description); + Assert.AreEqual(newEntry.QuoteCurrency, securityEntry.QuoteCurrency); + Assert.AreEqual(newEntry.ContractMultiplier, securityEntry.ContractMultiplier); + Assert.AreEqual(newEntry.MinimumPriceVariation, securityEntry.MinimumPriceVariation); + Assert.AreEqual(newEntry.LotSize, securityEntry.LotSize); + Assert.AreEqual(newEntry.MarketTicker, securityEntry.MarketTicker); + Assert.AreEqual(newEntry.MinimumOrderSize, securityEntry.MinimumOrderSize); + Assert.AreEqual(newEntry.PriceMagnifier, securityEntry.PriceMagnifier); + Assert.AreEqual(newEntry.StrikeMultiplier, securityEntry.StrikeMultiplier); + + // The old entry must be outdated + Assert.IsTrue(entry.Description != securityEntry.Description || + entry.QuoteCurrency != securityEntry.QuoteCurrency || + entry.ContractMultiplier != securityEntry.ContractMultiplier || + entry.MinimumPriceVariation != securityEntry.MinimumPriceVariation || + entry.LotSize != securityEntry.LotSize || + entry.MarketTicker != securityEntry.MarketTicker || + entry.MinimumOrderSize != securityEntry.MinimumOrderSize || + entry.PriceMagnifier != securityEntry.PriceMagnifier || + entry.StrikeMultiplier != securityEntry.StrikeMultiplier); + } + finally + { + realTimeHandler.Exit(); + } + } + + private class TestRealTimeHandler : LiveTradingRealTimeHandler + { + public ManualTimeProvider PublicTimeProvider { get; set; } = new ManualTimeProvider(); + + protected override ITimeProvider TimeProvider => PublicTimeProvider; + + public void ResetSymbolPropertiesDatabasePublic() + { + ResetSymbolPropertiesDatabase(); + } + } + } } } diff --git a/Tests/QuantConnect.Tests.csproj b/Tests/QuantConnect.Tests.csproj index 04d86fb0a8e3..acb371684612 100644 --- a/Tests/QuantConnect.Tests.csproj +++ b/Tests/QuantConnect.Tests.csproj @@ -634,5 +634,20 @@ PreserveNewest + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + diff --git a/Tests/TestData/dynamic-market-hours/modified-close/market-hours/market-hours-database.json b/Tests/TestData/dynamic-market-hours/modified-close/market-hours/market-hours-database.json new file mode 100644 index 000000000000..7b07ebd54bdb --- /dev/null +++ b/Tests/TestData/dynamic-market-hours/modified-close/market-hours/market-hours-database.json @@ -0,0 +1,49 @@ +{ + "entries": { + "Equity-usa-[*]": { + "dataTimeZone": "America/New_York", + "exchangeTimeZone": "America/New_York", + "sunday": [], + "monday": [ + { + "start": "09:30:00", + "end": "13:00:00", + "state": "market" + } + ], + "tuesday": [ + { + "start": "09:30:00", + "end": "13:00:00", + "state": "market" + } + ], + "wednesday": [ + { + "start": "09:30:00", + "end": "13:00:00", + "state": "market" + } + ], + "thursday": [ + { + "start": "09:30:00", + "end": "13:00:00", + "state": "market" + } + ], + "friday": [ + { + "start": "09:30:00", + "end": "13:00:00", + "state": "market" + } + ], + "saturday": [], + "holidays": [ + "12/25/2024" + ], + "earlyCloses": {} + } + } +} \ No newline at end of file diff --git a/Tests/TestData/dynamic-market-hours/modified-holidays/market-hours/market-hours-database.json b/Tests/TestData/dynamic-market-hours/modified-holidays/market-hours/market-hours-database.json new file mode 100644 index 000000000000..8b145d44844d --- /dev/null +++ b/Tests/TestData/dynamic-market-hours/modified-holidays/market-hours/market-hours-database.json @@ -0,0 +1,100 @@ +{ + "entries": { + "Equity-usa-[*]": { + "dataTimeZone": "America/New_York", + "exchangeTimeZone": "America/New_York", + "sunday": [], + "monday": [ + { + "start": "04:00:00", + "end": "09:30:00", + "state": "premarket" + }, + { + "start": "09:30:00", + "end": "16:00:00", + "state": "market" + }, + { + "start": "16:00:00", + "end": "20:00:00", + "state": "postmarket" + } + ], + "tuesday": [ + { + "start": "04:00:00", + "end": "09:30:00", + "state": "premarket" + }, + { + "start": "09:30:00", + "end": "16:00:00", + "state": "market" + }, + { + "start": "16:00:00", + "end": "20:00:00", + "state": "postmarket" + } + ], + "wednesday": [ + { + "start": "04:00:00", + "end": "09:30:00", + "state": "premarket" + }, + { + "start": "09:30:00", + "end": "16:00:00", + "state": "market" + }, + { + "start": "16:00:00", + "end": "20:00:00", + "state": "postmarket" + } + ], + "thursday": [ + { + "start": "04:00:00", + "end": "09:30:00", + "state": "premarket" + }, + { + "start": "09:30:00", + "end": "16:00:00", + "state": "market" + }, + { + "start": "16:00:00", + "end": "20:00:00", + "state": "postmarket" + } + ], + "friday": [ + { + "start": "04:00:00", + "end": "09:30:00", + "state": "premarket" + }, + { + "start": "09:30:00", + "end": "16:00:00", + "state": "market" + }, + { + "start": "16:00:00", + "end": "20:00:00", + "state": "postmarket" + } + ], + "saturday": [], + "holidays": [ + "12/13/2024", + "12/25/2024" + ], + "earlyCloses": {} + } + } +} \ No newline at end of file diff --git a/Tests/TestData/dynamic-market-hours/original/market-hours/market-hours-database.json b/Tests/TestData/dynamic-market-hours/original/market-hours/market-hours-database.json new file mode 100644 index 000000000000..0680b4d63c47 --- /dev/null +++ b/Tests/TestData/dynamic-market-hours/original/market-hours/market-hours-database.json @@ -0,0 +1,99 @@ +{ + "entries": { + "Equity-usa-[*]": { + "dataTimeZone": "America/New_York", + "exchangeTimeZone": "America/New_York", + "sunday": [], + "monday": [ + { + "start": "04:00:00", + "end": "09:30:00", + "state": "premarket" + }, + { + "start": "09:30:00", + "end": "16:00:00", + "state": "market" + }, + { + "start": "16:00:00", + "end": "20:00:00", + "state": "postmarket" + } + ], + "tuesday": [ + { + "start": "04:00:00", + "end": "09:30:00", + "state": "premarket" + }, + { + "start": "09:30:00", + "end": "16:00:00", + "state": "market" + }, + { + "start": "16:00:00", + "end": "20:00:00", + "state": "postmarket" + } + ], + "wednesday": [ + { + "start": "04:00:00", + "end": "09:30:00", + "state": "premarket" + }, + { + "start": "09:30:00", + "end": "16:00:00", + "state": "market" + }, + { + "start": "16:00:00", + "end": "20:00:00", + "state": "postmarket" + } + ], + "thursday": [ + { + "start": "04:00:00", + "end": "09:30:00", + "state": "premarket" + }, + { + "start": "09:30:00", + "end": "16:00:00", + "state": "market" + }, + { + "start": "16:00:00", + "end": "20:00:00", + "state": "postmarket" + } + ], + "friday": [ + { + "start": "04:00:00", + "end": "09:30:00", + "state": "premarket" + }, + { + "start": "09:30:00", + "end": "16:00:00", + "state": "market" + }, + { + "start": "16:00:00", + "end": "20:00:00", + "state": "postmarket" + } + ], + "saturday": [], + "holidays": [ + "12/25/2024" + ], + "earlyCloses": {} + } + } +} \ No newline at end of file diff --git a/Tests/TestData/dynamic-symbol-properties/modified/symbol-properties/symbol-properties-database.csv b/Tests/TestData/dynamic-symbol-properties/modified/symbol-properties/symbol-properties-database.csv new file mode 100644 index 000000000000..ec6486f8fe55 --- /dev/null +++ b/Tests/TestData/dynamic-symbol-properties/modified/symbol-properties/symbol-properties-database.csv @@ -0,0 +1,2 @@ +market,symbol,type,description,quote_currency,contract_multiplier,minimum_price_variation,lot_size,market_ticker,minimum_order_size,price_magnifier,strike_multiplier +usa,SPY,equity,,USD,1,0.001,10 diff --git a/Tests/TestData/dynamic-symbol-properties/original/symbol-properties/symbol-properties-database.csv b/Tests/TestData/dynamic-symbol-properties/original/symbol-properties/symbol-properties-database.csv new file mode 100644 index 000000000000..bf79f1a3c6c7 --- /dev/null +++ b/Tests/TestData/dynamic-symbol-properties/original/symbol-properties/symbol-properties-database.csv @@ -0,0 +1,2 @@ +market,symbol,type,description,quote_currency,contract_multiplier,minimum_price_variation,lot_size,market_ticker,minimum_order_size,price_magnifier,strike_multiplier +usa,SPY,equity,,USD,1,0.01,1