From e7361d64366cd8eaa9aee153ac50cf8654b4ea83 Mon Sep 17 00:00:00 2001 From: Marinovsky Date: Thu, 11 Jan 2024 17:19:54 -0500 Subject: [PATCH 01/16] Initial draft of the solution --- Engine/RealTime/LiveTradingRealTimeHandler.cs | 15 ++++- .../LiveTradingRealTimeHandlerTests.cs | 62 +++++++++++++++++++ 2 files changed, 74 insertions(+), 3 deletions(-) diff --git a/Engine/RealTime/LiveTradingRealTimeHandler.cs b/Engine/RealTime/LiveTradingRealTimeHandler.cs index 2bdc21adb423..793063957075 100644 --- a/Engine/RealTime/LiveTradingRealTimeHandler.cs +++ b/Engine/RealTime/LiveTradingRealTimeHandler.cs @@ -36,7 +36,7 @@ public class LiveTradingRealTimeHandler : BacktestingRealTimeHandler { private Thread _realTimeThread; private CancellationTokenSource _cancellationTokenSource = new(); - private static MarketHoursDatabase _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); + protected static MarketHoursDatabase MarketHoursDatabase = MarketHoursDatabase.FromDataFolder(); /// /// Boolean flag indicating thread state. @@ -124,6 +124,7 @@ private void RefreshMarketHoursToday(DateTime date) { var security = kvp.Value; + MarketHoursDatabase.Reset(); var marketHours = MarketToday(date, security.Symbol); security.Exchange.SetMarketHours(marketHours, date.DayOfWeek); var localMarketHours = security.Exchange.Hours.MarketHours[date.DayOfWeek]; @@ -185,8 +186,16 @@ private IEnumerable MarketToday(DateTime time, Symbol symbol yield break; } - var hours = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.ID.SecurityType); - foreach (var segment in hours.MarketHours[time.DayOfWeek].Segments) + var entry = MarketHoursDatabase.GetEntry(symbol.ID.Market, symbol, symbol.ID.SecurityType); + var securityExchangeHours = new SecurityExchangeHours( + entry.DataTimeZone, + entry.ExchangeHours.Holidays, + entry.ExchangeHours.MarketHours.ToDictionary(), + entry.ExchangeHours.EarlyCloses, + entry.ExchangeHours.LateOpens); + var hours = securityExchangeHours.GetMarketHours(time); + + foreach (var segment in hours.Segments) { yield return segment; } diff --git a/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs b/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs index 75a5660945a8..77a7841e6d7e 100644 --- a/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs +++ b/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs @@ -22,6 +22,14 @@ using QuantConnect.Lean.Engine.Results; using QuantConnect.Lean.Engine.RealTime; using QuantConnect.Tests.Engine.DataFeeds; +using QuantConnect.Algorithm; +using QuantConnect.AlgorithmFactory.Python.Wrappers; +using QuantConnect.Interfaces; +using QuantConnect.Tests.Common.Data.UniverseSelection; +using System.Linq; +using QuantConnect.Securities; +using QuantConnect.Tests.Common.Securities; +using System.Collections.Generic; namespace QuantConnect.Tests.Engine.RealTime { @@ -63,6 +71,33 @@ public void ThreadSafety() realTimeHandler.Exit(); } + [Test] + public void RefreshesMarketHoursCorrectly() + { + Security security; + var algorithm = new AlgorithmStub(); + security = algorithm.AddEquity("SPY"); + + var realTimeHandler = new TestLiveTradingRealTimeHandler(); + realTimeHandler.Setup(algorithm, + new AlgorithmNodePacket(PacketType.AlgorithmNode), + new BacktestingResultHandler(), + null, + new TestTimeLimitManager()); + + // Because neither implement EOD() deprecated it should be zero + var segments = security.Exchange.Hours.MarketHours[DateTime.UtcNow.AddDays(1).DayOfWeek].Segments.Count; + Assert.AreEqual(3, segments); + var securityExchangeHours = CreateExchangeHours(); + var entry = new MarketHoursDatabase.Entry(TimeZones.NewYork, securityExchangeHours); + var key = new SecurityDatabaseKey(Market.USA, null, SecurityType.Equity); + var mhdb = new MarketHoursDatabase(new Dictionary() { { key, entry} }); + realTimeHandler.SetMarketHoursDatabase(mhdb); + realTimeHandler.ScanPastEvents(DateTime.UtcNow.AddDays(1)); + segments = security.Exchange.Hours.MarketHours[DateTime.UtcNow.AddDays(1).DayOfWeek].Segments.Count; + Assert.AreEqual(1, segments); + } + private class TestTimeLimitManager : IIsolatorLimitResultProvider { public IsolatorLimitResult IsWithinLimit() @@ -78,5 +113,32 @@ public bool TryRequestAdditionalTime(int minutes) throw new NotImplementedException(); } } + + public class TestLiveTradingRealTimeHandler: LiveTradingRealTimeHandler + { + public void SetMarketHoursDatabase(MarketHoursDatabase marketHoursDatabase) + { + MarketHoursDatabase = marketHoursDatabase; + } + } + + public static SecurityExchangeHours CreateExchangeHours() + { + var sunday = LocalMarketHours.ClosedAllDay(DayOfWeek.Sunday); + var monday = new LocalMarketHours(DayOfWeek.Monday, new TimeSpan(9, 30, 0), new TimeSpan(16, 0, 0)); + var tuesday = new LocalMarketHours(DayOfWeek.Tuesday, new TimeSpan(9, 30, 0), new TimeSpan(16, 0, 0)); + var wednesday = new LocalMarketHours(DayOfWeek.Wednesday, new TimeSpan(9, 30, 0), new TimeSpan(16, 0, 0)); + var thursday = new LocalMarketHours(DayOfWeek.Thursday, new TimeSpan(9, 30, 0), new TimeSpan(16, 0, 0)); + var friday = new LocalMarketHours(DayOfWeek.Friday, new TimeSpan(9, 30, 0), new TimeSpan(16, 0, 0)); + var saturday = LocalMarketHours.ClosedAllDay(DayOfWeek.Saturday); + + var earlyCloses = new Dictionary { { DateTime.UtcNow.AddDays(1), new TimeSpan(13, 0, 0) } }; + var lateOpens = new Dictionary() { { DateTime.UtcNow.AddDays(1), new TimeSpan(10, 0, 0) } }; + var exchangeHours = new SecurityExchangeHours(TimeZones.NewYork, new List(), new[] + { + sunday, monday, tuesday, wednesday, thursday, friday, saturday + }.ToDictionary(x => x.DayOfWeek), earlyCloses, lateOpens); + return exchangeHours; + } } } From 9508edcc9bcaffce99a16db7e6f3f2692a38b564 Mon Sep 17 00:00:00 2001 From: Marinovsky Date: Fri, 12 Jan 2024 13:11:51 -0500 Subject: [PATCH 02/16] Add improvments --- Common/Securities/SecurityExchangeHours.cs | 5 + Engine/RealTime/LiveTradingRealTimeHandler.cs | 13 ++- .../LiveTradingRealTimeHandlerTests.cs | 108 ++++++++++++++---- 3 files changed, 97 insertions(+), 29 deletions(-) diff --git a/Common/Securities/SecurityExchangeHours.cs b/Common/Securities/SecurityExchangeHours.cs index 7bf09c7bb98b..4b952bb97ee2 100644 --- a/Common/Securities/SecurityExchangeHours.cs +++ b/Common/Securities/SecurityExchangeHours.cs @@ -435,6 +435,11 @@ private void SetMarketHoursForDay(DayOfWeek dayOfWeek, out LocalMarketHours loca /// The local date time to retrieve market hours for public LocalMarketHours GetMarketHours(DateTime localDateTime) { + if (_holidays.Contains(localDateTime.Ticks)) + { + return new LocalMarketHours(localDateTime.DayOfWeek); + } + LocalMarketHours marketHours; switch (localDateTime.DayOfWeek) { diff --git a/Engine/RealTime/LiveTradingRealTimeHandler.cs b/Engine/RealTime/LiveTradingRealTimeHandler.cs index 793063957075..cc6bf10561c3 100644 --- a/Engine/RealTime/LiveTradingRealTimeHandler.cs +++ b/Engine/RealTime/LiveTradingRealTimeHandler.cs @@ -36,7 +36,7 @@ public class LiveTradingRealTimeHandler : BacktestingRealTimeHandler { private Thread _realTimeThread; private CancellationTokenSource _cancellationTokenSource = new(); - protected static MarketHoursDatabase MarketHoursDatabase = MarketHoursDatabase.FromDataFolder(); + protected MarketHoursDatabase MarketHoursDatabase = MarketHoursDatabase.FromDataFolder(); /// /// Boolean flag indicating thread state. @@ -113,19 +113,20 @@ private void Run() } /// - /// Refresh the Today variable holding the market hours information + /// Refresh the market hours for each security in the given date /// + /// Each time this method is called, the MarketHoursDatabase is reset private void RefreshMarketHoursToday(DateTime date) { date = date.Date; + MarketHoursDatabase.Reset(); // update market hours for each security foreach (var kvp in Algorithm.Securities) { var security = kvp.Value; - MarketHoursDatabase.Reset(); - var marketHours = MarketToday(date, security.Symbol); + var marketHours = GetMarketHours(date, security.Symbol); security.Exchange.SetMarketHours(marketHours, date.DayOfWeek); var localMarketHours = security.Exchange.Hours.MarketHours[date.DayOfWeek]; Log.Trace($"LiveTradingRealTimeHandler.RefreshMarketHoursToday({security.Type}): Market hours set: Symbol: {security.Symbol} {localMarketHours} ({security.Exchange.Hours.TimeZone})"); @@ -176,9 +177,9 @@ public override void Exit() } /// - /// Get the calendar open hours for the date. + /// Get the market hours for the given symbol and date /// - private IEnumerable MarketToday(DateTime time, Symbol symbol) + private IEnumerable GetMarketHours(DateTime time, Symbol symbol) { if (Config.GetBool("force-exchange-always-open")) { diff --git a/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs b/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs index 77a7841e6d7e..0e6278406be2 100644 --- a/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs +++ b/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs @@ -71,8 +71,8 @@ public void ThreadSafety() realTimeHandler.Exit(); } - [Test] - public void RefreshesMarketHoursCorrectly() + [TestCaseSource(typeof(ExchangeHoursDataClass), nameof(ExchangeHoursDataClass.TestCases))] + public void RefreshesMarketHoursCorrectly(SecurityExchangeHours securityExchangeHours, MarketHoursSegment expectedSegment) { Security security; var algorithm = new AlgorithmStub(); @@ -85,17 +85,30 @@ public void RefreshesMarketHoursCorrectly() null, new TestTimeLimitManager()); - // Because neither implement EOD() deprecated it should be zero - var segments = security.Exchange.Hours.MarketHours[DateTime.UtcNow.AddDays(1).DayOfWeek].Segments.Count; - Assert.AreEqual(3, segments); - var securityExchangeHours = CreateExchangeHours(); + var time = DateTime.UtcNow.AddDays(1); var entry = new MarketHoursDatabase.Entry(TimeZones.NewYork, securityExchangeHours); var key = new SecurityDatabaseKey(Market.USA, null, SecurityType.Equity); var mhdb = new MarketHoursDatabase(new Dictionary() { { key, entry} }); realTimeHandler.SetMarketHoursDatabase(mhdb); - realTimeHandler.ScanPastEvents(DateTime.UtcNow.AddDays(1)); - segments = security.Exchange.Hours.MarketHours[DateTime.UtcNow.AddDays(1).DayOfWeek].Segments.Count; - Assert.AreEqual(1, segments); + realTimeHandler.ScanPastEvents(time); + Thread.Sleep(1000); + var marketHours = security.Exchange.Hours.MarketHours[time.DayOfWeek]; + var segment = marketHours.Segments.SingleOrDefault(); + if (segment == null) + { + Assert.AreEqual(expectedSegment, segment); + } + else + { + Assert.AreEqual(expectedSegment.Start, segment.Start); + Assert.AreEqual(expectedSegment.End, segment.End); + for (var hour = segment.Start; hour < segment.End; hour = hour.Add(TimeSpan.FromHours(1))) + { + Assert.IsTrue(marketHours.IsOpen(hour, false)); + } + Assert.AreEqual(expectedSegment.End, security.Exchange.Hours.GetNextMarketClose(time.Date, false).TimeOfDay); + Assert.AreEqual(expectedSegment.Start, security.Exchange.Hours.GetNextMarketOpen(time.Date, false).TimeOfDay); + } } private class TestTimeLimitManager : IIsolatorLimitResultProvider @@ -122,23 +135,72 @@ public void SetMarketHoursDatabase(MarketHoursDatabase marketHoursDatabase) } } - public static SecurityExchangeHours CreateExchangeHours() + public class ExchangeHoursDataClass { - var sunday = LocalMarketHours.ClosedAllDay(DayOfWeek.Sunday); - var monday = new LocalMarketHours(DayOfWeek.Monday, new TimeSpan(9, 30, 0), new TimeSpan(16, 0, 0)); - var tuesday = new LocalMarketHours(DayOfWeek.Tuesday, new TimeSpan(9, 30, 0), new TimeSpan(16, 0, 0)); - var wednesday = new LocalMarketHours(DayOfWeek.Wednesday, new TimeSpan(9, 30, 0), new TimeSpan(16, 0, 0)); - var thursday = new LocalMarketHours(DayOfWeek.Thursday, new TimeSpan(9, 30, 0), new TimeSpan(16, 0, 0)); - var friday = new LocalMarketHours(DayOfWeek.Friday, new TimeSpan(9, 30, 0), new TimeSpan(16, 0, 0)); - var saturday = LocalMarketHours.ClosedAllDay(DayOfWeek.Saturday); - - var earlyCloses = new Dictionary { { DateTime.UtcNow.AddDays(1), new TimeSpan(13, 0, 0) } }; - var lateOpens = new Dictionary() { { DateTime.UtcNow.AddDays(1), new TimeSpan(10, 0, 0) } }; - var exchangeHours = new SecurityExchangeHours(TimeZones.NewYork, new List(), new[] + private static LocalMarketHours _sunday = new LocalMarketHours(DayOfWeek.Sunday, new TimeSpan(9, 30, 0), new TimeSpan(16, 0, 0)); + private static LocalMarketHours _monday = new LocalMarketHours(DayOfWeek.Monday, new TimeSpan(9, 30, 0), new TimeSpan(16, 0, 0)); + private static LocalMarketHours _tuesday = new LocalMarketHours(DayOfWeek.Tuesday, new TimeSpan(9, 30, 0), new TimeSpan(16, 0, 0)); + private static LocalMarketHours _wednesday = new LocalMarketHours(DayOfWeek.Wednesday, new TimeSpan(9, 30, 0), new TimeSpan(16, 0, 0)); + private static LocalMarketHours _thursday = new LocalMarketHours(DayOfWeek.Thursday, new TimeSpan(9, 30, 0), new TimeSpan(16, 0, 0)); + private static LocalMarketHours _friday = new LocalMarketHours(DayOfWeek.Friday, new TimeSpan(9, 30, 0), new TimeSpan(16, 0, 0)); + private static LocalMarketHours _saturday = new LocalMarketHours(DayOfWeek.Saturday, new TimeSpan(9, 30, 0), new TimeSpan(16, 0, 0)); + + public static IEnumerable TestCases { - sunday, monday, tuesday, wednesday, thursday, friday, saturday + get + { + yield return new TestCaseData(CreateExchangeHoursWithEarlyCloseAndLateOpen(), new MarketHoursSegment(MarketHoursState.Market,new TimeSpan(10, 0, 0), new TimeSpan(13, 0, 0))); + yield return new TestCaseData(CreateExchangeHoursWithEarlyClose(), new MarketHoursSegment(MarketHoursState.Market, new TimeSpan(9, 30, 0), new TimeSpan(13, 0, 0))); + yield return new TestCaseData(CreateExchangeHoursWithLateOpen(), new MarketHoursSegment(MarketHoursState.Market, new TimeSpan(10, 0, 0), new TimeSpan(16, 0, 0))); + yield return new TestCaseData(CreateExchangeHoursWithHolidays(), null); + } + } + + private static SecurityExchangeHours CreateExchangeHoursWithEarlyCloseAndLateOpen() + { + var earlyCloses = new Dictionary { { DateTime.UtcNow.AddDays(1).Date, new TimeSpan(13, 0, 0) } }; + var lateOpens = new Dictionary() { { DateTime.UtcNow.AddDays(1).Date, new TimeSpan(10, 0, 0) } }; + var exchangeHours = new SecurityExchangeHours(TimeZones.NewYork, new List(), new[] + { + _sunday, _monday, _tuesday, _wednesday, _thursday, _friday, _saturday }.ToDictionary(x => x.DayOfWeek), earlyCloses, lateOpens); - return exchangeHours; + return exchangeHours; + } + + private static SecurityExchangeHours CreateExchangeHoursWithEarlyClose() + { + var earlyCloses = new Dictionary { { DateTime.UtcNow.AddDays(1).Date, new TimeSpan(13, 0, 0) } }; + var lateOpens = new Dictionary(); + var exchangeHours = new SecurityExchangeHours(TimeZones.NewYork, new List(), new[] + { + _sunday, _monday, _tuesday, _wednesday, _thursday, _friday, _saturday + }.ToDictionary(x => x.DayOfWeek), earlyCloses, lateOpens); + return exchangeHours; + } + + private static SecurityExchangeHours CreateExchangeHoursWithLateOpen() + { + var earlyCloses = new Dictionary(); + var lateOpens = new Dictionary() { { DateTime.UtcNow.AddDays(1).Date, new TimeSpan(10, 0, 0) } }; + var exchangeHours = new SecurityExchangeHours(TimeZones.NewYork, new List(), new[] + { + _sunday, _monday, _tuesday, _wednesday, _thursday, _friday, _saturday + }.ToDictionary(x => x.DayOfWeek), earlyCloses, lateOpens); + return exchangeHours; + } + + private static SecurityExchangeHours CreateExchangeHoursWithHolidays() + { + var earlyCloses = new Dictionary(); + var lateOpens = new Dictionary(); + var holidays = new List() { DateTime.UtcNow.AddDays(1).Date }; + var exchangeHours = new SecurityExchangeHours(TimeZones.NewYork, holidays, new[] + { + _sunday, _monday, _tuesday, _wednesday, _thursday, _friday, _saturday + }.ToDictionary(x => x.DayOfWeek), earlyCloses, lateOpens); + return exchangeHours; + } + } } } From 03ca812aca68846d2edd9a5daba984711baa7ce7 Mon Sep 17 00:00:00 2001 From: Marinovsky Date: Fri, 12 Jan 2024 13:13:28 -0500 Subject: [PATCH 03/16] nit changes --- Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs b/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs index 0e6278406be2..c7028ca688fb 100644 --- a/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs +++ b/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs @@ -22,13 +22,8 @@ using QuantConnect.Lean.Engine.Results; using QuantConnect.Lean.Engine.RealTime; using QuantConnect.Tests.Engine.DataFeeds; -using QuantConnect.Algorithm; -using QuantConnect.AlgorithmFactory.Python.Wrappers; -using QuantConnect.Interfaces; -using QuantConnect.Tests.Common.Data.UniverseSelection; using System.Linq; using QuantConnect.Securities; -using QuantConnect.Tests.Common.Securities; using System.Collections.Generic; namespace QuantConnect.Tests.Engine.RealTime From 458c9d673957477e0bd8983a652535d2ff0fab95 Mon Sep 17 00:00:00 2001 From: Marinovsky Date: Fri, 12 Jan 2024 15:34:53 -0500 Subject: [PATCH 04/16] Improve implementation --- Engine/RealTime/LiveTradingRealTimeHandler.cs | 2 +- .../LiveTradingRealTimeHandlerTests.cs | 59 +++++++++++++------ 2 files changed, 41 insertions(+), 20 deletions(-) diff --git a/Engine/RealTime/LiveTradingRealTimeHandler.cs b/Engine/RealTime/LiveTradingRealTimeHandler.cs index cc6bf10561c3..446fb52a5e3a 100644 --- a/Engine/RealTime/LiveTradingRealTimeHandler.cs +++ b/Engine/RealTime/LiveTradingRealTimeHandler.cs @@ -179,7 +179,7 @@ public override void Exit() /// /// Get the market hours for the given symbol and date /// - private IEnumerable GetMarketHours(DateTime time, Symbol symbol) + protected virtual IEnumerable GetMarketHours(DateTime time, Symbol symbol) { if (Config.GetBool("force-exchange-always-open")) { diff --git a/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs b/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs index c7028ca688fb..22cd8d08a1b5 100644 --- a/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs +++ b/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs @@ -66,6 +66,7 @@ public void ThreadSafety() realTimeHandler.Exit(); } + [NonParallelizable] [TestCaseSource(typeof(ExchangeHoursDataClass), nameof(ExchangeHoursDataClass.TestCases))] public void RefreshesMarketHoursCorrectly(SecurityExchangeHours securityExchangeHours, MarketHoursSegment expectedSegment) { @@ -85,25 +86,7 @@ public void RefreshesMarketHoursCorrectly(SecurityExchangeHours securityExchange var key = new SecurityDatabaseKey(Market.USA, null, SecurityType.Equity); var mhdb = new MarketHoursDatabase(new Dictionary() { { key, entry} }); realTimeHandler.SetMarketHoursDatabase(mhdb); - realTimeHandler.ScanPastEvents(time); - Thread.Sleep(1000); - var marketHours = security.Exchange.Hours.MarketHours[time.DayOfWeek]; - var segment = marketHours.Segments.SingleOrDefault(); - if (segment == null) - { - Assert.AreEqual(expectedSegment, segment); - } - else - { - Assert.AreEqual(expectedSegment.Start, segment.Start); - Assert.AreEqual(expectedSegment.End, segment.End); - for (var hour = segment.Start; hour < segment.End; hour = hour.Add(TimeSpan.FromHours(1))) - { - Assert.IsTrue(marketHours.IsOpen(hour, false)); - } - Assert.AreEqual(expectedSegment.End, security.Exchange.Hours.GetNextMarketClose(time.Date, false).TimeOfDay); - Assert.AreEqual(expectedSegment.Start, security.Exchange.Hours.GetNextMarketOpen(time.Date, false).TimeOfDay); - } + realTimeHandler.ScanAndAssertMarketHours(security, time, expectedSegment); } private class TestTimeLimitManager : IIsolatorLimitResultProvider @@ -124,10 +107,48 @@ public bool TryRequestAdditionalTime(int minutes) public class TestLiveTradingRealTimeHandler: LiveTradingRealTimeHandler { + private static AutoResetEvent OnSecurityUpdated = new AutoResetEvent(false); public void SetMarketHoursDatabase(MarketHoursDatabase marketHoursDatabase) { MarketHoursDatabase = marketHoursDatabase; } + + public void ScanAndAssertMarketHours(Security security, DateTime time, MarketHoursSegment expectedSegment) + { + OnSecurityUpdated.Reset(); + ScanPastEvents(time); + OnSecurityUpdated.WaitOne(); + AssertMarketHours(security, time, expectedSegment); + } + + protected override IEnumerable GetMarketHours(DateTime time, Symbol symbol) + { + var results = base.GetMarketHours(time, symbol); + OnSecurityUpdated.Set(); + return results; + } + + public void AssertMarketHours(Security security, DateTime time, MarketHoursSegment expectedSegment) + { + var marketHours = security.Exchange.Hours.MarketHours[time.DayOfWeek]; + var segment = marketHours.Segments.SingleOrDefault(); + + if (segment == null) + { + Assert.AreEqual(expectedSegment, segment); + } + else + { + Assert.AreEqual(expectedSegment.Start, segment.Start); + Assert.AreEqual(expectedSegment.End, segment.End); + for (var hour = segment.Start; hour < segment.End; hour = hour.Add(TimeSpan.FromHours(1))) + { + Assert.IsTrue(marketHours.IsOpen(hour, false)); + } + Assert.AreEqual(expectedSegment.End, security.Exchange.Hours.GetNextMarketClose(time.Date, false).TimeOfDay); + Assert.AreEqual(expectedSegment.Start, security.Exchange.Hours.GetNextMarketOpen(time.Date, false).TimeOfDay); + } + } } public class ExchangeHoursDataClass From 992d4cb6c56492973ce69d9ad9192ef55aa10af0 Mon Sep 17 00:00:00 2001 From: Marinovsky Date: Mon, 15 Jan 2024 12:29:37 -0500 Subject: [PATCH 05/16] Enhance unit tests --- Engine/RealTime/LiveTradingRealTimeHandler.cs | 2 +- .../LiveTradingRealTimeHandlerTests.cs | 20 ++++++++++--------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/Engine/RealTime/LiveTradingRealTimeHandler.cs b/Engine/RealTime/LiveTradingRealTimeHandler.cs index 446fb52a5e3a..a70a72d2def2 100644 --- a/Engine/RealTime/LiveTradingRealTimeHandler.cs +++ b/Engine/RealTime/LiveTradingRealTimeHandler.cs @@ -116,7 +116,7 @@ private void Run() /// Refresh the market hours for each security in the given date /// /// Each time this method is called, the MarketHoursDatabase is reset - private void RefreshMarketHoursToday(DateTime date) + protected void RefreshMarketHoursToday(DateTime date) { date = date.Date; MarketHoursDatabase.Reset(); diff --git a/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs b/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs index 22cd8d08a1b5..a34073ca5de5 100644 --- a/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs +++ b/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs @@ -81,12 +81,12 @@ public void RefreshesMarketHoursCorrectly(SecurityExchangeHours securityExchange null, new TestTimeLimitManager()); - var time = DateTime.UtcNow.AddDays(1); + var time = new DateTime(2023, 5, 28).Date; var entry = new MarketHoursDatabase.Entry(TimeZones.NewYork, securityExchangeHours); var key = new SecurityDatabaseKey(Market.USA, null, SecurityType.Equity); var mhdb = new MarketHoursDatabase(new Dictionary() { { key, entry} }); realTimeHandler.SetMarketHoursDatabase(mhdb); - realTimeHandler.ScanAndAssertMarketHours(security, time, expectedSegment); + realTimeHandler.TestRefreshMarketHoursToday(security, time, expectedSegment); } private class TestTimeLimitManager : IIsolatorLimitResultProvider @@ -113,10 +113,10 @@ public void SetMarketHoursDatabase(MarketHoursDatabase marketHoursDatabase) MarketHoursDatabase = marketHoursDatabase; } - public void ScanAndAssertMarketHours(Security security, DateTime time, MarketHoursSegment expectedSegment) + public void TestRefreshMarketHoursToday(Security security, DateTime time, MarketHoursSegment expectedSegment) { OnSecurityUpdated.Reset(); - ScanPastEvents(time); + RefreshMarketHoursToday(time); OnSecurityUpdated.WaitOne(); AssertMarketHours(security, time, expectedSegment); } @@ -148,6 +148,8 @@ public void AssertMarketHours(Security security, DateTime time, MarketHoursSegme Assert.AreEqual(expectedSegment.End, security.Exchange.Hours.GetNextMarketClose(time.Date, false).TimeOfDay); Assert.AreEqual(expectedSegment.Start, security.Exchange.Hours.GetNextMarketOpen(time.Date, false).TimeOfDay); } + + Exit(); } } @@ -174,8 +176,8 @@ public static IEnumerable TestCases private static SecurityExchangeHours CreateExchangeHoursWithEarlyCloseAndLateOpen() { - var earlyCloses = new Dictionary { { DateTime.UtcNow.AddDays(1).Date, new TimeSpan(13, 0, 0) } }; - var lateOpens = new Dictionary() { { DateTime.UtcNow.AddDays(1).Date, new TimeSpan(10, 0, 0) } }; + var earlyCloses = new Dictionary { { new DateTime(2023, 5, 28).Date, new TimeSpan(13, 0, 0) } }; + var lateOpens = new Dictionary() { { new DateTime(2023, 5, 28).Date, new TimeSpan(10, 0, 0) } }; var exchangeHours = new SecurityExchangeHours(TimeZones.NewYork, new List(), new[] { _sunday, _monday, _tuesday, _wednesday, _thursday, _friday, _saturday @@ -185,7 +187,7 @@ private static SecurityExchangeHours CreateExchangeHoursWithEarlyCloseAndLateOpe private static SecurityExchangeHours CreateExchangeHoursWithEarlyClose() { - var earlyCloses = new Dictionary { { DateTime.UtcNow.AddDays(1).Date, new TimeSpan(13, 0, 0) } }; + var earlyCloses = new Dictionary { { new DateTime(2023, 5, 28).Date, new TimeSpan(13, 0, 0) } }; var lateOpens = new Dictionary(); var exchangeHours = new SecurityExchangeHours(TimeZones.NewYork, new List(), new[] { @@ -197,7 +199,7 @@ private static SecurityExchangeHours CreateExchangeHoursWithEarlyClose() private static SecurityExchangeHours CreateExchangeHoursWithLateOpen() { var earlyCloses = new Dictionary(); - var lateOpens = new Dictionary() { { DateTime.UtcNow.AddDays(1).Date, new TimeSpan(10, 0, 0) } }; + var lateOpens = new Dictionary() { { new DateTime(2023, 5, 28).Date, new TimeSpan(10, 0, 0) } }; var exchangeHours = new SecurityExchangeHours(TimeZones.NewYork, new List(), new[] { _sunday, _monday, _tuesday, _wednesday, _thursday, _friday, _saturday @@ -209,7 +211,7 @@ private static SecurityExchangeHours CreateExchangeHoursWithHolidays() { var earlyCloses = new Dictionary(); var lateOpens = new Dictionary(); - var holidays = new List() { DateTime.UtcNow.AddDays(1).Date }; + var holidays = new List() { new DateTime(2023, 5, 28).Date }; var exchangeHours = new SecurityExchangeHours(TimeZones.NewYork, holidays, new[] { _sunday, _monday, _tuesday, _wednesday, _thursday, _friday, _saturday From e64d08915a1f4915e1fbacdfab2b9cc4851042b7 Mon Sep 17 00:00:00 2001 From: Marinovsky Date: Tue, 16 Jan 2024 12:04:35 -0500 Subject: [PATCH 06/16] Add one more unit test --- Engine/RealTime/LiveTradingRealTimeHandler.cs | 12 +- .../LiveTradingRealTimeHandlerTests.cs | 106 ++++++++++++++++-- 2 files changed, 110 insertions(+), 8 deletions(-) diff --git a/Engine/RealTime/LiveTradingRealTimeHandler.cs b/Engine/RealTime/LiveTradingRealTimeHandler.cs index a70a72d2def2..64e70b686fcf 100644 --- a/Engine/RealTime/LiveTradingRealTimeHandler.cs +++ b/Engine/RealTime/LiveTradingRealTimeHandler.cs @@ -119,7 +119,7 @@ private void Run() protected void RefreshMarketHoursToday(DateTime date) { date = date.Date; - MarketHoursDatabase.Reset(); + ResetMarketHoursDatabase(); // update market hours for each security foreach (var kvp in Algorithm.Securities) @@ -201,5 +201,15 @@ protected virtual IEnumerable GetMarketHours(DateTime time, yield return segment; } } + + /// + /// Resets the market hours database, forcing a reload when reused. + /// Called in tests where multiple algorithms are run sequentially, + /// and we need to guarantee that every test starts with the same environment. + /// + protected virtual void ResetMarketHoursDatabase() + { + MarketHoursDatabase.Reset(); + } } } diff --git a/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs b/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs index a34073ca5de5..b27539b8d6dc 100644 --- a/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs +++ b/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs @@ -25,6 +25,14 @@ using System.Linq; using QuantConnect.Securities; using System.Collections.Generic; +using QuantConnect.Data.Market; +using QuantConnect.Lean.Engine.TransactionHandlers; +using Moq; +using QuantConnect.Brokerages.Backtesting; +using static QuantConnect.Tests.Engine.BrokerageTransactionHandlerTests.BrokerageTransactionHandlerTests; +using QuantConnect.Orders; +using System.Reflection; +using QuantConnect.Lean.Engine.HistoricalData; namespace QuantConnect.Tests.Engine.RealTime { @@ -81,7 +89,7 @@ public void RefreshesMarketHoursCorrectly(SecurityExchangeHours securityExchange null, new TestTimeLimitManager()); - var time = new DateTime(2023, 5, 28).Date; + var time = new DateTime(2023, 5, 30).Date; var entry = new MarketHoursDatabase.Entry(TimeZones.NewYork, securityExchangeHours); var key = new SecurityDatabaseKey(Market.USA, null, SecurityType.Equity); var mhdb = new MarketHoursDatabase(new Dictionary() { { key, entry} }); @@ -89,6 +97,57 @@ public void RefreshesMarketHoursCorrectly(SecurityExchangeHours securityExchange realTimeHandler.TestRefreshMarketHoursToday(security, time, expectedSegment); } + [Test] + public void ResetMarketHoursCorrectly() + { + var algorithm = new TestAlgorithm { HistoryProvider = new FakeHistoryProvider() }; + algorithm.SubscriptionManager.SetDataManager(new DataManagerStub(algorithm)); + algorithm.SetCash(100000); + algorithm.SetStartDate(2023, 5, 30); + algorithm.SetEndDate(2023, 5, 30); + var security = algorithm.AddEquity("SPY"); + security.Exchange = new SecurityExchange(SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork)); + var symbol = security.Symbol; + algorithm.SetFinishedWarmingUp(); + + var handleOptionNotification = typeof(BrokerageTransactionHandler).GetMethod("HandleOptionNotification", BindingFlags.NonPublic | BindingFlags.Instance); + + var transactionHandler = new TestBrokerageTransactionHandler(); + var broker = new BacktestingBrokerage(algorithm); + transactionHandler.Initialize(algorithm, broker, new BacktestingResultHandler()); + + // Creates a market order + security.SetMarketPrice(new TradeBar(new DateTime(2023, 5, 30), symbol, 280m, 280m, 280m, 280m, 100)); + + var orderRequest = new SubmitOrderRequest(OrderType.Market, security.Type, security.Symbol, 1, 0, 0, new DateTime(2023, 5, 30), "TestTag1"); + + var orderProcessorMock = new Mock(); + orderProcessorMock.Setup(m => m.GetOrderTicket(It.IsAny())).Returns(new OrderTicket(algorithm.Transactions, orderRequest)); + algorithm.Transactions.SetOrderProcessor(orderProcessorMock.Object); + var orderTicket = transactionHandler.Process(orderRequest); + transactionHandler.HandleOrderRequest(orderRequest); + Assert.IsTrue(orderTicket.Status == OrderStatus.Submitted); + broker.Scan(); + Assert.IsTrue(orderTicket.Status == OrderStatus.Filled); + + var realTimeHandler = new TestLiveTradingRealTimeHandlerReset(); + realTimeHandler.Setup(algorithm, + new AlgorithmNodePacket(PacketType.AlgorithmNode), + new BacktestingResultHandler(), + null, + new TestTimeLimitManager()); + realTimeHandler.AddRefreshHoursScheduledEvent(); + + orderRequest = new SubmitOrderRequest(OrderType.Market, security.Type, security.Symbol, 1, 0, 0, new DateTime(2023, 5, 30), "TestTag2"); + orderRequest.SetOrderId(2); + orderTicket = transactionHandler.Process(orderRequest); + transactionHandler.HandleOrderRequest(orderRequest); + Assert.IsTrue(orderTicket.Status == OrderStatus.Submitted); + broker.Scan(); + Assert.IsTrue(orderTicket.Status != OrderStatus.Filled); + realTimeHandler.Exit(); + } + private class TestTimeLimitManager : IIsolatorLimitResultProvider { public IsolatorLimitResult IsWithinLimit() @@ -153,6 +212,39 @@ public void AssertMarketHours(Security security, DateTime time, MarketHoursSegme } } + public class TestLiveTradingRealTimeHandlerReset : LiveTradingRealTimeHandler + { + private static AutoResetEvent OnSecurityUpdated = new AutoResetEvent(false); + + public void AddRefreshHoursScheduledEvent() + { + Add(new ScheduledEvent("RefreshHours", new[] { new DateTime(2023, 6, 29) }, (name, triggerTime) => + { + // refresh market hours from api every day + RefreshMarketHoursToday((new DateTime(2023, 5, 30)).Date); + })); + OnSecurityUpdated.Reset(); + SetTime(DateTime.UtcNow); + OnSecurityUpdated.WaitOne(); + Exit(); + } + + protected override IEnumerable GetMarketHours(DateTime time, Symbol symbol) + { + var results = base.GetMarketHours(time, symbol); + OnSecurityUpdated.Set(); + return results; + } + + protected override void ResetMarketHoursDatabase() + { + var entry = new MarketHoursDatabase.Entry(TimeZones.NewYork, ExchangeHoursDataClass.CreateExchangeHoursWithHolidays()); + var key = new SecurityDatabaseKey(Market.USA, null, SecurityType.Equity); + var mhdb = new MarketHoursDatabase(new Dictionary() { { key, entry } }); + MarketHoursDatabase = mhdb; + } + } + public class ExchangeHoursDataClass { private static LocalMarketHours _sunday = new LocalMarketHours(DayOfWeek.Sunday, new TimeSpan(9, 30, 0), new TimeSpan(16, 0, 0)); @@ -176,8 +268,8 @@ public static IEnumerable TestCases private static SecurityExchangeHours CreateExchangeHoursWithEarlyCloseAndLateOpen() { - var earlyCloses = new Dictionary { { new DateTime(2023, 5, 28).Date, new TimeSpan(13, 0, 0) } }; - var lateOpens = new Dictionary() { { new DateTime(2023, 5, 28).Date, new TimeSpan(10, 0, 0) } }; + var earlyCloses = new Dictionary { { new DateTime(2023, 5, 30).Date, new TimeSpan(13, 0, 0) } }; + var lateOpens = new Dictionary() { { new DateTime(2023, 5, 30).Date, new TimeSpan(10, 0, 0) } }; var exchangeHours = new SecurityExchangeHours(TimeZones.NewYork, new List(), new[] { _sunday, _monday, _tuesday, _wednesday, _thursday, _friday, _saturday @@ -187,7 +279,7 @@ private static SecurityExchangeHours CreateExchangeHoursWithEarlyCloseAndLateOpe private static SecurityExchangeHours CreateExchangeHoursWithEarlyClose() { - var earlyCloses = new Dictionary { { new DateTime(2023, 5, 28).Date, new TimeSpan(13, 0, 0) } }; + var earlyCloses = new Dictionary { { new DateTime(2023, 5, 30).Date, new TimeSpan(13, 0, 0) } }; var lateOpens = new Dictionary(); var exchangeHours = new SecurityExchangeHours(TimeZones.NewYork, new List(), new[] { @@ -199,7 +291,7 @@ private static SecurityExchangeHours CreateExchangeHoursWithEarlyClose() private static SecurityExchangeHours CreateExchangeHoursWithLateOpen() { var earlyCloses = new Dictionary(); - var lateOpens = new Dictionary() { { new DateTime(2023, 5, 28).Date, new TimeSpan(10, 0, 0) } }; + var lateOpens = new Dictionary() { { new DateTime(2023, 5, 30).Date, new TimeSpan(10, 0, 0) } }; var exchangeHours = new SecurityExchangeHours(TimeZones.NewYork, new List(), new[] { _sunday, _monday, _tuesday, _wednesday, _thursday, _friday, _saturday @@ -207,11 +299,11 @@ private static SecurityExchangeHours CreateExchangeHoursWithLateOpen() return exchangeHours; } - private static SecurityExchangeHours CreateExchangeHoursWithHolidays() + public static SecurityExchangeHours CreateExchangeHoursWithHolidays() { var earlyCloses = new Dictionary(); var lateOpens = new Dictionary(); - var holidays = new List() { new DateTime(2023, 5, 28).Date }; + var holidays = new List() { new DateTime(2023, 5, 30).Date }; var exchangeHours = new SecurityExchangeHours(TimeZones.NewYork, holidays, new[] { _sunday, _monday, _tuesday, _wednesday, _thursday, _friday, _saturday From 9f2134d74b210694dc75b0b6f578564d2c0ca2ba Mon Sep 17 00:00:00 2001 From: Marinovsky Date: Tue, 16 Jan 2024 14:02:37 -0500 Subject: [PATCH 07/16] Fix failing tests and update MHDB --- Data/market-hours/market-hours-database.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/Data/market-hours/market-hours-database.json b/Data/market-hours/market-hours-database.json index d23233b9fd5e..8d298f1f594d 100644 --- a/Data/market-hours/market-hours-database.json +++ b/Data/market-hours/market-hours-database.json @@ -1170,7 +1170,6 @@ "9/7/2020", "9/6/2021", "9/5/2022", - "9/4/2023", "9/2/2024", "9/11/2001", "9/12/2001", @@ -1476,7 +1475,6 @@ "9/7/2020", "9/6/2021", "9/5/2022", - "9/4/2023", "9/2/2024", "9/11/2001", "9/12/2001", @@ -1782,7 +1780,6 @@ "9/7/2020", "9/6/2021", "9/5/2022", - "9/4/2023", "9/2/2024", "9/11/2001", "9/12/2001", From 3550d72687eaa3d524227c19f3f3a6874f697860 Mon Sep 17 00:00:00 2001 From: Marinovsky Date: Thu, 18 Jan 2024 10:42:11 -0500 Subject: [PATCH 08/16] Add requested changes --- Engine/RealTime/LiveTradingRealTimeHandler.cs | 8 +------- Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs | 9 +++++---- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/Engine/RealTime/LiveTradingRealTimeHandler.cs b/Engine/RealTime/LiveTradingRealTimeHandler.cs index 64e70b686fcf..aa3bcba7447d 100644 --- a/Engine/RealTime/LiveTradingRealTimeHandler.cs +++ b/Engine/RealTime/LiveTradingRealTimeHandler.cs @@ -188,13 +188,7 @@ protected virtual IEnumerable GetMarketHours(DateTime time, } var entry = MarketHoursDatabase.GetEntry(symbol.ID.Market, symbol, symbol.ID.SecurityType); - var securityExchangeHours = new SecurityExchangeHours( - entry.DataTimeZone, - entry.ExchangeHours.Holidays, - entry.ExchangeHours.MarketHours.ToDictionary(), - entry.ExchangeHours.EarlyCloses, - entry.ExchangeHours.LateOpens); - var hours = securityExchangeHours.GetMarketHours(time); + var hours = entry.ExchangeHours.GetMarketHours(time); foreach (var segment in hours.Segments) { diff --git a/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs b/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs index b27539b8d6dc..b342cae94e58 100644 --- a/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs +++ b/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs @@ -78,9 +78,8 @@ public void ThreadSafety() [TestCaseSource(typeof(ExchangeHoursDataClass), nameof(ExchangeHoursDataClass.TestCases))] public void RefreshesMarketHoursCorrectly(SecurityExchangeHours securityExchangeHours, MarketHoursSegment expectedSegment) { - Security security; var algorithm = new AlgorithmStub(); - security = algorithm.AddEquity("SPY"); + var security = algorithm.AddEquity("SPY"); var realTimeHandler = new TestLiveTradingRealTimeHandler(); realTimeHandler.Setup(algorithm, @@ -145,7 +144,9 @@ public void ResetMarketHoursCorrectly() Assert.IsTrue(orderTicket.Status == OrderStatus.Submitted); broker.Scan(); Assert.IsTrue(orderTicket.Status != OrderStatus.Filled); + realTimeHandler.Exit(); + broker.Dispose(); } private class TestTimeLimitManager : IIsolatorLimitResultProvider @@ -192,7 +193,7 @@ public void AssertMarketHours(Security security, DateTime time, MarketHoursSegme var marketHours = security.Exchange.Hours.MarketHours[time.DayOfWeek]; var segment = marketHours.Segments.SingleOrDefault(); - if (segment == null) + if (expectedSegment == null) { Assert.AreEqual(expectedSegment, segment); } @@ -212,7 +213,7 @@ public void AssertMarketHours(Security security, DateTime time, MarketHoursSegme } } - public class TestLiveTradingRealTimeHandlerReset : LiveTradingRealTimeHandler + private class TestLiveTradingRealTimeHandlerReset : LiveTradingRealTimeHandler { private static AutoResetEvent OnSecurityUpdated = new AutoResetEvent(false); From b845ec24d3510d086baff8ad3e35a6f6e8e15a41 Mon Sep 17 00:00:00 2001 From: Marinovsky Date: Thu, 18 Jan 2024 16:26:47 -0500 Subject: [PATCH 09/16] Nit change --- Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs b/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs index b342cae94e58..3d5b835737a4 100644 --- a/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs +++ b/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs @@ -112,7 +112,7 @@ public void ResetMarketHoursCorrectly() var handleOptionNotification = typeof(BrokerageTransactionHandler).GetMethod("HandleOptionNotification", BindingFlags.NonPublic | BindingFlags.Instance); var transactionHandler = new TestBrokerageTransactionHandler(); - var broker = new BacktestingBrokerage(algorithm); + using var broker = new BacktestingBrokerage(algorithm); transactionHandler.Initialize(algorithm, broker, new BacktestingResultHandler()); // Creates a market order @@ -146,7 +146,6 @@ public void ResetMarketHoursCorrectly() Assert.IsTrue(orderTicket.Status != OrderStatus.Filled); realTimeHandler.Exit(); - broker.Dispose(); } private class TestTimeLimitManager : IIsolatorLimitResultProvider From 3456231e97b79ef89e2eaf7045a343248290be91 Mon Sep 17 00:00:00 2001 From: Marinovsky Date: Fri, 19 Jan 2024 10:36:00 -0500 Subject: [PATCH 10/16] Add tweaks to MarketHoursDatabase class --- Common/Securities/MarketHoursDatabase.cs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/Common/Securities/MarketHoursDatabase.cs b/Common/Securities/MarketHoursDatabase.cs index 376fa4008a09..ebe2e0eca6bb 100644 --- a/Common/Securities/MarketHoursDatabase.cs +++ b/Common/Securities/MarketHoursDatabase.cs @@ -36,7 +36,7 @@ public class MarketHoursDatabase private static MarketHoursDatabase _alwaysOpenMarketHoursDatabase; private static readonly object DataFolderMarketHoursDatabaseLock = new object(); - private readonly Dictionary _entries; + private Dictionary _entries; /// /// Gets all the exchange hours held by this provider @@ -116,6 +116,14 @@ public static void Reset() } } + public void CheckAndResetEntries() + { + if (_dataFolderMarketHoursDatabase == null) + { + _entries = FromDataFolder().ExchangeHoursListing.ToDictionary(); + } + } + /// /// Gets the instance of the class produced by reading in the market hours /// data found in /Data/market-hours/ @@ -163,6 +171,7 @@ public static MarketHoursDatabase FromFile(string path) /// The entry matching the specified market/symbol/security-type public virtual Entry SetEntry(string market, string symbol, SecurityType securityType, SecurityExchangeHours exchangeHours, DateTimeZone dataTimeZone = null) { + CheckAndResetEntries(); dataTimeZone = dataTimeZone ?? exchangeHours.TimeZone; var key = new SecurityDatabaseKey(market, symbol, securityType); var entry = new Entry(dataTimeZone, exchangeHours); @@ -194,6 +203,7 @@ public virtual Entry SetEntryAlwaysOpen(string market, string symbol, SecurityTy /// The entry matching the specified market/symbol/security-type public virtual Entry GetEntry(string market, string symbol, SecurityType securityType) { + CheckAndResetEntries(); Entry entry; // Fall back on the Futures MHDB entry if the FOP lookup failed. // Some FOPs have the same symbol properties as their futures counterparts. @@ -232,6 +242,7 @@ public virtual Entry GetEntry(string market, string symbol, SecurityType securit /// True if the entry was present, else false public bool TryGetEntry(string market, Symbol symbol, SecurityType securityType, out Entry entry) { + CheckAndResetEntries(); return TryGetEntry(market, GetDatabaseSymbolKey(symbol), securityType, out entry); } @@ -245,6 +256,7 @@ public bool TryGetEntry(string market, Symbol symbol, SecurityType securityType, /// True if the entry was present, else false public bool TryGetEntry(string market, string symbol, SecurityType securityType, out Entry entry) { + CheckAndResetEntries(); var symbolKey = new SecurityDatabaseKey(market, symbol, securityType); return _entries.TryGetValue(symbolKey, out entry) // now check with null symbol key @@ -312,6 +324,7 @@ public static string GetDatabaseSymbolKey(Symbol symbol) /// True if an entry is found, otherwise false protected bool ContainsKey(SecurityDatabaseKey key) { + CheckAndResetEntries(); return _entries.ContainsKey(key); } From c0bb8500e2e3777194aa9860c3b0055dc05702be Mon Sep 17 00:00:00 2001 From: Marinovsky Date: Fri, 19 Jan 2024 12:47:36 -0500 Subject: [PATCH 11/16] Solve bug and add unit tests --- Common/Securities/MarketHoursDatabase.cs | 17 +++++++---------- .../Securities/MarketHoursDatabaseTests.cs | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/Common/Securities/MarketHoursDatabase.cs b/Common/Securities/MarketHoursDatabase.cs index ebe2e0eca6bb..19879bce94a4 100644 --- a/Common/Securities/MarketHoursDatabase.cs +++ b/Common/Securities/MarketHoursDatabase.cs @@ -37,6 +37,7 @@ public class MarketHoursDatabase private static readonly object DataFolderMarketHoursDatabaseLock = new object(); private Dictionary _entries; + private Dictionary _customEntries = new(); /// /// Gets all the exchange hours held by this provider @@ -116,12 +117,12 @@ public static void Reset() } } - public void CheckAndResetEntries() + public void ReloadEntries() { - if (_dataFolderMarketHoursDatabase == null) - { - _entries = FromDataFolder().ExchangeHoursListing.ToDictionary(); - } + Reset(); + var newEntries = FromDataFolder().ExchangeHoursListing.ToDictionary(); + newEntries = newEntries.Concat(_customEntries).ToDictionary(); + _entries = newEntries; } /// @@ -171,11 +172,11 @@ public static MarketHoursDatabase FromFile(string path) /// The entry matching the specified market/symbol/security-type public virtual Entry SetEntry(string market, string symbol, SecurityType securityType, SecurityExchangeHours exchangeHours, DateTimeZone dataTimeZone = null) { - CheckAndResetEntries(); dataTimeZone = dataTimeZone ?? exchangeHours.TimeZone; var key = new SecurityDatabaseKey(market, symbol, securityType); var entry = new Entry(dataTimeZone, exchangeHours); _entries[key] = entry; + _customEntries[key] = entry; return entry; } @@ -203,7 +204,6 @@ public virtual Entry SetEntryAlwaysOpen(string market, string symbol, SecurityTy /// The entry matching the specified market/symbol/security-type public virtual Entry GetEntry(string market, string symbol, SecurityType securityType) { - CheckAndResetEntries(); Entry entry; // Fall back on the Futures MHDB entry if the FOP lookup failed. // Some FOPs have the same symbol properties as their futures counterparts. @@ -242,7 +242,6 @@ public virtual Entry GetEntry(string market, string symbol, SecurityType securit /// True if the entry was present, else false public bool TryGetEntry(string market, Symbol symbol, SecurityType securityType, out Entry entry) { - CheckAndResetEntries(); return TryGetEntry(market, GetDatabaseSymbolKey(symbol), securityType, out entry); } @@ -256,7 +255,6 @@ public bool TryGetEntry(string market, Symbol symbol, SecurityType securityType, /// True if the entry was present, else false public bool TryGetEntry(string market, string symbol, SecurityType securityType, out Entry entry) { - CheckAndResetEntries(); var symbolKey = new SecurityDatabaseKey(market, symbol, securityType); return _entries.TryGetValue(symbolKey, out entry) // now check with null symbol key @@ -324,7 +322,6 @@ public static string GetDatabaseSymbolKey(Symbol symbol) /// True if an entry is found, otherwise false protected bool ContainsKey(SecurityDatabaseKey key) { - CheckAndResetEntries(); return _entries.ContainsKey(key); } diff --git a/Tests/Common/Securities/MarketHoursDatabaseTests.cs b/Tests/Common/Securities/MarketHoursDatabaseTests.cs index 66da30fb9939..273c8f51c293 100644 --- a/Tests/Common/Securities/MarketHoursDatabaseTests.cs +++ b/Tests/Common/Securities/MarketHoursDatabaseTests.cs @@ -363,6 +363,21 @@ public void CustomEntriesStoredAndFetched() Assert.AreSame(entry, fetchedEntry); } + [Test] + public void CustomEntriesAreNotLostWhenReset() + { + var database = MarketHoursDatabase.FromDataFolder(); + var ticker = "UWU"; + var hours = SecurityExchangeHours.AlwaysOpen(TimeZones.Berlin); + var entry = database.SetEntry(Market.USA, ticker, SecurityType.Base, hours); + + MarketHoursDatabase.Entry returnedEntry; + Assert.IsTrue(database.TryGetEntry(Market.USA, ticker, SecurityType.Base, out returnedEntry)); + Assert.AreEqual(returnedEntry, entry); + database.ReloadEntries(); + Assert.IsTrue(database.TryGetEntry(Market.USA, ticker, SecurityType.Base, out returnedEntry)); + } + private static MarketHoursDatabase GetMarketHoursDatabase(string file) { return MarketHoursDatabase.FromFile(file); From f0c6a0902c9ef2cd5cd8b7dd653d0dd2c269b0b2 Mon Sep 17 00:00:00 2001 From: Marinovsky Date: Fri, 19 Jan 2024 13:20:57 -0500 Subject: [PATCH 12/16] Update unit tests --- Engine/RealTime/LiveTradingRealTimeHandler.cs | 2 +- .../RealTime/LiveTradingRealTimeHandlerTests.cs | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/Engine/RealTime/LiveTradingRealTimeHandler.cs b/Engine/RealTime/LiveTradingRealTimeHandler.cs index aa3bcba7447d..12750c895c2d 100644 --- a/Engine/RealTime/LiveTradingRealTimeHandler.cs +++ b/Engine/RealTime/LiveTradingRealTimeHandler.cs @@ -203,7 +203,7 @@ protected virtual IEnumerable GetMarketHours(DateTime time, /// protected virtual void ResetMarketHoursDatabase() { - MarketHoursDatabase.Reset(); + MarketHoursDatabase.ReloadEntries(); } } } diff --git a/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs b/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs index 3d5b835737a4..b30e3a74a9e5 100644 --- a/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs +++ b/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs @@ -167,9 +167,21 @@ public bool TryRequestAdditionalTime(int minutes) public class TestLiveTradingRealTimeHandler: LiveTradingRealTimeHandler { private static AutoResetEvent OnSecurityUpdated = new AutoResetEvent(false); + private MarketHoursDatabase newMarketHoursDatabase; public void SetMarketHoursDatabase(MarketHoursDatabase marketHoursDatabase) { - MarketHoursDatabase = marketHoursDatabase; + newMarketHoursDatabase = marketHoursDatabase; + } + protected override void ResetMarketHoursDatabase() + { + if (newMarketHoursDatabase != null) + { + MarketHoursDatabase = newMarketHoursDatabase; + } + else + { + base.ResetMarketHoursDatabase(); + } } public void TestRefreshMarketHoursToday(Security security, DateTime time, MarketHoursSegment expectedSegment) From 0c7a91b2f11e7de83c0e26ff6985d02148fe7ab7 Mon Sep 17 00:00:00 2001 From: Marinovsky Date: Mon, 22 Jan 2024 11:52:52 -0500 Subject: [PATCH 13/16] Address requested changes --- Common/Securities/MarketHoursDatabase.cs | 7 +++++-- Tests/Common/Securities/MarketHoursDatabaseTests.cs | 1 + Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs | 1 - 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Common/Securities/MarketHoursDatabase.cs b/Common/Securities/MarketHoursDatabase.cs index 19879bce94a4..7f63f3660f46 100644 --- a/Common/Securities/MarketHoursDatabase.cs +++ b/Common/Securities/MarketHoursDatabase.cs @@ -117,11 +117,14 @@ public static void Reset() } } + /// + /// Reload entries dictionary from MHDB file and merge them with previous custom ones + /// public void ReloadEntries() { Reset(); - var newEntries = FromDataFolder().ExchangeHoursListing.ToDictionary(); - newEntries = newEntries.Concat(_customEntries).ToDictionary(); + var fileEntries = FromDataFolder().ExchangeHoursListing; + var newEntries = fileEntries.Concat(_customEntries).ToDictionary(); _entries = newEntries; } diff --git a/Tests/Common/Securities/MarketHoursDatabaseTests.cs b/Tests/Common/Securities/MarketHoursDatabaseTests.cs index 273c8f51c293..70b4ae631432 100644 --- a/Tests/Common/Securities/MarketHoursDatabaseTests.cs +++ b/Tests/Common/Securities/MarketHoursDatabaseTests.cs @@ -376,6 +376,7 @@ public void CustomEntriesAreNotLostWhenReset() Assert.AreEqual(returnedEntry, entry); database.ReloadEntries(); Assert.IsTrue(database.TryGetEntry(Market.USA, ticker, SecurityType.Base, out returnedEntry)); + Assert.AreEqual(returnedEntry, entry); } private static MarketHoursDatabase GetMarketHoursDatabase(string file) diff --git a/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs b/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs index b30e3a74a9e5..a5d12d63c46c 100644 --- a/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs +++ b/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs @@ -74,7 +74,6 @@ public void ThreadSafety() realTimeHandler.Exit(); } - [NonParallelizable] [TestCaseSource(typeof(ExchangeHoursDataClass), nameof(ExchangeHoursDataClass.TestCases))] public void RefreshesMarketHoursCorrectly(SecurityExchangeHours securityExchangeHours, MarketHoursSegment expectedSegment) { From f8c320853f14320c7d989be33bd5e2678c07409c Mon Sep 17 00:00:00 2001 From: Marinovsky Date: Mon, 22 Jan 2024 13:04:00 -0500 Subject: [PATCH 14/16] Handle potential bug --- Common/Securities/MarketHoursDatabase.cs | 2 +- .../Securities/MarketHoursDatabaseTests.cs | 30 ++++++++++++++----- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/Common/Securities/MarketHoursDatabase.cs b/Common/Securities/MarketHoursDatabase.cs index 7f63f3660f46..50d095af2e3e 100644 --- a/Common/Securities/MarketHoursDatabase.cs +++ b/Common/Securities/MarketHoursDatabase.cs @@ -123,7 +123,7 @@ public static void Reset() public void ReloadEntries() { Reset(); - var fileEntries = FromDataFolder().ExchangeHoursListing; + var fileEntries = FromDataFolder().ExchangeHoursListing.Where(x => !_customEntries.ContainsKey(x.Key)); var newEntries = fileEntries.Concat(_customEntries).ToDictionary(); _entries = newEntries; } diff --git a/Tests/Common/Securities/MarketHoursDatabaseTests.cs b/Tests/Common/Securities/MarketHoursDatabaseTests.cs index 70b4ae631432..d1a31a071558 100644 --- a/Tests/Common/Securities/MarketHoursDatabaseTests.cs +++ b/Tests/Common/Securities/MarketHoursDatabaseTests.cs @@ -363,20 +363,34 @@ public void CustomEntriesStoredAndFetched() Assert.AreSame(entry, fetchedEntry); } - [Test] - public void CustomEntriesAreNotLostWhenReset() + [TestCase("UWU", SecurityType.Base)] + [TestCase("SPX", SecurityType.Index)] + public void CustomEntriesAreNotLostWhenReset(string ticker, SecurityType securityType) { var database = MarketHoursDatabase.FromDataFolder(); - var ticker = "UWU"; - var hours = SecurityExchangeHours.AlwaysOpen(TimeZones.Berlin); - var entry = database.SetEntry(Market.USA, ticker, SecurityType.Base, hours); + var hours = SecurityExchangeHours.AlwaysOpen(TimeZones.Chicago); + var entry = database.SetEntry(Market.USA, ticker, securityType, hours); MarketHoursDatabase.Entry returnedEntry; - Assert.IsTrue(database.TryGetEntry(Market.USA, ticker, SecurityType.Base, out returnedEntry)); + Assert.IsTrue(database.TryGetEntry(Market.USA, ticker, securityType, out returnedEntry)); Assert.AreEqual(returnedEntry, entry); - database.ReloadEntries(); - Assert.IsTrue(database.TryGetEntry(Market.USA, ticker, SecurityType.Base, out returnedEntry)); + Assert.DoesNotThrow(() => database.ReloadEntries()); + Assert.IsTrue(database.TryGetEntry(Market.USA, ticker, securityType, out returnedEntry)); + Assert.AreEqual(returnedEntry, entry); + } + + [Test] + public void ReloadEntriesDoesNotFailWhenRepeatedEntry() + { + var database = MarketHoursDatabase.FromDataFolder(); + var ticker = "SPX"; + var hours = SecurityExchangeHours.AlwaysOpen(TimeZones.Chicago); + var entry = database.SetEntry(Market.USA, ticker, SecurityType.Index, hours); + + MarketHoursDatabase.Entry returnedEntry; + Assert.IsTrue(database.TryGetEntry(Market.USA, ticker, SecurityType.Index, out returnedEntry)); Assert.AreEqual(returnedEntry, entry); + Assert.DoesNotThrow(() => database.ReloadEntries()); } private static MarketHoursDatabase GetMarketHoursDatabase(string file) From 86ecdb3509b4fab81335c699e8dc79330ad2fb35 Mon Sep 17 00:00:00 2001 From: Marinovsky Date: Mon, 22 Jan 2024 13:05:49 -0500 Subject: [PATCH 15/16] Nit change --- .../Common/Securities/MarketHoursDatabaseTests.cs | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/Tests/Common/Securities/MarketHoursDatabaseTests.cs b/Tests/Common/Securities/MarketHoursDatabaseTests.cs index d1a31a071558..9fe15c99c0d0 100644 --- a/Tests/Common/Securities/MarketHoursDatabaseTests.cs +++ b/Tests/Common/Securities/MarketHoursDatabaseTests.cs @@ -379,20 +379,6 @@ public void CustomEntriesAreNotLostWhenReset(string ticker, SecurityType securit Assert.AreEqual(returnedEntry, entry); } - [Test] - public void ReloadEntriesDoesNotFailWhenRepeatedEntry() - { - var database = MarketHoursDatabase.FromDataFolder(); - var ticker = "SPX"; - var hours = SecurityExchangeHours.AlwaysOpen(TimeZones.Chicago); - var entry = database.SetEntry(Market.USA, ticker, SecurityType.Index, hours); - - MarketHoursDatabase.Entry returnedEntry; - Assert.IsTrue(database.TryGetEntry(Market.USA, ticker, SecurityType.Index, out returnedEntry)); - Assert.AreEqual(returnedEntry, entry); - Assert.DoesNotThrow(() => database.ReloadEntries()); - } - private static MarketHoursDatabase GetMarketHoursDatabase(string file) { return MarketHoursDatabase.FromFile(file); From 50bf94af86782d172aa11b4244efefa1a36d8e30 Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Mon, 22 Jan 2024 18:11:04 -0300 Subject: [PATCH 16/16] Minor review --- Common/Securities/MarketHoursDatabase.cs | 4 ++-- Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Common/Securities/MarketHoursDatabase.cs b/Common/Securities/MarketHoursDatabase.cs index 50d095af2e3e..10bba4f99315 100644 --- a/Common/Securities/MarketHoursDatabase.cs +++ b/Common/Securities/MarketHoursDatabase.cs @@ -37,7 +37,7 @@ public class MarketHoursDatabase private static readonly object DataFolderMarketHoursDatabaseLock = new object(); private Dictionary _entries; - private Dictionary _customEntries = new(); + private readonly Dictionary _customEntries = new(); /// /// Gets all the exchange hours held by this provider @@ -123,7 +123,7 @@ public static void Reset() public void ReloadEntries() { Reset(); - var fileEntries = FromDataFolder().ExchangeHoursListing.Where(x => !_customEntries.ContainsKey(x.Key)); + var fileEntries = FromDataFolder()._entries.Where(x => !_customEntries.ContainsKey(x.Key)); var newEntries = fileEntries.Concat(_customEntries).ToDictionary(); _entries = newEntries; } diff --git a/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs b/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs index a5d12d63c46c..3004a9778460 100644 --- a/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs +++ b/Tests/Engine/RealTime/LiveTradingRealTimeHandlerTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -36,7 +36,7 @@ namespace QuantConnect.Tests.Engine.RealTime { - [TestFixture, Parallelizable(ParallelScope.All)] + [TestFixture] public class LiveTradingRealTimeHandlerTests { [Test]