From fabf1191b70a770b083f499a10a73be9365acfcb Mon Sep 17 00:00:00 2001 From: Martin-Molinero Date: Thu, 13 Feb 2025 18:40:55 -0300 Subject: [PATCH] Improve live trading streaming packets (#8584) * Improve live trading streaming packets * Further improvements * Add test for default empty holding - Improve deserialization --- Brokerages/Brokerage.cs | 5 + Common/Extensions.cs | 25 ++++ Common/Global.cs | 136 ++++++++++++++++++--- Common/Messages/Messages.QuantConnect.cs | 11 +- Common/Packets/LiveResultPacket.cs | 21 ---- Engine/Results/LiveTradingResultHandler.cs | 7 +- Tests/Common/HoldingTests.cs | 70 +++++++++++ Tests/Common/Util/ExtensionsTests.cs | 10 ++ 8 files changed, 241 insertions(+), 44 deletions(-) diff --git a/Brokerages/Brokerage.cs b/Brokerages/Brokerage.cs index 641e9606c631..0a2532201b40 100644 --- a/Brokerages/Brokerage.cs +++ b/Brokerages/Brokerage.cs @@ -334,6 +334,11 @@ protected virtual List GetAccountHoldings(Dictionary br if (brokerageData != null && brokerageData.Remove("live-holdings", out var value) && !string.IsNullOrEmpty(value)) { + if (Log.DebuggingEnabled) + { + Log.Debug($"Brokerage.GetAccountHoldings(): raw value: {value}"); + } + // remove the key, we really only want to return the cached value on the first request var result = JsonConvert.DeserializeObject>(value); if (result == null) diff --git a/Common/Extensions.cs b/Common/Extensions.cs index 51f0d4396e9a..457a2c58cf57 100644 --- a/Common/Extensions.cs +++ b/Common/Extensions.cs @@ -1309,6 +1309,31 @@ public static decimal SmartRounding(this decimal input) return input.RoundToSignificantDigits(7).Normalize(); } + /// + /// Provides global smart rounding to a shorter version + /// + public static decimal SmartRoundingShort(this decimal input) + { + input = Normalize(input); + if (input <= 1) + { + // 0.99 > input + return input; + } + else if (input <= 10) + { + // 1.01 to 9.99 + return Math.Round(input, 2); + } + else if (input <= 100) + { + // 99.9 to 10.1 + return Math.Round(input, 1); + } + // 100 to inf + return Math.Truncate(input); + } + /// /// Casts the specified input value to a decimal while acknowledging the overflow conditions /// diff --git a/Common/Global.cs b/Common/Global.cs index 3f90aa8cabfc..9dd4c65900e3 100644 --- a/Common/Global.cs +++ b/Common/Global.cs @@ -15,6 +15,8 @@ using System; using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using System.ComponentModel; using QuantConnect.Securities; using Newtonsoft.Json.Converters; using System.Runtime.Serialization; @@ -60,48 +62,104 @@ public static class DateFormat /// /// Singular holding of assets from backend live nodes: /// - [JsonObject] + [JsonConverter(typeof(HoldingJsonConverter))] public class Holding { + private decimal? _conversionRate; + private decimal _marketValue; + private decimal _unrealizedPnl; + private decimal _unrealizedPnLPercent; + /// Symbol of the Holding: - [JsonProperty(PropertyName = "symbol")] + [JsonIgnore] public Symbol Symbol { get; set; } = Symbol.Empty; /// Type of the security - [JsonProperty(PropertyName = "type")] + [JsonIgnore] public SecurityType Type => Symbol.SecurityType; /// The currency symbol of the holding, such as $ - [JsonProperty(PropertyName = "currencySymbol")] + [DefaultValue("$")] + [JsonProperty(PropertyName = "c", DefaultValueHandling = DefaultValueHandling.Ignore)] public string CurrencySymbol { get; set; } /// Average Price of our Holding in the currency the symbol is traded in - [JsonProperty(PropertyName = "averagePrice", DefaultValueHandling = DefaultValueHandling.Ignore)] + [JsonConverter(typeof(DecimalJsonConverter))] + [JsonProperty(PropertyName = "a", DefaultValueHandling = DefaultValueHandling.Ignore)] public decimal AveragePrice { get; set; } /// Quantity of Symbol We Hold. - [JsonProperty(PropertyName = "quantity", DefaultValueHandling = DefaultValueHandling.Ignore)] + [JsonConverter(typeof(DecimalJsonConverter))] + [JsonProperty(PropertyName = "q", DefaultValueHandling = DefaultValueHandling.Ignore)] public decimal Quantity { get; set; } /// Current Market Price of the Asset in the currency the symbol is traded in - [JsonProperty(PropertyName = "marketPrice", DefaultValueHandling = DefaultValueHandling.Ignore)] + [JsonConverter(typeof(DecimalJsonConverter))] + [JsonProperty(PropertyName = "p", DefaultValueHandling = DefaultValueHandling.Ignore)] public decimal MarketPrice { get; set; } /// Current market conversion rate into the account currency - [JsonProperty(PropertyName = "conversionRate", DefaultValueHandling = DefaultValueHandling.Ignore)] - public decimal? ConversionRate { get; set; } + [JsonConverter(typeof(DecimalJsonConverter))] + [JsonProperty(PropertyName = "r", DefaultValueHandling = DefaultValueHandling.Ignore)] + public decimal? ConversionRate + { + get + { + return _conversionRate; + } + set + { + if (value != 1) + { + _conversionRate = value; + } + } + } /// Current market value of the holding - [JsonProperty(PropertyName = "marketValue", DefaultValueHandling = DefaultValueHandling.Ignore)] - public decimal MarketValue { get; set; } + [JsonConverter(typeof(DecimalJsonConverter))] + [JsonProperty(PropertyName = "v", DefaultValueHandling = DefaultValueHandling.Ignore)] + public decimal MarketValue + { + get + { + return _marketValue; + } + set + { + _marketValue = value.SmartRoundingShort(); + } + } /// Current unrealized P/L of the holding - [JsonProperty(PropertyName = "unrealizedPnl", DefaultValueHandling = DefaultValueHandling.Ignore)] - public decimal UnrealizedPnL { get; set; } + [JsonConverter(typeof(DecimalJsonConverter))] + [JsonProperty(PropertyName = "u", DefaultValueHandling = DefaultValueHandling.Ignore)] + public decimal UnrealizedPnL + { + get + { + return _unrealizedPnl; + } + set + { + _unrealizedPnl = value.SmartRoundingShort(); + } + } /// Current unrealized P/L % of the holding - [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] - public decimal UnrealizedPnLPercent { get; set; } + [JsonConverter(typeof(DecimalJsonConverter))] + [JsonProperty(PropertyName = "up", DefaultValueHandling = DefaultValueHandling.Ignore)] + public decimal UnrealizedPnLPercent + { + get + { + return _unrealizedPnLPercent; + } + set + { + _unrealizedPnLPercent = value.SmartRoundingShort(); + } + } /// Create a new default holding: public Holding() @@ -159,6 +217,54 @@ public override string ToString() { return Messages.Holding.ToString(this); } + + private class DecimalJsonConverter : JsonConverter + { + public override bool CanRead => false; + public override bool CanConvert(Type objectType) => typeof(decimal) == objectType; + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + writer.WriteRawValue(((decimal)value).NormalizeToStr()); + } + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + throw new NotImplementedException(); + } + } + private class HoldingJsonConverter : JsonConverter + { + public override bool CanWrite => false; + public override bool CanConvert(Type objectType) => typeof(Holding) == objectType; + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + var jObject = JObject.Load(reader); + var result = new Holding + { + Symbol = jObject["symbol"]?.ToObject() ?? jObject["Symbol"]?.ToObject() ?? Symbol.Empty, + CurrencySymbol = jObject["c"]?.Value() ?? jObject["currencySymbol"]?.Value() ?? jObject["CurrencySymbol"]?.Value() ?? string.Empty, + AveragePrice = jObject["a"]?.Value() ?? jObject["averagePrice"]?.Value() ?? jObject["AveragePrice"]?.Value() ?? 0, + Quantity = jObject["q"]?.Value() ?? jObject["quantity"]?.Value() ?? jObject["Quantity"]?.Value() ?? 0, + MarketPrice = jObject["p"]?.Value() ?? jObject["marketPrice"]?.Value() ?? jObject["MarketPrice"]?.Value() ?? 0, + ConversionRate = jObject["r"]?.Value() ?? jObject["conversionRate"]?.Value() ?? jObject["ConversionRate"]?.Value() ?? null, + MarketValue = jObject["v"]?.Value() ?? jObject["marketValue"]?.Value() ?? jObject["MarketValue"]?.Value() ?? 0, + UnrealizedPnL = jObject["u"]?.Value() ?? jObject["unrealizedPnl"]?.Value() ?? jObject["UnrealizedPnl"]?.Value() ?? 0, + UnrealizedPnLPercent = jObject["up"]?.Value() ?? jObject["unrealizedPnLPercent"]?.Value() ?? jObject["UnrealizedPnLPercent"]?.Value() ?? 0, + }; + if (!result.ConversionRate.HasValue) + { + result.ConversionRate = 1; + } + if (string.IsNullOrEmpty(result.CurrencySymbol)) + { + result.CurrencySymbol = "$"; + } + return result; + } + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + throw new NotImplementedException(); + } + } } /// diff --git a/Common/Messages/Messages.QuantConnect.cs b/Common/Messages/Messages.QuantConnect.cs index 8fe731449c55..1d2a8bd8cf06 100644 --- a/Common/Messages/Messages.QuantConnect.cs +++ b/Common/Messages/Messages.QuantConnect.cs @@ -470,10 +470,15 @@ public static class Holding [MethodImpl(MethodImplOptions.AggressiveInlining)] public static string ToString(QuantConnect.Holding instance) { - var value = Invariant($@"{instance.Symbol.Value}: {instance.Quantity} @ { - instance.CurrencySymbol}{instance.AveragePrice} - Market: {instance.CurrencySymbol}{instance.MarketPrice}"); + var currencySymbol = instance.CurrencySymbol; + if (string.IsNullOrEmpty(currencySymbol)) + { + currencySymbol = "$"; + } + var value = Invariant($@"{instance.Symbol?.Value}: {instance.Quantity} @ { + currencySymbol}{instance.AveragePrice} - Market: {currencySymbol}{instance.MarketPrice}"); - if (instance.ConversionRate != 1m) + if (instance.ConversionRate.HasValue && instance.ConversionRate != 1m) { value += Invariant($" - Conversion: {instance.ConversionRate}"); } diff --git a/Common/Packets/LiveResultPacket.cs b/Common/Packets/LiveResultPacket.cs index 6564f1f00fb3..f054c61beb87 100644 --- a/Common/Packets/LiveResultPacket.cs +++ b/Common/Packets/LiveResultPacket.cs @@ -39,31 +39,16 @@ public class LiveResultPacket : Packet /// public int ProjectId { get; set; } - /// - /// User session Id who issued the result packet - /// - public string SessionId { get; set; } = string.Empty; - /// /// Live Algorithm Id (DeployId) for this result packet /// public string DeployId { get; set; } = string.Empty; - /// - /// Compile Id algorithm which generated this result packet - /// - public string CompileId { get; set; } = string.Empty; - /// /// Result data object for this result packet /// public LiveResult Results { get; set; } = new LiveResult(); - /// - /// Processing time / running time for the live algorithm. - /// - public double ProcessingTime { get; set; } - /// /// Default constructor for JSON Serialization /// @@ -80,15 +65,12 @@ public LiveResultPacket(string json) try { var packet = JsonConvert.DeserializeObject(json); - CompileId = packet.CompileId; Channel = packet.Channel; - SessionId = packet.SessionId; DeployId = packet.DeployId; Type = packet.Type; UserId = packet.UserId; ProjectId = packet.ProjectId; Results = packet.Results; - ProcessingTime = packet.ProcessingTime; } catch (Exception err) { @@ -106,13 +88,10 @@ public LiveResultPacket(LiveNodePacket job, LiveResult results) { try { - SessionId = job.SessionId; - CompileId = job.CompileId; DeployId = job.DeployId; Results = results; UserId = job.UserId; ProjectId = job.ProjectId; - SessionId = job.SessionId; Channel = job.Channel; } catch (Exception err) { diff --git a/Engine/Results/LiveTradingResultHandler.cs b/Engine/Results/LiveTradingResultHandler.cs index 590e1feadeb3..7def310bd722 100644 --- a/Engine/Results/LiveTradingResultHandler.cs +++ b/Engine/Results/LiveTradingResultHandler.cs @@ -218,7 +218,6 @@ private void Update() Log.Debug("LiveTradingResultHandler.Update(): End build delta charts"); //Profit loss changes, get the banner statistics, summary information on the performance for the headers. - var deltaStatistics = new Dictionary(); var serverStatistics = GetServerStatistics(utcNow); var holdings = GetHoldings(Algorithm.Securities.Values, Algorithm.SubscriptionManager.SubscriptionDataConfigService); @@ -232,7 +231,7 @@ private void Update() // since we're sending multiple packets, let's do it async and forget about it // chart data can get big so let's break them up into groups - var splitPackets = SplitPackets(deltaCharts, deltaOrders, holdings, Algorithm.Portfolio.CashBook, deltaStatistics, runtimeStatistics, serverStatistics, deltaOrderEvents); + var splitPackets = SplitPackets(deltaCharts, deltaOrders, holdings, Algorithm.Portfolio.CashBook, runtimeStatistics, serverStatistics, deltaOrderEvents); foreach (var liveResultPacket in splitPackets) { @@ -256,6 +255,7 @@ private void Update() var orderEvents = GetOrderEventsToStore(); + var deltaStatistics = new Dictionary(); var orders = new Dictionary(TransactionHandler.Orders); var complete = new LiveResultPacket(_job, new LiveResult(new LiveResultParameters(chartComplete, orders, Algorithm.Transactions.TransactionRecord, holdings, Algorithm.Portfolio.CashBook, deltaStatistics, runtimeStatistics, orderEvents, serverStatistics, state: GetAlgorithmState()))); StoreResult(complete); @@ -469,7 +469,6 @@ private IEnumerable SplitPackets(Dictionary del Dictionary deltaOrders, Dictionary holdings, CashBook cashbook, - Dictionary deltaStatistics, SortedDictionary runtimeStatistics, Dictionary serverStatistics, List deltaOrderEvents) @@ -514,7 +513,6 @@ private IEnumerable SplitPackets(Dictionary del new LiveResultPacket(_job, new LiveResult { Holdings = holdings, CashBook = cashbook}), new LiveResultPacket(_job, new LiveResult { - Statistics = deltaStatistics, RuntimeStatistics = runtimeStatistics, ServerStatistics = serverStatistics }) @@ -824,7 +822,6 @@ protected void SendFinalResult() result = LiveResultPacket.CreateEmpty(_job); result.Results.State = endState; } - result.ProcessingTime = (endTime - StartTime).TotalSeconds; StoreInsights(); diff --git a/Tests/Common/HoldingTests.cs b/Tests/Common/HoldingTests.cs index 8c50dd91e94c..a73400a3ace9 100644 --- a/Tests/Common/HoldingTests.cs +++ b/Tests/Common/HoldingTests.cs @@ -15,9 +15,11 @@ using System; using NUnit.Framework; +using Newtonsoft.Json; using QuantConnect.Securities; using QuantConnect.Data.Market; using QuantConnect.Tests.Engine.DataFeeds; +using System.Collections.Generic; namespace QuantConnect.Tests.Common { @@ -66,5 +68,73 @@ public void PriceRounding(SecurityType securityType) Assert.AreEqual(10.0000m, holding.AveragePrice); } } + + [Test] + public void RoundTrip() + { + var algo = new AlgorithmStub(); + var security = algo.AddEquity("SPY"); + security.SetMarketPrice(new Tick(new DateTime(2022, 01, 04), security.Symbol, 10.0001m, 10.0001m)); + security.Holdings.SetHoldings(10.1000000000m, 10); + + var holding = new Holding(security); + + var result = JsonConvert.SerializeObject(holding); + + Assert.AreEqual("{\"a\":10.1,\"q\":10,\"p\":10,\"v\":100,\"u\":-2,\"up\":-1.98}", result); + + var deserialized = JsonConvert.DeserializeObject(result); + + Assert.AreEqual(deserialized.AveragePrice, holding.AveragePrice); + Assert.AreEqual(deserialized.Quantity, holding.Quantity); + Assert.AreEqual(deserialized.MarketPrice, holding.MarketPrice); + Assert.AreEqual(deserialized.MarketValue, holding.MarketValue); + Assert.AreEqual(deserialized.UnrealizedPnL, holding.UnrealizedPnL); + Assert.AreEqual(deserialized.UnrealizedPnLPercent, holding.UnrealizedPnLPercent); + + Assert.AreEqual(": 10 @ $10.1 - Market: $10", deserialized.ToString()); + } + + [TestCase(true)] + [TestCase(false)] + public void BackwardsCompatible(bool upperCase) + { + string source; + if (upperCase) + { + source = "{\"Symbol\":{\"value\":\"A\",\"id\":\"A RPTMYV3VC57P\",\"permtick\":\"A\"},\"Type\":1,\"CurrencySymbol\":\"$\",\"AveragePrice\":148.34,\"Quantity\":192.0," + + "\"MarketPrice\":145.21,\"ConversionRate\":1.0,\"MarketValue\":27880.3200,\"UnrealizedPnl\":-601.96,\"UnrealizedPnLPercent\":-2.11}"; + } + else + { + source = "{\"symbol\":{\"value\":\"A\",\"id\":\"A RPTMYV3VC57P\",\"permtick\":\"A\"},\"type\":1,\"currencySymbol\":\"$\",\"averagePrice\":148.34,\"quantity\":192.0," + + "\"marketPrice\":145.21,\"conversionRate\":1.0,\"marketValue\":27880.3200,\"unrealizedPnl\":-601.96,\"unrealizedPnLPercent\":-2.11}"; + } + var deserialized = JsonConvert.DeserializeObject(source); + + Assert.IsNotNull(deserialized.Symbol); + Assert.AreEqual("A", deserialized.Symbol.Value); + Assert.AreEqual(148.34, deserialized.AveragePrice); + Assert.AreEqual(192, deserialized.Quantity); + Assert.AreEqual(145.21, deserialized.MarketPrice); + Assert.AreEqual(27880, deserialized.MarketValue); + Assert.AreEqual(-601.96, deserialized.UnrealizedPnL); + Assert.AreEqual(-2.11, deserialized.UnrealizedPnLPercent); + } + + [Test] + public void DefaultHoldings() + { + var algo = new AlgorithmStub(); + var security = algo.AddEquity("SPY"); + var holding = new Holding(security); + + var result = JsonConvert.SerializeObject(new Dictionary { { security.Symbol.ID.ToString(), holding } }); + + Assert.AreEqual("{\"SPY R735QTJ8XC9X\":{}}", result); + + var deserialized = JsonConvert.DeserializeObject>(result); + Assert.AreEqual(0, deserialized[security.Symbol.ID.ToString()].AveragePrice); + } } } diff --git a/Tests/Common/Util/ExtensionsTests.cs b/Tests/Common/Util/ExtensionsTests.cs index a2877f99a1a1..e0449b9ab968 100644 --- a/Tests/Common/Util/ExtensionsTests.cs +++ b/Tests/Common/Util/ExtensionsTests.cs @@ -1114,6 +1114,16 @@ public void NormalizeDecimalReturnsNoTrailingZeros(decimal input, string expecte Assert.AreEqual(expectedOutput, output.ToStringInvariant()); } + [TestCase(0.072842, "0.072842")] + [TestCase(7.5819999, "7.58")] + [TestCase(54.1119999, "54.1")] + [TestCase(1152280.01234568423, "1152280")] + public void SmartRoundingShort(decimal input, string expectedOutput) + { + var output = input.SmartRoundingShort().ToStringInvariant(); + Assert.AreEqual(expectedOutput, output); + } + [Test] [TestCase(0.072842, 3, "0.0728")] [TestCase(0.0019999, 2, "0.0020")]