From 93699f733f2b00b4861b71e646b775407ff48b91 Mon Sep 17 00:00:00 2001 From: Martin-Molinero Date: Wed, 17 Nov 2021 16:17:39 -0300 Subject: [PATCH 1/3] Adding support for FTX and Kraken brokerages - Add support for FTX and Kraken brokerages --- README.md | 20 +++- announcements.json | 4 + lean/commands/live.py | 76 ++++++++++++++- lean/components/cloud/data_downloader.py | 21 ++-- lean/components/util/logger.py | 5 +- lean/constants.py | 6 ++ lean/models/brokerages/cloud/__init__.py | 6 +- lean/models/brokerages/cloud/ftx.py | 55 +++++++++++ lean/models/brokerages/cloud/kraken.py | 55 +++++++++++ lean/models/brokerages/local/__init__.py | 14 ++- lean/models/brokerages/local/ftx.py | 119 +++++++++++++++++++++++ lean/models/brokerages/local/kraken.py | 107 ++++++++++++++++++++ 12 files changed, 465 insertions(+), 23 deletions(-) create mode 100644 lean/models/brokerages/cloud/ftx.py create mode 100644 lean/models/brokerages/cloud/kraken.py create mode 100644 lean/models/brokerages/local/ftx.py create mode 100644 lean/models/brokerages/local/kraken.py diff --git a/README.md b/README.md index 40508192..69fdeba9 100644 --- a/README.md +++ b/README.md @@ -204,7 +204,7 @@ Usage: lean cloud live [OPTIONS] PROJECT events and --notify-insights. Options: - --brokerage [Paper Trading|Interactive Brokers|Tradier|OANDA|Bitfinex|Coinbase Pro|Binance] + --brokerage [Paper Trading|Interactive Brokers|Tradier|OANDA|Bitfinex|Coinbase Pro|Binance|Kraken|FTX] The brokerage to use --ib-user-name TEXT Your Interactive Brokers username --ib-account TEXT Your Interactive Brokers account id @@ -720,9 +720,9 @@ Options: -d, --detach Run the live deployment in a detached Docker container and return immediately --gui Enable monitoring and controlling of the deployment via the local GUI --gui-organization TEXT The name or id of the organization with the local GUI module subscription - --brokerage [Paper Trading|Interactive Brokers|Tradier|OANDA|Bitfinex|Coinbase Pro|Binance|Zerodha|Terminal Link|Atreyu|Trading Technologies] + --brokerage [Paper Trading|Interactive Brokers|Tradier|OANDA|Bitfinex|Coinbase Pro|Binance|Zerodha|Terminal Link|Atreyu|Trading Technologies|Kraken|FTX] The brokerage to use - --data-feed [Interactive Brokers|Tradier|OANDA|Bitfinex|Coinbase Pro|Binance|Zerodha|Terminal Link|Trading Technologies|Custom data only|IQFeed] + --data-feed [Interactive Brokers|Tradier|OANDA|Bitfinex|Coinbase Pro|Binance|Zerodha|Terminal Link|Trading Technologies|Custom data only|Kraken|FTX|IQFeed] The data feed to use --ib-user-name TEXT Your Interactive Brokers username --ib-account TEXT Your Interactive Brokers account id @@ -764,7 +764,8 @@ Options: --iqfeed-password TEXT Your IQFeed password --iqfeed-product-name TEXT The product name of your IQFeed developer account --iqfeed-version TEXT The product version of your IQFeed developer account - --terminal-link-organization TEXT The name or id of the organization with the Terminal Link module subscription + --terminal-link-organization TEXT + The name or id of the organization with the Terminal Link module subscription --bloomberg-environment [Production|Beta] The environment to run in --bloomberg-server-host TEXT The host of the Bloomberg server @@ -810,6 +811,15 @@ Options: --tt-order-routing-host TEXT The host of the order routing server --tt-order-routing-port TEXT The port of the order routing server --tt-log-fix-messages BOOLEAN Whether FIX messages should be logged + --kraken-organization TEXT The name or id of the organization with the kraken module subscription + --kraken-api-key TEXT Your Kraken API key + --kraken-api-secret TEXT Your Kraken API secret + --kraken-verification-tier TEXT + Your Kraken Verification Tier + --ftx-organization TEXT The name or id of the organization with the FTX module subscription + --ftx-api-key TEXT Your FTX API key + --ftx-api-secret TEXT Your FTX API secret + --ftx-account-tier TEXT Your FTX Account Tier --release Compile C# projects in release configuration instead of debug --image TEXT The LEAN engine image to use (defaults to quantconnect/lean:latest) --update Pull the LEAN engine image before starting live trading @@ -1031,6 +1041,6 @@ If you need to add dependencies, first update `setup.py` (if it is a production The automated tests can be ran by running `pytest`. The filesystem and HTTP requests are mocked when running tests to make sure they run in an isolated environment. -To update the commands reference part of the readme run `python scripts/readme.py` from the root of the project. +Can build the lean CLI by running `python setup.py sdist bdist_wheel` from the root of the project and to install it `pip install --force-reinstall dist/lean-dev-py3-none-any.whl`. To update the commands reference part of the readme run `python scripts/readme.py` from the root of the project, after installing the new version. Maintainers can publish new releases by pushing a Git tag containing the new version to GitHub. This will trigger a GitHub Actions workflow which releases the current `master` branch to PyPI with the value of the tag as version. Make sure the version is not prefixed with "v". diff --git a/announcements.json b/announcements.json index 4ddfc7bb..bfcd5864 100644 --- a/announcements.json +++ b/announcements.json @@ -1,5 +1,9 @@ { "announcements": [ + { + "date": "2021-11-17", + "message": "We've added support for Kraken and FTX brokerages" + }, { "date": "2021-06-21", "message": "We've just released a new opening video of the CLI:\nhttps://youtube.com/watch?v=QJibe1XpP-U" diff --git a/lean/commands/live.py b/lean/commands/live.py index 0b0fcd87..d2fadbc6 100644 --- a/lean/commands/live.py +++ b/lean/commands/live.py @@ -35,6 +35,8 @@ from lean.models.brokerages.local.tradier import TradierBrokerage, TradierDataFeed from lean.models.brokerages.local.trading_technologies import TradingTechnologiesBrokerage, TradingTechnologiesDataFeed from lean.models.brokerages.local.zerodha import ZerodhaBrokerage, ZerodhaDataFeed +from lean.models.brokerages.local.kraken import KrakenBrokerage, KrakenDataFeed +from lean.models.brokerages.local.ftx import FTXBrokerage, FTXDataFeed from lean.models.errors import MoreInfoError from lean.models.logger import Option @@ -59,7 +61,9 @@ "tt-market-data-host", "tt-market-data-port", "tt-order-routing-sender-comp-id", "tt-order-routing-target-comp-id", "tt-order-routing-host", "tt-order-routing-port", - "tt-log-fix-messages"] + "tt-log-fix-messages"], + "KrakenBrokerage": ["kraken-api-key", "kraken-api-secret", "kraken-verification-tier"], + "FTXBrokerage": ["ftx-api-key", "ftx-api-secret", "ftx-account-tier"] } # Data queue handler -> required configuration properties @@ -74,7 +78,9 @@ "ZerodhaBrokerage": _required_brokerage_properties["ZerodhaBrokerage"] + ["zerodha-history-subscription"], "BloombergBrokerage": _required_brokerage_properties["BloombergBrokerage"], "TradingTechnologiesBrokerage": _required_brokerage_properties["TradingTechnologiesBrokerage"], - "QuantConnect.ToolBox.IQFeed.IQFeedDataQueueHandler": ["iqfeed-iqconnect", "iqfeed-productName", "iqfeed-version"] + "QuantConnect.ToolBox.IQFeed.IQFeedDataQueueHandler": ["iqfeed-iqconnect", "iqfeed-productName", "iqfeed-version"], + "KrakenBrokerage": _required_brokerage_properties["KrakenBrokerage"], + "FTXBrokerage": _required_brokerage_properties["FTXBrokerage"] } _environment_skeleton = { @@ -526,6 +532,38 @@ def _get_default_value(key: str) -> Optional[Any]: type=bool, default=lambda: _get_default_value("tt-log-fix-messages"), help="Whether FIX messages should be logged") +@click.option("--kraken-organization", + type=str, + default=lambda: _get_default_value("job-organization-id"), + help="The name or id of the organization with the kraken module subscription") +@click.option("--kraken-api-key", + type=str, + default=lambda: _get_default_value("kraken-api-key"), + help="Your Kraken API key") +@click.option("--kraken-api-secret", + type=str, + default=lambda: _get_default_value("kraken-api-secret"), + help="Your Kraken API secret") +@click.option("--kraken-verification-tier", + type=str, + default=lambda: _get_default_value("kraken-verification-tier"), + help="Your Kraken Verification Tier") +@click.option("--ftx-organization", + type=str, + default=lambda: _get_default_value("job-organization-id"), + help="The name or id of the organization with the FTX module subscription") +@click.option("--ftx-api-key", + type=str, + default=lambda: _get_default_value("ftx-api-key"), + help="Your FTX API key") +@click.option("--ftx-api-secret", + type=str, + default=lambda: _get_default_value("ftx-api-secret"), + help="Your FTX API secret") +@click.option("--ftx-account-tier", + type=str, + default=lambda: _get_default_value("ftx-account-tier"), + help="Your FTX Account Tier") @click.option("--release", is_flag=True, default=False, @@ -611,6 +649,14 @@ def live(project: Path, tt_order_routing_host: Optional[str], tt_order_routing_port: Optional[str], tt_log_fix_messages: Optional[bool], + kraken_organization: Optional[str], + kraken_api_key: Optional[str], + kraken_api_secret: Optional[str], + kraken_verification_tier: Optional[str], + ftx_organization: Optional[str], + ftx_api_key: Optional[str], + ftx_api_secret: Optional[str], + ftx_account_tier: Optional[str], release: bool, image: Optional[str], update: bool) -> None: @@ -767,6 +813,18 @@ def live(project: Path, tt_order_routing_host, tt_order_routing_port, tt_log_fix_messages) + elif brokerage == KrakenBrokerage.get_name(): + ensure_options(["kraken_api_key", "kraken_api_secret", "kraken_verification_tier"]) + brokerage_configurer = KrakenBrokerage(_get_organization_id(kraken_organization, "Kraken"), + kraken_api_key, + kraken_api_secret, + kraken_verification_tier) + elif brokerage == FTXBrokerage.get_name(): + ensure_options(["ftx_api_key", "ftx_api_secret", "ftx_account_tier"]) + brokerage_configurer = FTXBrokerage(_get_organization_id(ftx_organization, "FTX"), + ftx_api_key, + ftx_api_secret, + ftx_account_tier) if data_feed == InteractiveBrokersDataFeed.get_name(): ensure_options(["ib_user_name", "ib_account", "ib_password", "ib_enable_delayed_streaming_data"]) @@ -872,6 +930,20 @@ def live(project: Path, iqfeed_password, iqfeed_product_name, iqfeed_version) + elif data_feed == KrakenDataFeed.get_name(): + ensure_options(["kraken_api_key", "kraken_api_secret", "kraken_verification_tier"]) + data_feed_configurer = KrakenDataFeed( + KrakenBrokerage(_get_organization_id(kraken_organization, "Kraken"), + kraken_api_key, + kraken_api_secret, + kraken_verification_tier)) + elif data_feed == FTXDataFeed.get_name(): + ensure_options(["ftx_api_key", "ftx_api_secret", "ftx_account_tier"]) + data_feed_configurer = FTXDataFeed( + FTXBrokerage(_get_organization_id(ftx_organization, "FTX"), + ftx_api_key, + ftx_api_secret, + ftx_account_tier)) environment_name = "lean-cli" lean_config = lean_config_manager.get_complete_lean_config(environment_name, algorithm_file, None) diff --git a/lean/components/cloud/data_downloader.py b/lean/components/cloud/data_downloader.py index 28be8ded..003c0d76 100644 --- a/lean/components/cloud/data_downloader.py +++ b/lean/components/cloud/data_downloader.py @@ -55,6 +55,17 @@ def download_files(self, data_files: List[Any], overwrite: bool, organization_id lambda: progress.update(progress_task, advance=1)) for data_file in data_files) + # update our config after we download all files, and not in parallel! + for relative_file in data_files: + if "/map_files/map_files_" in relative_file and relative_file.endswith(".zip"): + self._lean_config_manager.set_properties({ + "map-file-provider": "QuantConnect.Data.Auxiliary.LocalZipMapFileProvider" + }) + if "/factor_files/factor_files_" in relative_file and relative_file.endswith(".zip"): + self._lean_config_manager.set_properties({ + "factor-file-provider": "QuantConnect.Data.Auxiliary.LocalZipFactorFileProvider" + }) + progress.stop() except KeyboardInterrupt as e: progress.stop() @@ -98,14 +109,4 @@ def _download_file(self, with local_path.open("wb+") as f: f.write(file_content) - if relative_file.startswith("equity/usa/map_files/map_files_") and relative_file.endswith(".zip"): - self._lean_config_manager.set_properties({ - "map-file-provider": "QuantConnect.Data.Auxiliary.LocalZipMapFileProvider" - }) - - if relative_file.startswith("equity/usa/factor_files/factor_files_") and relative_file.endswith(".zip"): - self._lean_config_manager.set_properties({ - "factor-file-provider": "QuantConnect.Data.Auxiliary.LocalZipFactorFileProvider" - }) - callback() diff --git a/lean/components/util/logger.py b/lean/components/util/logger.py index 9f4bdf59..0e4ae046 100644 --- a/lean/components/util/logger.py +++ b/lean/components/util/logger.py @@ -78,13 +78,14 @@ def progress(self, prefix: str = "", suffix: str = "{task.percentage:0.0f}%") -> progress.start() return progress - def prompt_list(self, text: str, options: List[Option]) -> Any: + def prompt_list(self, text: str, options: List[Option], default: Optional[str] = None) -> Any: """Asks the user to select an option from a list of possible options. The user will not be prompted for input if there is only a single option. :param text: the text to display before prompting :param options: the available options + :param default: the default value if no input is given :return: the chosen option's id """ if len(options) == 1: @@ -96,7 +97,7 @@ def prompt_list(self, text: str, options: List[Option]) -> Any: self.info(f"{i + 1}) {option.label}") while True: - user_input = click.prompt("Enter an option", type=str) + user_input = click.prompt("Enter an option", type=str, default=default, show_default=True) try: index = int(user_input) diff --git a/lean/constants.py b/lean/constants.py index 37e23923..c3f1d619 100644 --- a/lean/constants.py +++ b/lean/constants.py @@ -81,6 +81,12 @@ # The product id of the Atreyu module ATREYU_PRODUCT_ID = 65 +# The product id of the Kraken module +KRAKEN_PRODUCT_ID = 130 + +# The product id of the FTX module +FTX_PRODUCT_ID = 138 + # The product ids for which a valid subscription is seen as a valid GUI module subscription GUI_PRODUCT_SUBSCRIPTION_IDS = [119, 120] diff --git a/lean/models/brokerages/cloud/__init__.py b/lean/models/brokerages/cloud/__init__.py index 01b91f9d..e80b5830 100644 --- a/lean/models/brokerages/cloud/__init__.py +++ b/lean/models/brokerages/cloud/__init__.py @@ -18,6 +18,8 @@ from lean.models.brokerages.cloud.oanda import OANDABrokerage from lean.models.brokerages.cloud.paper_trading import PaperTradingBrokerage from lean.models.brokerages.cloud.tradier import TradierBrokerage +from lean.models.brokerages.cloud.kraken import KrakenBrokerage +from lean.models.brokerages.cloud.ftx import FTXBrokerage all_cloud_brokerages = [ PaperTradingBrokerage, @@ -26,5 +28,7 @@ OANDABrokerage, BitfinexBrokerage, CoinbaseProBrokerage, - BinanceBrokerage + BinanceBrokerage, + KrakenBrokerage, + FTXBrokerage ] diff --git a/lean/models/brokerages/cloud/ftx.py b/lean/models/brokerages/cloud/ftx.py new file mode 100644 index 00000000..c38cd3a9 --- /dev/null +++ b/lean/models/brokerages/cloud/ftx.py @@ -0,0 +1,55 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict + +import click + +from lean.components.util.logger import Logger +from lean.models.brokerages.cloud.base import CloudBrokerage + + +class FTXBrokerage(CloudBrokerage): + """A CloudBrokerage implementation for FTX.""" + + def __init__(self, api_key: str, secret_key: str, account_tier: str) -> None: + self._api_key = api_key + self._secret_key = secret_key + self._account_tier = account_tier + + @classmethod + def get_id(cls) -> str: + return "FTXBrokerage" + + @classmethod + def get_name(cls) -> str: + return "FTX" + + @classmethod + def build(cls, logger: Logger) -> CloudBrokerage: + logger.info(""" +Create an API key by logging in and accessing the FTX Profile page (https://ftx.com/profile). + """.strip()) + + api_key = click.prompt("API key") + secret_key = logger.prompt_password("Secret key") + account_tier = click.prompt("Account Tier") + + return FTXBrokerage(api_key, secret_key, account_tier) + + def _get_settings(self) -> Dict[str, str]: + return { + "key": self._api_key, + "secret": self._secret_key, + "accountTier": self._account_tier + } diff --git a/lean/models/brokerages/cloud/kraken.py b/lean/models/brokerages/cloud/kraken.py new file mode 100644 index 00000000..82b7365e --- /dev/null +++ b/lean/models/brokerages/cloud/kraken.py @@ -0,0 +1,55 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict + +import click + +from lean.components.util.logger import Logger +from lean.models.brokerages.cloud.base import CloudBrokerage + + +class KrakenBrokerage(CloudBrokerage): + """A CloudBrokerage implementation for Kraken.""" + + def __init__(self, api_key: str, secret_key: str, verification_tier: str) -> None: + self._api_key = api_key + self._secret_key = secret_key + self._verification_tier = verification_tier + + @classmethod + def get_id(cls) -> str: + return "KrakenBrokerage" + + @classmethod + def get_name(cls) -> str: + return "Kraken" + + @classmethod + def build(cls, logger: Logger) -> CloudBrokerage: + logger.info(""" +Create an API key by logging in and accessing the Kraken API Management page (https://www.kraken.com/u/security/api). + """.strip()) + + api_key = click.prompt("API key") + secret_key = logger.prompt_password("Secret key") + verification_tier = click.prompt("Verification Tier") + + return KrakenBrokerage(api_key, secret_key, verification_tier) + + def _get_settings(self) -> Dict[str, str]: + return { + "key": self._api_key, + "secret": self._secret_key, + "verificationTier": self._verification_tier + } diff --git a/lean/models/brokerages/local/__init__.py b/lean/models/brokerages/local/__init__.py index 8b1a520a..b9421d82 100644 --- a/lean/models/brokerages/local/__init__.py +++ b/lean/models/brokerages/local/__init__.py @@ -29,6 +29,8 @@ from lean.models.brokerages.local.tradier import TradierBrokerage, TradierDataFeed from lean.models.brokerages.local.trading_technologies import TradingTechnologiesBrokerage, TradingTechnologiesDataFeed from lean.models.brokerages.local.zerodha import ZerodhaBrokerage, ZerodhaDataFeed +from lean.models.brokerages.local.kraken import KrakenBrokerage, KrakenDataFeed +from lean.models.brokerages.local.ftx import FTXBrokerage, FTXDataFeed from lean.models.config import LeanConfigConfigurer all_local_brokerages = [ @@ -42,7 +44,9 @@ ZerodhaBrokerage, TerminalLinkBrokerage, AtreyuBrokerage, - TradingTechnologiesBrokerage + TradingTechnologiesBrokerage, + KrakenBrokerage, + FTXBrokerage ] all_local_data_feeds = [ @@ -55,7 +59,9 @@ ZerodhaDataFeed, TerminalLinkDataFeed, TradingTechnologiesDataFeed, - CustomDataOnlyDataFeed + CustomDataOnlyDataFeed, + KrakenDataFeed, + FTXDataFeed ] local_brokerage_data_feeds: Dict[Type[LocalBrokerage], List[Type[LeanConfigConfigurer]]] = { @@ -69,7 +75,9 @@ ZerodhaBrokerage: [ZerodhaDataFeed], TerminalLinkBrokerage: [TerminalLinkDataFeed], AtreyuBrokerage: [x for x in all_local_data_feeds if x != CustomDataOnlyDataFeed], - TradingTechnologiesBrokerage: [TradingTechnologiesDataFeed] + TradingTechnologiesBrokerage: [TradingTechnologiesDataFeed], + KrakenBrokerage: [KrakenDataFeed], + FTXBrokerage: [FTXDataFeed] } if container.platform_manager().is_host_windows() or os.environ.get("__README__", "false") == "true": diff --git a/lean/models/brokerages/local/ftx.py b/lean/models/brokerages/local/ftx.py new file mode 100644 index 00000000..107ba3d2 --- /dev/null +++ b/lean/models/brokerages/local/ftx.py @@ -0,0 +1,119 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any, Dict + +import click + +from lean.components.util.logger import Logger +from lean.constants import FTX_PRODUCT_ID +from lean.container import container +from lean.models.brokerages.local.base import LocalBrokerage +from lean.models.config import LeanConfigConfigurer +from lean.models.logger import Option + + +class FTXBrokerage(LocalBrokerage): + """A LocalBrokerage implementation for the FTX brokerage.""" + + _is_module_installed = False + + def __init__(self, organization_id: str, api_key: str, api_secret: str, account_tier: str) -> None: + self._api_key = api_key + self._api_secret = api_secret + self._account_tier = account_tier + self._organization_id = organization_id + + @classmethod + def get_name(cls) -> str: + return "FTX" + + @classmethod + def _build(cls, lean_config: Dict[str, Any], logger: Logger) -> LocalBrokerage: + api_client = container.api_client() + + organizations = api_client.organizations.get_all() + options = [Option(id=organization.id, label=organization.name) for organization in organizations] + + organization_id = logger.prompt_list( + "Select the organization with the FTX module subscription", + options + ) + + logger.info(""" +Create an API key by logging in and accessing the FTX Profile page (https://ftx.com/profile). + """.strip()) + + api_key = click.prompt("API key", cls._get_default(lean_config, "ftx-api-key")) + api_secret = logger.prompt_password("API secret", cls._get_default(lean_config, "ftx-api-secret")) + + account_tier = logger.prompt_list( + "Select the Account Tier", + [Option(id="Tier1", label="Tier1"), + Option(id="Tier2", label="Tier2"), + Option(id="Tier3", label="Tier3"), + Option(id="Tier4", label="Tier4"), + Option(id="Tier5", label="Tier5"), + Option(id="Tier6", label="Tier6"), + Option(id="VIP1", label="VIP1"), + Option(id="VIP2", label="VIP2"), + Option(id="VIP3", label="VIP3"), + Option(id="MM1", label="MM1"), + Option(id="MM2", label="MM2"), + Option(id="MM3", label="MM3")], + cls._get_default(lean_config, "ftx-account-tier") + ) + + return FTXBrokerage(organization_id, api_key, api_secret, account_tier) + + def _configure_environment(self, lean_config: Dict[str, Any], environment_name: str) -> None: + self.ensure_module_installed() + + lean_config["environments"][environment_name]["live-mode-brokerage"] = "FTXBrokerage" + lean_config["environments"][environment_name]["transaction-handler"] = \ + "QuantConnect.Lean.Engine.TransactionHandlers.BrokerageTransactionHandler" + + def configure_credentials(self, lean_config: Dict[str, Any]) -> None: + lean_config["ftx-api-key"] = self._api_key + lean_config["ftx-api-secret"] = self._api_secret + lean_config["ftx-account-tier"] = self._account_tier + lean_config["job-organization-id"] = self._organization_id + + self._save_properties(lean_config, ["job-organization-id", "ftx-api-key", "ftx-api-secret", "ftx-account-tier"]) + + def ensure_module_installed(self) -> None: + if not self._is_module_installed: + container.module_manager().install_module(FTX_PRODUCT_ID, self._organization_id) + self._is_module_installed = True + +class FTXDataFeed(LeanConfigConfigurer): + """A LeanConfigConfigurer implementation for the FTX data feed.""" + + def __init__(self, brokerage: FTXBrokerage) -> None: + self._brokerage = brokerage + + @classmethod + def get_name(cls) -> str: + return FTXBrokerage.get_name() + + @classmethod + def build(cls, lean_config: Dict[str, Any], logger: Logger) -> LeanConfigConfigurer: + return FTXDataFeed(FTXBrokerage.build(lean_config, logger)) + + def configure(self, lean_config: Dict[str, Any], environment_name: str) -> None: + self._brokerage.ensure_module_installed() + + lean_config["environments"][environment_name]["data-queue-handler"] = "FTXBrokerage" + lean_config["environments"][environment_name]["history-provider"] = "BrokerageHistoryProvider" + + self._brokerage.configure_credentials(lean_config) diff --git a/lean/models/brokerages/local/kraken.py b/lean/models/brokerages/local/kraken.py new file mode 100644 index 00000000..0b828ceb --- /dev/null +++ b/lean/models/brokerages/local/kraken.py @@ -0,0 +1,107 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any, Dict + +import click + +from lean.components.util.logger import Logger +from lean.constants import KRAKEN_PRODUCT_ID +from lean.container import container +from lean.models.brokerages.local.base import LocalBrokerage +from lean.models.config import LeanConfigConfigurer +from lean.models.logger import Option + + +class KrakenBrokerage(LocalBrokerage): + """A LocalBrokerage implementation for the Kraken brokerage.""" + + _is_module_installed = False + + def __init__(self, organization_id: str, api_key: str, api_secret: str, verification_tier: str) -> None: + self._api_key = api_key + self._api_secret = api_secret + self._organization_id = organization_id + self._verification_tier = verification_tier + + @classmethod + def get_name(cls) -> str: + return "Kraken" + + @classmethod + def _build(cls, lean_config: Dict[str, Any], logger: Logger) -> LocalBrokerage: + api_client = container.api_client() + + organizations = api_client.organizations.get_all() + options = [Option(id=organization.id, label=organization.name) for organization in organizations] + + organization_id = logger.prompt_list( + "Select the organization with the Kraken module subscription", + options + ) + + logger.info(""" +Create an API key by logging in and accessing the Kraken API Management page (https://www.kraken.com/u/security/api). + """.strip()) + + api_key = click.prompt("API key", cls._get_default(lean_config, "kraken-api-key")) + api_secret = logger.prompt_password("API secret", cls._get_default(lean_config, "kraken-api-secret")) + + verification_tier = logger.prompt_list("Select the Verification Tier", + [Option(id="Starter", label="Starter"), Option(id="Intermediate", label="Intermediate"), Option(id="Pro", label="Pro")], + cls._get_default(lean_config, "kraken-verification-tier") + ) + + return KrakenBrokerage(organization_id, api_key, api_secret, verification_tier) + + def _configure_environment(self, lean_config: Dict[str, Any], environment_name: str) -> None: + self.ensure_module_installed() + + lean_config["environments"][environment_name]["live-mode-brokerage"] = "KrakenBrokerage" + lean_config["environments"][environment_name]["transaction-handler"] = \ + "QuantConnect.Lean.Engine.TransactionHandlers.BrokerageTransactionHandler" + + def configure_credentials(self, lean_config: Dict[str, Any]) -> None: + lean_config["kraken-api-key"] = self._api_key + lean_config["kraken-api-secret"] = self._api_secret + lean_config["job-organization-id"] = self._organization_id + lean_config["kraken-verification-tier"] = self._verification_tier + + self._save_properties(lean_config, ["job-organization-id", "kraken-api-key", "kraken-api-secret", "kraken-verification-tier"]) + + def ensure_module_installed(self) -> None: + if not self._is_module_installed: + container.module_manager().install_module(KRAKEN_PRODUCT_ID, self._organization_id) + self._is_module_installed = True + +class KrakenDataFeed(LeanConfigConfigurer): + """A LeanConfigConfigurer implementation for the Kraken data feed.""" + + def __init__(self, brokerage: KrakenBrokerage) -> None: + self._brokerage = brokerage + + @classmethod + def get_name(cls) -> str: + return KrakenBrokerage.get_name() + + @classmethod + def build(cls, lean_config: Dict[str, Any], logger: Logger) -> LeanConfigConfigurer: + return KrakenDataFeed(KrakenBrokerage.build(lean_config, logger)) + + def configure(self, lean_config: Dict[str, Any], environment_name: str) -> None: + self._brokerage.ensure_module_installed() + + lean_config["environments"][environment_name]["data-queue-handler"] = "KrakenBrokerage" + lean_config["environments"][environment_name]["history-provider"] = "BrokerageHistoryProvider" + + self._brokerage.configure_credentials(lean_config) From f68588cab1d1b55c33dfef5644e203537d19ec35 Mon Sep 17 00:00:00 2001 From: Martin-Molinero Date: Thu, 18 Nov 2021 12:18:38 -0300 Subject: [PATCH 2/3] Add log for dev version --- lean/commands/gui/start.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lean/commands/gui/start.py b/lean/commands/gui/start.py index ceccf7e0..32dc4ef7 100644 --- a/lean/commands/gui/start.py +++ b/lean/commands/gui/start.py @@ -178,7 +178,8 @@ def start(organization: Optional[str], # Install the CLI in the GUI container run_options["commands"].append("pip uninstall -y lean") if lean.__version__ == "dev": - lean_cli_dir = Path(__file__).absolute().parent.parent.parent.parent + lean_cli_dir = str(Path(__file__).absolute().parent.parent.parent.parent) + logger.info(f"Detected lean dev version. Will mount local folder '{lean_cli_dir}' as /lean-cli") run_options["volumes"][str(lean_cli_dir)] = { "bind": "/lean-cli", "mode": "rw" From 0b69fba3b2079538c7563f3d6228157de846f857 Mon Sep 17 00:00:00 2001 From: Martin-Molinero Date: Thu, 18 Nov 2021 18:31:49 -0300 Subject: [PATCH 3/3] Refresh MHDB and SPDB file databases --- lean/components/api/data_client.py | 8 ++++++ lean/components/cloud/data_downloader.py | 34 +++++++++++++++++++++--- lean/container.py | 1 + 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/lean/components/api/data_client.py b/lean/components/api/data_client.py index 29eb990e..38655f5c 100644 --- a/lean/components/api/data_client.py +++ b/lean/components/api/data_client.py @@ -46,6 +46,14 @@ def download_file(self, file_path: str, organization_id: str) -> bytes: return self._http_client.get(data["link"]).content + def download_public_file(self, data_endpoint: str) -> bytes: + """Downloads the content of a downloadable public file. + + :param data_endpoint: the url of the public file + :return: the content of the file + """ + return self._http_client.get(data_endpoint).content + def list_files(self, prefix: str) -> List[str]: """Lists all remote files with a given prefix. diff --git a/lean/components/cloud/data_downloader.py b/lean/components/cloud/data_downloader.py index 003c0d76..ac989239 100644 --- a/lean/components/cloud/data_downloader.py +++ b/lean/components/cloud/data_downloader.py @@ -13,6 +13,7 @@ import multiprocessing from pathlib import Path +from datetime import * from typing import Any, List, Callable from joblib import delayed, Parallel @@ -23,6 +24,12 @@ from lean.models.errors import RequestFailedError +def _store_local_file(file_content: bytes, file_path: Path): + file_path.parent.mkdir(parents=True, exist_ok=True) + with file_path.open("wb+") as f: + f.write(file_content) + + class DataDownloader: """The DataDownloader is responsible for downloading data from QuantConnect Datasets.""" @@ -37,6 +44,28 @@ def __init__(self, logger: Logger, api_client: APIClient, lean_config_manager: L self._api_client = api_client self._lean_config_manager = lean_config_manager + def update_database_files(self): + """Will update lean data folder database files if required + + """ + try: + now = datetime.now() + config = self._lean_config_manager.get_lean_config() + last_update = config["file-database-last-update"] if "file-database-last-update" in config else '' + if not last_update or now - datetime.strptime(last_update, '%m/%d/%Y') > timedelta(days=1): + data_dir = self._lean_config_manager.get_data_directory() + + _store_local_file(self._api_client.data.download_public_file( + "https://raw.githubusercontent.com/QuantConnect/Lean/master/Data/symbol-properties/symbol-properties-database.csv"), + data_dir / "symbol-properties" / "symbol-properties-database.csv") + _store_local_file(self._api_client.data.download_public_file( + "https://raw.githubusercontent.com/QuantConnect/Lean/master/Data/market-hours/market-hours-database.json"), + data_dir / "market-hours" / "market-hours-database.json") + + self._lean_config_manager.set_properties({"file-database-last-update": now.strftime('%m/%d/%Y')}) + except Exception as e: + self._logger.error(str(e)) + def download_files(self, data_files: List[Any], overwrite: bool, organization_id: str) -> None: """Downloads files from QuantConnect Datasets to the local data directory. @@ -105,8 +134,5 @@ def _download_file(self, callback() return - local_path.parent.mkdir(parents=True, exist_ok=True) - with local_path.open("wb+") as f: - f.write(file_content) - + _store_local_file(file_content, local_path) callback() diff --git a/lean/container.py b/lean/container.py index 953314d8..0d35a7ed 100644 --- a/lean/container.py +++ b/lean/container.py @@ -116,3 +116,4 @@ class Container(DeclarativeContainer): container = Container() +container.data_downloader().update_database_files()