Merge branch 'QuantConnect:master' into master

README.md

@@ -204,7 +204,7 @@ Usage: lean cloud live [OPTIONS] PROJECT
   events and --notify-insights.
 
 Options:
-  --brokerage [Paper Trading|Interactive Brokers|Tradier|OANDA|Bitfinex|Coinbase Pro|Binance]
+  --brokerage [Paper Trading|Interactive Brokers|Tradier|OANDA|Bitfinex|Coinbase Pro|Binance|Kraken|FTX]
                                   The brokerage to use
   --ib-user-name TEXT             Your Interactive Brokers username
   --ib-account TEXT               Your Interactive Brokers account id
@@ -720,9 +720,9 @@ Options:
   -d, --detach                    Run the live deployment in a detached Docker container and return immediately
   --gui                           Enable monitoring and controlling of the deployment via the local GUI
   --gui-organization TEXT         The name or id of the organization with the local GUI module subscription
-  --brokerage [Paper Trading|Interactive Brokers|Tradier|OANDA|Bitfinex|Coinbase Pro|Binance|Zerodha|Terminal Link|Atreyu|Trading Technologies]
+  --brokerage [Paper Trading|Interactive Brokers|Tradier|OANDA|Bitfinex|Coinbase Pro|Binance|Zerodha|Terminal Link|Atreyu|Trading Technologies|Kraken|FTX]
                                   The brokerage to use
-  --data-feed [Interactive Brokers|Tradier|OANDA|Bitfinex|Coinbase Pro|Binance|Zerodha|Terminal Link|Trading Technologies|Custom data only|IQFeed]
+  --data-feed [Interactive Brokers|Tradier|OANDA|Bitfinex|Coinbase Pro|Binance|Zerodha|Terminal Link|Trading Technologies|Custom data only|Kraken|FTX|IQFeed]
                                   The data feed to use
   --ib-user-name TEXT             Your Interactive Brokers username
   --ib-account TEXT               Your Interactive Brokers account id
@@ -764,7 +764,8 @@ Options:
   --iqfeed-password TEXT          Your IQFeed password
   --iqfeed-product-name TEXT      The product name of your IQFeed developer account
   --iqfeed-version TEXT           The product version of your IQFeed developer account
-  --terminal-link-organization TEXT   The name or id of the organization with the Terminal Link module subscription
+  --terminal-link-organization TEXT
+                                  The name or id of the organization with the Terminal Link module subscription
   --bloomberg-environment [Production|Beta]
                                   The environment to run in
   --bloomberg-server-host TEXT    The host of the Bloomberg server
@@ -810,6 +811,15 @@ Options:
   --tt-order-routing-host TEXT    The host of the order routing server
   --tt-order-routing-port TEXT    The port of the order routing server
   --tt-log-fix-messages BOOLEAN   Whether FIX messages should be logged
+  --kraken-organization TEXT      The name or id of the organization with the kraken module subscription
+  --kraken-api-key TEXT           Your Kraken API key
+  --kraken-api-secret TEXT        Your Kraken API secret
+  --kraken-verification-tier TEXT
+                                  Your Kraken Verification Tier
+  --ftx-organization TEXT         The name or id of the organization with the FTX module subscription
+  --ftx-api-key TEXT              Your FTX API key
+  --ftx-api-secret TEXT           Your FTX API secret
+  --ftx-account-tier TEXT         Your FTX Account Tier
   --release                       Compile C# projects in release configuration instead of debug
   --image TEXT                    The LEAN engine image to use (defaults to quantconnect/lean:latest)
   --update                        Pull the LEAN engine image before starting live trading
@@ -1031,6 +1041,6 @@ If you need to add dependencies, first update `setup.py` (if it is a production
 
 The automated tests can be ran by running `pytest`. The filesystem and HTTP requests are mocked when running tests to make sure they run in an isolated environment.
 
-To update the commands reference part of the readme run `python scripts/readme.py` from the root of the project.
+Can build the lean CLI by running `python setup.py sdist bdist_wheel` from the root of the project and to install it `pip install --force-reinstall dist/lean-dev-py3-none-any.whl`. To update the commands reference part of the readme run `python scripts/readme.py` from the root of the project, after installing the new version.
 
 Maintainers can publish new releases by pushing a Git tag containing the new version to GitHub. This will trigger a GitHub Actions workflow which releases the current `master` branch to PyPI with the value of the tag as version. Make sure the version is not prefixed with "v".

announcements.json

@@ -1,5 +1,9 @@
 {
     "announcements": [
+        {
+            "date": "2021-11-17",
+            "message": "We've added support for Kraken and FTX brokerages"
+        },
         {
             "date": "2021-06-21",
             "message": "We've just released a new opening video of the CLI:\nhttps://youtube.com/watch?v=QJibe1XpP-U"

lean/commands/gui/start.py

@@ -178,7 +178,8 @@ def start(organization: Optional[str],
     # Install the CLI in the GUI container
     run_options["commands"].append("pip uninstall -y lean")
     if lean.__version__ == "dev":
-        lean_cli_dir = Path(__file__).absolute().parent.parent.parent.parent
+        lean_cli_dir = str(Path(__file__).absolute().parent.parent.parent.parent)
+        logger.info(f"Detected lean dev version. Will mount local folder '{lean_cli_dir}' as /lean-cli")
         run_options["volumes"][str(lean_cli_dir)] = {
             "bind": "/lean-cli",
             "mode": "rw"

lean/commands/live.py

@@ -35,6 +35,8 @@
 from lean.models.brokerages.local.tradier import TradierBrokerage, TradierDataFeed
 from lean.models.brokerages.local.trading_technologies import TradingTechnologiesBrokerage, TradingTechnologiesDataFeed
 from lean.models.brokerages.local.zerodha import ZerodhaBrokerage, ZerodhaDataFeed
+from lean.models.brokerages.local.kraken import KrakenBrokerage, KrakenDataFeed
+from lean.models.brokerages.local.ftx import FTXBrokerage, FTXDataFeed
 from lean.models.errors import MoreInfoError
 from lean.models.logger import Option
 
@@ -59,7 +61,9 @@
                                      "tt-market-data-host", "tt-market-data-port",
                                      "tt-order-routing-sender-comp-id", "tt-order-routing-target-comp-id",
                                      "tt-order-routing-host", "tt-order-routing-port",
-                                     "tt-log-fix-messages"]
+                                     "tt-log-fix-messages"],
+    "KrakenBrokerage": ["kraken-api-key", "kraken-api-secret", "kraken-verification-tier"],
+    "FTXBrokerage": ["ftx-api-key", "ftx-api-secret", "ftx-account-tier"]
 }
 
 # Data queue handler -> required configuration properties
@@ -74,7 +78,9 @@
     "ZerodhaBrokerage": _required_brokerage_properties["ZerodhaBrokerage"] + ["zerodha-history-subscription"],
     "BloombergBrokerage": _required_brokerage_properties["BloombergBrokerage"],
     "TradingTechnologiesBrokerage": _required_brokerage_properties["TradingTechnologiesBrokerage"],
-    "QuantConnect.ToolBox.IQFeed.IQFeedDataQueueHandler": ["iqfeed-iqconnect", "iqfeed-productName", "iqfeed-version"]
+    "QuantConnect.ToolBox.IQFeed.IQFeedDataQueueHandler": ["iqfeed-iqconnect", "iqfeed-productName", "iqfeed-version"],
+    "KrakenBrokerage": _required_brokerage_properties["KrakenBrokerage"],
+    "FTXBrokerage": _required_brokerage_properties["FTXBrokerage"]
 }
 
 _environment_skeleton = {
@@ -526,6 +532,38 @@ def _get_default_value(key: str) -> Optional[Any]:
               type=bool,
               default=lambda: _get_default_value("tt-log-fix-messages"),
               help="Whether FIX messages should be logged")
+@click.option("--kraken-organization",
+              type=str,
+              default=lambda: _get_default_value("job-organization-id"),
+              help="The name or id of the organization with the kraken module subscription")
+@click.option("--kraken-api-key",
+              type=str,
+              default=lambda: _get_default_value("kraken-api-key"),
+              help="Your Kraken API key")
+@click.option("--kraken-api-secret",
+              type=str,
+              default=lambda: _get_default_value("kraken-api-secret"),
+              help="Your Kraken API secret")
+@click.option("--kraken-verification-tier",
+              type=str,
+              default=lambda: _get_default_value("kraken-verification-tier"),
+              help="Your Kraken Verification Tier")
+@click.option("--ftx-organization",
+              type=str,
+              default=lambda: _get_default_value("job-organization-id"),
+              help="The name or id of the organization with the FTX module subscription")
+@click.option("--ftx-api-key",
+              type=str,
+              default=lambda: _get_default_value("ftx-api-key"),
+              help="Your FTX API key")
+@click.option("--ftx-api-secret",
+              type=str,
+              default=lambda: _get_default_value("ftx-api-secret"),
+              help="Your FTX API secret")
+@click.option("--ftx-account-tier",
+              type=str,
+              default=lambda: _get_default_value("ftx-account-tier"),
+              help="Your FTX Account Tier")
 @click.option("--release",
               is_flag=True,
               default=False,
@@ -611,6 +649,14 @@ def live(project: Path,
          tt_order_routing_host: Optional[str],
          tt_order_routing_port: Optional[str],
          tt_log_fix_messages: Optional[bool],
+         kraken_organization: Optional[str],
+         kraken_api_key: Optional[str],
+         kraken_api_secret: Optional[str],
+         kraken_verification_tier: Optional[str],
+         ftx_organization: Optional[str],
+         ftx_api_key: Optional[str],
+         ftx_api_secret: Optional[str],
+         ftx_account_tier: Optional[str],
          release: bool,
          image: Optional[str],
          update: bool) -> None:
@@ -767,6 +813,18 @@ def live(project: Path,
                                                                 tt_order_routing_host,
                                                                 tt_order_routing_port,
                                                                 tt_log_fix_messages)
+        elif brokerage == KrakenBrokerage.get_name():
+            ensure_options(["kraken_api_key", "kraken_api_secret", "kraken_verification_tier"])
+            brokerage_configurer = KrakenBrokerage(_get_organization_id(kraken_organization, "Kraken"),
+                                                   kraken_api_key,
+                                                   kraken_api_secret,
+                                                   kraken_verification_tier)
+        elif brokerage == FTXBrokerage.get_name():
+            ensure_options(["ftx_api_key", "ftx_api_secret", "ftx_account_tier"])
+            brokerage_configurer = FTXBrokerage(_get_organization_id(ftx_organization, "FTX"),
+                                                ftx_api_key,
+                                                ftx_api_secret,
+                                                ftx_account_tier)
 
         if data_feed == InteractiveBrokersDataFeed.get_name():
             ensure_options(["ib_user_name", "ib_account", "ib_password", "ib_enable_delayed_streaming_data"])
@@ -872,6 +930,20 @@ def live(project: Path,
                                                   iqfeed_password,
                                                   iqfeed_product_name,
                                                   iqfeed_version)
+        elif data_feed == KrakenDataFeed.get_name():
+            ensure_options(["kraken_api_key", "kraken_api_secret", "kraken_verification_tier"])
+            data_feed_configurer = KrakenDataFeed(
+                KrakenBrokerage(_get_organization_id(kraken_organization, "Kraken"),
+                                kraken_api_key,
+                                kraken_api_secret,
+                                kraken_verification_tier))
+        elif data_feed == FTXDataFeed.get_name():
+            ensure_options(["ftx_api_key", "ftx_api_secret", "ftx_account_tier"])
+            data_feed_configurer = FTXDataFeed(
+                FTXBrokerage(_get_organization_id(ftx_organization, "FTX"),
+                             ftx_api_key,
+                             ftx_api_secret,
+                             ftx_account_tier))
 
         environment_name = "lean-cli"
         lean_config = lean_config_manager.get_complete_lean_config(environment_name, algorithm_file, None)

lean/components/api/data_client.py

@@ -46,6 +46,14 @@ def download_file(self, file_path: str, organization_id: str) -> bytes:
 
         return self._http_client.get(data["link"]).content
 
+    def download_public_file(self, data_endpoint: str) -> bytes:
+        """Downloads the content of a downloadable public file.
+
+        :param data_endpoint: the url of the public file
+        :return: the content of the file
+        """
+        return self._http_client.get(data_endpoint).content
+
     def list_files(self, prefix: str) -> List[str]:
         """Lists all remote files with a given prefix.
 

lean/components/cloud/data_downloader.py

@@ -13,6 +13,7 @@
 
 import multiprocessing
 from pathlib import Path
+from datetime import *
 from typing import Any, List, Callable
 
 from joblib import delayed, Parallel
@@ -23,6 +24,12 @@
 from lean.models.errors import RequestFailedError
 
 
+def _store_local_file(file_content: bytes, file_path: Path):
+    file_path.parent.mkdir(parents=True, exist_ok=True)
+    with file_path.open("wb+") as f:
+        f.write(file_content)
+
+
 class DataDownloader:
     """The DataDownloader is responsible for downloading data from QuantConnect Datasets."""
 
@@ -37,6 +44,28 @@ def __init__(self, logger: Logger, api_client: APIClient, lean_config_manager: L
         self._api_client = api_client
         self._lean_config_manager = lean_config_manager
 
+    def update_database_files(self):
+        """Will update lean data folder database files if required
+
+        """
+        try:
+            now = datetime.now()
+            config = self._lean_config_manager.get_lean_config()
+            last_update = config["file-database-last-update"] if "file-database-last-update" in config else ''
+            if not last_update or now - datetime.strptime(last_update, '%m/%d/%Y') > timedelta(days=1):
+                data_dir = self._lean_config_manager.get_data_directory()
+
+                _store_local_file(self._api_client.data.download_public_file(
+                    "https://raw.githubusercontent.com/QuantConnect/Lean/master/Data/symbol-properties/symbol-properties-database.csv"),
+                    data_dir / "symbol-properties" / "symbol-properties-database.csv")
+                _store_local_file(self._api_client.data.download_public_file(
+                    "https://raw.githubusercontent.com/QuantConnect/Lean/master/Data/market-hours/market-hours-database.json"),
+                    data_dir / "market-hours" / "market-hours-database.json")
+
+                self._lean_config_manager.set_properties({"file-database-last-update": now.strftime('%m/%d/%Y')})
+        except Exception as e:
+            self._logger.error(str(e))
+
     def download_files(self, data_files: List[Any], overwrite: bool, organization_id: str) -> None:
         """Downloads files from QuantConnect Datasets to the local data directory.
 
@@ -55,6 +84,17 @@ def download_files(self, data_files: List[Any], overwrite: bool, organization_id
                                                   lambda: progress.update(progress_task, advance=1))
                      for data_file in data_files)
 
+            # update our config after we download all files, and not in parallel!
+            for relative_file in data_files:
+                if "/map_files/map_files_" in relative_file and relative_file.endswith(".zip"):
+                    self._lean_config_manager.set_properties({
+                        "map-file-provider": "QuantConnect.Data.Auxiliary.LocalZipMapFileProvider"
+                    })
+                if "/factor_files/factor_files_" in relative_file and relative_file.endswith(".zip"):
+                    self._lean_config_manager.set_properties({
+                        "factor-file-provider": "QuantConnect.Data.Auxiliary.LocalZipFactorFileProvider"
+                    })
+
             progress.stop()
         except KeyboardInterrupt as e:
             progress.stop()
@@ -94,18 +134,5 @@ def _download_file(self,
             callback()
             return
 
-        local_path.parent.mkdir(parents=True, exist_ok=True)
-        with local_path.open("wb+") as f:
-            f.write(file_content)
-
-        if relative_file.startswith("equity/usa/map_files/map_files_") and relative_file.endswith(".zip"):
-            self._lean_config_manager.set_properties({
-                "map-file-provider": "QuantConnect.Data.Auxiliary.LocalZipMapFileProvider"
-            })
-
-        if relative_file.startswith("equity/usa/factor_files/factor_files_") and relative_file.endswith(".zip"):
-            self._lean_config_manager.set_properties({
-                "factor-file-provider": "QuantConnect.Data.Auxiliary.LocalZipFactorFileProvider"
-            })
-
+        _store_local_file(file_content, local_path)
         callback()

lean/components/util/logger.py

@@ -78,13 +78,14 @@ def progress(self, prefix: str = "", suffix: str = "{task.percentage:0.0f}%") ->
         progress.start()
         return progress
 
-    def prompt_list(self, text: str, options: List[Option]) -> Any:
+    def prompt_list(self, text: str, options: List[Option], default: Optional[str] = None) -> Any:
         """Asks the user to select an option from a list of possible options.
 
         The user will not be prompted for input if there is only a single option.
 
         :param text: the text to display before prompting
         :param options: the available options
+        :param default: the default value if no input is given
         :return: the chosen option's id
         """
         if len(options) == 1:
@@ -96,7 +97,7 @@ def prompt_list(self, text: str, options: List[Option]) -> Any:
             self.info(f"{i + 1}) {option.label}")
 
         while True:
-            user_input = click.prompt("Enter an option", type=str)
+            user_input = click.prompt("Enter an option", type=str, default=default, show_default=True)
 
             try:
                 index = int(user_input)

lean/constants.py

@@ -81,6 +81,12 @@
 # The product id of the Atreyu module
 ATREYU_PRODUCT_ID = 65
 
+# The product id of the Kraken module
+KRAKEN_PRODUCT_ID = 130
+
+# The product id of the FTX module
+FTX_PRODUCT_ID = 138
+
 # The product ids for which a valid subscription is seen as a valid GUI module subscription
 GUI_PRODUCT_SUBSCRIPTION_IDS = [119, 120]
 

lean/container.py

@@ -116,3 +116,4 @@ class Container(DeclarativeContainer):
 
 
 container = Container()
+container.data_downloader().update_database_files()

lean/models/brokerages/cloud/__init__.py

@@ -18,6 +18,8 @@
 from lean.models.brokerages.cloud.oanda import OANDABrokerage
 from lean.models.brokerages.cloud.paper_trading import PaperTradingBrokerage
 from lean.models.brokerages.cloud.tradier import TradierBrokerage
+from lean.models.brokerages.cloud.kraken import KrakenBrokerage
+from lean.models.brokerages.cloud.ftx import FTXBrokerage
 
 all_cloud_brokerages = [
     PaperTradingBrokerage,
@@ -26,5 +28,7 @@
     OANDABrokerage,
     BitfinexBrokerage,
     CoinbaseProBrokerage,
-    BinanceBrokerage
+    BinanceBrokerage,
+    KrakenBrokerage,
+    FTXBrokerage
 ]