🦙 feat(trade): alpaca 🦙 (#244)

* progress #minor

* improve error handling for release

* progress

* add workflow

* fix lint issue and change to day order

* add test

* add another test

* add another test

* finish tests, consider BTC if QQQ fails

* split string

* make notional string

* try again

* improve sleep_until fx

* optimize portfolio once a month

* run ohlc on dev env

* add timezone for sleep_until

.github/workflows/build.yml

@@ -25,6 +25,8 @@ env:
   AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
   PREF_EXCHANGE: ${{ secrets.PREF_EXCHANGE }}
   GLASSNODE_PASS: ${{ secrets.GLASSNODE_PASS }}
+  ALPACA_PAPER: ${{ secrets.ALPACA_PAPER }}
+  ALPACA_PAPER_SECRET: ${{ secrets.ALPACA_PAPER_SECRET }}
 
 jobs:
   build:

.github/workflows/deps.yml

@@ -28,6 +28,9 @@ env:
   PREF_EXCHANGE: ${{ secrets.PREF_EXCHANGE }}
   TEST: true
   GLASSNODE_PASS: ${{ secrets.GLASSNODE_PASS }}
+  ALPACA_PAPER: ${{ secrets.ALPACA_PAPER }}
+  ALPACA_PAPER_SECRET: ${{ secrets.ALPACA_PAPER_SECRET }}
+  S3_BUCKET: ${{ secrets.S3_DEV_BUCKET }}
 
 jobs:
   build:
@@ -77,8 +80,6 @@ jobs:
           chrome-version: stable
 
       - name: Run all unit tests
-        env:
-          S3_BUCKET: ${{ secrets.S3_DEV_BUCKET }}
         run: coverage run -m pytest -vv -s
 
       - name: Generate test coverage report
@@ -113,6 +114,10 @@ jobs:
         run: |
           python util/decrypt.py encrypted/create_model.py
           python util/decrypt.py encrypted/predict_signal.py
+          python util/decrypt.py encrypted/optimize_portfolio.py
+
+      - name: Optimize portfolio
+        run: python encrypted/optimize_portfolio.py
 
       - name: Create model
         run: python encrypted/create_model.py

.github/workflows/optimize.yml

@@ -0,0 +1,55 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Optimize Portfolio
+
+on:
+  schedule:
+    - cron: "30 15 1-7 * *"
+    # 11:30am EDT 
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+
+      - name: Cache pip dependencies
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8 pytest coverage
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+
+      - name: Decrypt script
+        env:
+          RH_PASSWORD: ${{ secrets.RH_PASSWORD }}
+          SALT: ${{ secrets.SALT }}
+        run: |
+          python util/decrypt.py encrypted/optimize_portfolio.py
+
+      - name: Optimize portfolio
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+          S3_BUCKET: ${{ secrets.S3_BUCKET }}
+          ALPACA: ${{ secrets.ALPACA }}
+          ALPACA_SECRET: ${{ secrets.ALPACA_SECRET }}
+        run: |
+          python encrypted/predict_signal.py
+

.github/workflows/sandbox.yml

@@ -83,8 +83,8 @@ jobs:
       # - name: Update splits
       #   run: python scripts/update_splits.py
 
-      # - name: Update OHLC
-      #   run: python scripts/update_ohlc.py
+      - name: Update OHLC
+        run: python scripts/update_ohlc.py
 
       # - name: Update intraday
       #   run: python scripts/update_intraday.py
@@ -106,6 +106,10 @@ jobs:
         run: |
           python util/decrypt.py encrypted/create_model.py
           python util/decrypt.py encrypted/predict_signal.py
+          python util/decrypt.py encrypted/optimize_portfolio.py
+
+      - name: Optimize portfolio
+        run: python encrypted/optimize_portfolio.py
 
       - name: Create model
         run: python encrypted/create_model.py

encrypted/optimize_portfolio.py.encrypted

@@ -0,0 +1 @@
+gAAAAABnidf7E2zHGfugNoEBHFFZ8L49YLV1d0CGbrYfR0IolJcmouZdaNVpy7_FuXQP1fhe0yTJBjxoB2Rz6YWXguXNIE0x8zPAviu03WV-vEwoS2iRMx6Ey5pex1SZUO_VutH72GEaTNT5NVztLk8dt7z60Zc9is-m1_4KywNmekm1Qqgv0cGSOULgquLYbly0MVSrR2xIoEphohO4jLLj3Uzo7s7qWQ08V2pCqUrxFiNHRzLTDM3Pb-hX6kTtJIDDtjS_CoztTvWbxvuwjMSTqmrEyVt9BoKRfMjA2CERuk8L-kBuyrpekQCs1OdkMKU7ZVCB8NtutElK66W1DcoPhlYIbhN5dNoo2UcKiL73VEuse7XWyZHOM47CaQO3qWJV1o9q2N7AqO9ytOGf8yHWOaw4hwXfsXvjQlUY_9magkPJtHOz_3R67tryE--nv_wHEXXHI7S4A3sJ-Dtqr-c7DmKL1IpGeAIHGrhiLNYI-T9AT1gQyl1KWPdgy1OdVLSgnCBtcxwYvktcHbrAwLxX3_nuhNSYNtFBzpIo0SVFhPPDfaceYzZyBvLnitcKS3Ecz7AgzZv3_CV4qCh4ZBRlHqU7exdT99kPzKrAVE7bsSmPj-OP3dUT5cC99PaMNpPmYlDYp5FEIjhG8wWabcUFQqnZOXmNBDAjS3m2hhNAsp3wqKBQAwqExOAWeYW-k9iArvYqggboRPDBTmVxHqBqy-hnKHooDPegmkKjKqgz0gcDJ_96-1Q88RUOUHvr_ML2o_IiNdowJs9IBunmBn4vvPxp9g3FhCv-lv_DS5VsLQZjdO-xEqo-wJ3GSq7jt8iFM4F3_kHyT9yYgxUimBEjHlYkzcfuXhvhwY7mtzC_kGPkKwTpaPqkSuEgzRcn7gryb55nhVoE2ofosF9ndJXPTsGlcHU3Fze3ILdz5R19DVQ_xBfNii4Z7GlA-H66CS8key1WGi6OnE_kfnU0o9O-wMlOJ1bXX9p2O6mdgSHzzuRb1FNPtTeHiFKEV95tdTb3bQyj_deOdVUKpoHxMjemRoitFetp5tjtTj_4zHdOOQdku7Jn-7_r96UTX1ZcwJ5VO54B_H9uZyyh6dbgF-aZE2f-v0LwMeEqPFSvEo9g9DfH-YKxmYeahL5D7RU_fVVmG6tUDAIjlsi5YLCl0QeLES5r5uE1QStYmGnx5VFGrbPld3P6wMCO2OP23xmyPeXLDVO-WCvAqWE9chAPs5x8sDHbseEgTBxwlnRpFM629iKA0jIgCwxj2ZBhu9aAUaI-zdGzH_VX-RUjqyD5R2kNxfkhaB53lfkwVWOuv1WD8uBhuiSH0U-YDLQYsL-irBc1X08Gucl2otfqaqICBUH5KSaFN6w-Q_PfHAU2K7R-qSVNVUBA_XyowDfa6jPaEDY7NHNMQ2jRytBM0uT2GfIokzHVp0CWnkQUOG7pG0uuH17Egcdgtysxiws_-rMpU6CYV1ow2ebowA8_l9G_D9TeN0V7-3466Dgk5nC59Ahary2OaAgwQlK8sNKJs-7MiMHiX9JcMRIC_VJiN6sPAo-rmEFyXrEaYJQ3k6yJ3-_DPNbK29LAOugYfAIKvJnhX88_xqCS-jSJF26-sisX_70jrMUZsgphqo_zkxet7FZZJwrhBObBwqncqmuTJLfSChDhU2guTD6bnnDDzM_FTeksnl38RkKkhR7GIExMEbJyn5RhPyxt_jJdpatOp_fsJbnbZ3bGZBPIscOD47KkuDZYwK4Mau1547Hx4qrLF40yP78PSQcPwv5l2-XU-skptvdrJ-sopdgnM563-lh6neOdXQVVQJVlfRs6ERRcITaUc0wyjrQn0szFYrCEoFSdagTdK0JmYBzZFBOu9-leXaDgMVaeJGguFLJDcctg-4XbHD-5CxOpbQoMLo0OQyBQlJQBJWauyp4s8fF5nVxTa92VKpKptAr9Z56Dd6NAlzXYpILgTOurkMwEoRR_47lk6ir4XqAyuy_mKnSaAO02vLgTF6lcUGLrTHDgviFXSre9eTcZ4ZnirVktXgAcbppbrNxYR8dMF0tclIVbNJvJV5UWP9EavKZimOcuHSQEb6yOyxd5rV9XU0sT-J7YV04-rIdD2twRKhaPhHhwfBq-LUD9czeh8wD1dVuARL7CZ1fuwcusyTve75RLv3VYhosyy_hddNRBEfeyS_O0442Iy8X84iX02Fd40Al3Dkd6gJpgCAoQuPWKvfrll_B9RCVuVWKSiXzlupqiefT-IxvoD2Z7onqYHxolNVNEdQHWavgYGdulQ1pOTnKE4sdQw2N54eGE1ZqKbItd97O_JO4Vwz42Nwt5P0GCfFVSMAmk63KW--aEX3Cl9XceP83-GOb9n_d_tKvfoiUoFl8IPTbWh47CxtouV0H1p-kCx_st1UkxdNhwebZhvvCjYembh4l0CL0rofJOXZsEyprh5YytFLPzgqFKlECtMcMbiZpdcH1oHLzqa5ZmrQq1U1yfw5Lj8z6EMx0nxkgAQY4t56Orkfdt0joGxlnSj8LKnfw2rGLDids2sMFXk_rwdpFAgkj8Cm-XdRREIZC6CV1yCzyDZMfIHtnYkUN2qa5etXeseShfGOOaGiNZOahORjZtvUmE15s-Lw2-LTuJQux0u6bWGT2ix37M3pqkpkiujY11dsr3JywwIjPx1DvS_kSafM1eDdWtTkCD6kMBlagvS5EYrnHDvc_IfMov_aq9D6EPhC2tZB78_Spb9FVoLYPE_DEhqSAKz9yVMTt_9gkxcHuCjn37zTkPwTDHCTIYp-Mp-slvaMLZ0ys8Cml8_FE9y9djaZ9uP5WdUZrxIStpz7k4qgcruuCJhVRzBfuBGXkNTMUoERwTxt-6wxzBQaakakeL2yABezEMngJh6LCJXuexu3FFDT7RbzhfQj-k-tY1suT5vcuad8YN8XkFiJhNB0hElxIUqCEerC7G18-443hb7FMErOdvuKSv7yzXB1aNd7M_Pfdr920T2brBNfFdhYBqyNIk7HUloM7Rq4ipDrcH_6dLc3R6aIJLDQV4-Ynk40kzA7fd0fgoBtbRvhvVUvoEe6aF8W8BraJ1F02iJ0-1ZZIofnagQEp4oh_mKVx8CR0uTeDT4mcyT-kQf4zZ_ub7PwZMGRtxmpaBhRpfB_4_8EqHgRA8s8AW5k3fBbCDMzH-O9stWFwNwMkKkS2YPilwwsbg2rfMWHbRJEHt_YrM_dzh_JokPjOEYi-jI8mYvQ1YPSX-pRQqfxDsaAfXCNUezb92rIKkrJB6YzKY9tgn0nu7JtD1laxYbNts19WpcL8vzWmr-qRMmRoFdUxFdbImdxqsVXiCvZdt1fSaMq9WdybgClqh15rHf67CeKLQfEMoa7KCwArc8BfQziurWMQk9mrUvAyBVFB7lm4oKLD1UqBj4cOienTNpYqnN4C7l9DypB3SNDhSsvKsF8T1YW57dT7Qt2HdIKq1_xjjlTWhtJQmFvJ8nYlNUgT27ajbtbxszk32X1Ial2ZZeGfc39UWE7tRrHaGtdkioAHrZ8b0az-fc32nVkvaVG-M1NWXFcrB3qTfIAj9IspOM9UFs0dh2-2M1J0MNH-RgLvpi2td6PEQu3X3KkCW_Vesd1dzjUTPdMqZ5g9Az1zYCDn-t4WADiFBi09P3uE5gHIKgHH3024-u15zavm16j8DrZ9x7fmSSc5YjGr3MqDcQ8hR8E-BPofO-II_VYz-aJJ8j7FPShRuUwgUN0XwEI4cviFtj6HQCTywK5r8izPZoi7cNN2Adl8eEq-HNtKQwhZayYhfs0IMpGXNYs9jipmecu15uCV4p_ocuimtSyn48PzMNGEAWYI3e8Mezlpjz_HA68iIy-N8PHcHCqji09j95imhD7OC0nXfwa3_QlXcVs3RblonZ0PlWyoUNAHZ9hBBzrOkOuNH5H5KECgVUliacmeQ63rvQPrfY_9w2XWy8rQAbrsLM86qfXjY0cyv31aSBEZ1pxsYeF-CRzJeAI6ggWH0cYBeYLJsLZN7sdtKOpHSB5HTGG0EAsqifmie5XTffvRCTc1lNX4XsE_AbLDo6Z63A1HpGhGbwxDL7w-RwPm9t4vuA5ztspow6TnuV6pU7VXDjfQz2Z447bMZaWNzYYumVf_TPzSzpN7oIpVzLToYTOpyIH2a1ZIMD2wrw-QazK4MiMtcR5ag_5KGZF1y5ra20AhgcbPYPSFPwhs=

hyperdrive/Calculus.py

@@ -24,8 +24,8 @@ def find_centroid(self, points, method='mean'):
             ]
         return [self.avg(component) for component in components]
 
-    def delta(self, series):
-        return series / series.shift() - 1
+    def delta(self, series, window=1):
+        return series / series.shift(window) - 1
 
     def roll(self, series, window):
         return series.rolling(window).mean()

hyperdrive/Constants.py

@@ -70,6 +70,7 @@ def get_env_bool(var_name):
 
 # Time
 TZ = timezone('US/Eastern')
+UTC = timezone('UTC')
 DATE_FMT = '%Y-%m-%d'
 TIME_FMT = '%H:%M'
 PRECISE_TIME_FMT = '%H:%M:%S'
@@ -103,7 +104,7 @@ def get_env_bool(var_name):
 
 # Model
 MAX_MODEL_AGE_DAYS = 90  # 3 months
-MIN_MODEL_ACCURACY = 0.95
+MIN_MODEL_ACCURACY = 0.925
 
 # Misc
 POLY_CRYPTO_SYMBOLS = [
@@ -233,6 +234,12 @@ def get_orders_path(self):
             'orders.csv'
         )
 
+    def get_new_orders_path(self, provider):
+        return os.path.join(
+            DATA_DIR,
+            f'{provider}.csv'
+        )
+
     def get_api_path(self, endpoint):
         return os.path.join(
             DATA_DIR,

hyperdrive/DataSource.py

@@ -458,8 +458,7 @@ def get_ndx(self, date=datetime.now()):
         return self.standardize_ndx(df)
 
 
-class Alpaca(MarketData):
-    # AlpacaData
+class AlpacaData(MarketData):
     def __init__(
             self,
             token=os.environ.get('ALPACA'),

hyperdrive/Exchange.py

@@ -17,6 +17,61 @@ def create_pair(self, base, quote):
         return f'{base}{quote}'
 
 
+class AlpacaEx(CEX):
+    def __init__(
+            self,
+            token=os.environ.get('ALPACA'),
+            secret=os.environ.get('ALPACA_SECRET'),
+            paper=False
+    ):
+        super().__init__()
+        self.base = (f'https://{"paper-" if paper or C.TEST else ""}'
+                     'api.alpaca.markets')
+        self.version = 'v2'
+        self.token = os.environ.get(
+            'ALPACA_PAPER') if paper or C.TEST else token
+        self.secret = os.environ.get(
+            'ALPACA_PAPER_SECRET') if paper or C.TEST else secret
+        if not (self.token and self.secret):
+            raise Exception('missing Alpaca credentials')
+
+    def make_request(self, method, route, payload={}):
+        parts = [self.base, self.version, route]
+        url = '/'.join(parts)
+        headers = {
+            "accept": "application/json",
+            "APCA-API-KEY-ID": self.token,
+            "APCA-API-SECRET-KEY": self.secret
+        }
+        response = requests.request(method, url, json=payload, headers=headers)
+        if response.ok:
+            return response.json()
+        else:
+            raise Exception(response.text)
+
+    def get_positions(self):
+        return self.make_request('GET', 'positions')
+
+    def close_position(self, symbol):
+        return self.make_request('DELETE', f'positions/{symbol}')
+
+    def get_order(self, id):
+        return self.make_request('GET', f'orders/{id}')
+
+    def get_account(self):
+        return self.make_request('GET', 'account')
+
+    def create_order(self, symbol, side, notional):
+        payload = {
+            'symbol': symbol,
+            'side': side.lower(),
+            'type': 'market',
+            'notional': str(notional),
+            'time_in_force': 'day'
+        }
+        return self.make_request('POST', 'orders', payload)
+
+
 class Kraken(CEX):
     def __init__(self, key=None, secret=None, test=False):
         super().__init__()

hyperdrive/TimeMachine.py

@@ -1,6 +1,6 @@
 from time import sleep
 from datetime import datetime, timedelta
-from Constants import TZ, DATE_FMT, TIME_FMT, PRECISE_TIME_FMT
+from Constants import TZ, UTC, DATE_FMT, TIME_FMT, PRECISE_TIME_FMT
 
 
 class TimeTraveller:
@@ -68,10 +68,10 @@ def combine_date_time(self, date, time):
     def get_diff(self, t1, t2):
         return abs((t1 - t2).total_seconds())
 
-    def sleep_until(self, time):
+    def sleep_until(self, time, tz=UTC):
         # time could be "00:00"
-        curr = datetime.utcnow()
-        prev_sched = datetime.combine(curr.date(), self.get_time(time))
+        curr = datetime.now(tz)
+        prev_sched = datetime.combine(curr.date(), self.get_time(time), tz)
         next_sched = prev_sched + timedelta(days=1)
 
         prev_diff = self.get_diff(curr, prev_sched)
@@ -81,7 +81,7 @@ def sleep_until(self, time):
         diff = self.get_diff(curr, sched) if sched > curr else 0
 
         while diff > 0:
-            curr = datetime.utcnow()
+            curr = datetime.now(tz)
             diff = self.get_diff(curr, sched) if sched > curr else 0
             sleep(diff)
 

scripts/update_hist_ohlc.py

@@ -2,33 +2,33 @@
 import sys
 from multiprocessing import Process
 sys.path.append('hyperdrive')
-from DataSource import Polygon, Alpaca  # noqa autopep8
+from DataSource import Polygon, AlpacaData  # noqa autopep8
 from Constants import PathFinder  # noqa autopep8
 import Constants as C  # noqa autopep8
 
-alpc = Alpaca(paper=C.TEST)
+alpc = AlpacaData(paper=C.TEST)
 poly = Polygon(os.environ['POLYGON'])
 stock_symbols = poly.get_symbols()
 poly_symbols = stock_symbols + C.POLY_CRYPTO_SYMBOLS
 alpc_symbols = set(alpc.get_ndx()[C.SYMBOL]).union(stock_symbols)
-timeframe = '2m'
+timeframe = '10y'
 
 # Double redundancy
 # 1st pass
 
 
-def update_poly_ohlc():
-    for symbol in poly_symbols:
-        filename = PathFinder().get_ohlc_path(
-            symbol=symbol, provider=poly.provider)
-        try:
-            poly.save_ohlc(symbol=symbol, timeframe=timeframe)
-        except Exception as e:
-            print(f'Polygon.io OHLC update failed for {symbol}.')
-            print(e)
-        finally:
-            if C.CI and os.path.exists(filename):
-                os.remove(filename)
+# def update_poly_ohlc():
+#     for symbol in poly_symbols:
+#         filename = PathFinder().get_ohlc_path(
+#             symbol=symbol, provider=poly.provider)
+#         try:
+#             poly.save_ohlc(symbol=symbol, timeframe=timeframe)
+#         except Exception as e:
+#             print(f'Polygon.io OHLC update failed for {symbol}.')
+#             print(e)
+#         finally:
+#             if C.CI and os.path.exists(filename):
+#                 os.remove(filename)
 
 # 2nd pass
 
@@ -47,9 +47,10 @@ def update_alpc_ohlc():
                 os.remove(filename)
 
 
-p1 = Process(target=update_poly_ohlc)
-p2 = Process(target=update_alpc_ohlc)
-p1.start()
-p2.start()
-p1.join()
-p2.join()
+if __name__ == '__main__':
+    # p1 = Process(target=update_poly_ohlc)
+    p2 = Process(target=update_alpc_ohlc)
+    # p1.start()
+    p2.start()
+    # p1.join()
+    p2.join()

scripts/update_ohlc.py

@@ -2,12 +2,12 @@
 import sys
 from multiprocessing import Process, Value
 sys.path.append('hyperdrive')
-from DataSource import Polygon, Alpaca  # noqa autopep8
+from DataSource import Polygon, AlpacaData  # noqa autopep8
 from Constants import PathFinder  # noqa autopep8
 import Constants as C  # noqa autopep8
 
 counter = Value('i', 0)
-alpc = Alpaca(paper=C.TEST)
+alpc = AlpacaData(paper=C.TEST)
 poly = Polygon(os.environ['POLYGON'])
 stock_symbols = poly.get_symbols()
 poly_symbols = stock_symbols + C.POLY_CRYPTO_SYMBOLS

setup.py

@@ -12,9 +12,12 @@ def get_version():
     token = os.environ.get('GITHUB')
     headers = {'Authorization': f'token {token}'}
     response = requests.get(url, headers=headers if token else None)
-    data = response.json()
-    version = data['tag_name'].replace('v', '')
-    return version
+    if response.ok:
+        data = response.json()
+        version = data['tag_name'].replace('v', '')
+        return version
+    else:
+        raise Exception(response.text)
 
 
 def get_requirements():

test/test_DataSource.py

@@ -4,7 +4,7 @@
 from random import choice
 import pandas as pd
 from hyperdrive.DataSource import MarketData, Indices, Polygon, \
-                        LaborStats, Glassnode, Alpaca  # noqa autopep8
+                        LaborStats, Glassnode, AlpacaData  # noqa autopep8
 import hyperdrive.Constants as C  # noqa autopep8
 from hyperdrive.Workflow import Flow  # noqa autopep8
 from hyperdrive.Utils import SwissArmyKnife  # noqa autopep8
@@ -14,7 +14,7 @@
 knife = SwissArmyKnife()
 md = knife.use_dev(MarketData())
 idc = knife.use_dev(Indices())
-alpc = knife.use_dev(Alpaca(paper=True))
+alpc = knife.use_dev(AlpacaData(paper=True))
 poly = knife.use_dev(Polygon())
 bls = knife.use_dev(LaborStats())
 glass = knife.use_dev(Glassnode(use_cookies=True))
@@ -277,7 +277,7 @@ def test_get_ndx(self):
 
 class TestAlpaca:
     def test_init(self):
-        assert isinstance(alpc, Alpaca)
+        assert isinstance(alpc, AlpacaData)
         assert hasattr(alpc, 'base')
         assert hasattr(alpc, 'token')
         assert hasattr(alpc, 'secret')