💯 feat(data): nasdaq 💯 (#240)

* progress

* finish save_ndx fx

* debug

* delete debug logs

* update files to remove iex

* add workflow and script for indices + update rh lib

* lint

* remove stocktwits

* test failures

* poly

* remove sentiment fxs

* fixes?

* edit test_save_splits

* change split timeframe

* fix splits for polygon data

* add more tests

* add dep

* fix standardize fx

* fix test #minor

* delete update sentiment

.github/workflows/build.yml

@@ -13,9 +13,6 @@ env:
   RH_USERNAME: ${{ secrets.RH_USERNAME }}
   RH_PASSWORD: ${{ secrets.RH_PASSWORD }}
   RH_2FA: ${{ secrets.RH_2FA }}
-  IEXCLOUD: ${{ secrets.IEXCLOUD }}
-  IEXCLOUD_SANDBOX: ${{ secrets.IEXCLOUD_SANDBOX }}
-  STOCKTWITS: ${{ secrets.STOCKTWITS }}
   GLASSNODE: ${{ secrets.GLASSNODE }}
   BLS: ${{ secrets.BLS }}
   BINANCE_TESTNET_KEY: ${{ secrets.BINANCE_TESTNET_KEY }}

.github/workflows/deps.yml

@@ -15,9 +15,6 @@ env:
   RH_USERNAME: ${{ secrets.RH_USERNAME }}
   RH_PASSWORD: ${{ secrets.RH_PASSWORD }}
   RH_2FA: ${{ secrets.RH_2FA }}
-  IEXCLOUD: ${{ secrets.IEXCLOUD }}
-  IEXCLOUD_SANDBOX: ${{ secrets.IEXCLOUD_SANDBOX }}
-  STOCKTWITS: ${{ secrets.STOCKTWITS }}
   GLASSNODE: ${{ secrets.GLASSNODE }}
   BLS: ${{ secrets.BLS }}
   BINANCE_TESTNET_KEY: ${{ secrets.BINANCE_TESTNET_KEY }}
@@ -99,9 +96,6 @@ jobs:
       # - name: Update OHLC
       #   run: python scripts/update_ohlc.py
 
-      - name: Update social sentiment
-        run: python scripts/update_sentiment.py
-
       # - name: Update intraday
       #   run: python scripts/update_intraday.py
 

.github/workflows/dividends.yml

@@ -39,7 +39,6 @@ jobs:
 
       - name: Update dividends
         env:
-          IEXCLOUD: ${{ secrets.IEXCLOUD }}
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}

.github/workflows/sentiment2.yml → .github/workflows/indices.yml

@@ -1,12 +1,12 @@
 # This workflow will automatically update data files
 # For more information see: https://help.github.com/en/actions/reference/events-that-trigger-workflows#scheduled-events-schedule
 
-name: Social Sentiment (2)
+name: Indices
 
 on:
   schedule:
-    - cron: "30 10 * * *"
-    # 6:30am EST
+    - cron: "00 22 * * *"
+    # 5:00pm EST
   workflow_dispatch:
 
 jobs:
@@ -37,12 +37,10 @@ jobs:
           python -m pip install --upgrade pip
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
 
-      - name: Update social sentiment
+      - name: Update indices
         env:
-          BATCH: ${{ 2 }}
-          STOCKTWITS: ${{ secrets.STOCKTWITS }}
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
           S3_BUCKET: ${{ secrets.S3_BUCKET }}
-        run: python scripts/update_sentiment.py
+        run: python scripts/update_indices.py

.github/workflows/intraday.yml

@@ -39,7 +39,6 @@ jobs:
 
       - name: Update Intraday OHLC
         env:
-          IEXCLOUD: ${{ secrets.IEXCLOUD }}
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}

.github/workflows/ohlc.yml

@@ -39,7 +39,6 @@ jobs:
 
       - name: Update OHLC
         env:
-          IEXCLOUD: ${{ secrets.IEXCLOUD }}
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}

.github/workflows/sandbox.yml

@@ -13,9 +13,6 @@ env:
   RH_USERNAME: ${{ secrets.RH_USERNAME }}
   RH_PASSWORD: ${{ secrets.RH_PASSWORD }}
   RH_2FA: ${{ secrets.RH_2FA }}
-  IEXCLOUD: ${{ secrets.IEXCLOUD }}
-  IEXCLOUD_SANDBOX: ${{ secrets.IEXCLOUD_SANDBOX }}
-  STOCKTWITS: ${{ secrets.STOCKTWITS }}
   GLASSNODE: ${{ secrets.GLASSNODE }}
   S3_BUCKET: ${{ secrets.S3_DEV_BUCKET }}
   POLYGON: ${{ secrets.POLYGON }}
@@ -87,9 +84,6 @@ jobs:
       # - name: Update OHLC
       #   run: python scripts/update_ohlc.py
 
-      - name: Update social sentiment
-        run: python scripts/update_sentiment.py
-
       # - name: Update intraday
       #   run: python scripts/update_intraday.py
 

.github/workflows/splits.yml

@@ -39,7 +39,6 @@ jobs:
 
       - name: Update splits
         env:
-          IEXCLOUD: ${{ secrets.IEXCLOUD }}
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}

README.md

@@ -44,25 +44,25 @@ Most secrets must be passed as environment variables. Future updates will allow
 
 Pre-requisites:
 
-- an IEXCloud or Polygon API key
+- a Polygon API key
 - an AWS account and an S3 bucket
 
 Environment Variables:
 
-- `IEXCLOUD` or `POLYGON`
+- `POLYGON`
 - `AWS_ACCESS_KEY_ID`
 - `AWS_SECRET_ACCESS_KEY`
 - `AWS_DEFAULT_REGION`
 - `S3_BUCKET`
 
 ```
 from hyperdrive import DataSource
-from DataSource import IEXCloud
+from DataSource import Polygon
 
-# Your IEXCloud API token must be an environment variable (accessible in os.environ['IEXCLOUD'])
+# Your Polygon API token must be an environment variable (accessible in os.environ['POLYGON'])
 
-iex = IEXCloud()
-df = iex.get_ohlc(symbol='TSLA', timeframe='7d')
+poly = Polygon()
+df = poly.get_ohlc(symbol='TSLA', timeframe='7d')
 print(df)
 ```
 
@@ -77,36 +77,36 @@ Output:
 2867 2021-11-16  1003.31  1057.1999  1002.18  1054.73  26542359
 ```
 
-Although this function won't save data to the S3 bucket, hyperdrive checks the S3 bucket with key `data/ohlc/iexcloud/TSLA.csv` to see if any cached data exists to correct for inconsistencies in values and column names. -->
+Although this function won't save data to the S3 bucket, hyperdrive checks the S3 bucket with key `data/ohlc/polygon/TSLA.csv` to see if any cached data exists to correct for inconsistencies in values and column names. -->
 
 ### 1. Storing data
 
 Pre-requisites:
 
-- an IEXCloud or Polygon API key
+- a Polygon API key
 - an AWS account and an S3 bucket
 
 Environment Variables:
 
-- `IEXCLOUD` or `POLYGON`
+- `POLYGON`
 - `AWS_ACCESS_KEY_ID`
 - `AWS_SECRET_ACCESS_KEY`
 - `AWS_DEFAULT_REGION`
 - `S3_BUCKET`
 
 ```
 from hyperdrive import DataSource
-from DataSource import IEXCloud, MarketData
+from DataSource import Polygon, MarketData
 
-# IEXCloud API token loaded as an environment variable (os.environ['IEXCLOUD'])
+# Polygon API token loaded as an environment variable (os.environ['POLYGON'])
 
 symbol = 'TSLA'
 timeframe = '7d'
 
 md = MarketData()
-iex = IEXCloud()
+poly = Polygon()
 
-iex.save_ohlc(symbol=symbol, timeframe=timeframe)
+poly.save_ohlc(symbol=symbol, timeframe=timeframe)
 df = md.get_ohlc(symbol=symbol, timeframe=timeframe)
 
 print(df)
@@ -267,11 +267,11 @@ Use the scripts provided in the [`scripts/`](https://github.com/suchak1/hyperdri
 Available data collection functions:
 
 - [x] [![Symbols](https://github.com/suchak1/hyperdrive/workflows/Symbols/badge.svg)](https://github.com/suchak1/hyperdrive/actions?query=workflow%3ASymbols) (from Robinhood)
-- [x] [![OHLC](https://github.com/suchak1/hyperdrive/workflows/OHLC/badge.svg)](https://github.com/suchak1/hyperdrive/actions?query=workflow%3AOHLC) (from IEXCloud and Polygon)
-- [x] [![Intraday](https://github.com/suchak1/hyperdrive/workflows/Intraday/badge.svg)](https://github.com/suchak1/hyperdrive/actions?query=workflow%3AIntraday) (from IEXCloud and Polygon)
-- [x] [![Dividends](https://github.com/suchak1/hyperdrive/workflows/Dividends/badge.svg)](https://github.com/suchak1/hyperdrive/actions?query=workflow%3ADividends) (from IEXCloud and Polygon)
-- [x] [![Splits](https://github.com/suchak1/hyperdrive/workflows/Splits/badge.svg)](https://github.com/suchak1/hyperdrive/actions?query=workflow%3ASplits) (from IEXCloud and Polygon)
-- [x] [![Social Sentiment](<https://github.com/suchak1/hyperdrive/workflows/Social%20Sentiment%20(1)/badge.svg>)](https://github.com/suchak1/hyperdrive/actions?query=workflow%3A%22Social+Sentiment+%281%29%22) (from StockTwits)
+- [x] [![OHLC](https://github.com/suchak1/hyperdrive/workflows/OHLC/badge.svg)](https://github.com/suchak1/hyperdrive/actions?query=workflow%3AOHLC) (from Polygon)
+- [x] [![Intraday](https://github.com/suchak1/hyperdrive/workflows/Intraday/badge.svg)](https://github.com/suchak1/hyperdrive/actions?query=workflow%3AIntraday) (from Polygon)
+- [x] [![Dividends](https://github.com/suchak1/hyperdrive/workflows/Dividends/badge.svg)](https://github.com/suchak1/hyperdrive/actions?query=workflow%3ADividends) (from Polygon)
+- [x] [![Splits](https://github.com/suchak1/hyperdrive/workflows/Splits/badge.svg)](https://github.com/suchak1/hyperdrive/actions?query=workflow%3ASplits) (from Polygon)
+<!-- - [x] [![Social Sentiment](<https://github.com/suchak1/hyperdrive/workflows/Social%20Sentiment%20(1)/badge.svg>)](https://github.com/suchak1/hyperdrive/actions?query=workflow%3A%22Social+Sentiment+%281%29%22) (from StockTwits) -->
 - [x] [![Unemployment](https://github.com/suchak1/hyperdrive/workflows/Unemployment/badge.svg)](https://github.com/suchak1/hyperdrive/actions?query=workflow%3AUnemployment) (from the Bureau of Labor Statistics)
 
 ---

hyperdrive/Calculus.py

@@ -38,14 +38,14 @@ def smooth(self, series, window, order):
         )
 
     def derive(self, y, x=np.array([0, 1])):
-        if type(x) == pd.Series:
+        if isinstance(x, pd.Series):
             x = x.to_numpy()
         x = x.astype('float64')
         x_delta = (x[1] - x[0])
         return np.gradient(y, x_delta)
 
     def cv(self, x, ddof=0):
-        if type(x) == pd.Series:
+        if isinstance(x, pd.Series):
             axis = 0
         else:
             axis = 1

hyperdrive/Constants.py

@@ -31,17 +31,14 @@ def get_env_bool(var_name):
 OHLC_DIR = 'ohlc'
 SENT_DIR = 'sentiment'
 INTRA_DIR = 'intraday'
+IDX_DIR = 'indices'
 # providers
-IEX_DIR = 'iexcloud'
 POLY_DIR = 'polygon'
-TWIT_DIR = 'stocktwits'
 # models
 MODELS_DIR = 'models'
 
 folders = {
-    'iexcloud': IEX_DIR,
     'polygon': POLY_DIR,
-    'stocktwits': TWIT_DIR
 }
 
 # Column Names
@@ -79,7 +76,6 @@ def get_env_bool(var_name):
 POS = 'Pos'
 NEG = 'Neg'
 DELTA = 'Delta'
-TWIT_RATE = 175
 
 # Unemployment
 UN_RATE = 'UnRate'
@@ -152,7 +148,7 @@ def get_symbols_path(self):
             'symbols.csv'
         )
 
-    def get_dividends_path(self, symbol, provider='iexcloud'):
+    def get_dividends_path(self, symbol, provider=POLY_DIR):
         # given a symbol
         # return the path to its csv
         return os.path.join(
@@ -162,7 +158,7 @@ def get_dividends_path(self, symbol, provider='iexcloud'):
             f'{symbol.upper()}.csv'
         )
 
-    def get_splits_path(self, symbol, provider='iexcloud'):
+    def get_splits_path(self, symbol, provider=POLY_DIR):
         # given a symbol
         # return the path to its stock splits
         return os.path.join(
@@ -172,7 +168,7 @@ def get_splits_path(self, symbol, provider='iexcloud'):
             f'{symbol.upper()}.csv'
         )
 
-    def get_ohlc_path(self, symbol, provider='iexcloud'):
+    def get_ohlc_path(self, symbol, provider=POLY_DIR):
         # given a symbol
         # return the path to its ohlc data
         return os.path.join(
@@ -182,17 +178,7 @@ def get_ohlc_path(self, symbol, provider='iexcloud'):
             f'{symbol.upper()}.csv'
         )
 
-    def get_sentiment_path(self, symbol, provider='stocktwits'):
-        # given a symbol
-        # return the path to its social sentiment data
-        return os.path.join(
-            DATA_DIR,
-            SENT_DIR,
-            folders[provider],
-            f'{symbol.upper()}.csv'
-        )
-
-    def get_intraday_path(self, symbol, date, provider='iexcloud'):
+    def get_intraday_path(self, symbol, date, provider=POLY_DIR):
         # given a symbol,
         # return the path to its intraday ohlc data
         return os.path.join(
@@ -248,6 +234,13 @@ def get_api_path(self, endpoint):
             f'{endpoint}.json',
         )
 
+    def get_ndx_path(self):
+        return os.path.join(
+            DATA_DIR,
+            IDX_DIR,
+            'ndx.csv'
+        )
+
     def get_all_paths(self, path, truncate=False):
         # given a path, get all sub paths
         paths = []