oct1 commit

.vscode/settings.json

@@ -1,7 +1,19 @@
 {
     "cSpell.words": [
+        "alphavantage",
+        "AROON",
+        "COMOD",
+        "DEMA",
         "FXAIX",
+        "MACDEXT",
+        "ndxt",
+        "STOCH",
+        "STOCHF",
+        "STOCHRSI",
+        "TEMA",
+        "TRIMA",
         "VFAIX",
+        "WILLR",
         "yfinance"
     ]
 }

Pipfile

@@ -9,7 +9,6 @@ pandas = "*"
 requests = "*"
 lxml = "*"
 yfinance = {extras = ["nospam", "repair"], version = "*"}
-secedgar = {git = "https://github.com/sec-edgar/sec-edgar.git"}
 pandas-datareader = "*"
 fastapi = "*"
 uvicorn = {extras = ["standard"], version = "*"}

api/api.py

@@ -3,13 +3,18 @@
 from fastapi import FastAPI
 from datetime import datetime
 import pandas as pd
+import logging
+from glob import glob
+
+logger = logging.getLogger(__name__)
 
 # import pandas_datareader as pdr
 import json
 import os
 
 # import yfinance as yf
 
+TODAY = datetime.today().strftime("%Y-%m-%d")
 app = FastAPI()
 
 DATA_DIR = os.getenv("DATA_DIR", "/data")
@@ -20,15 +25,23 @@ async def root():
     return {"message": "OK"}
 
 
-@app.get("/ticker/{ticker}")
-async def get_ticker(ticker):
-    print(f"Getting stock data for {ticker}")
-
-    # IF DATA FOUND - RETURN THIS
+@app.get("/symbol/{symbol}")
+async def get_symbol(symbol):
+    print(f"Getting symbol data for {symbol}")
+    # symbol = symbol.strip().replace("/", "").upper()
     try:
-        with open(f"{DATA_DIR}/{ticker}.json", "w") as file:
-            file.write(json.dumps(data))
+        fname = f"{DATA_DIR}/MIDAS/{TODAY}-{symbol}.json"
+        print(f"Looking for {fname}")
+        with open(fname, "r") as file:
+            data = json.load(file)
 
         return data
-    except:
-        logger.info("No ticker found")
+    except Exception as err:
+        logger.info("No symbol found")
+        return {"message": f"No symbol found {err}"}
+
+
+@app.get("/files")
+async def get_files():
+    files = glob(f"{DATA_DIR}/MIDAS/*")
+    return json.dumps(files)

cwe_dl.py

@@ -0,0 +1,45 @@
+import requests
+import pandas as pd
+from io import StringIO
+import re
+from bs4 import BeautifulSoup
+
+
+BASE = "https://cwe.mitre.org"
+URL = f"{BASE}/data/downloads.html"
+
+
+def main():
+    headers = {
+        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/91.0.4472.101 Safari/537.3"
+    }
+    # fname = f"{DATA_DIR}/500.csv"
+
+    # if os.path.exists(fname):
+    #     logger.info("File exists %s", fname)
+    #     return
+
+    # Fetch the page content
+    response = requests.get(URL, headers=headers, timeout=30)
+
+    # Use pandas to read the HTML tables
+    tables = pd.read_html(response.text)
+    table1 = tables[0]
+    table2 = tables[1]
+    table3 = tables[2]
+    table4 = tables[3]
+    print(table1, table2, table3, table4)
+    # soup = BeautifulSoup(response.text)
+
+    # for link in soup.findAll('a', attrs={'href': re.compile('.csv.zip')}):
+    #     fname = link.get("href").split('/')[-1]
+    #     link_abs = f'{BASE}{link.get("href")}'
+    #     print(link_abs)
+    #     with open(fname, 'wb') as f:
+    #         f.write(requests.get(link_abs).content)
+
+
+if __name__ == "__main__":
+    main()

docker-compose.yml

@@ -6,7 +6,7 @@ services:
       context: .
       dockerfile: Dockerfile
     volumes:
-      - /Volumes/DATA/data/finance:/data
+      - /Volumes/data/finance:/data
     environment:
       - WORKERS=1
     entrypoint: ["python", "worker_stocks.py"]
@@ -18,14 +18,14 @@ services:
       context: ./api
       dockerfile: Dockerfile
     volumes:
-      - /Volumes/DATA/data/finance:/data
+      - /Volumes/data/finance:/data
     entrypoint: [
         "uvicorn",
         "--host",
         "0.0.0.0",
         "--port",
         "8000",
-        "api.api:app", #{dir}/{file}:app
+        "api.api:app", # {dir} / {file}:app
         "--reload",
       ]
     networks:
@@ -52,7 +52,7 @@ services:
       context: .
       dockerfile: Dockerfile
     volumes:
-      - /Volumes/DATA/data/finance:/data
+      - /Volumes/data/finance:/data
     environment:
       - WORKERS=10
     entrypoint: ["python", "worker_sec.py"]
@@ -65,7 +65,7 @@ services:
       context: .
       dockerfile: Dockerfile
     volumes:
-      - /Users/jon/Data/FINANCE:/data
+      - /Volumes/data/finance:/data
     environment:
       - WORKERS=2
     # ports: