Skip to content

Commit 70a3bc2

Browse files
jon-the-devjon-the-dev
committed
dropped old compose
1 parent 8214017 commit 70a3bc2

21 files changed

+1895
-3542
lines changed

‎.vscode/settings.json

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"cSpell.words": [
3+
"FXAIX",
4+
"VFAIX",
5+
"yfinance"
6+
]
7+
}

‎500.csv

-504
This file was deleted.

‎Dockerfile

+31-9
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,37 @@
1-
FROM ubuntu:24.04
1+
FROM python:3.12-slim as base
22

3-
MAINTAINER "[email protected]"
43

5-
RUN apt-get update -y && \
6-
apt-get install -y python3-pip
4+
# Setup env
5+
ENV LANG C.UTF-8
6+
ENV LC_ALL C.UTF-8
7+
ENV PYTHONDONTWRITEBYTECODE 1
8+
ENV PYTHONFAULTHANDLER 1
79

8-
# We copy just the requirements.txt first to leverage Docker cache
9-
COPY ./requirements.txt /app/requirements.txt
10+
FROM base AS python-deps
1011

11-
COPY ./app/* /app
12+
# Install pipenv and compilation dependencies
13+
RUN pip install pipenv
14+
RUN apt-get update && apt-get install -y --no-install-recommends gcc git
1215

13-
WORKDIR /app
16+
# Install python dependencies in /.venv
17+
COPY Pipfile .
18+
COPY Pipfile.lock .
19+
RUN PIPENV_VENV_IN_PROJECT=1 pipenv install --deploy
1420

15-
RUN pip install -r /app/requirements.txt
21+
FROM base AS runtime
22+
23+
# Copy virtual env from python-deps stage
24+
COPY --from=python-deps /.venv /.venv
25+
ENV PATH="/.venv/bin:$PATH"
26+
27+
# Create and switch to a new user
28+
RUN useradd --create-home appuser
29+
WORKDIR /home/appuser
30+
USER appuser
31+
32+
# Install application into container
33+
COPY . .
34+
35+
# Run the application
36+
# ENTRYPOINT ["python","get_data.py"]
37+
# CMD ["--force-refresh"]

‎Dockerfile.orig

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
FROM ubuntu:24.04
2+
3+
LABEL maintainer="[email protected]"
4+
5+
RUN apt-get update -y && \
6+
apt-get install -y python3-pip
7+
8+
# We copy just the requirements.txt first to leverage Docker cache
9+
COPY ./requirements.txt /app/requirements.txt
10+
11+
COPY ./app/* /app
12+
13+
WORKDIR /app
14+
15+
RUN pip install -r /app/requirements.txt

‎Makefile

+4-3
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11

22

33
run:
4-
op run --env-file=.env pipenv run streamlit run app.py
4+
op run --env-file=.env pipenv run streamlit run midas.py
55

6-
get-data:
7-
pipenv run python grab_data.py
6+
7+
# get-data:
8+
# pipenv run python grab_data.py

‎Pipfile

+2-1
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,9 @@ fastapi = "*"
1515
uvicorn = {extras = ["standard"], version = "*"}
1616
matplotlib = "*"
1717
pylint = "*"
18+
sec-edgar-downloader = "*"
1819

1920
[dev-packages]
2021

2122
[requires]
22-
python_version = "3.11"
23+
python_version = "3.12"

‎Pipfile.lock

-2,483
This file was deleted.

‎api/Dockerfile

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
FROM python:3.12-slim as base
2+
3+
4+
# Setup env
5+
ENV LANG C.UTF-8
6+
ENV LC_ALL C.UTF-8
7+
ENV PYTHONDONTWRITEBYTECODE 1
8+
ENV PYTHONFAULTHANDLER 1
9+
10+
FROM base AS python-deps
11+
12+
# Install pipenv and compilation dependencies
13+
RUN pip install pipenv
14+
RUN apt-get update && apt-get install -y --no-install-recommends gcc git
15+
16+
# Install python dependencies in /.venv
17+
COPY Pipfile .
18+
# COPY Pipfile.lock .
19+
RUN PIPENV_VENV_IN_PROJECT=1 pipenv install --deploy
20+
21+
FROM base AS runtime
22+
23+
# Copy virtual env from python-deps stage
24+
COPY --from=python-deps /.venv /.venv
25+
ENV PATH="/.venv/bin:$PATH"
26+
27+
# Create and switch to a new user
28+
RUN useradd --create-home appuser
29+
WORKDIR /home/appuser
30+
USER appuser
31+
32+
# Install application into container
33+
COPY . ./api
34+
35+
# Run the application
36+
# ENTRYPOINT ["python","get_data.py"]
37+
# CMD ["--force-refresh"]

‎api/Pipfile

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
[[source]]
2+
url = "https://pypi.org/simple"
3+
verify_ssl = true
4+
name = "pypi"
5+
6+
[packages]
7+
fastapi = "*"
8+
pandas = "*"
9+
uvicorn = "*"
10+
11+
[dev-packages]
12+
13+
[requires]
14+
python_version = "3.12"

‎api/api.py

+12-202
Original file line numberDiff line numberDiff line change
@@ -1,224 +1,34 @@
11
"""api.py"""
2+
23
from fastapi import FastAPI
34
from datetime import datetime
45
import pandas as pd
5-
import pandas_datareader as pdr
6-
import json
7-
import yfinance as yf
8-
9-
app = FastAPI()
10-
11-
@app.get("/")
12-
async def root():
13-
return {"message": "Hello World"}
14-
15-
16-
async def get_econ_data():
17-
# FIXME: This is not working
18-
start = datetime(1960, 1, 1)
19-
end = datetime(2024, 4, 22)
20-
21-
# Retrieve the data for each feature
22-
data_gdp = pdr.DataReader("GDP", "fred", start, end)["GDP"]
23-
data_gdp.index = pd.to_datetime(data_gdp.index)
24-
25-
data_cpi = pdr.DataReader("CPIAUCSL", "fred", start, end)["CPIAUCSL"]
26-
data_cpi.index = pd.to_datetime(data_cpi.index)
27-
28-
data_stock = pdr.DataReader("SPASTT01USM661N", "fred", start, end)[
29-
"SPASTT01USM661N"
30-
]
31-
data_stock.index = pd.to_datetime(data_stock.index)
32-
33-
data_pce = pdr.DataReader("PCE", "fred", start, end)["PCE"]
34-
data_pce.index = pd.to_datetime(data_pce.index)
356

36-
data_govs = pdr.DataReader("FGEXPND", "fred", start, end)["FGEXPND"]
37-
data_govs.index = pd.to_datetime(data_govs.index)
7+
# import pandas_datareader as pdr
8+
import json
9+
import os
3810

39-
data_binv = pdr.DataReader("W987RC1Q027SBEA", "fred", start, end)["W987RC1Q027SBEA"]
40-
data_binv.index = pd.to_datetime(data_binv.index)
11+
# import yfinance as yf
4112

42-
data_em = pdr.DataReader("PAYEMS", "fred", start, end)["PAYEMS"]
43-
data_em.index = pd.to_datetime(data_em.index)
13+
app = FastAPI()
4414

45-
data_unem = pdr.DataReader("ICSA", "fred", start, end)["ICSA"]
46-
data_unem.index = pd.to_datetime(data_unem.index)
15+
DATA_DIR = os.getenv("DATA_DIR", "/data")
4716

48-
# Combine the features into a single DataFrame
49-
data = pd.DataFrame(
50-
{
51-
"data_gdp": data_gdp,
52-
"data_cpi": data_cpi,
53-
"data_stock": data_stock,
54-
"data_pce": data_pce,
55-
"data_govs": data_govs,
56-
"data_binv": data_binv,
57-
"data_em": data_em,
58-
"data_unem": data_unem,
59-
}
60-
)
6117

62-
# Remove rows with missing values
63-
data = data.dropna()
18+
@app.get("/")
19+
async def root():
20+
return {"message": "OK"}
6421

65-
return data
6622

6723
@app.get("/ticker/{ticker}")
6824
async def get_ticker(ticker):
6925
print(f"Getting stock data for {ticker}")
7026

7127
# IF DATA FOUND - RETURN THIS
7228
try:
73-
with open(f"data/{ticker}.json", "w") as file:
29+
with open(f"{DATA_DIR}/{ticker}.json", "w") as file:
7430
file.write(json.dumps(data))
7531

7632
return data
7733
except:
78-
logger.info('No ticker found - downloading data...')
79-
80-
data = {}
81-
try:
82-
stock_ticker = yf.Ticker(ticker)
83-
except Exception as err:
84-
print("Error: %s", err)
85-
86-
# get all stock info
87-
try:
88-
data["info"] = stock_ticker.info
89-
except Exception as err:
90-
print("Error: %s", err)
91-
92-
try:
93-
# get historical market data
94-
mo1 = stock_ticker.history(period="1mo")
95-
data["1mo_hist"] = pd.DataFrame(mo1).to_json()
96-
except Exception as err:
97-
print("Error: %s", err)
98-
99-
try:
100-
# show meta information about the history (requires history() to be called first)
101-
data["history_metadata"] = stock_ticker.history_metadata
102-
except Exception as err:
103-
print("Error: %s", err)
104-
105-
try:
106-
# show actions (dividends, splits, capital gains)
107-
data["actions"] = pd.DataFrame(stock_ticker.actions).to_json()
108-
data["dividends"] = pd.DataFrame(stock_ticker.dividends).to_json()
109-
data["splits"] = pd.DataFrame(stock_ticker.splits).to_json()
110-
except Exception as err:
111-
print("Error: %s", err)
112-
try:
113-
data["capital_gains"] = pd.DataFrame(
114-
stock_ticker.capital_gains
115-
).to_json() # only for mutual funds & etfs
116-
except Exception as err:
117-
print("Error: %s", err)
118-
119-
try:
120-
# show share count
121-
get_shares_full = stock_ticker.get_shares_full(start="2022-01-01", end=None)
122-
df = pd.DataFrame(get_shares_full)
123-
df.reset_index(inplace=True)
124-
data["get_shares_full"] = pd.DataFrame(df).to_json()
125-
except Exception as err:
126-
print("Error: %s", err)
127-
128-
# show financials:
129-
# - income statement
130-
131-
try:
132-
income_stmt = stock_ticker.income_stmt
133-
data["income_stmt"] = pd.DataFrame(income_stmt).to_json()
134-
135-
data["quarterly_income_stmt"] = pd.DataFrame(
136-
stock_ticker.quarterly_income_stmt
137-
).to_json()
138-
# - balance sheet
139-
data["balance_sheet"] = pd.DataFrame(stock_ticker.balance_sheet).to_json()
140-
data["quarterly_balance_sheet"] = pd.DataFrame(
141-
stock_ticker.quarterly_balance_sheet
142-
).to_json()
143-
# - cash flow statement
144-
data["cashflow"] = pd.DataFrame(stock_ticker.cashflow).to_json()
145-
data["quarterly_cashflow"] = pd.DataFrame(
146-
stock_ticker.quarterly_cashflow
147-
).to_json()
148-
# see `Ticker.get_income_stmt()` for more options
149-
except Exception as err:
150-
print("Error: %s", err)
151-
152-
try:
153-
# show holders
154-
data["major_holders"] = pd.DataFrame(stock_ticker.major_holders).to_json()
155-
data["institutional_holders"] = pd.DataFrame(
156-
stock_ticker.institutional_holders
157-
).to_json()
158-
data["mutualfund_holders"] = pd.DataFrame(
159-
stock_ticker.mutualfund_holders
160-
).to_json()
161-
data["insider_transactions"] = pd.DataFrame(
162-
stock_ticker.insider_transactions
163-
).to_json()
164-
data["insider_purchases"] = pd.DataFrame(
165-
stock_ticker.insider_purchases
166-
).to_json()
167-
data["insider_roster_holders"] = pd.DataFrame(
168-
stock_ticker.insider_roster_holders
169-
).to_json()
170-
except Exception as err:
171-
print("Error: %s", err)
172-
173-
try:
174-
# show recommendations
175-
data["recommendations"] = pd.DataFrame(stock_ticker.recommendations).to_json()
176-
data["recommendations_summary"] = pd.DataFrame(
177-
stock_ticker.recommendations_summary
178-
).to_json()
179-
data["upgrades_downgrades"] = pd.DataFrame(
180-
stock_ticker.upgrades_downgrades
181-
).to_json()
182-
except Exception as err:
183-
print("Error: %s", err)
184-
185-
# Show future and historic earnings dates, returns at most next 4 quarters and last 8 quarters by default.
186-
# Note: If more are needed use stock_ticker.get_earnings_dates(limit=XX) with increased limit argument.
187-
# data['earnings_dates'] = pd.DataFrame(stock_ticker.earnings_dates).to_json()
188-
189-
# show ISIN code - *experimental*
190-
# ISIN = International Securities Identification Number
191-
try:
192-
data["isin"] = stock_ticker.isin
193-
except Exception as err:
194-
print("Error: %s", err)
195-
196-
try:
197-
# show options expirations
198-
data["options"] = stock_ticker.options
199-
200-
# FIXME: Get the options data into the json dict so we can publish to mongo
201-
# data['options_data'] = {}
202-
# for opt in data['options']:
203-
# print(f'Getting options for {opt}')
204-
# # data['options_data'][opt] = stock_ticker.option_chain(opt)
205-
# # data['options_data'][opt] = stock_ticker.option_chain(opt)
206-
# data['options_data'][f'{opt}_calls'] = pd.DataFrame(stock_ticker.option_chain(opt)).to_json()
207-
# data['options_data'][f'{opt}_puts'] = pd.DataFrame(stock_ticker.option_chain(opt)).to_json()
208-
except Exception as err:
209-
print("Error: %s", err)
210-
211-
try:
212-
# show news
213-
data["news"] = stock_ticker.news
214-
except Exception as err:
215-
print("Error: %s", err)
216-
217-
# TODO: This is only used when debuggins Json not Serializable errors
218-
# with open(f'data/{symbol}.txt', 'w') as f:
219-
# print(data, file=f)
220-
221-
with open(f"data/{ticker}.json", "w") as file:
222-
file.write(json.dumps(data))
223-
224-
return data
34+
logger.info("No ticker found")

‎api/requirements.txt

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
fastapi
2+
pandas

0 commit comments

Comments
 (0)