diff --git a/.gitignore b/.gitignore index 8c16c01..1a53303 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# Orderbook Instances +temporary_model_storage/ + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/projects/orderbook/app.py b/projects/orderbook/app.py index 1a274d8..6e15185 100644 --- a/projects/orderbook/app.py +++ b/projects/orderbook/app.py @@ -1,24 +1,26 @@ -import os +import os, glob from flask import Flask, jsonify, request from db_config import get_db_connection from docker_utils import build_docker_image, run_docker_container, stop_docker_container +from github_utils import recursive_repo_clone ORDERBOOKS_TABLE_NAME = 'order_books_v2' app = Flask(__name__) -""" -This endpoint creates an orderbook instance. It expects the following arguments: - - name: unique name of algorithm - - tickerstotrack: tickers (e.g. (AAPL, GOOG)) - - algo_path: path to algorithm from the projects directory (ex. harv-extension') - - updatetime: time interval for updates (minutes) - - end: lifespan of instance (days) -It creates an entry in the database for the orderbook, as well as generates a Docker image, saved as a tar file in [TODO: directory] -""" + @app.route('/create_orderbook', methods=['GET']) def create_orderbook(): + """ + This endpoint creates an orderbook instance. It expects the following arguments: + - name: unique orderbook name + - tickerstotrack: tickers (e.g. (AAPL, GOOG)) + - algo_path: GitHub URL. Specific branch and filepath are supported, but optional. (e.g. 'https://github.com/Wat-Street/money-making/tree/main/projects/orderbook_test_model') + - updatetime: time interval for updates (minutes) + - end: lifespan of instance (days) + It creates an entry in the database for the orderbook, as well as generates a Docker image, saved as a tar file in [TODO: directory] + """ name = request.args.get('name') tickers_to_track = request.args.get('tickerstotrack', '').split(',') algo_path = request.args.get('algo_path') @@ -30,25 +32,38 @@ def create_orderbook(): return jsonify({"error": "Missing required parameters"}), 400 try: + # pull algorithm into local + temp_model_store = "temporary_model_storage" + recursive_repo_clone(algo_path, temp_model_store) + print(f'Successfully pulled algo {name} repo to temporary model storage') + # paths to pull algorithm and store image - path_to_algo = f"../{algo_path}" - path_to_image = f"docker_images/{name}.tar" + path_to_algo = f"{temp_model_store}" + path_to_image_store = f"docker_images/{name}.tar" - # check if image with this name already exists. If not, build it from the path_to_algo. - if not os.path.exists(path_to_image): - # build docker image - image = build_docker_image(name, path_to_algo) - - # save image as .tar to path_to_image - with open(path_to_image, 'wb') as image_tar: - for chunk in image.save(): - image_tar.write(chunk) - print(f"Saved Docker image for '{name}' to {path_to_image}") + # check if image with this name already exists. If so, delete it first. + # Then, build it from the path_to_algo. + if os.path.exists(path_to_image_store): + os.remove(path_to_image_store) + + # build docker image + image = build_docker_image(name, path_to_algo) + + # save image as .tar to path_to_image_store + with open(path_to_image_store, 'wb') as image_tar: + for chunk in image.save(): + image_tar.write(chunk) + print(f"Saved Docker image for '{name}' to {path_to_image_store}") + + # delete the temporary model storage folder after image build + for file in glob.glob('{temp_model_store}/*'): + os.remove(file) + print(f'Successfully removed contents of temporary model storage') # save the order book in the database conn = get_db_connection() cur = conn.cursor() - cur.execute( + cur.execute( # TODO sqlalchemy f""" INSERT INTO {ORDERBOOKS_TABLE_NAME} (name, tickers_to_track, algo_link, update_time, end_duration) VALUES ('{name}', ARRAY {tickers_to_track}, '{algo_path}', '{update_time}', '{end_duration}') @@ -65,13 +80,13 @@ def create_orderbook(): return jsonify({"error": str(e)}), 500 -""" -This endpoint allows you to view an order book. -Expects: name of algorithm. -Returns: a json containing the trades, worth, and balance of the order book. -""" @app.route("/view_orderbook", methods=["GET"]) def view_orderbook(): + """ + This endpoint allows you to view an order book. + Expects: name of algorithm. + Returns: a json containing the trades, worth, and balance of the order book. + """ name = request.args.get('name') # retrieve order book from database @@ -91,13 +106,13 @@ def view_orderbook(): return {"error": "Order book not found"}, 404 -""" -This endpoint deletes an orderbook instance. -Expects: name of algorithm. -This function deletes the orderbook instance from the database. The image persists in the docker_images folder. -""" @app.route("/delete_orderbook", methods=['GET']) def delete_orderbook(): + """ + This endpoint deletes an orderbook instance. + Expects: name of algorithm. + This function deletes the orderbook instance from the database. The image persists in the docker_images folder. + """ name = request.args.get('name') conn = get_db_connection() @@ -122,4 +137,4 @@ def delete_orderbook(): if __name__ == "__main__": - app.run() \ No newline at end of file + app.run() diff --git a/projects/orderbook/github_utils.py b/projects/orderbook/github_utils.py new file mode 100644 index 0000000..73253c2 --- /dev/null +++ b/projects/orderbook/github_utils.py @@ -0,0 +1,62 @@ +import fsspec +from pathlib import Path + + +def extract_components(url: str) -> list: + """ + expected URL input and examples: + - root directory on default (main) branch: https://github.com/Wat-Street/money-making + - root directory on a specific branch: https://github.com/Wat-Street/money-making/tree/branch_name + - a folder on a specific branch: https://github.com/Wat-Street/money-making/tree/branch_name/projects/orderbook + + from a Github URL, extract: + - organization (ex. 'Wat-Street') + - repo (ex.'money-making') + - ref (the branch, ex. 'main') **optional** default 'main' + - file path (ex. 'projects/orderbook_test-model') **optional** default root directory + """ + GH_DOMAIN = 'github.com' + parts = url.strip().split('/') + + gh_domain = parts[2] + organization = parts[3] + repository = parts[4] + branch = 'main' + filepath = '' + + if gh_domain.lower() != GH_DOMAIN: + # make sure this is a github link + raise Exception('Expected Github domain. Received a domain at: {gh_domain}') + + if len(parts) == 5: + # root directory on default (main) branch + return organization, repository, branch, filepath + + branch = parts[6] + + if len(parts) == 7: + # root directory on a specific branch + return organization, repository, branch, filepath + + # a folder on a specific branch + filepath = '/'.join(parts[7:]) + return organization, repository, branch, filepath + + +def recursive_repo_clone(url: str, destination_folder:str = "temporary-storage"): + FILESYSTEM_PROTOCOL = "github" + + organization, repository, branch, filepath = extract_components(url) + + destination = Path.cwd() / destination_folder + destination.mkdir(exist_ok=True, parents=True) + fs = fsspec.filesystem(FILESYSTEM_PROTOCOL, org=organization, repo=repository, ref=branch) + fs.get(fs.ls(filepath), destination.as_posix(), recursive=True) + + +if __name__ == '__main__': + url = 'https://github.com/Wat-Street/money-making/tree/main/projects/orderbook_test_model' + url2 = 'https://github.com/Wat-Street/money-making' + url3 = 'https://github.com/Wat-Street/money-making/tree/harv-extension' + recursive_repo_clone(url) +