From a5d799b92fa4dc2a60e70508dd1a8bfaa5f455e7 Mon Sep 17 00:00:00 2001 From: reebxu459 Date: Thu, 16 Jan 2025 01:18:42 -0500 Subject: [PATCH 1/4] github utils function definition --- projects/orderbook/github_utils.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 projects/orderbook/github_utils.py diff --git a/projects/orderbook/github_utils.py b/projects/orderbook/github_utils.py new file mode 100644 index 0000000..5f98e85 --- /dev/null +++ b/projects/orderbook/github_utils.py @@ -0,0 +1,15 @@ +import os +import subprocess +import shutil + +def clone_directory_from_gh(repo_url, commit_id, target_directory, subdirectory): + """ + Clone a subdirectory from a GitHub repo at a specific commit ID. + Args: + repo_url (str): URL of the GitHub repository. + commit_id (str): Commit hash to checkout. + target_directory (str): Destination directory to copy the subdirectory into. + subdirectory (str): Subdirectory within the repository to copy. + """ + pass + From 47c3f7b141711994f235edc032fad8921c081733 Mon Sep 17 00:00:00 2001 From: reebxu459 Date: Tue, 28 Jan 2025 23:58:24 -0500 Subject: [PATCH 2/4] github repo clone script --- projects/orderbook/github_utils.py | 69 +++++++++++++++++++++++++----- 1 file changed, 58 insertions(+), 11 deletions(-) diff --git a/projects/orderbook/github_utils.py b/projects/orderbook/github_utils.py index 5f98e85..73253c2 100644 --- a/projects/orderbook/github_utils.py +++ b/projects/orderbook/github_utils.py @@ -1,15 +1,62 @@ -import os -import subprocess -import shutil +import fsspec +from pathlib import Path -def clone_directory_from_gh(repo_url, commit_id, target_directory, subdirectory): + +def extract_components(url: str) -> list: """ - Clone a subdirectory from a GitHub repo at a specific commit ID. - Args: - repo_url (str): URL of the GitHub repository. - commit_id (str): Commit hash to checkout. - target_directory (str): Destination directory to copy the subdirectory into. - subdirectory (str): Subdirectory within the repository to copy. + expected URL input and examples: + - root directory on default (main) branch: https://github.com/Wat-Street/money-making + - root directory on a specific branch: https://github.com/Wat-Street/money-making/tree/branch_name + - a folder on a specific branch: https://github.com/Wat-Street/money-making/tree/branch_name/projects/orderbook + + from a Github URL, extract: + - organization (ex. 'Wat-Street') + - repo (ex.'money-making') + - ref (the branch, ex. 'main') **optional** default 'main' + - file path (ex. 'projects/orderbook_test-model') **optional** default root directory """ - pass + GH_DOMAIN = 'github.com' + parts = url.strip().split('/') + + gh_domain = parts[2] + organization = parts[3] + repository = parts[4] + branch = 'main' + filepath = '' + + if gh_domain.lower() != GH_DOMAIN: + # make sure this is a github link + raise Exception('Expected Github domain. Received a domain at: {gh_domain}') + + if len(parts) == 5: + # root directory on default (main) branch + return organization, repository, branch, filepath + + branch = parts[6] + + if len(parts) == 7: + # root directory on a specific branch + return organization, repository, branch, filepath + + # a folder on a specific branch + filepath = '/'.join(parts[7:]) + return organization, repository, branch, filepath + + +def recursive_repo_clone(url: str, destination_folder:str = "temporary-storage"): + FILESYSTEM_PROTOCOL = "github" + + organization, repository, branch, filepath = extract_components(url) + + destination = Path.cwd() / destination_folder + destination.mkdir(exist_ok=True, parents=True) + fs = fsspec.filesystem(FILESYSTEM_PROTOCOL, org=organization, repo=repository, ref=branch) + fs.get(fs.ls(filepath), destination.as_posix(), recursive=True) + + +if __name__ == '__main__': + url = 'https://github.com/Wat-Street/money-making/tree/main/projects/orderbook_test_model' + url2 = 'https://github.com/Wat-Street/money-making' + url3 = 'https://github.com/Wat-Street/money-making/tree/harv-extension' + recursive_repo_clone(url) From ba258921d188dbc2da8957e48fc909b59f8409cc Mon Sep 17 00:00:00 2001 From: reebxu459 Date: Wed, 29 Jan 2025 00:46:05 -0500 Subject: [PATCH 3/4] integrated repo pulling into app code --- .gitignore | 3 +++ projects/orderbook/app.py | 44 ++++++++++++++++++++++++++------------- 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index 8c16c01..1a53303 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# Orderbook Instances +temporary_model_storage/ + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/projects/orderbook/app.py b/projects/orderbook/app.py index 1a274d8..787b104 100644 --- a/projects/orderbook/app.py +++ b/projects/orderbook/app.py @@ -1,7 +1,8 @@ -import os +import os, glob from flask import Flask, jsonify, request from db_config import get_db_connection from docker_utils import build_docker_image, run_docker_container, stop_docker_container +from github_utils import recursive_repo_clone ORDERBOOKS_TABLE_NAME = 'order_books_v2' @@ -10,9 +11,9 @@ """ This endpoint creates an orderbook instance. It expects the following arguments: - - name: unique name of algorithm + - name: unique orderbook name - tickerstotrack: tickers (e.g. (AAPL, GOOG)) - - algo_path: path to algorithm from the projects directory (ex. harv-extension') + - algo_path: GitHub URL. Specific branch and filepath are supported, but optional. (e.g. 'https://github.com/Wat-Street/money-making/tree/main/projects/orderbook_test_model') - updatetime: time interval for updates (minutes) - end: lifespan of instance (days) It creates an entry in the database for the orderbook, as well as generates a Docker image, saved as a tar file in [TODO: directory] @@ -30,20 +31,33 @@ def create_orderbook(): return jsonify({"error": "Missing required parameters"}), 400 try: + # pull algorithm into local + destination_path = "temporary_model_storage" + recursive_repo_clone(algo_path, destination_path) + print(f'Successfully pulled algo {name} repo to temporary model storage') + # paths to pull algorithm and store image - path_to_algo = f"../{algo_path}" + path_to_algo = f"{destination_path}" path_to_image = f"docker_images/{name}.tar" - # check if image with this name already exists. If not, build it from the path_to_algo. - if not os.path.exists(path_to_image): - # build docker image - image = build_docker_image(name, path_to_algo) - - # save image as .tar to path_to_image - with open(path_to_image, 'wb') as image_tar: - for chunk in image.save(): - image_tar.write(chunk) - print(f"Saved Docker image for '{name}' to {path_to_image}") + # check if image with this name already exists. If so, delete it first. + # Then, build it from the path_to_algo. + if os.path.exists(path_to_image): + os.remove(path_to_image) + + # build docker image + image = build_docker_image(name, path_to_algo) + + # save image as .tar to path_to_image + with open(path_to_image, 'wb') as image_tar: + for chunk in image.save(): + image_tar.write(chunk) + print(f"Saved Docker image for '{name}' to {path_to_image}") + + # delete the temporary model storage folder after image build + for file in glob.glob('temporary_model_storage/*'): + os.remove(file) + print(f'Successfully removed contents of temporary model storage') # save the order book in the database conn = get_db_connection() @@ -122,4 +136,4 @@ def delete_orderbook(): if __name__ == "__main__": - app.run() \ No newline at end of file + app.run() From f5dc761dc64bd897bd79432fa9801b75accfc6bf Mon Sep 17 00:00:00 2001 From: reebxu459 Date: Wed, 29 Jan 2025 00:59:11 -0500 Subject: [PATCH 4/4] format fix --- projects/orderbook/app.py | 61 ++++++++++++++++++++------------------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/projects/orderbook/app.py b/projects/orderbook/app.py index 787b104..6e15185 100644 --- a/projects/orderbook/app.py +++ b/projects/orderbook/app.py @@ -9,17 +9,18 @@ app = Flask(__name__) -""" -This endpoint creates an orderbook instance. It expects the following arguments: - - name: unique orderbook name - - tickerstotrack: tickers (e.g. (AAPL, GOOG)) - - algo_path: GitHub URL. Specific branch and filepath are supported, but optional. (e.g. 'https://github.com/Wat-Street/money-making/tree/main/projects/orderbook_test_model') - - updatetime: time interval for updates (minutes) - - end: lifespan of instance (days) -It creates an entry in the database for the orderbook, as well as generates a Docker image, saved as a tar file in [TODO: directory] -""" + @app.route('/create_orderbook', methods=['GET']) def create_orderbook(): + """ + This endpoint creates an orderbook instance. It expects the following arguments: + - name: unique orderbook name + - tickerstotrack: tickers (e.g. (AAPL, GOOG)) + - algo_path: GitHub URL. Specific branch and filepath are supported, but optional. (e.g. 'https://github.com/Wat-Street/money-making/tree/main/projects/orderbook_test_model') + - updatetime: time interval for updates (minutes) + - end: lifespan of instance (days) + It creates an entry in the database for the orderbook, as well as generates a Docker image, saved as a tar file in [TODO: directory] + """ name = request.args.get('name') tickers_to_track = request.args.get('tickerstotrack', '').split(',') algo_path = request.args.get('algo_path') @@ -32,37 +33,37 @@ def create_orderbook(): try: # pull algorithm into local - destination_path = "temporary_model_storage" - recursive_repo_clone(algo_path, destination_path) + temp_model_store = "temporary_model_storage" + recursive_repo_clone(algo_path, temp_model_store) print(f'Successfully pulled algo {name} repo to temporary model storage') # paths to pull algorithm and store image - path_to_algo = f"{destination_path}" - path_to_image = f"docker_images/{name}.tar" + path_to_algo = f"{temp_model_store}" + path_to_image_store = f"docker_images/{name}.tar" # check if image with this name already exists. If so, delete it first. # Then, build it from the path_to_algo. - if os.path.exists(path_to_image): - os.remove(path_to_image) + if os.path.exists(path_to_image_store): + os.remove(path_to_image_store) # build docker image image = build_docker_image(name, path_to_algo) - # save image as .tar to path_to_image - with open(path_to_image, 'wb') as image_tar: + # save image as .tar to path_to_image_store + with open(path_to_image_store, 'wb') as image_tar: for chunk in image.save(): image_tar.write(chunk) - print(f"Saved Docker image for '{name}' to {path_to_image}") + print(f"Saved Docker image for '{name}' to {path_to_image_store}") # delete the temporary model storage folder after image build - for file in glob.glob('temporary_model_storage/*'): + for file in glob.glob('{temp_model_store}/*'): os.remove(file) print(f'Successfully removed contents of temporary model storage') # save the order book in the database conn = get_db_connection() cur = conn.cursor() - cur.execute( + cur.execute( # TODO sqlalchemy f""" INSERT INTO {ORDERBOOKS_TABLE_NAME} (name, tickers_to_track, algo_link, update_time, end_duration) VALUES ('{name}', ARRAY {tickers_to_track}, '{algo_path}', '{update_time}', '{end_duration}') @@ -79,13 +80,13 @@ def create_orderbook(): return jsonify({"error": str(e)}), 500 -""" -This endpoint allows you to view an order book. -Expects: name of algorithm. -Returns: a json containing the trades, worth, and balance of the order book. -""" @app.route("/view_orderbook", methods=["GET"]) def view_orderbook(): + """ + This endpoint allows you to view an order book. + Expects: name of algorithm. + Returns: a json containing the trades, worth, and balance of the order book. + """ name = request.args.get('name') # retrieve order book from database @@ -105,13 +106,13 @@ def view_orderbook(): return {"error": "Order book not found"}, 404 -""" -This endpoint deletes an orderbook instance. -Expects: name of algorithm. -This function deletes the orderbook instance from the database. The image persists in the docker_images folder. -""" @app.route("/delete_orderbook", methods=['GET']) def delete_orderbook(): + """ + This endpoint deletes an orderbook instance. + Expects: name of algorithm. + This function deletes the orderbook instance from the database. The image persists in the docker_images folder. + """ name = request.args.get('name') conn = get_db_connection()