Cloning a GitHub Directory into Project

.gitignore

@@ -1,3 +1,6 @@
+# Orderbook Instances
+temporary_model_storage/
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

projects/orderbook/app.py

@@ -1,24 +1,26 @@
-import os
+import os, glob
 from flask import Flask, jsonify, request
 from db_config import get_db_connection
 from docker_utils import build_docker_image, run_docker_container, stop_docker_container
+from github_utils import recursive_repo_clone
 
 ORDERBOOKS_TABLE_NAME = 'order_books_v2'
 
 
 app = Flask(__name__)
 
-"""
-This endpoint creates an orderbook instance. It expects the following arguments:
-    - name: unique name of algorithm
-    - tickerstotrack: tickers (e.g. (AAPL, GOOG))
-    - algo_path: path to algorithm from the projects directory (ex. harv-extension')
-    - updatetime: time interval for updates (minutes)
-    - end: lifespan of instance (days)
-It creates an entry in the database for the orderbook, as well as generates a Docker image, saved as a tar file in [TODO: directory]
-"""
+
 @app.route('/create_orderbook', methods=['GET'])
 def create_orderbook():
+    """
+    This endpoint creates an orderbook instance. It expects the following arguments:
+        - name: unique orderbook name
+        - tickerstotrack: tickers (e.g. (AAPL, GOOG))
+        - algo_path: GitHub URL. Specific branch and filepath are supported, but optional. (e.g. 'https://github.com/Wat-Street/money-making/tree/main/projects/orderbook_test_model')
+        - updatetime: time interval for updates (minutes)
+        - end: lifespan of instance (days)
+    It creates an entry in the database for the orderbook, as well as generates a Docker image, saved as a tar file in [TODO: directory]
+    """
     name = request.args.get('name')
     tickers_to_track = request.args.get('tickerstotrack', '').split(',')
     algo_path = request.args.get('algo_path')
@@ -30,25 +32,38 @@ def create_orderbook():
         return jsonify({"error": "Missing required parameters"}), 400
 
     try:
+        # pull algorithm into local
+        temp_model_store = "temporary_model_storage"
+        recursive_repo_clone(algo_path, temp_model_store)
+        print(f'Successfully pulled algo {name} repo to temporary model storage')
+
         # paths to pull algorithm and store image
-        path_to_algo = f"../{algo_path}"
-        path_to_image = f"docker_images/{name}.tar"
+        path_to_algo = f"{temp_model_store}"
+        path_to_image_store = f"docker_images/{name}.tar"
 
-        # check if image with this name already exists. If not, build it from the path_to_algo.
-        if not os.path.exists(path_to_image):
-            # build docker image
-            image = build_docker_image(name, path_to_algo)
-
-            # save image as .tar to path_to_image
-            with open(path_to_image, 'wb') as image_tar:
-                for chunk in image.save():
-                    image_tar.write(chunk)
-            print(f"Saved Docker image for '{name}' to {path_to_image}")
+        # check if image with this name already exists. If so, delete it first.
+        # Then, build it from the path_to_algo.
+        if os.path.exists(path_to_image_store):
+            os.remove(path_to_image_store)
+
+        # build docker image
+        image = build_docker_image(name, path_to_algo)
+
+        # save image as .tar to path_to_image_store
+        with open(path_to_image_store, 'wb') as image_tar:
+            for chunk in image.save():
+                image_tar.write(chunk)
+        print(f"Saved Docker image for '{name}' to {path_to_image_store}")
+
+        # delete the temporary model storage folder after image build
+        for file in glob.glob('{temp_model_store}/*'):
+            os.remove(file)
+        print(f'Successfully removed contents of temporary model storage')
 
         # save the order book in the database
         conn = get_db_connection()
         cur = conn.cursor()
-        cur.execute(
+        cur.execute( # TODO sqlalchemy
             f"""
                 INSERT INTO {ORDERBOOKS_TABLE_NAME} (name, tickers_to_track, algo_link, update_time, end_duration)
                 VALUES ('{name}', ARRAY {tickers_to_track}, '{algo_path}', '{update_time}', '{end_duration}')
@@ -65,13 +80,13 @@ def create_orderbook():
         return jsonify({"error": str(e)}), 500
 
 
-"""
-This endpoint allows you to view an order book.
-Expects: name of algorithm.
-Returns: a json containing the trades, worth, and balance of the order book.
-"""
 @app.route("/view_orderbook", methods=["GET"])
 def view_orderbook():
+    """
+    This endpoint allows you to view an order book.
+    Expects: name of algorithm.
+    Returns: a json containing the trades, worth, and balance of the order book.
+    """
     name = request.args.get('name')
 
     # retrieve order book from database
@@ -91,13 +106,13 @@ def view_orderbook():
     return {"error": "Order book not found"}, 404
 
 
-"""
-This endpoint deletes an orderbook instance.
-Expects: name of algorithm.
-This function deletes the orderbook instance from the database. The image persists in the docker_images folder.
-"""
 @app.route("/delete_orderbook", methods=['GET'])
 def delete_orderbook():
+    """
+    This endpoint deletes an orderbook instance.
+    Expects: name of algorithm.
+    This function deletes the orderbook instance from the database. The image persists in the docker_images folder.
+    """
     name = request.args.get('name')
 
     conn = get_db_connection()
@@ -122,4 +137,4 @@ def delete_orderbook():
 
 
 if __name__ == "__main__":
-    app.run()
+    app.run()

projects/orderbook/github_utils.py

@@ -0,0 +1,62 @@
+import fsspec
+from pathlib import Path
+
+
+def extract_components(url: str) -> list:
+    """
+    expected URL input and examples: 
+    - root directory on default (main) branch: https://github.com/Wat-Street/money-making
+    - root directory on a specific branch: https://github.com/Wat-Street/money-making/tree/branch_name
+    - a folder on a specific branch: https://github.com/Wat-Street/money-making/tree/branch_name/projects/orderbook
+
+    from a Github URL, extract:
+    - organization (ex. 'Wat-Street')
+    - repo (ex.'money-making')
+    - ref (the branch, ex. 'main') **optional** default 'main'
+    - file path (ex. 'projects/orderbook_test-model') **optional** default root directory
+    """
+    GH_DOMAIN = 'github.com'
+    parts = url.strip().split('/')
+
+    gh_domain = parts[2]
+    organization = parts[3]
+    repository = parts[4]
+    branch = 'main'
+    filepath = ''
+
+    if gh_domain.lower() != GH_DOMAIN:
+        # make sure this is a github link
+        raise Exception('Expected Github domain. Received a domain at: {gh_domain}')
+
+    if len(parts) == 5:
+        # root directory on default (main) branch
+        return organization, repository, branch, filepath
+
+    branch = parts[6]
+
+    if len(parts) == 7:
+        # root directory on a specific branch
+        return organization, repository, branch, filepath
+
+    # a folder on a specific branch
+    filepath = '/'.join(parts[7:])
+    return organization, repository, branch, filepath
+
+
+def recursive_repo_clone(url: str, destination_folder:str = "temporary-storage"):
+    FILESYSTEM_PROTOCOL = "github"
+
+    organization, repository, branch, filepath = extract_components(url)
+
+    destination = Path.cwd() / destination_folder
+    destination.mkdir(exist_ok=True, parents=True)
+    fs = fsspec.filesystem(FILESYSTEM_PROTOCOL, org=organization, repo=repository, ref=branch)
+    fs.get(fs.ls(filepath), destination.as_posix(), recursive=True)
+
+
+if __name__ == '__main__':
+    url = 'https://github.com/Wat-Street/money-making/tree/main/projects/orderbook_test_model'
+    url2 = 'https://github.com/Wat-Street/money-making'
+    url3 = 'https://github.com/Wat-Street/money-making/tree/harv-extension'
+    recursive_repo_clone(url)
+