From bddce12e570a117c8c83c759e5a932e0e99d3871 Mon Sep 17 00:00:00 2001 From: Radis Toubalidis <72709550+radistoubalidis@users.noreply.github.com> Date: Wed, 7 Sep 2022 12:24:27 +0300 Subject: [PATCH 01/21] Enrich the CML Action using Jupytext (#176) * reproduce FLINT.Cloud apis * example sim report using cml * Update cml-report.yml * Update cml-report.yml * Update cml-report.yml * Update cml-report.yml * test if cml is setup right * add cml script * changed file paths to not show local directories * update cml report * setup GCBM summary * setup GCBM summary * setup GCBM summary * setup GCBM summary * setup GCBM summary * setup GCBM summary * setup GCBM summary * setup GCBM summary * update python script * run model summary * add sim start and end dates * delete irrelevant files * convert annual_summaries.py to py:percent format Signed-off-by: radis toubalidis * converts and run annual_summaries.py as notebook usinng jupytext Signed-off-by: radis toubalidis * test if action works with jupytext Signed-off-by: radis toubalidis * test if action works with jupytext Signed-off-by: radis toubalidis * test if action works with jupytext Signed-off-by: radis toubalidis * test if action works with jupytext (try 4) Signed-off-by: radis toubalidis * test if action works with jupytext (try 5) Signed-off-by: radis toubalidis * test if action works with jupytext (fixed typo on 5, try 6) Signed-off-by: radis toubalidis * test if action works with jupytext (fixed typo on 7) Signed-off-by: radis toubalidis * test if action works with jupytext (try 8) Signed-off-by: radis toubalidis * test if action works with jupytext (try 9) Signed-off-by: radis toubalidis * test if action works with jupytext (try 10) Signed-off-by: radis toubalidis * test if action works with jupytext (try 11) Signed-off-by: radis toubalidis * test if action works with jupytext (try 12) Signed-off-by: radis toubalidis * check why cml comment is 404 not found Signed-off-by: radis toubalidis * uncomment labeling filter Signed-off-by: radis toubalidis * adds --uptate option to cml send-comment command Signed-off-by: radis toubalidis * test cml action with --update Signed-off-by: radis toubalidis * test cml action with --update Signed-off-by: radis toubalidis * test action with GITHUB_SHA var Signed-off-by: radis toubalidis * test action with GITHUB_SHA var (fix typo) Signed-off-by: radis toubalidis * test cml action without --pr option Signed-off-by: radis toubalidis * run workflow only on labels * reproduce FLINT.Cloud apis * example sim report using cml * Update cml-report.yml * Update cml-report.yml * Update cml-report.yml * Update cml-report.yml * test if cml is setup right * add cml script * changed file paths to not show local directories * update cml report * setup GCBM summary * setup GCBM summary * setup GCBM summary * setup GCBM summary * setup GCBM summary * setup GCBM summary * setup GCBM summary * setup GCBM summary * update python script * run model summary * add sim start and end dates * delete irrelevant files * convert annual_summaries.py to py:percent format Signed-off-by: radis toubalidis * converts and run annual_summaries.py as notebook usinng jupytext Signed-off-by: radis toubalidis * test if action works with jupytext Signed-off-by: radis toubalidis * test if action works with jupytext Signed-off-by: radis toubalidis * test if action works with jupytext Signed-off-by: radis toubalidis * test if action works with jupytext (try 4) Signed-off-by: radis toubalidis * test if action works with jupytext (try 5) Signed-off-by: radis toubalidis * test if action works with jupytext (fixed typo on 5, try 6) Signed-off-by: radis toubalidis * test if action works with jupytext (fixed typo on 7) Signed-off-by: radis toubalidis * test if action works with jupytext (try 8) Signed-off-by: radis toubalidis * test if action works with jupytext (try 9) Signed-off-by: radis toubalidis * test if action works with jupytext (try 10) Signed-off-by: radis toubalidis * test if action works with jupytext (try 11) Signed-off-by: radis toubalidis * test if action works with jupytext (try 12) Signed-off-by: radis toubalidis * check why cml comment is 404 not found Signed-off-by: radis toubalidis * uncomment labeling filter Signed-off-by: radis toubalidis * adds --uptate option to cml send-comment command Signed-off-by: radis toubalidis * test cml action with --update Signed-off-by: radis toubalidis * test cml action with --update Signed-off-by: radis toubalidis * test action with GITHUB_SHA var Signed-off-by: radis toubalidis * test action with GITHUB_SHA var (fix typo) Signed-off-by: radis toubalidis * test cml action without --pr option Signed-off-by: radis toubalidis * run workflow only on labels * fix lint Signed-off-by: radis toubalidis Co-authored-by: HarshCasper Signed-off-by: Freeman --- .github/workflows/cml-report.yml | 17 +++++++----- local/rest_api_gcbm/GCBM_summary.txt | 37 +++++++++++++++++++++++++ local/rest_api_gcbm/annual_summaries.py | 12 ++++++++ local/rest_api_gcbm/app.py | 8 ++++-- 4 files changed, 64 insertions(+), 10 deletions(-) create mode 100644 local/rest_api_gcbm/GCBM_summary.txt diff --git a/.github/workflows/cml-report.yml b/.github/workflows/cml-report.yml index 2f8c2612..7024fa05 100644 --- a/.github/workflows/cml-report.yml +++ b/.github/workflows/cml-report.yml @@ -49,7 +49,7 @@ jobs: python3 GCBM.CompileResults/compileresults.py sqlite:///tests/output/gcbm_output.db --output_db sqlite:///tests/output/compiled_simulation_output.db - name: Install dependencies - run: pip3 install pandas matplotlib + run: pip3 install pandas matplotlib jupytext - name: Setup CML uses: iterative/setup-cml@v1 @@ -58,9 +58,12 @@ jobs: env: repo_token: ${{ secrets.GITHUB_TOKEN }} run: | - cd local/rest_api_gcbm - python3 annual_summaries.py - echo "# GCBM Model" > report.md - echo "## Total Biomass Curve" >> report.md - cml-publish tests/output/total_biomass_mt.png --md >> report.md - cml-send-comment --pr report.md + cd local/rest_api_gcbm + python3 annual_summaries.py + echo "# GCBM Model" > report.md + echo "## Code" >> report.md + jupytext --to md --output - annual_summaries.py >> report.md + # this command takes each cell of the notebook and adds it to report.md as markdown + echo "## Total Biomass Curve" >> report.md + cml publish tests/output/total_biomass_mt.png --md >> report.md + cml send-comment --update report.md diff --git a/local/rest_api_gcbm/GCBM_summary.txt b/local/rest_api_gcbm/GCBM_summary.txt new file mode 100644 index 00000000..fba7f691 --- /dev/null +++ b/local/rest_api_gcbm/GCBM_summary.txt @@ -0,0 +1,37 @@ +| Data Files | +|:----------------------------------------------------------| +| @\input\simulation_name\bounding_box.tiff | +| @\input\simulation_name\Classifier1_moja.json | +| @\input\simulation_name\Classifier1_moja.tiff | +| @\input\simulation_name\Classifier2_moja.json | +| @\input\simulation_name\Classifier2_moja.tiff | +| @\input\simulation_name\disturbances_2011_moja.json | +| @\input\simulation_name\disturbances_2011_moja.tiff | +| @\input\simulation_name\disturbances_2012_moja.json | +| @\input\simulation_name\disturbances_2012_moja.tiff | +| @\input\simulation_name\disturbances_2013_moja.json | +| @\input\simulation_name\disturbances_2013_moja.tiff | +| @\input\simulation_name\disturbances_2014_moja.json | +| @\input\simulation_name\disturbances_2014_moja.tiff | +| @\input\simulation_name\disturbances_2015_moja.json | +| @\input\simulation_name\disturbances_2015_moja.tiff | +| @\input\simulation_name\disturbances_2016_moja.json | +| @\input\simulation_name\disturbances_2016_moja.tiff | +| @\input\simulation_name\disturbances_2018_moja.json | +| @\input\simulation_name\disturbances_2018_moja.tiff | +| @\input\simulation_name\initial_age_moja.json | +| @\input\simulation_name\initial_age_moja.tiff | +| @\input\simulation_name\mean_annual_temperature_moja.json | +| @\input\simulation_name\mean_annual_temperature_moja.tiff | +| @\input\simulation_name\study_area.json || Config files | +|:------------------------------------------------| +| @\input\simulation_name\gcbm_config.cfg | +| @\input\simulation_name\internal_variables.json | +| @\input\simulation_name\localdomain.json | +| @\input\simulation_name\logging.conf | +| @\input\simulation_name\modules_cbm.json | +| @\input\simulation_name\modules_output.json | +| @\input\simulation_name\pools_cbm.json | +| @\input\simulation_name\provider_config.json | +| @\input\simulation_name\spinup.json | +| @\input\simulation_name\variables.json | \ No newline at end of file diff --git a/local/rest_api_gcbm/annual_summaries.py b/local/rest_api_gcbm/annual_summaries.py index a8e4aa83..c6c0d8a7 100644 --- a/local/rest_api_gcbm/annual_summaries.py +++ b/local/rest_api_gcbm/annual_summaries.py @@ -1,6 +1,14 @@ +# %% [markdown] +# # Dependencies + +# %% import sqlite3 import pandas as pd +# %% [markdown] +# # Connect to Database and Define SQL Query + +# %% conn = sqlite3.connect("tests/output/compiled_simulation_output.db") query = f""" @@ -14,6 +22,10 @@ ORDER BY years.year """ +# %% [markdown] +# Run Query and Generate plot + +# %% df = pd.read_sql_query(query, conn) ax = df.plot.line("year") ax.figure.savefig("tests/output/total_biomass_mt.png", dpi=300) diff --git a/local/rest_api_gcbm/app.py b/local/rest_api_gcbm/app.py index 0bb104ef..52d0c854 100644 --- a/local/rest_api_gcbm/app.py +++ b/local/rest_api_gcbm/app.py @@ -629,7 +629,8 @@ def send_table(): resp[table_name[0]] = schema return resp, 200 -@app.route("/gcbm/table/rename", methods=['POST']) + +@app.route("/gcbm/table/rename", methods=["POST"]) def rename_table(): """ Rename a table @@ -653,13 +654,14 @@ def rename_table(): try: connection = sqlite3.connect(input_dir + "gcbm_input.db") cursor = connection.cursor() - previous_name = request.form.get('previous') - new_name = request.form.get('new') + previous_name = request.form.get("previous") + new_name = request.form.get("new") cursor.execute(f"ALTER TABLE {previous_name} rename to {new_name}") return {"status": 1} except Exception as exception: return {"status": 0, "error": str(exception)} + @app.route("/gcbm/dynamic", methods=["POST"]) def gcbm_dynamic(): """ From 65135c8247e39acb116105d909dc481cc3111fdb Mon Sep 17 00:00:00 2001 From: Mohd Jasir Khan <53305380+khanjasir90@users.noreply.github.com> Date: Sat, 10 Sep 2022 13:24:19 +0530 Subject: [PATCH 02/21] =?UTF-8?q?chore:=20moved=20get=5Fconfig=5Ftemplates?= =?UTF-8?q?,=20get=5Fmodules=5Fcbm=5Fconfig,=20get=5Fpr=E2=80=A6=20(#181)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor: moved get_config_templates, get_modules_cbm_config, get_provider_config to preprocess.py file Signed-off-by: khanjasir90 * app.py and preprocess.py linted Signed-off-by: khanjasir90 Signed-off-by: khanjasir90 Signed-off-by: Freeman --- local/rest_api_gcbm/app.py | 351 +---------------------------- local/rest_api_gcbm/preprocess.py | 354 ++++++++++++++++++++++++++++++ 2 files changed, 355 insertions(+), 350 deletions(-) create mode 100644 local/rest_api_gcbm/preprocess.py diff --git a/local/rest_api_gcbm/app.py b/local/rest_api_gcbm/app.py index 52d0c854..c7a5daf9 100644 --- a/local/rest_api_gcbm/app.py +++ b/local/rest_api_gcbm/app.py @@ -21,6 +21,7 @@ from flask import jsonify from config_table import rename_columns import sqlite3 +from preprocess import get_config_templates, get_modules_cbm_config, get_provider_config flask.helpers._endpoint_from_view_func = flask.scaffold._endpoint_from_view_func from flask_restful import Resource, Api, reqparse @@ -230,356 +231,6 @@ def gcbm_upload(): } -def get_config_templates(input_dir): - if not os.path.exists(f"{input_dir}/templates"): - shutil.copytree( - f"{os.getcwd()}/templates", f"{input_dir}/templates", dirs_exist_ok=False - ) - - -# TODO: there needs to be a link between the files configured here append -# the ["vars"] attribute of modules_cbm.json -> CBMDisturbanceListener -# current hack is to drop the last five characters, but thats very fragile -def get_modules_cbm_config(input_dir): - with open(f"{input_dir}/templates/modules_cbm.json", "r+") as modules_cbm_config: - disturbances = [] - data = json.load(modules_cbm_config) - for file in os.listdir(f"{input_dir}/disturbances/"): - disturbances.append( - file.split(".")[0][:-5] - ) # drop `_moja` to match modules_cbm.json template - modules_cbm_config.seek(0) - data["Modules"]["CBMDisturbanceListener"]["settings"]["vars"] = disturbances - json.dump(data, modules_cbm_config, indent=4) - modules_cbm_config.truncate() - - -def get_provider_config(input_dir): - with open(f"{input_dir}/templates/provider_config.json", "r+") as provider_config: - lst = [] - data = json.load(provider_config) - - for file in os.listdir(f"{input_dir}/db/"): - d = dict() - d["path"] = file - d["type"] = "SQLite" - data["Providers"]["SQLite"] = d - provider_config.seek(0) - - for file in os.listdir(f"{input_dir}/disturbances/"): - d = dict() - d["name"] = file[:-10] - d["layer_path"] = file - d["layer_prefix"] = file[:-5] - lst.append(d) - provider_config.seek(0) - data["Providers"]["RasterTiled"]["layers"] = lst - - for file in os.listdir(f"{input_dir}/classifiers/"): - d = dict() - d["name"] = file[:-10] - d["layer_path"] = file - d["layer_prefix"] = file[:-5] - lst.append(d) - provider_config.seek(0) - data["Providers"]["RasterTiled"]["layers"] = lst - - for file in os.listdir(f"{input_dir}/miscellaneous/"): - d = dict() - d["name"] = file[:-10] - d["layer_path"] = file - d["layer_prefix"] = file[:-5] - lst.append(d) - provider_config.seek(0) - data["Providers"]["RasterTiled"]["layers"] = lst - - Rasters = [] - Rastersm = [] - nodatam = [] - nodata = [] - cellLatSize = [] - cellLonSize = [] - paths = [] - - for root, dirs, files in os.walk(os.path.abspath(f"{input_dir}/disturbances/")): - for file in files: - fp = os.path.join(root, file) - Rasters.append(fp) - paths.append(fp) - - for root, dirs, files in os.walk(os.path.abspath(f"{input_dir}/classifiers/")): - for file in files: - fp1 = os.path.join(root, file) - Rasters.append(fp1) - paths.append(fp1) - - for nd in Rasters: - img = rst.open(nd) - t = img.transform - x = t[0] - y = -t[4] - n = img.nodata - cellLatSize.append(x) - cellLonSize.append(y) - nodata.append(n) - - result = all(element == cellLatSize[0] for element in cellLatSize) - if result: - cellLat = x - cellLon = y - nd = n - blockLat = x * 400 - blockLon = y * 400 - tileLat = x * 4000 - tileLon = y * 4000 - else: - print("Corrupt files") - - provider_config.seek(0) - - data["Providers"]["RasterTiled"]["cellLonSize"] = cellLon - data["Providers"]["RasterTiled"]["cellLatSize"] = cellLat - data["Providers"]["RasterTiled"]["blockLonSize"] = blockLon - data["Providers"]["RasterTiled"]["blockLatSize"] = blockLat - data["Providers"]["RasterTiled"]["tileLatSize"] = tileLat - data["Providers"]["RasterTiled"]["tileLonSize"] = tileLon - - json.dump(data, provider_config, indent=4) - provider_config.truncate() - - dictionary = { - "layer_type": "GridLayer", - "layer_data": "Byte", - "nodata": nd, - "tileLatSize": tileLat, - "tileLonSize": tileLon, - "blockLatSize": blockLat, - "blockLonSize": blockLon, - "cellLatSize": cellLat, - "cellLonSize": cellLon, - } - - # should be able to accept variable number of inputs, but requires - # means for user to specify/verify correct ["attributes"] - def get_input_layers(): - for root, dirs, files in os.walk( - os.path.abspath(f"{input_dir}/miscellaneous/") - ): - for file in files: - fp2 = os.path.join(root, file) - Rastersm.append(fp2) - - for i in Rastersm: - img = rst.open(i) - d = img.nodata - nodatam.append(d) - - with open( - f"{input_dir}/initial_age_moja.json", "w", encoding="utf8" - ) as json_file: - dictionary["layer_type"] = "GridLayer" - dictionary["layer_data"] = "Int16" - dictionary["nodata"] = nodatam[1] - json.dump(dictionary, json_file, indent=4) - - with open( - f"{input_dir}/mean_annual_temperature_moja.json", "w", encoding="utf8" - ) as json_file: - dictionary["layer_type"] = "GridLayer" - dictionary["layer_data"] = "Float32" - dictionary["nodata"] = nodatam[0] - json.dump(dictionary, json_file, indent=4) - - with open( - f"{input_dir}/Classifier1_moja.json", "w", encoding="utf8" - ) as json_file: - dictionary["layer_type"] = "GridLayer" - dictionary["layer_data"] = "Byte" - dictionary["nodata"] = nd - dictionary["attributes"] = { - "1": "TA", - "2": "BP", - "3": "BS", - "4": "JP", - "5": "WS", - "6": "WB", - "7": "BF", - "8": "GA", - } - json.dump(dictionary, json_file, indent=4) - - with open( - f"{input_dir}/Classifier2_moja.json", "w", encoding="utf8" - ) as json_file: - dictionary["layer_type"] = "GridLayer" - dictionary["layer_data"] = "Byte" - dictionary["nodata"] = nd - dictionary["attributes"] = {"1": "5", "2": "6", "3": "7", "4": "8"} - json.dump(dictionary, json_file, indent=4) - - with open( - f"{input_dir}/disturbances_2011_moja.json", "w", encoding="utf8" - ) as json_file: - dictionary["layer_data"] = "Byte" - dictionary["nodata"] = nd - dictionary["attributes"] = { - "1": {"year": 2011, "disturbance_type": "Wildfire", "transition": 1} - } - json.dump(dictionary, json_file, indent=4) - - with open( - f"{input_dir}/disturbances_2012_moja.json", "w", encoding="utf8" - ) as json_file: - dictionary["attributes"] = { - "1": {"year": 2012, "disturbance_type": "Wildfire", "transition": 1} - } - json.dump(dictionary, json_file, indent=4) - - with open( - f"{input_dir}/disturbances_2013_moja.json", "w", encoding="utf8" - ) as json_file: - dictionary["attributes"] = { - "1": { - "year": 2013, - "disturbance_type": "Mountain pine beetle — Very severe impact", - "transition": 1, - }, - "2": { - "year": 2013, - "disturbance_type": "Wildfire", - "transition": 1, - }, - } - json.dump(dictionary, json_file, indent=4) - - with open( - f"{input_dir}/disturbances_2014_moja.json", "w", encoding="utf8" - ) as json_file: - dictionary["attributes"] = { - "1": { - "year": 2014, - "disturbance_type": "Mountain pine beetle — Very severe impact", - "transition": 1, - } - } - json.dump(dictionary, json_file, indent=4) - - with open( - f"{input_dir}/disturbances_2015_moja.json", "w", encoding="utf8" - ) as json_file: - dictionary["attributes"] = { - "1": {"year": 2016, "disturbance_type": "Wildfire", "transition": 1} - } - json.dump(dictionary, json_file, indent=4) - - with open( - f"{input_dir}/disturbances_2016_moja.json", "w", encoding="utf8" - ) as json_file: - dictionary["attributes"] = { - "1": {"year": 2016, "disturbance_type": "Wildfire", "transition": 1} - } - json.dump(dictionary, json_file, indent=4) - - with open( - f"{input_dir}/disturbances_2018_moja.json", "w", encoding="utf8" - ) as json_file: - dictionary["attributes"] = { - "1": { - "year": 2018, - "disturbance_type": "Mountain pine beetle — Low impact", - "transition": 1, - } - } - json.dump(dictionary, json_file, indent=4) - - get_input_layers() - - def get_study_area(): - study_area = { - "tile_size": tileLat, - "block_size": blockLat, - "tiles": [ - { - "x": int(t[2]), - "y": int(t[5]), - "index": 12674, - } - ], - "pixel_size": cellLat, - "layers": [], - } - - with open(f"{input_dir}/study_area.json", "w", encoding="utf") as json_file: - list = [] - - for file in os.listdir(f"{input_dir}/miscellaneous/"): - d1 = dict() - d1["name"] = file[:-10] - d1["type"] = "VectorLayer" - list.append(d1) - study_area["layers"] = list - - for file in os.listdir(f"{input_dir}/classifiers/"): - d1 = dict() - d1["name"] = file[:-10] - d1["type"] = "VectorLayer" - d1["tags"] = ["classifier"] - list.append(d1) - study_area["layers"] = list - - for file in os.listdir(f"{input_dir}/disturbances/"): - d1 = dict() - d1["name"] = file[:-10] - d1["type"] = "DisturbanceLayer" - d1["tags"] = ["disturbance"] - list.append(d1) - study_area["layers"] = list - - json.dump(study_area, json_file, indent=4) - - get_study_area() - - for root, dirs, files in os.walk(os.path.abspath(f"{input_dir}/disturbances/")): - for file in files: - fp = os.path.join(root, file) - Rasters.append(fp) - paths.append(fp) - - for root, dirs, files in os.walk(os.path.abspath(f"{input_dir}/classifiers/")): - for file in files: - fp1 = os.path.join(root, file) - Rasters.append(fp1) - paths.append(fp1) - - for root, dirs, files in os.walk( - os.path.abspath(f"{input_dir}/miscellaneous/") - ): - for file in files: - fp2 = os.path.join(root, file) - paths.append(fp2) - - for root, dirs, files in os.walk(os.path.abspath(f"{input_dir}/templates/")): - for file in files: - fp3 = os.path.join(root, file) - paths.append(fp3) - - for root, dirs, files in os.walk(os.path.abspath(f"{input_dir}/db/")): - for file in files: - fp4 = os.path.join(root, file) - paths.append(fp4) - - # copy files to input directory - for i in paths: - shutil.copy2(i, (f"{input_dir}")) - - # delete folders from input directory - shutil.rmtree((f"{input_dir}/disturbances/")) - shutil.rmtree((f"{input_dir}/templates/")) - shutil.rmtree((f"{input_dir}/classifiers/")) - shutil.rmtree((f"{input_dir}/miscellaneous/")) - shutil.rmtree((f"{input_dir}/db/")) - - @app.route("/config", methods=["POST"]) def config_table(): obj = request.get_json() diff --git a/local/rest_api_gcbm/preprocess.py b/local/rest_api_gcbm/preprocess.py new file mode 100644 index 00000000..261d2ed8 --- /dev/null +++ b/local/rest_api_gcbm/preprocess.py @@ -0,0 +1,354 @@ +import os +import shutil +import json +import rasterio as rst + + +def get_config_templates(input_dir): + if not os.path.exists(f"{input_dir}/templates"): + shutil.copytree( + f"{os.getcwd()}/templates", f"{input_dir}/templates", dirs_exist_ok=False + ) + + +# TODO: there needs to be a link between the files configured here append +# the ["vars"] attribute of modules_cbm.json -> CBMDisturbanceListener +# current hack is to drop the last five characters, but thats very fragile +def get_modules_cbm_config(input_dir): + with open(f"{input_dir}/templates/modules_cbm.json", "r+") as modules_cbm_config: + disturbances = [] + data = json.load(modules_cbm_config) + for file in os.listdir(f"{input_dir}/disturbances/"): + disturbances.append( + file.split(".")[0][:-5] + ) # drop `_moja` to match modules_cbm.json template + modules_cbm_config.seek(0) + data["Modules"]["CBMDisturbanceListener"]["settings"]["vars"] = disturbances + json.dump(data, modules_cbm_config, indent=4) + modules_cbm_config.truncate() + + +def get_provider_config(input_dir): + with open(f"{input_dir}/templates/provider_config.json", "r+") as provider_config: + lst = [] + data = json.load(provider_config) + + for file in os.listdir(f"{input_dir}/db/"): + d = dict() + d["path"] = file + d["type"] = "SQLite" + data["Providers"]["SQLite"] = d + provider_config.seek(0) + + for file in os.listdir(f"{input_dir}/disturbances/"): + d = dict() + d["name"] = file[:-10] + d["layer_path"] = file + d["layer_prefix"] = file[:-5] + lst.append(d) + provider_config.seek(0) + data["Providers"]["RasterTiled"]["layers"] = lst + + for file in os.listdir(f"{input_dir}/classifiers/"): + d = dict() + d["name"] = file[:-10] + d["layer_path"] = file + d["layer_prefix"] = file[:-5] + lst.append(d) + provider_config.seek(0) + data["Providers"]["RasterTiled"]["layers"] = lst + + for file in os.listdir(f"{input_dir}/miscellaneous/"): + d = dict() + d["name"] = file[:-10] + d["layer_path"] = file + d["layer_prefix"] = file[:-5] + lst.append(d) + provider_config.seek(0) + data["Providers"]["RasterTiled"]["layers"] = lst + + Rasters = [] + Rastersm = [] + nodatam = [] + nodata = [] + cellLatSize = [] + cellLonSize = [] + paths = [] + + for root, dirs, files in os.walk(os.path.abspath(f"{input_dir}/disturbances/")): + for file in files: + fp = os.path.join(root, file) + Rasters.append(fp) + paths.append(fp) + + for root, dirs, files in os.walk(os.path.abspath(f"{input_dir}/classifiers/")): + for file in files: + fp1 = os.path.join(root, file) + Rasters.append(fp1) + paths.append(fp1) + + for nd in Rasters: + img = rst.open(nd) + t = img.transform + x = t[0] + y = -t[4] + n = img.nodata + cellLatSize.append(x) + cellLonSize.append(y) + nodata.append(n) + + result = all(element == cellLatSize[0] for element in cellLatSize) + if result: + cellLat = x + cellLon = y + nd = n + blockLat = x * 400 + blockLon = y * 400 + tileLat = x * 4000 + tileLon = y * 4000 + else: + print("Corrupt files") + + provider_config.seek(0) + + data["Providers"]["RasterTiled"]["cellLonSize"] = cellLon + data["Providers"]["RasterTiled"]["cellLatSize"] = cellLat + data["Providers"]["RasterTiled"]["blockLonSize"] = blockLon + data["Providers"]["RasterTiled"]["blockLatSize"] = blockLat + data["Providers"]["RasterTiled"]["tileLatSize"] = tileLat + data["Providers"]["RasterTiled"]["tileLonSize"] = tileLon + + json.dump(data, provider_config, indent=4) + provider_config.truncate() + + dictionary = { + "layer_type": "GridLayer", + "layer_data": "Byte", + "nodata": nd, + "tileLatSize": tileLat, + "tileLonSize": tileLon, + "blockLatSize": blockLat, + "blockLonSize": blockLon, + "cellLatSize": cellLat, + "cellLonSize": cellLon, + } + + # should be able to accept variable number of inputs, but requires + # means for user to specify/verify correct ["attributes"] + def get_input_layers(): + for root, dirs, files in os.walk( + os.path.abspath(f"{input_dir}/miscellaneous/") + ): + for file in files: + fp2 = os.path.join(root, file) + Rastersm.append(fp2) + + for i in Rastersm: + img = rst.open(i) + d = img.nodata + nodatam.append(d) + + with open( + f"{input_dir}/initial_age_moja.json", "w", encoding="utf8" + ) as json_file: + dictionary["layer_type"] = "GridLayer" + dictionary["layer_data"] = "Int16" + dictionary["nodata"] = nodatam[1] + json.dump(dictionary, json_file, indent=4) + + with open( + f"{input_dir}/mean_annual_temperature_moja.json", "w", encoding="utf8" + ) as json_file: + dictionary["layer_type"] = "GridLayer" + dictionary["layer_data"] = "Float32" + dictionary["nodata"] = nodatam[0] + json.dump(dictionary, json_file, indent=4) + + with open( + f"{input_dir}/Classifier1_moja.json", "w", encoding="utf8" + ) as json_file: + dictionary["layer_type"] = "GridLayer" + dictionary["layer_data"] = "Byte" + dictionary["nodata"] = nd + dictionary["attributes"] = { + "1": "TA", + "2": "BP", + "3": "BS", + "4": "JP", + "5": "WS", + "6": "WB", + "7": "BF", + "8": "GA", + } + json.dump(dictionary, json_file, indent=4) + + with open( + f"{input_dir}/Classifier2_moja.json", "w", encoding="utf8" + ) as json_file: + dictionary["layer_type"] = "GridLayer" + dictionary["layer_data"] = "Byte" + dictionary["nodata"] = nd + dictionary["attributes"] = {"1": "5", "2": "6", "3": "7", "4": "8"} + json.dump(dictionary, json_file, indent=4) + + with open( + f"{input_dir}/disturbances_2011_moja.json", "w", encoding="utf8" + ) as json_file: + dictionary["layer_data"] = "Byte" + dictionary["nodata"] = nd + dictionary["attributes"] = { + "1": {"year": 2011, "disturbance_type": "Wildfire", "transition": 1} + } + json.dump(dictionary, json_file, indent=4) + + with open( + f"{input_dir}/disturbances_2012_moja.json", "w", encoding="utf8" + ) as json_file: + dictionary["attributes"] = { + "1": {"year": 2012, "disturbance_type": "Wildfire", "transition": 1} + } + json.dump(dictionary, json_file, indent=4) + + with open( + f"{input_dir}/disturbances_2013_moja.json", "w", encoding="utf8" + ) as json_file: + dictionary["attributes"] = { + "1": { + "year": 2013, + "disturbance_type": "Mountain pine beetle — Very severe impact", + "transition": 1, + }, + "2": { + "year": 2013, + "disturbance_type": "Wildfire", + "transition": 1, + }, + } + json.dump(dictionary, json_file, indent=4) + + with open( + f"{input_dir}/disturbances_2014_moja.json", "w", encoding="utf8" + ) as json_file: + dictionary["attributes"] = { + "1": { + "year": 2014, + "disturbance_type": "Mountain pine beetle — Very severe impact", + "transition": 1, + } + } + json.dump(dictionary, json_file, indent=4) + + with open( + f"{input_dir}/disturbances_2015_moja.json", "w", encoding="utf8" + ) as json_file: + dictionary["attributes"] = { + "1": {"year": 2016, "disturbance_type": "Wildfire", "transition": 1} + } + json.dump(dictionary, json_file, indent=4) + + with open( + f"{input_dir}/disturbances_2016_moja.json", "w", encoding="utf8" + ) as json_file: + dictionary["attributes"] = { + "1": {"year": 2016, "disturbance_type": "Wildfire", "transition": 1} + } + json.dump(dictionary, json_file, indent=4) + + with open( + f"{input_dir}/disturbances_2018_moja.json", "w", encoding="utf8" + ) as json_file: + dictionary["attributes"] = { + "1": { + "year": 2018, + "disturbance_type": "Mountain pine beetle — Low impact", + "transition": 1, + } + } + json.dump(dictionary, json_file, indent=4) + + get_input_layers() + + def get_study_area(): + study_area = { + "tile_size": tileLat, + "block_size": blockLat, + "tiles": [ + { + "x": int(t[2]), + "y": int(t[5]), + "index": 12674, + } + ], + "pixel_size": cellLat, + "layers": [], + } + + with open(f"{input_dir}/study_area.json", "w", encoding="utf") as json_file: + list = [] + + for file in os.listdir(f"{input_dir}/miscellaneous/"): + d1 = dict() + d1["name"] = file[:-10] + d1["type"] = "VectorLayer" + list.append(d1) + study_area["layers"] = list + + for file in os.listdir(f"{input_dir}/classifiers/"): + d1 = dict() + d1["name"] = file[:-10] + d1["type"] = "VectorLayer" + d1["tags"] = ["classifier"] + list.append(d1) + study_area["layers"] = list + + for file in os.listdir(f"{input_dir}/disturbances/"): + d1 = dict() + d1["name"] = file[:-10] + d1["type"] = "DisturbanceLayer" + d1["tags"] = ["disturbance"] + list.append(d1) + study_area["layers"] = list + + json.dump(study_area, json_file, indent=4) + + get_study_area() + + for root, dirs, files in os.walk(os.path.abspath(f"{input_dir}/disturbances/")): + for file in files: + fp = os.path.join(root, file) + Rasters.append(fp) + paths.append(fp) + + for root, dirs, files in os.walk(os.path.abspath(f"{input_dir}/classifiers/")): + for file in files: + fp1 = os.path.join(root, file) + Rasters.append(fp1) + paths.append(fp1) + + for root, dirs, files in os.walk( + os.path.abspath(f"{input_dir}/miscellaneous/") + ): + for file in files: + fp2 = os.path.join(root, file) + paths.append(fp2) + + for root, dirs, files in os.walk(os.path.abspath(f"{input_dir}/templates/")): + for file in files: + fp3 = os.path.join(root, file) + paths.append(fp3) + + for root, dirs, files in os.walk(os.path.abspath(f"{input_dir}/db/")): + for file in files: + fp4 = os.path.join(root, file) + paths.append(fp4) + + # copy files to input directory + for i in paths: + shutil.copy2(i, (f"{input_dir}")) + + # delete folders from input directory + shutil.rmtree((f"{input_dir}/disturbances/")) + shutil.rmtree((f"{input_dir}/templates/")) + shutil.rmtree((f"{input_dir}/classifiers/")) + shutil.rmtree((f"{input_dir}/miscellaneous/")) + shutil.rmtree((f"{input_dir}/db/")) From 9545161b9236c69be94936e2a15bb3a60ff6a289 Mon Sep 17 00:00:00 2001 From: Sanjay Singh Rajpoot <67458417+SanjaySinghRajpoot@users.noreply.github.com> Date: Fri, 16 Sep 2022 18:48:55 +0530 Subject: [PATCH 03/21] feat: added Furo theme, GCBM dev docs and Restructured deployments docs (#164) * feat: added fluro and GCBM dev docs Signed-off-by: SanjaySinghRajpoot * chores: furo added to requirements Signed-off-by: SanjaySinghRajpoot * chores: restructed Deployment docs Signed-off-by: SanjaySinghRajpoot Signed-off-by: SanjaySinghRajpoot Signed-off-by: Freeman --- .vscode/settings.json | 3 + docs/conf.py | 4 +- docs/deployment/azure.rst | 225 +++++++++++++++++++++++++++ docs/deployment/google_cloud.rst | 31 ++++ docs/deployment/index.rst | 255 +------------------------------ docs/development/gcbm_guide.rst | 19 +++ docs/development/index.rst | 1 + docs/requirements.txt | 1 + 8 files changed, 286 insertions(+), 253 deletions(-) create mode 100644 .vscode/settings.json create mode 100644 docs/deployment/azure.rst create mode 100644 docs/deployment/google_cloud.rst create mode 100644 docs/development/gcbm_guide.rst diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..a7d0fc7b --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "esbonio.sphinx.confDir": "" +} \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index 2e7ccfa2..bfc95995 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -18,7 +18,7 @@ # -- Project information ----------------------------------------------------- project = "FLINT.Cloud Technical Guide" -copyright = "2021, Sneha Mishra, Arnav Tiwari" +copyright = "2022, Sneha Mishra, Arnav Tiwari" author = u"Sneha Mishra, Arnav Tiwari & contributors" @@ -50,7 +50,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = "sphinx_rtd_theme" +html_theme = "furo" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, diff --git a/docs/deployment/azure.rst b/docs/deployment/azure.rst new file mode 100644 index 00000000..5086a24c --- /dev/null +++ b/docs/deployment/azure.rst @@ -0,0 +1,225 @@ +.. _Deployment: + +Azure +----- + +To deploy FLINT.Cloud to Azure, we recommend using the Azure App Service +with a custom Microsoft Container Registry container built from our +``local`` directory. This setup uses `Azure CLI`_ to +deploy the infrastructure and the applications to be used. You need to +sign in to the Azure CLI by using the ``az login`` command. To +finish the authentication process, follow the steps displayed in your +terminal. To build images, we use `Docker`_ and then push them to `Azure Container Registry`_. + +Download the project +~~~~~~~~~~~~~~~~~~~~ + +Clone the repository: + +.. code:: sh + + git clone https://github.com/moja-global/flint.cloud + +Navigate to the ``local`` directory: + +.. code:: sh + + sh local + +Build the images locally +~~~~~~~~~~~~~~~~~~~~~~~~ + +To build the ``rest_api_gcbm`` image locally, run the following command: + +.. code:: sh + + pushd rest_api_gcbm + docker build --build-arg BUILD_TYPE=RELEASE --build-arg NUM_CPU=4 -t gcbm-api . + popd + +To build the ``rest_api_flint.example`` image locally, run the following +command: + +.. code:: sh + + pushd rest_api_flint.example + docker build -t flint-api . + popd + +Create a resource group +~~~~~~~~~~~~~~~~~~~~~~~ + +To push images onto and deploy containers with the Azure App +Service, you need to first prepare some resources. Start by +creating a resource group that will collect all your +resources. + +.. code:: sh + + az group create --name myResourceGroup --location centralindia + +You can change the ``--location`` value to specify a region near you. + +Create a Container Registry +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can now push the image to Azure Container Registry so that App +Service can deploy it. Create an Azure Container Registry to push your +images to: + +.. code:: sh + + az acr create --name --resource-group myResourceGroup --sku Basic --admin-enabled true + +Replace ```` with a suitable name for your registry. The +name must contain only letters and numbers, and must be unique across all +of Azure. + +Retrieve your credentials for the Container Registry: + +.. code:: sh + + az acr credential show --resource-group myResourceGroup --name + +Use the ``docker login`` command to sign in to the container registry: + +.. code:: sh + + docker login .azurecr.io --username + +Replace ```` and ```` with values from +the previous steps. When prompted, type in one of the passwords from the +previous step. + +Tag the images with the registry name: + +.. code:: sh + + docker tag rest_api_gcbm .azurecr.io/rest_api_gcbm:latest + docker tag rest_api_flint.example .azurecr.io/rest_api_flint.example:latest + +Use the docker push command to push the image to the registry: + +.. code:: sh + + docker push .azurecr.io/rest_api_gcbm:latest + docker push .azurecr.io/rest_api_flint.example:latest + +Use the ``az acr repository list`` command to verify that the push was +successful: + +.. code:: sh + + az acr repository list -n + +Deploy the image from registry +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To deploy a container to Azure App Service, you first create a web app +on App Service, then connect the web app to the container registry. When +the web app starts, App Service automatically pulls the image from the +registry. + +Create an App Service plan using the ``az appservice plan create`` +command: + +.. code:: sh + + az appservice plan create --name myAppServicePlan --resource-group myResourceGroup --is-linux + +Create the web app with the ``az webpp create`` command. Since we are +deploying two images to two different web apps, you need to +enter these commands twice. To deploy ``rest_api_gcbm`` to the first +web app and ``rest_api_flint.example`` to the second web app, run the +following commands: + +.. code:: sh + + az webapp create --resource-group myResourceGroup --plan myAppServicePlan --name --deployment-container-image-name .azurecr.io/rest_api_gcbm:latest + az webapp create --resource-group myResourceGroup --plan myAppServicePlan --name --deployment-container-image-name .azurecr.io/rest_api_flint.example:latest + +Use the ``az webapp config appsettings set`` to set the +``WEBSITES_PORT`` environment variable. In +our case, the port to be exposed is ``8080``. + +.. code:: sh + + az webapp config appsettings set --resource-group myResourceGroup --name --settings WEBSITES_PORT=8080 + az webapp config appsettings set --resource-group myResourceGroup --name --settings WEBSITES_PORT=8080 + +Enable the system-assigned managed identity for the web app by using the +``az webapp identity assign`` command: + +.. code:: sh + + az webapp identity assign --resource-group myResourceGroup --name --query principalId --output tsv + az webapp identity assign --resource-group myResourceGroup --name --query principalId --output tsv + +Replace ```` with the name you used in the previous step. The +output of the command (filtered by the ``--query`` and ``--output`` +arguments) is the service principal of the assigned identity. + +Retrieve your subscription ID with the ``az account show`` command, +which you need in the next step: + +.. code:: sh + + az account show --query id --output tsv + +Grant the managed identity permission to access the container registry: + +.. code:: sh + + az role assignment create --assignee --scope /subscriptions//resourceGroups/myResourceGroup/providers/Microsoft.ContainerRegistry/registries/ --role "AcrPull" + +Replace the following values: + +- ```` with the service principal ID from the + ``az webapp identity assign`` command. +- ```` with the name of your container registry. +- ```` with the subscription ID retrieved from the + ``az account show`` command. + +Make sure the above steps are repeated for both of the apps that you are +going to deploy. Configure your app to use the managed identity to pull +from Azure Container Registry. + +.. code:: sh + + az resource update --ids /subscriptions//resourceGroups/myResourceGroup/providers/Microsoft.Web/sites//config/web --set properties.acrUseManagedIdentityCreds=True + az resource update --ids /subscriptions//resourceGroups/myResourceGroup/providers/Microsoft.Web/sites//config/web --set properties.acrUseManagedIdentityCreds=True + +Replace the following values: + +- ```` with the subscription ID retrieved from the az + account show command. +- ```` with the name of your web app. + +Deploy the image +~~~~~~~~~~~~~~~~ + +Use the ``az webapp config container set`` command to specify the +container registry and the image to deploy for the web app: + +.. code:: sh + + az webapp config container set --name --resource-group myResourceGroup --docker-custom-image-name .azurecr.io/rest_api_gcbm:latest --docker-registry-server-url https://.azurecr.io + az webapp config container set --name --resource-group myResourceGroup --docker-custom-image-name .azurecr.io/rest_api_flint.example:latest --docker-registry-server-url https://.azurecr.io + +Replace ```` and ```` with the name of your web +app, and replace all instances of ```` with the name of your +registry. When the ``az webapp config container set`` command completes, +the web app is running in the container on App Service. + +To test the app, browse to ``https://.azurewebsites.net``, +replacing ```` with the name of your web app. To clean up the +resources, you only need to delete the resource group that contains +them: + +.. code:: sh + + az group delete --name myResourceGroup + +.. _Azure CLI: https://docs.microsoft.com/en-us/cli/azure/ +.. _Docker: https://www.docker.com/ +.. _Azure Container Registry: https://azure.microsoft.com/en-in/services/container-registry/ diff --git a/docs/deployment/google_cloud.rst b/docs/deployment/google_cloud.rst new file mode 100644 index 00000000..d1f4023c --- /dev/null +++ b/docs/deployment/google_cloud.rst @@ -0,0 +1,31 @@ +.. _Deployment: + +Google Cloud Platform +--------------------- + +To deploy FLINT.Cloud to GCP, we provide a +production grade setup using a Layered Architecture setup on top of the +Google Cloud. In this setup we use `Terraform`_, an +Infrastructure-as-a-Code tool, to deploy the infrastructure and the +applications we want to use. + +To deploy the FLINT.Cloud to GCP, follow these steps: + +1. Create a GCP service account with project owner permissions in your + project. This is used by Terraform to provision all the necessary + resources. +2. Copy ``main.tf`` from the ``layered`` directory of this repository to + your Cloud Console machine. +3. In ``main.tf``, change the project variable to your ``project ID``. + Change any other variables, if necessary. +4. Download the key in JSON format for the service account created in ``Step 1`` + to your project's Cloud Console machine. Rename it + to ``service_account.json``. +5. Run ``terraform apply``. After this command finishes, it should + output the URL to FLINT Cloud (ingress). + +To tear down the infrastructure and delete the application, run +``terraform destroy`` in the same directory where ``main.tf`` is +present. If this fails, run it again. + +.. _Terraform: https://www.terraform.io/ \ No newline at end of file diff --git a/docs/deployment/index.rst b/docs/deployment/index.rst index ede831a1..a3647887 100644 --- a/docs/deployment/index.rst +++ b/docs/deployment/index.rst @@ -10,256 +10,9 @@ APIs that are used by FLINT-specific applications, like FLINT-UI. This guide assumes that you have some knowledge of the concepts of GCP and Azure, and that you have successfully deployed an application before. -Google Cloud Platform ---------------------- -To deploy FLINT.Cloud to GCP, we provide a -production grade setup using a Layered Architecture setup on top of the -Google Cloud. In this setup we use `Terraform`_, an -Infrastructure-as-a-Code tool, to deploy the infrastructure and the -applications we want to use. +.. toctree:: + :maxdepth: 1 -To deploy the FLINT.Cloud to GCP, follow these steps: - -1. Create a GCP service account with project owner permissions in your - project. This is used by Terraform to provision all the necessary - resources. -2. Copy ``main.tf`` from the ``layered`` directory of this repository to - your Cloud Console machine. -3. In ``main.tf``, change the project variable to your ``project ID``. - Change any other variables, if necessary. -4. Download the key in JSON format for the service account created in ``Step 1`` - to your project's Cloud Console machine. Rename it - to ``service_account.json``. -5. Run ``terraform apply``. After this command finishes, it should - output the URL to FLINT Cloud (ingress). - -To tear down the infrastructure and delete the application, run -``terraform destroy`` in the same directory where ``main.tf`` is -present. If this fails, run it again. - -.. _Terraform: https://www.terraform.io/ - -Azure ------ - -To deploy FLINT.Cloud to Azure, we recommend using the Azure App Service -with a custom Microsoft Container Registry container built from our -``local`` directory. This setup uses `Azure CLI`_ to -deploy the infrastructure and the applications to be used. You need to -sign in to the Azure CLI by using the ``az login`` command. To -finish the authentication process, follow the steps displayed in your -terminal. To build images, we use `Docker`_ and then push them to `Azure Container Registry`_. - -Download the project -~~~~~~~~~~~~~~~~~~~~ - -Clone the repository: - -.. code:: sh - - git clone https://github.com/moja-global/flint.cloud - -Navigate to the ``local`` directory: - -.. code:: sh - - sh local - -Build the images locally -~~~~~~~~~~~~~~~~~~~~~~~~ - -To build the ``rest_api_gcbm`` image locally, run the following command: - -.. code:: sh - - pushd rest_api_gcbm - docker build --build-arg BUILD_TYPE=RELEASE --build-arg NUM_CPU=4 -t gcbm-api . - popd - -To build the ``rest_api_flint.example`` image locally, run the following -command: - -.. code:: sh - - pushd rest_api_flint.example - docker build -t flint-api . - popd - -Create a resource group -~~~~~~~~~~~~~~~~~~~~~~~ - -To push images onto and deploy containers with the Azure App -Service, you need to first prepare some resources. Start by -creating a resource group that will collect all your -resources. - -.. code:: sh - - az group create --name myResourceGroup --location centralindia - -You can change the ``--location`` value to specify a region near you. - -Create a Container Registry -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -You can now push the image to Azure Container Registry so that App -Service can deploy it. Create an Azure Container Registry to push your -images to: - -.. code:: sh - - az acr create --name --resource-group myResourceGroup --sku Basic --admin-enabled true - -Replace ```` with a suitable name for your registry. The -name must contain only letters and numbers, and must be unique across all -of Azure. - -Retrieve your credentials for the Container Registry: - -.. code:: sh - - az acr credential show --resource-group myResourceGroup --name - -Use the ``docker login`` command to sign in to the container registry: - -.. code:: sh - - docker login .azurecr.io --username - -Replace ```` and ```` with values from -the previous steps. When prompted, type in one of the passwords from the -previous step. - -Tag the images with the registry name: - -.. code:: sh - - docker tag rest_api_gcbm .azurecr.io/rest_api_gcbm:latest - docker tag rest_api_flint.example .azurecr.io/rest_api_flint.example:latest - -Use the docker push command to push the image to the registry: - -.. code:: sh - - docker push .azurecr.io/rest_api_gcbm:latest - docker push .azurecr.io/rest_api_flint.example:latest - -Use the ``az acr repository list`` command to verify that the push was -successful: - -.. code:: sh - - az acr repository list -n - -Deploy the image from registry -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -To deploy a container to Azure App Service, you first create a web app -on App Service, then connect the web app to the container registry. When -the web app starts, App Service automatically pulls the image from the -registry. - -Create an App Service plan using the ``az appservice plan create`` -command: - -.. code:: sh - - az appservice plan create --name myAppServicePlan --resource-group myResourceGroup --is-linux - -Create the web app with the ``az webpp create`` command. Since we are -deploying two images to two different web apps, you need to -enter these commands twice. To deploy ``rest_api_gcbm`` to the first -web app and ``rest_api_flint.example`` to the second web app, run the -following commands: - -.. code:: sh - - az webapp create --resource-group myResourceGroup --plan myAppServicePlan --name --deployment-container-image-name .azurecr.io/rest_api_gcbm:latest - az webapp create --resource-group myResourceGroup --plan myAppServicePlan --name --deployment-container-image-name .azurecr.io/rest_api_flint.example:latest - -Use the ``az webapp config appsettings set`` to set the -``WEBSITES_PORT`` environment variable. In -our case, the port to be exposed is ``8080``. - -.. code:: sh - - az webapp config appsettings set --resource-group myResourceGroup --name --settings WEBSITES_PORT=8080 - az webapp config appsettings set --resource-group myResourceGroup --name --settings WEBSITES_PORT=8080 - -Enable the system-assigned managed identity for the web app by using the -``az webapp identity assign`` command: - -.. code:: sh - - az webapp identity assign --resource-group myResourceGroup --name --query principalId --output tsv - az webapp identity assign --resource-group myResourceGroup --name --query principalId --output tsv - -Replace ```` with the name you used in the previous step. The -output of the command (filtered by the ``--query`` and ``--output`` -arguments) is the service principal of the assigned identity. - -Retrieve your subscription ID with the ``az account show`` command, -which you need in the next step: - -.. code:: sh - - az account show --query id --output tsv - -Grant the managed identity permission to access the container registry: - -.. code:: sh - - az role assignment create --assignee --scope /subscriptions//resourceGroups/myResourceGroup/providers/Microsoft.ContainerRegistry/registries/ --role "AcrPull" - -Replace the following values: - -- ```` with the service principal ID from the - ``az webapp identity assign`` command. -- ```` with the name of your container registry. -- ```` with the subscription ID retrieved from the - ``az account show`` command. - -Make sure the above steps are repeated for both of the apps that you are -going to deploy. Configure your app to use the managed identity to pull -from Azure Container Registry. - -.. code:: sh - - az resource update --ids /subscriptions//resourceGroups/myResourceGroup/providers/Microsoft.Web/sites//config/web --set properties.acrUseManagedIdentityCreds=True - az resource update --ids /subscriptions//resourceGroups/myResourceGroup/providers/Microsoft.Web/sites//config/web --set properties.acrUseManagedIdentityCreds=True - -Replace the following values: - -- ```` with the subscription ID retrieved from the az - account show command. -- ```` with the name of your web app. - -Deploy the image -~~~~~~~~~~~~~~~~ - -Use the ``az webapp config container set`` command to specify the -container registry and the image to deploy for the web app: - -.. code:: sh - - az webapp config container set --name --resource-group myResourceGroup --docker-custom-image-name .azurecr.io/rest_api_gcbm:latest --docker-registry-server-url https://.azurecr.io - az webapp config container set --name --resource-group myResourceGroup --docker-custom-image-name .azurecr.io/rest_api_flint.example:latest --docker-registry-server-url https://.azurecr.io - -Replace ```` and ```` with the name of your web -app, and replace all instances of ```` with the name of your -registry. When the ``az webapp config container set`` command completes, -the web app is running in the container on App Service. - -To test the app, browse to ``https://.azurewebsites.net``, -replacing ```` with the name of your web app. To clean up the -resources, you only need to delete the resource group that contains -them: - -.. code:: sh - - az group delete --name myResourceGroup - -.. _Azure CLI: https://docs.microsoft.com/en-us/cli/azure/ -.. _Docker: https://www.docker.com/ -.. _Azure Container Registry: https://azure.microsoft.com/en-in/services/container-registry/ + google_cloud + azure \ No newline at end of file diff --git a/docs/development/gcbm_guide.rst b/docs/development/gcbm_guide.rst new file mode 100644 index 00000000..7d7fdcc6 --- /dev/null +++ b/docs/development/gcbm_guide.rst @@ -0,0 +1,19 @@ +.. _DevelopmentSetup: + +GCBM Local Development Guide +###################### + +To start Development of GCBM, you can run the GCBM local-run container by pushing the following commands in the root directory. + +.. code:: sh + + docker compose up + +This will take a few minutes to download and install all the required dependenceis once done, Navigate to http://localhost:8080/ in the browser. You can stop the running container by pushing the following command: + +.. code:: sh + + docker compose down + +This will stop the docker container instance. + diff --git a/docs/development/index.rst b/docs/development/index.rst index ab3d15a1..bcfca435 100644 --- a/docs/development/index.rst +++ b/docs/development/index.rst @@ -19,3 +19,4 @@ Contents: :maxdepth: 1 git_and_github_guide + gcbm_guide \ No newline at end of file diff --git a/docs/requirements.txt b/docs/requirements.txt index 5f7da1fd..a702c9b7 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -2,3 +2,4 @@ sphinx_copybutton==0.3.1 sphinx-tabs recommonmark sphinx_rtd_theme +furo From 2a803b6957ffd3815221be77c739ddc3fd76fdad Mon Sep 17 00:00:00 2001 From: Sanjay Singh Rajpoot <67458417+SanjaySinghRajpoot@users.noreply.github.com> Date: Fri, 16 Sep 2022 19:08:28 +0530 Subject: [PATCH 04/21] feat: autoapi implemented to document APIs (#173) * feat: autoapi implemented * chore: autoapi added to requirements Signed-off-by: SanjaySinghRajpoot * CI fix Signed-off-by: SanjaySinghRajpoot Signed-off-by: SanjaySinghRajpoot Signed-off-by: Freeman --- docs/conf.py | 15 ++++++++++++++- docs/requirements.txt | 2 ++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index bfc95995..0990b51c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -18,7 +18,8 @@ # -- Project information ----------------------------------------------------- project = "FLINT.Cloud Technical Guide" -copyright = "2022, Sneha Mishra, Arnav Tiwari" + +copyright = "2022, Moja Global" author = u"Sneha Mishra, Arnav Tiwari & contributors" @@ -56,3 +57,15 @@ # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ["_static"] + +# Auto API options +autoapi_type = "python" +autoapi_dirs = ["../local/rest_api_gcbm", "../local/tests"] +autoapi_options = [ + "members", + "undoc-members", + "show-inheritance", + "show-module-summary", + "special-members", + "imported-members", +] diff --git a/docs/requirements.txt b/docs/requirements.txt index a702c9b7..fd276638 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -2,4 +2,6 @@ sphinx_copybutton==0.3.1 sphinx-tabs recommonmark sphinx_rtd_theme +sphinx-autoapi furo + From e4d2e62d77048d6aa9daa284a2baf839a1447c68 Mon Sep 17 00:00:00 2001 From: Amit <51860725+Crystalsage@users.noreply.github.com> Date: Sat, 17 Sep 2022 08:01:28 +0530 Subject: [PATCH 05/21] docs: Add responses to the GCBM docs (#184) docs: Add expected response codes on sucess to the GCBM docs Adds response codes to the GCBM curl documentation. Also changes the format of the document to be more navigable. Signed-off-by: Amit Signed-off-by: Amit Co-authored-by: Namya LG <53875297+Namyalg@users.noreply.github.com> Signed-off-by: Freeman --- local/rest_api_gcbm/curl.md | 193 +++++++++++++++++++++++++----------- 1 file changed, 136 insertions(+), 57 deletions(-) diff --git a/local/rest_api_gcbm/curl.md b/local/rest_api_gcbm/curl.md index c3775baf..0c3832d3 100644 --- a/local/rest_api_gcbm/curl.md +++ b/local/rest_api_gcbm/curl.md @@ -1,57 +1,136 @@ -

Endpoints

- -1. `/gcbm/new/` - - The title of a new simulation can be passed or the default title `simulation` will be used. (i.e the default title of the simulation is `simulation`, here the title `run4` is used) - - ``` - curl -d "title=run4" -X POST http://localhost:8080/gcbm/new - ```` - -2. `/gcbm/upload` - - Go to the location of the unzipped file `GCBM_New_Demo_Run.zip` - - ``` - cd path-to-unzipped-file - ``` - - All the inputs are uploaded via a `POST` request - - ``` - curl -F disturbances='@disturbances/disturbances_2011_moja.tiff' \ - -F disturbances='@disturbances/disturbances_2012_moja.tiff' \ - -F disturbances='@disturbances/disturbances_2013_moja.tiff' \ - -F disturbances='@disturbances/disturbances_2014_moja.tiff' \ - -F disturbances='@disturbances/disturbances_2015_moja.tiff' \ - -F disturbances='@disturbances/disturbances_2016_moja.tiff' \ - -F disturbances='@disturbances/disturbances_2018_moja.tiff' \ - -F classifiers='@classifiers/Classifier1_moja.tiff' \ - -F classifiers='@classifiers/Classifier2_moja.tiff' \ - -F db='@db/gcbm_input.db' \ - -F miscellaneous='@miscellaneous/initial_age_moja.tiff' \ - -F miscellaneous='@miscellaneous/mean_annual_temperature_moja.tiff' \ - -F title="run4" \ - http://localhost:8080/gcbm/upload - - ``` -3. `/gcbm/dynamic` - - ``` - curl -d "title=run4" -X POST http://localhost:8080/gcbm/dynamic - ``` - -4. `/gcbm/status` - ``` - curl -d "title=run4" -X POST http://localhost:8080/gcbm/status - ``` -5. `/gcbm/download` - A file named `output.zip` will be obtained. This file contains the outputs generated, which can be analysed on unzipping. - - ``` - curl -d "title=run4" -X POST http://localhost:8080/gcbm/download -L -o run4.zip - ``` -6. `/gcbm/list` - ``` - curl http://localhost:8080/gcbm/list - ``` +

GCBM API Endpoints

+ +### Create a new GCBM simulation +The title of a new simulation can be passed or the default title `simulation` will be used. (i.e the default title of the simulation is `simulation`, here the title `run4` is used) + +| **Method** | **Endpoint** | **Response code** | +|--- | ---| --- | +| POST| `/gcbm/new` | 200 OK (Success) | + + +Example: +``` +curl -d "title=run4" -X POST http://localhost:8080/gcbm/new +``` + +Expected response (Success): +```json +{ + "data": "New simulation started. Please move on to the next stage for uploading files at /gcbm/upload." +} +``` + +### Upload configuration files +Upload all of the files required to run the simulation. +| **Method** | **Endpoint** | **Response codes** | +|--- | ---| --- | +| POST| `/gcbm/upload` | 200 OK (Success), 400 (Error) | + +Example: + +Go to the location of the unzipped file `GCBM_New_Demo_Run.zip` + +``` +cd path-to-unzipped-file +``` + +All the inputs are uploaded via a `POST` request + +``` +curl -F disturbances='@disturbances/disturbances_2011_moja.tiff' \ + -F disturbances='@disturbances/disturbances_2012_moja.tiff' \ + -F disturbances='@disturbances/disturbances_2013_moja.tiff' \ + -F disturbances='@disturbances/disturbances_2014_moja.tiff' \ + -F disturbances='@disturbances/disturbances_2015_moja.tiff' \ + -F disturbances='@disturbances/disturbances_2016_moja.tiff' \ + -F disturbances='@disturbances/disturbances_2018_moja.tiff' \ + -F classifiers='@classifiers/Classifier1_moja.tiff' \ + -F classifiers='@classifiers/Classifier2_moja.tiff' \ + -F db='@db/gcbm_input.db' \ + -F miscellaneous='@miscellaneous/initial_age_moja.tiff' \ + -F miscellaneous='@miscellaneous/mean_annual_temperature_moja.tiff' \ + -F title="run4" \ + http://localhost:8080/gcbm/upload +``` + +Expected response (Success): +```json +{ + "data": "All files uploaded succesfully. Proceed to the next step of the API at gcbm/dynamic." +} +``` + +### Run the simulation +After uploading your files, you can run the simulation through this endpoint. + +| **Method** | **Endpoint** | **Response code** | +|--- | ---| --- | +| POST | `/gcbm/dynamic` | 200 OK (Success) | + + +Example: + +``` +curl -d "title=run4" -X POST http://localhost:8080/gcbm/dynamic +``` + +Expected response (Success): +```json +{ + "status": "Run started" +} +``` + +### Get status of the simulation + +| **Method** | **Endpoint** | **Response code** | +|--- | ---| --- | +| POST | `/gcbm/status` | 200 OK (Success) | + + +``` +curl -d "title=run4" -X POST http://localhost:8080/gcbm/status +``` + + +Expected response (In progress): +```json +{ + "finished": "In Progress" +} +``` + +Expected response (When simulation completes): +```json +{ + "finished": "Output is ready to download at gcbm/download" +} +``` + +### Download the result of simulation +| **Method** | **Endpoint** | **Response code** | +|--- | ---| --- | +| POST | `/gcbm/download` | 200 OK (Success) | + +A file named `output.zip` will be obtained. This file contains the outputs generated, which can be analysed on unzipping. + +``` +curl -d "title=run4" -X POST http://localhost:8080/gcbm/download -L -o run4.zip +``` + +### List all of the simulations +| **Method** | **Endpoint** | **Response code** | +|--- | ---| --- | +| GET | `/gcbm/list` | 200 OK (Success) | + +``` +curl http://localhost:8080/gcbm/list +``` + +Expected response (Success): +```json +{ + "data": ["run4"], + "message": "To create a new simulation, create a request at gcbm/new. To access the results of the existing simulations, create a request at gcbm/download." +} +``` From 7ecf76a9b4866576b48ffedb57d42bdab43b9723 Mon Sep 17 00:00:00 2001 From: Mohd Jasir Khan <53305380+khanjasir90@users.noreply.github.com> Date: Sun, 2 Oct 2022 18:01:59 +0530 Subject: [PATCH 06/21] feat: added endpoint to return .json for the input .tiff file (#185) * feat: added endpoint to return .json for the input .tiff file Signed-off-by: khanjasir90 * feat: updated endpoint with specified changes and some constraints Signed-off-by: khanjasir90 * feat: formatted code with black Signed-off-by: khanjasir90 * updated minor typos Signed-off-by: khanjasir90 Signed-off-by: khanjasir90 Co-authored-by: Namya LG <53875297+Namyalg@users.noreply.github.com> Signed-off-by: Freeman --- local/rest_api_gcbm/app.py | 52 +++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/local/rest_api_gcbm/app.py b/local/rest_api_gcbm/app.py index c7a5daf9..f19a03ad 100644 --- a/local/rest_api_gcbm/app.py +++ b/local/rest_api_gcbm/app.py @@ -323,7 +323,7 @@ def gcbm_dynamic(): responses: 200: parameters: - - in: body + - in: body name: title required: true schema: @@ -350,6 +350,56 @@ def gcbm_dynamic(): return {"status": "Run started"}, 200 +@app.route("/gcbm/getConfig", methods=["POST"]) +def getConfig(): + """ + Return .json for the input .tiff files + --- + tags: + - gcbm + responses: + 200: + parameters: + - in: body + name: title + required: true + schema: + type: string + description: Name of the Simulation + name: file_name + required: true + schema: + type: string + description: Name of the File + """ + # Default title = Simulation + title = request.form.get("title").strip() + file_name = request.form.get("file_name").strip() + input_dir = f"{os.getcwd()}/input/{title}" + + # check if title is empty + if title == "": + return {"error": "No Simulation name specified"}, 400 + + # check if file_name is empty + if file_name == "": + return {"error": "No file name specified"}, 400 + + # Check if simulation exists or not + if not os.path.exists(f"{input_dir}"): + return {"error": "Simulation with the name " + title + " doesn't exists"}, 400 + + input_dir_file = f"{input_dir}/{file_name}.json" + # Check if file exists or not + if not os.path.exists(f"{input_dir_file}"): + return {"error": "File with name " + file_name + " doesn't exists"}, 400 + + # Return the json for the corresponding file name + file_obj = open(f"{input_dir_file}") + obj = json.load(file_obj) + return {"data": obj}, 200 + + def launch_run(title, input_dir): s = time.time() logging.debug("Starting run") From 61a459b798471434006fc69fcacdda36f644194b Mon Sep 17 00:00:00 2001 From: Sanjay Singh Rajpoot <67458417+SanjaySinghRajpoot@users.noreply.github.com> Date: Sat, 8 Oct 2022 13:00:57 +0530 Subject: [PATCH 07/21] Updated curl cmds for the GCBM dynamic endpoints (#194) * feat: updated curl cmds Signed-off-by: SanjaySinghRajpoot * db added and doc formated Signed-off-by: SanjaySinghRajpoot * example text fixed Signed-off-by: SanjaySinghRajpoot Signed-off-by: SanjaySinghRajpoot Signed-off-by: Freeman --- local/rest_api_gcbm/curl.md | 196 +++++++++++++++++++++++++++++++----- 1 file changed, 169 insertions(+), 27 deletions(-) diff --git a/local/rest_api_gcbm/curl.md b/local/rest_api_gcbm/curl.md index 0c3832d3..e8b32653 100644 --- a/local/rest_api_gcbm/curl.md +++ b/local/rest_api_gcbm/curl.md @@ -1,29 +1,32 @@

GCBM API Endpoints

### Create a new GCBM simulation -The title of a new simulation can be passed or the default title `simulation` will be used. (i.e the default title of the simulation is `simulation`, here the title `run4` is used) -| **Method** | **Endpoint** | **Response code** | -|--- | ---| --- | -| POST| `/gcbm/new` | 200 OK (Success) | +The title of a new simulation can be passed or the default title `simulation` will be used. (i.e the default title of the simulation is `simulation`, here the title `run4` is used) +| **Method** | **Endpoint** | **Response code** | +| ---------- | ------------ | ----------------- | +| POST | `/gcbm/new` | 200 OK (Success) | Example: + ``` curl -d "title=run4" -X POST http://localhost:8080/gcbm/new ``` Expected response (Success): + ```json { - "data": "New simulation started. Please move on to the next stage for uploading files at /gcbm/upload." + "data": "New simulation started. Please move on to the next stage for uploading files at /gcbm/upload." } ``` -### Upload configuration files +### Upload configuration files + Upload all of the files required to run the simulation. -| **Method** | **Endpoint** | **Response codes** | -|--- | ---| --- | +| **Method** | **Endpoint** | **Response codes** | +|--- | ---| --- | | POST| `/gcbm/upload` | 200 OK (Success), 400 (Error) | Example: @@ -54,19 +57,154 @@ curl -F disturbances='@disturbances/disturbances_2011_moja.tiff' \ ``` Expected response (Success): + ```json { "data": "All files uploaded succesfully. Proceed to the next step of the API at gcbm/dynamic." } ``` -### Run the simulation -After uploading your files, you can run the simulation through this endpoint. +You can upload all the simulation files at once or upload them separately. To upload files separately follow the commands below. + +### Upload Disturbances file + +Upload Disturbances file to run the simulation. +| **Method** | **Endpoint** | **Response codes** | +|--- | ---| --- | +| POST| `/gcbm/upload/disturbances` | 200 OK (Success), 400 (Error) | + +Example: + +Go to the location of the unzipped file `GCBM_New_Demo_Run.zip` + +``` +cd path-to-unzipped-file +``` + +Upload all the files present in the disturbances folder. + +``` +curl -F disturbances='@disturbances/disturbances_2011_moja.tiff' \ + -F disturbances='@disturbances/disturbances_2012_moja.tiff' \ + -F disturbances='@disturbances/disturbances_2013_moja.tiff' \ + -F disturbances='@disturbances/disturbances_2014_moja.tiff' \ + -F disturbances='@disturbances/disturbances_2015_moja.tiff' \ + -F disturbances='@disturbances/disturbances_2016_moja.tiff' \ + -F disturbances='@disturbances/disturbances_2018_moja.tiff' \ + -F title="run4" \ + http://localhost:8080/gcbm/upload/disturbances +``` + +Expected response (Success): + +```json +{ + "data": "Disturbances file uploaded succesfully. Proceed to the next step." +} +``` + +### Upload Classifiers file + +Upload Classifiers file to run the simulation. +| **Method** | **Endpoint** | **Response codes** | +|--- | ---| --- | +| POST| `/gcbm/upload/classifiers` | 200 OK (Success), 400 (Error) | + +Example: + +Go to the location of the unzipped file `GCBM_New_Demo_Run.zip` + +``` +cd path-to-unzipped-file +``` + +Upload all the files present in the disturbances folder. + +``` +curl -F classifiers='@classifiers/Classifier1_moja.tiff' \ + -F classifiers='@classifiers/Classifier2_moja.tiff' \ + -F title="run4" \ + http://localhost:8080/gcbm/upload/classifiers +``` + +Expected response (Success): + +```json +{ + "data": "Classifiers file uploaded succesfully. Proceed to the next step." +} +``` -| **Method** | **Endpoint** | **Response code** | -|--- | ---| --- | -| POST | `/gcbm/dynamic` | 200 OK (Success) | +### Upload Miscellaneous file +Upload Miscellaneous file to run the simulation. +| **Method** | **Endpoint** | **Response codes** | +|--- | ---| --- | +| POST| `/gcbm/upload/miscellaneous` | 200 OK (Success), 400 (Error) | + +Example: + +Go to the location of the unzipped file `GCBM_New_Demo_Run.zip` + +``` +cd path-to-unzipped-file +``` + +Upload all the files present in the disturbances folder. + +``` +curl -F miscellaneous='@miscellaneous/initial_age_moja.tiff' \ + -F miscellaneous='@miscellaneous/mean_annual_temperature_moja.tiff' \ + -F title="run4" \ + http://localhost:8080/gcbm/upload/miscellaneous +``` + +Expected response (Success): + +```json +{ + "data": "Miscellaneous file uploaded succesfully. Proceed to the next step." +} +``` + +### Upload db file + +Upload db file to run the simulation. +| **Method** | **Endpoint** | **Response codes** | +|--- | ---| --- | +| POST| `/gcbm/upload/db` | 200 OK (Success), 400 (Error) | + +Example: + +Go to the location of the unzipped file `GCBM_New_Demo_Run.zip` + +``` +cd path-to-unzipped-file +``` + +Upload all the files present in the disturbances folder. + +``` +curl -F db='@db/gcbm_input.db' \ + -F title="run4" \ + http://localhost:8080/gcbm/upload/db +``` + +Expected response (Success): + +```json +{ + "data": "db file uploaded succesfully. Proceed to the next step." +} +``` + +### Run the simulation + +After uploading your files, you can run the simulation through this endpoint. + +| **Method** | **Endpoint** | **Response code** | +| ---------- | --------------- | ----------------- | +| POST | `/gcbm/dynamic` | 200 OK (Success) | Example: @@ -75,25 +213,25 @@ curl -d "title=run4" -X POST http://localhost:8080/gcbm/dynamic ``` Expected response (Success): + ```json { "status": "Run started" } ``` -### Get status of the simulation - -| **Method** | **Endpoint** | **Response code** | -|--- | ---| --- | -| POST | `/gcbm/status` | 200 OK (Success) | +### Get status of the simulation +| **Method** | **Endpoint** | **Response code** | +| ---------- | -------------- | ----------------- | +| POST | `/gcbm/status` | 200 OK (Success) | ``` curl -d "title=run4" -X POST http://localhost:8080/gcbm/status ``` - Expected response (In progress): + ```json { "finished": "In Progress" @@ -101,16 +239,18 @@ Expected response (In progress): ``` Expected response (When simulation completes): + ```json { "finished": "Output is ready to download at gcbm/download" } ``` -### Download the result of simulation -| **Method** | **Endpoint** | **Response code** | -|--- | ---| --- | -| POST | `/gcbm/download` | 200 OK (Success) | +### Download the result of simulation + +| **Method** | **Endpoint** | **Response code** | +| ---------- | ---------------- | ----------------- | +| POST | `/gcbm/download` | 200 OK (Success) | A file named `output.zip` will be obtained. This file contains the outputs generated, which can be analysed on unzipping. @@ -118,16 +258,18 @@ A file named `output.zip` will be obtained. This file contains the outputs gener curl -d "title=run4" -X POST http://localhost:8080/gcbm/download -L -o run4.zip ``` -### List all of the simulations -| **Method** | **Endpoint** | **Response code** | -|--- | ---| --- | -| GET | `/gcbm/list` | 200 OK (Success) | +### List all of the simulations + +| **Method** | **Endpoint** | **Response code** | +| ---------- | ------------ | ----------------- | +| GET | `/gcbm/list` | 200 OK (Success) | ``` curl http://localhost:8080/gcbm/list ``` Expected response (Success): + ```json { "data": ["run4"], From bcf0c3f5b32c4bb5fcb2cfdf5b756cdd7d362ea0 Mon Sep 17 00:00:00 2001 From: temitayo Date: Sun, 16 Oct 2022 00:47:24 +0100 Subject: [PATCH 08/21] chore:removed /rothc endpoint description from swagger documentation of rest api gcbm (#195) Signed-off-by: olalekan temitayo Signed-off-by: olalekan temitayo Signed-off-by: Freeman --- local/rest_api_gcbm/static/swagger.json | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/local/rest_api_gcbm/static/swagger.json b/local/rest_api_gcbm/static/swagger.json index 6c8535d0..87010cd7 100644 --- a/local/rest_api_gcbm/static/swagger.json +++ b/local/rest_api_gcbm/static/swagger.json @@ -98,20 +98,6 @@ ] } }, - "/rothc": { - "post": { - "description": "", - "responses": { - "200": { - "description": "RothC based example FLINT" - } - }, - "summary": "Get RothC example of FLINT", - "tags": [ - "rothc" - ] - } - }, "/version": { "get": { "description": "", From 420d80a40b0e9eb6de81b7cbe431503d9eae6159 Mon Sep 17 00:00:00 2001 From: Saurabh Suchak <91744743+Saurabh-Suchak@users.noreply.github.com> Date: Sun, 16 Oct 2022 15:54:36 +0530 Subject: [PATCH 09/21] Improved Documentation for GCBM REST API (#144) * Update README.md * Update README.md * Update README.md * Update README.md * Update README.md * Update README.md * Update README.md * changed the endpoint name * Update app.py * Update README.md Co-authored-by: Harsh Mishra Signed-off-by: Freeman --- local/rest_api_gcbm/README.md | 103 +++++++++++++++++++++++++--------- 1 file changed, 77 insertions(+), 26 deletions(-) diff --git a/local/rest_api_gcbm/README.md b/local/rest_api_gcbm/README.md index fae9e057..98170e3a 100644 --- a/local/rest_api_gcbm/README.md +++ b/local/rest_api_gcbm/README.md @@ -1,38 +1,89 @@ -# FLINT.Cloud - -### GCBM local-run REST API Setup - -Run the GCBM local-run container by pushing the following command: +# GCBM REST API Setup + +The simulation is currently supported on Linux/macOS. To run the GCBM simulation locally, execute the following steps: + +1. Clone the `FLINT.Cloud` repository using the command: + + ```bash + git clone https://github.com/moja-global/FLINT.Cloud.git + ``` + +2. Navigate to the `rest_api_gcbm` directory: + ```bash + cd FLINT.Cloud/local/rest_api_gcbm + ``` -```bash -docker-compose up -``` +3. Build the docker image with the image name `gcbm-api`: + ```bash + docker build --build-arg BUILD_TYPE=RELEASE --build-arg NUM_CPU=4 -t gcbm-api . + ``` -Navigate to http://localhost:8080/ in the browser. You can stop the running container by pushing the following command: +4. Create a container using the `gcbm-api` image and start it: + ```bash + docker run --rm -p 8080:8080 gcbm-api + ``` -```bash -docker-compose down -``` +Navigate to http://localhost:8080/ in the browser to test the various endpoints available. -Currently the REST API has the following endpoints available for access:- +## Endpoints + +The GCBM REST API has the following endpoints available for access: | Endpoint | Functionality | Method | | :----------------: | :----------------: | :----------------: | -| **\help\all** | This endpoint produces a help message with information on all options for moja.CLI | `GET` -| **\help\arg** | This endpoint produces a help message with information on option arg for moja.CLI. | `GET` -| **\gcbm\new** | This endpoint creates a new directory to store input and output files of the simulation. Parameter to be passed in the body is the title of the new simulation or default value simulation will be used. | `POST` | -| **\gcbm\upload** | This endpoint is used to upload the input files: config_files, input and database. Remember to upload the files in the body from the GCBM folder which is available in the root of this repository for setup and use. A directory /input will be created to store the specified files. | `POST` | -| **\gcbm\dynamic** | This endpoint runs simulation in a thread and generates an output zip file in /output directory. The process may take a while. Parameter is the title of the simulation. | `POST` | -| **\gcbm\status** | This endpoint is employed to check the status of the simulation. It sends a message 'Output is ready to download' to notify that the output zip file is generated. Parameter is the title of the simulation. | `POST` -| **\gcbm\download** | This endpoint is used to download the output zip file. Parameter is the title of the simulation. | `POST` -| **\gcbm\list** | This endpoint retrieves the complete list of simulations that are created using /new. | `GET` +| **/help/all** | This endpoint produces a help message with information on all options for moja.CLI | `GET` +| **/help/arg** | This endpoint produces a help message with information on option arg for moja.CLI. | `GET` +| **/gcbm/new** | This endpoint creates a new directory to store input and output files of the simulation. Parameter to be passed in the body is the title of the new simulation or default value simulation will be used. | `POST` | +| **/gcbm/upload** | This endpoint is used to upload the input files: config_files, input and database. Remember to upload the files in the body from the GCBM folder which is available in the root of this repository for setup and use. A directory /input will be created to store the specified files. | `POST` | +| **/gcbm/run** | This endpoint runs simulation in a thread and generates an output zip file in /output directory. The process may take a while. Parameter is the title of the simulation. | `POST` | +| **/gcbm/status** | This endpoint is employed to check the status of the simulation. It sends a message 'Output is ready to download' to notify that the output zip file is generated. Parameter is the title of the simulation. | `POST` +| **/gcbm/download** | This endpoint is used to download the output zip file. Parameter is the title of the simulation. | `POST` +| **/gcbm/list** | This endpoint retrieves the complete list of simulations that are created using /new. | `GET` + +The inputs are contained in `GCBM_New_Demo_Run.zip`, present in the root of the directory. This file must be unzipped for further usage. Once the container is up and running, the following methods can be used to interact with the endpoints: + +1. A sample collection is available [in our Postaman collection](https://github.com/nynaalekhya/FLINT.Cloud/blob/local-gcbm-run/rest_local_run/local_run.postman_collection). Import the collection to Postman and run the endpoints. -The inputs are contained in `GCBM_Demo_Run.zip`, present in the root of the directory. This file must be unzipped for further usage. +2. The endpoints can be interacted with using `cURL`. `cURL` is used in command lines or scripts to transfer data. Find out more about curl [on the official website](https://curl.se/). Commands using `cURL` can be found [in our documentation](curl.md) -

Once the container is up and running, the following methods can be used to interact with the endpoints

+## Running a GCBM simulation -1. A sample Postman collection is available [here](https://github.com/nynaalekhya/FLINT.Cloud/blob/local-gcbm-run/rest_local_run/local_run.postman_collection). Import the collection to Postman and run the endpoints. +To run the endpoints and start a new GCBM simulation : + +1. Unzip `GCBM_New_Demo_Run` and open a terminal inside the folder and run the following curl commands: + +2. Create a new simulation (the default title of the simulation is `simulation`; Here the title `run4` is used): + ```bash + curl -d "title=run4" -X POST http://localhost:8080/gcbm/new + ```` -2. The endpoints can be interacted with using `cURL`. -curl is used in command lines or scripts to transfer data. Find out more about curl [here](https://curl.se/). Commands using cURL can be found [here](curl.md) +3. Upload the input files: + ```bash + curl -F disturbances='@disturbances/disturbances_2011_moja.tiff' \ + -F disturbances='@disturbances/disturbances_2012_moja.tiff' \ + -F disturbances='@disturbances/disturbances_2013_moja.tiff' \ + -F disturbances='@disturbances/disturbances_2014_moja.tiff' \ + -F disturbances='@disturbances/disturbances_2015_moja.tiff' \ + -F disturbances='@disturbances/disturbances_2016_moja.tiff' \ + -F disturbances='@disturbances/disturbances_2018_moja.tiff' \ + -F classifiers='@classifiers/Classifier1_moja.tiff' \ + -F classifiers='@classifiers/Classifier2_moja.tiff' \ + -F db='@db/gcbm_input.db' \ + -F miscellaneous='@miscellaneous/initial_age_moja.tiff' \ + -F miscellaneous='@miscellaneous/mean_annual_temperature_moja.tiff' \ + -F title="run4" \ + http://localhost:8080/gcbm/upload + ``` +4. To start the simulation, run: + ```bash + curl -d "title=run4" -X POST http://localhost:8080/gcbm/run + ``` + + It should take around 35-40 minutes to finish running and even less depending on your local machine specifications. The end message should be `SQLite insert complete`. The message would be printed on the terminal used to execute Docker commands. + + +5. To download the simulation outputs, run the following curl command 3-5 minutes after the simulation has finished running: + ```bash + curl -d "title=run4" -X POST http://localhost:8080/gcbm/download -L -o run4.zip + ``` From 6f843c697dce5412c11f063fd27b2442fa057731 Mon Sep 17 00:00:00 2001 From: Freeman Date: Fri, 28 Oct 2022 00:13:44 +0100 Subject: [PATCH 10/21] GCBM Refactor idea Draft Signed-off-by: Freeman --- local/rest_api_skeleton/Endpoints/__init__.py | 0 .../Endpoints/gcbm_endpoints.py | 125 +++ local/rest_api_skeleton/Helpers/__init__.py | 0 .../rest_api_skeleton/Helpers/for_requests.py | 26 + local/rest_api_skeleton/Helpers/preprocess.py | 277 ++++++ local/rest_api_skeleton/__init__.py | 0 local/rest_api_skeleton/app.py | 16 + local/rest_api_skeleton/models/__init__.py | 0 local/rest_api_skeleton/templates/__init__.py | 0 .../templates/gcbm_config.cfg | 7 + .../templates/internal_variables.json | 34 + .../templates/localdomain.json | 31 + .../rest_api_skeleton/templates/logging.conf | 17 + .../templates/modules_cbm.json | 59 ++ .../templates/modules_output.json | 816 ++++++++++++++++++ .../templates/pools_cbm.json | 34 + .../templates/provider_config.json | 55 ++ local/rest_api_skeleton/templates/spinup.json | 56 ++ .../templates/variables.json | 257 ++++++ 19 files changed, 1810 insertions(+) create mode 100644 local/rest_api_skeleton/Endpoints/__init__.py create mode 100644 local/rest_api_skeleton/Endpoints/gcbm_endpoints.py create mode 100644 local/rest_api_skeleton/Helpers/__init__.py create mode 100644 local/rest_api_skeleton/Helpers/for_requests.py create mode 100644 local/rest_api_skeleton/Helpers/preprocess.py create mode 100644 local/rest_api_skeleton/__init__.py create mode 100644 local/rest_api_skeleton/app.py create mode 100644 local/rest_api_skeleton/models/__init__.py create mode 100644 local/rest_api_skeleton/templates/__init__.py create mode 100644 local/rest_api_skeleton/templates/gcbm_config.cfg create mode 100644 local/rest_api_skeleton/templates/internal_variables.json create mode 100644 local/rest_api_skeleton/templates/localdomain.json create mode 100644 local/rest_api_skeleton/templates/logging.conf create mode 100644 local/rest_api_skeleton/templates/modules_cbm.json create mode 100644 local/rest_api_skeleton/templates/modules_output.json create mode 100644 local/rest_api_skeleton/templates/pools_cbm.json create mode 100644 local/rest_api_skeleton/templates/provider_config.json create mode 100644 local/rest_api_skeleton/templates/spinup.json create mode 100644 local/rest_api_skeleton/templates/variables.json diff --git a/local/rest_api_skeleton/Endpoints/__init__.py b/local/rest_api_skeleton/Endpoints/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local/rest_api_skeleton/Endpoints/gcbm_endpoints.py b/local/rest_api_skeleton/Endpoints/gcbm_endpoints.py new file mode 100644 index 00000000..46b865a0 --- /dev/null +++ b/local/rest_api_skeleton/Endpoints/gcbm_endpoints.py @@ -0,0 +1,125 @@ +from flask_restful import Resource +import os +from threading import Thread +from Helpers.preprocess import DisturbanceConfig, launch_run, ClassifierConfig, MiscellaneousConfig +from Helpers.for_requests import title_query, miscellaneous_query, disturbance_query, classifier_query + +class disturbance(Resource): + def post(self): + title = title_query().get("title") or "simulation" + input_dir = f"{os.getcwd()}/input/{title}" + if not os.path.exists(f"{input_dir}"): + os.makedirs(f"{input_dir}") + if not os.path.exists(f"{input_dir}/disturbances"): + os.makedirs(f"{input_dir}/disturbances") + + # store disturbances file in a new folder + disturbances = disturbance_query() + if not disturbances: + return {"error": "Missing disturbances file"}, 400 + for file in disturbances.get("disturbances"): + file.save(f"{input_dir}/disturbances/{file.filename}") + try: + disturb = DisturbanceConfig(input_dir, file.filename, disturbances.get("attributes")) + disturb() + except Exception as e: + return e + disturb.flatten_directory("disturbances") + return {"data": "Disturbances file uploaded succesfully. Proceed to the next step."} + +class classifier(Resource): + def post(self): + # Get the title from the payload + title = title_query().get("title") or "simulation" + + # Check for project directory else create one + input_dir = f"{os.getcwd()}/input/{title}" + if not os.path.exists(f"{input_dir}"): + os.makedirs(f"{input_dir}") + + # input files follow a strict structure + if not os.path.exists(f"{input_dir}/classifiers"): + os.makedirs(f"{input_dir}/classifiers") + + # store disturbances file in a new folder + classifiers = classifier_query() + if not classifiers: + return {"error": "Missing classifiers file"}, 400 + + for file in classifiers.get("classifiers"): + file.save(f"{input_dir}/classifiers/{file.filename}") + try: + classify = ClassifierConfig(input_dir, file.filename, classifiers.get("attributes")) + classify() + except Exception as e: + return e + classify.flatten_directory("classifiers") + return {"data": "Classifiers file uploaded succesfully. Proceed to the next step."} + +class Database(Resource): + def post(self): + pass + +class miscellaneous(Resource): + def post(self): + # Get the title from the payload + title = title_query().get("title") or "simulation" + + # Check for project directory else create one + input_dir = f"{os.getcwd()}/input/{title}" + if not os.path.exists(f"{input_dir}"): + os.makedirs(f"{input_dir}") + + # input files follow a strict structure + if not os.path.exists(f"{input_dir}/miscellaneous"): + os.makedirs(f"{input_dir}/miscellaneous") + + # store miscellaneous file in a new folder + mis = miscellaneous_query() + if not mis: + return {"error": "Missing classifiers file"}, 400 + + for file in mis.get("miscellaneous"): + file.save(f"{input_dir}/miscellaneous/{file.filename}") + try: + miscel = MiscellaneousConfig(input_dir, file.filename, mis.get("attributes")) + miscel() + except Exception as e: + return e + miscel.flatten_directory("miscellaneous") + return {"data": "Classifiers file uploaded succesfully. Proceed to the next step."} + +class title(Resource): + def post(self): + # Default title = simulation + title = title_query().get("title") or "simulation" + # Sanitize title + title = "".join(c for c in title if c.isalnum()) + # input_dir = f"{title}" + input_dir = f"{os.getcwd()}/input/{title}" + if not os.path.exists(f"{input_dir}"): + os.makedirs(f"{input_dir}") + message = "New simulation started. Please move on to the next stage for uploading files at /gcbm/upload." + else: + message = "Simulation already exists. Please check the list of simulations present before proceeding with a new simulation at gcbm/list. You may also download the input and output files for this simulation at gcbm/download sending parameter title in the body." + + return {"data": message}, 200 + +class Run(Resource): + """THIS ENDPOINT WILL BE ABLE TO RUN A SIMULATION + GOAL IS TO EQUIP IT TO BE ABLE TO RUN MORE THAN ONE SIMULATIONS AT A TIME""" + def post(self): + title = title_query().get("title") or "simulation" + + # Sanitize title + title = "".join(c for c in title if c.isalnum()) + input_dir = f"{os.getcwd()}/input/{title}" + + + if not os.path.exists(f"{input_dir}"): + os.makedirs(f"{input_dir}") + + thread = Thread(target=launch_run, kwargs={"title": title, "input_dir": input_dir}) + thread.start() + + return {"status": "Run started"}, 200 diff --git a/local/rest_api_skeleton/Helpers/__init__.py b/local/rest_api_skeleton/Helpers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local/rest_api_skeleton/Helpers/for_requests.py b/local/rest_api_skeleton/Helpers/for_requests.py new file mode 100644 index 00000000..db53093d --- /dev/null +++ b/local/rest_api_skeleton/Helpers/for_requests.py @@ -0,0 +1,26 @@ +from flask_restful import reqparse +from werkzeug.datastructures import FileStorage + +def title_query(): + query = reqparse.RequestParser() + query.add_argument("title", required=True, location="form" ) + return query.parse_args() + +def classifier_query(): + query = reqparse.RequestParser() + query.add_argument("classifiers",type=FileStorage, required=True, action="append", location="files" ) + query.add_argument("attributes", location= "form") + return query.parse_args() + +def disturbance_query(): + query = reqparse.RequestParser() + query.add_argument("disturbances",type=FileStorage, required=True, action="append", location="files" ) + query.add_argument("attributes", location= "form") + return query.parse_args() + +def miscellaneous_query(): + query = reqparse.RequestParser() + query.add_argument("miscellaneous",type=FileStorage, required=True, action="append", location="files" ) + query.add_argument("attributes", location= "form") + return query.parse_args() + diff --git a/local/rest_api_skeleton/Helpers/preprocess.py b/local/rest_api_skeleton/Helpers/preprocess.py new file mode 100644 index 00000000..157a0081 --- /dev/null +++ b/local/rest_api_skeleton/Helpers/preprocess.py @@ -0,0 +1,277 @@ +import os, time, subprocess +import shutil +import json +import rasterio as rst + +def launch_run(title, input_dir): + s = time.time() + with open(f"{input_dir}/gcbm_logs.csv", "w+") as f: + res = subprocess.Popen( + [ + "/opt/gcbm/moja.cli", + "--config_file", + "gcbm_config.cfg", + "--config_provider", + "provider_config.json", + ], + stdout=f, + cwd=f"{input_dir}", + ) + (_, err) = res.communicate() + + if not os.path.exists(f"{input_dir}/output"): + return "OK" + + # cut and paste output folder to app/output/simulation_name + shutil.copytree(f"{input_dir}/output", (f"{os.getcwd()}/output/{title}")) + shutil.make_archive( + f"{os.getcwd()}/output/{title}", "zip", f"{os.getcwd()}/output/{title}" + ) + shutil.rmtree((f"{input_dir}/output")) + e = time.time() + + response = { + "exitCode": res.returncode, + "execTime": e - s, + "response": "Operation executed successfully. Downloadable links for input and output are attached in the response. Alternatively, you may also download this simulation input and output results by making a request at gcbm/download with the title in the body.", + } + +class Configs: + + def __init__(self, input_dir) -> None: + self.input_dir = input_dir + self.Rasters = [] + self.Rastersm = [] + self.nodatam = [] + self.nodata = [] + self.cellLatSize = [] + self.cellLonSize = [] + self.paths = [] + self.lst = [] + self.provider_config = open(f"{os.getcwd()}/templates/provider_config.json", "r+") + + def get_config_templates(self): + if not os.path.exists(f"{self.input_dir}/templates"): + shutil.copytree( + f"{os.getcwd()}/templates", f"{self.input_dir}/templates", dirs_exist_ok=False + ) + self.provider_config = open(f"{self.input_dir}/templates/provider_config.json", "r+") + + def get_modules_cbm_config(self): + with open(f"{self.input_dir}/templates/modules_cbm.json", "r+") as modules_cbm_config: + data = json.load(modules_cbm_config) + disturbances = [file.split(".")[0][:-5] for file in os.listdir(f"{self.input_dir}/disturbances/")] # drop `_moja` to match modules_cbm.json template + modules_cbm_config.seek(0) + data["Modules"]["CBMDisturbanceListener"]["settings"]["vars"] = disturbances + json.dump(data, modules_cbm_config, indent=4) + modules_cbm_config.truncate() + + # for database input + def database_writes(self): + data = json.load(self.provider_config) + for file in os.listdir(f"{self.input_dir}/db/"): + data["Providers"]["SQLite"] = {"type": "SQLite", "path": file } + self.provider_config.seek(0) + + + def write_configs(self, config_type : str): + data = json.load(self.provider_config) + for file in os.listdir(f"{self.input_dir}/{config_type}/"): + d = dict() + d["name"] = file[:-10] + d["layer_path"] = file + d["layer_prefix"] = file[:-5] + self.lst.append(d) + if config_type == "disturbances" or config_type == "classifiers": + for root, _, files in os.walk( + os.path.abspath(f"{self.input_dir}/{config_type}/") + ): + for file in files: + fp = os.path.join(root, file) + self.Rasters.append(fp) + self.paths.append(fp) + for self.nd in self.Rasters: + img = rst.open(self.nd) + t = img.transform + x = t[0] + y = -t[4] + n = img.nodata + self.cellLatSize.append(x) + self.cellLonSize.append(y) + self.nodata.append(n) + result = all(element == self.cellLatSize[0] for element in self.cellLatSize) + if result: + cellLat = x + cellLon = y + self.nd = n + blockLat = x * 400 + blockLon = y * 400 + tileLat = x * 400 + tileLon = y * 4000 + else: + print("Corrupt files") + + self.provider_config.seek(0) + new_values = {"cellLonSize": cellLon,"cellLatSize": cellLat, "blockLonSize": blockLon, "blockLatSize": blockLat, "tileLatSize": tileLat, "tileLonSize": tileLon} + data["Providers"]["RasterTiled"]["layers"] = self.lst + data["Providers"]["RasterTiled"].update(new_values) + + json.dump(data, self.provider_config, indent=4) + self.provider_config.truncate() + + self.dictionary = { + "layer_type": "GridLayer", + "layer_data": "Byte", + "nodata": self.nd, + "tileLatSize": tileLat, + "tileLonSize": tileLon, + "blockLatSize": blockLat, + "blockLonSize": blockLon, + "cellLatSize": cellLat, + "cellLonSize": cellLon, + } + + self.study_area = { + "tile_size": tileLat, + "block_size": blockLat, + "tiles": [ + { + "x": int(t[2]), + "y": int(t[5]), + "index": 12674, + } + ], + "pixel_size": cellLat, + "layers": [], + } + + def add_file_to_path(self, config_type): + for root, _, files in os.walk(os.path.abspath(f"{self.input_dir}/{config_type}/")): + for file in files: + fp = os.path.join(root, file) + self.paths.append(fp) + + def copy_directory(self): + for i in self.paths: + shutil.copy2(i, (f"{self.input_dir}")) + + def flatten_directory(self, config_type): + shutil.rmtree((f"{self.input_dir}/{config_type}/")) + +class DisturbanceConfig(Configs): + def __init__(self, input_dir, config_file : str, attribute : dict = None ) -> None: + super().__init__(input_dir) + self.config_file = config_file + self.attribute = attribute + + def disturbances_special(self): + with open(f"{self.input_dir}/{self.config_file}", "w", encoding="utf8") as json_file: + self.dictionary["attributes"] = self.attribute + json.dump(self.dictionary, json_file, indent=4) + with open( + f"{self.input_dir}/study_area.json", "w", encoding="utf" + ) as json_file: + study_area = [] + self.study_area["layers"] = study_area + json.dump(self.study_area, json_file, indent=4) + for file in os.listdir(f"{self.input_dir}/disturbances/"): + study_area.append( + { + "name": file[:10], + "type": "DisturbanceLayer", + "tags": ["disturbance"], + } + ) + + self.study_area["layers"] = study_area + json.dump(self.study_area, json_file, indent=4) + + def __call__(self): + self.get_config_templates() + self.get_modules_cbm_config() + self.write_configs("disturbances") + self.disturbances_special() + self.add_file_to_path("disturbances") + self.copy_directory() + +class ClassifierConfig(Configs): + def __init__(self, input_dir, config_file : str, attribute : dict ) -> None: + super().__init__(input_dir) + self.config_file = config_file + self.attribute = attribute + + def classifier_special(self): + with open(f"{self.input_dir}/{self.config_file}", "w", encoding="utf8") as json_file: + self.dictionary["attributes"] = self.attribute + json.dump(self.dictionary, json_file, indent=4) + + with open( + f"{self.input_dir}/study_area.json", "w", encoding="utf" + ) as json_file: + study_area = [] + for file in os.listdir(f"{self.input_dir}/classifiers/"): + study_area.append( + {"name": file[:10], "type": "VectorLayer", "tags": ["classifier"]} + ) + + self.study_area["layers"] = study_area + json.dump(self.study_area, json_file, indent=4) + + def __call__(self): + self.get_config_templates() + self.get_modules_cbm_config() + self.write_configs("classifiers") + self.classifier_special() + self.add_file_to_path("classifiers") + self.copy_directory() + +class MiscellaneousConfig(Configs): + def __init__(self, input_dir, config_file : str, attribute : dict ) -> None: + super().__init__(input_dir) + self.config_file = config_file + self.attribute = attribute + + def miscellaneous_special(self): + for root, _, files in os.walk( + os.path.abspath(f"{self.input_dir}/miscellaneous/") +): + for file in files: + fp2 = os.path.join(root, file) + self.Rastersm.append(fp2) + + for i in self.Rastersm: + img = rst.open(i) + d = img.nodata + self.nodatam.append(d) + """this is an experimental thing""" + with open(f"{self.input_dir}/{self.config_file}", "w", encoding="utf8") as json_file: + self.dictionary["layer_type"] = "GridLayer" + if self.config_file == "mean_annual_temperature_moja.json": + self.dictionary["layer_data"] = "Float32" + else: + self.dictionary["layer_data"] = "Int16" + self.dictionary["nodata"] = 32767 + json.dump(self.dictionary, json_file, indent=4) + + # for study area + with open( + f"{self.input_dir}/study_area.json", "w", encoding="utf" + ) as json_file: + study_area = [] + + for file in os.listdir(f"{self.input_dir}/miscellaneous/"): + study_area.append({"name": file[:10], "type": "VectorLayer"}) + + self.study_area["layers"] = study_area + json.dump(self.study_area, json_file, indent=4) + + def __call__(self): + self.get_config_templates() + self.get_modules_cbm_config() + self.write_configs("miscellaneous") + self.classifier_special() + self.add_file_to_path("miscellaneous") + self.copy_directory() + self.flatten_directory("miscellaneous") + + diff --git a/local/rest_api_skeleton/__init__.py b/local/rest_api_skeleton/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local/rest_api_skeleton/app.py b/local/rest_api_skeleton/app.py new file mode 100644 index 00000000..c5502bc0 --- /dev/null +++ b/local/rest_api_skeleton/app.py @@ -0,0 +1,16 @@ +from flask import Flask +from flask_restful import Api +from Endpoints.gcbm_endpoints import disturbance, Run, title, classifier, miscellaneous + +app = Flask(__name__) +api = Api() + +api.add_resource(disturbance, "/gcbm/upload/disturbances") +api.add_resource(title, "/gcbm/create") +api.add_resource(classifier, "/gcbm/upload/classifies") +api.add_resource(miscellaneous, "/gcbm/upload/miscellaneous") +api.add_resource(Run, "/gcbm/run") + + +if __name__ == "__main__": + app.run(debug=True) diff --git a/local/rest_api_skeleton/models/__init__.py b/local/rest_api_skeleton/models/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local/rest_api_skeleton/templates/__init__.py b/local/rest_api_skeleton/templates/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local/rest_api_skeleton/templates/gcbm_config.cfg b/local/rest_api_skeleton/templates/gcbm_config.cfg new file mode 100644 index 00000000..e6640ac9 --- /dev/null +++ b/local/rest_api_skeleton/templates/gcbm_config.cfg @@ -0,0 +1,7 @@ +config=localdomain.json +config=pools_cbm.json +config=modules_cbm.json +config=modules_output.json +config=spinup.json +config=variables.json +config=internal_variables.json diff --git a/local/rest_api_skeleton/templates/internal_variables.json b/local/rest_api_skeleton/templates/internal_variables.json new file mode 100644 index 00000000..d9f71537 --- /dev/null +++ b/local/rest_api_skeleton/templates/internal_variables.json @@ -0,0 +1,34 @@ +{ + "Variables": { + "spatialLocationInfo": { + "flintdata": { + "type": "SpatialLocationInfo", + "library": "internal.flint", + "settings": {} + } + }, + "simulateLandUnit": true, + "is_decaying": true, + "spinup_moss_only": false, + "run_peatland": false, + "peatlandId": -1, + "is_forest": true, + "run_moss": false, + "run_delay": false, + "landUnitBuildSuccess": true, + "regen_delay": 0, + "age": 0, + "tileIndex": 0, + "blockIndex": 0, + "cellIndex": 0, + "LandUnitId": -1, + "landUnitArea": 0, + "classifier_set": {}, + "localDomainId": 0, + "LocalDomainId": 1, + "age_class": 0, + "historic_land_class": "FL", + "current_land_class": "FL", + "unfccc_land_class": "UNFCCC_FL_R_FL" + } +} \ No newline at end of file diff --git a/local/rest_api_skeleton/templates/localdomain.json b/local/rest_api_skeleton/templates/localdomain.json new file mode 100644 index 00000000..5bb04f33 --- /dev/null +++ b/local/rest_api_skeleton/templates/localdomain.json @@ -0,0 +1,31 @@ +{ + "Libraries": { + "moja.modules.cbm": "external", + "moja.modules.gdal": "external" + }, + "LocalDomain": { + "start_date": "2010/01/01", + "end_date": "2021/01/01", + "landUnitBuildSuccess": "landUnitBuildSuccess", + "simulateLandUnit": "simulateLandUnit", + "sequencer_library": "moja.modules.cbm", + "sequencer": "CBMSequencer", + "timing": "annual", + "type": "spatial_tiled", + "landscape": { + "provider": "RasterTiled", + "num_threads": 4, + "tiles": [ + { + "x": -106, + "y": 55, + "index": 12674 + } + ], + "x_pixels": 4000, + "y_pixels": 4000, + "tile_size_x": 1.0, + "tile_size_y": 1.0 + } + } +} \ No newline at end of file diff --git a/local/rest_api_skeleton/templates/logging.conf b/local/rest_api_skeleton/templates/logging.conf new file mode 100644 index 00000000..2da5fec5 --- /dev/null +++ b/local/rest_api_skeleton/templates/logging.conf @@ -0,0 +1,17 @@ +[Core] +DisableLogging=false + +[Sinks.console] +Destination=Console +Asynchronous=false +AutoFlush=true +Format="<%TimeStamp%> (%Severity%) - %Message%" +Filter="%Severity% >= info" + +[Sinks.file] +Destination=TextFile +FileName="output/simulation.log" +Asynchronous=false +AutoFlush=true +Format="<%TimeStamp%> (%Severity%) - %Message%" +Filter="%Severity% >= debug" diff --git a/local/rest_api_skeleton/templates/modules_cbm.json b/local/rest_api_skeleton/templates/modules_cbm.json new file mode 100644 index 00000000..0b237195 --- /dev/null +++ b/local/rest_api_skeleton/templates/modules_cbm.json @@ -0,0 +1,59 @@ +{ + "Modules": { + "CBMBuildLandUnitModule": { + "order": 1, + "library": "moja.modules.cbm" + }, + "CBMSequencer": { + "order": 2, + "library": "moja.modules.cbm" + }, + "CBMDisturbanceListener": { + "enabled": true, + "order": 3, + "library": "moja.modules.cbm", + "settings": { + "vars": [ + + ] + } + }, + "CBMDisturbanceEventModule": { + "enabled": true, + "order": 4, + "library": "moja.modules.cbm" + }, + "CBMTransitionRulesModule": { + "enabled": true, + "order": 5, + "library": "moja.modules.cbm" + }, + "CBMLandClassTransitionModule": { + "enabled": true, + "order": 6, + "library": "moja.modules.cbm" + }, + "CBMGrowthModule": { + "enabled": true, + "order": 7, + "library": "moja.modules.cbm" + }, + "CBMDecayModule": { + "enabled": true, + "order": 8, + "library": "moja.modules.cbm", + "settings": { + "extra_decay_removals": false + } + }, + "CBMAgeIndicators": { + "enabled": true, + "order": 9, + "library": "moja.modules.cbm" + }, + "TransactionManagerAfterSubmitModule": { + "order": 10, + "library": "internal.flint" + } + } +} \ No newline at end of file diff --git a/local/rest_api_skeleton/templates/modules_output.json b/local/rest_api_skeleton/templates/modules_output.json new file mode 100644 index 00000000..cad01b28 --- /dev/null +++ b/local/rest_api_skeleton/templates/modules_output.json @@ -0,0 +1,816 @@ +{ + "Modules": { + "WriteVariableGeotiff": { + "enabled": true, + "order": 11, + "library": "moja.modules.gdal", + "settings": { + "items": [ + { + "data_name": "Age", + "enabled": true, + "variable_data_type": "Int16", + "on_notification": "OutputStep", + "variable_name": "age" + }, + { + "pool_name": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther" + ], + "data_name": "AG_Biomass_C", + "enabled": true, + "variable_data_type": "float", + "on_notification": "OutputStep" + }, + { + "pool_name": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots", + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ], + "data_name": "Total_Ecosystem_C", + "enabled": true, + "variable_data_type": "float", + "on_notification": "OutputStep" + }, + { + "pool_name": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots" + ], + "data_name": "Total_Biomass_C", + "enabled": true, + "variable_data_type": "float", + "on_notification": "OutputStep" + }, + { + "pool_name": [ + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ], + "data_name": "Dead_Organic_Matter_C", + "enabled": true, + "variable_data_type": "float", + "on_notification": "OutputStep" + }, + { + "pool_name": [ + "BelowGroundVeryFastSoil", + "BelowGroundSlowSoil" + ], + "data_name": "Soil_C", + "enabled": true, + "variable_data_type": "float", + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": { + "to": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots" + ], + "from": [ + "Atmosphere" + ] + }, + "data_name": "NPP", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": [ + { + "to": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots" + ], + "from": [ + "Atmosphere" + ] + }, + { + "subtract": true, + "from": [ + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ], + "to": [ + "CO2", + "CH4", + "CO" + ], + "flux_source": "annual_process" + } + ], + "data_name": "NEP", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": [ + { + "flux_source": "annual_process", + "from": [ + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ], + "to": [ + "CO2", + "CH4", + "CO" + ] + } + ], + "data_name": "Decomp_Releases", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": [ + { + "to": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots" + ], + "from": [ + "Atmosphere" + ] + }, + { + "subtract": true, + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ], + "to": [ + "Products" + ], + "flux_source": "disturbance" + }, + { + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots", + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ], + "subtract": true, + "to": [ + "CO2", + "CH4", + "CO" + ] + } + ], + "data_name": "NBP", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": [ + { + "to": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots" + ], + "from": [ + "Atmosphere" + ] + }, + { + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots", + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ], + "subtract": true, + "to": [ + "CO2", + "CH4", + "CO" + ] + }, + { + "subtract": true, + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ], + "to": [ + "Products" + ], + "flux_source": "disturbance" + } + ], + "data_name": "Delta_Total_Ecosystem", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": [ + { + "to": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots" + ], + "from": [ + "Atmosphere" + ] + }, + { + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots" + ], + "subtract": true, + "to": [ + "CO2", + "CH4", + "CO" + ] + }, + { + "subtract": true, + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots" + ], + "to": [ + "Products" + ], + "flux_source": "disturbance" + }, + { + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots" + ], + "subtract": true, + "to": [ + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ] + } + ], + "data_name": "Delta_Total_Biomass", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": [ + { + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots" + ], + "to": [ + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ] + }, + { + "from": [ + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ], + "subtract": true, + "to": [ + "CO2", + "CH4", + "CO", + "Products" + ] + } + ], + "data_name": "Delta_Total_DOM", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": { + "to": [ + "CO2", + "CH4", + "CO" + ], + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots", + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ] + }, + "data_name": "Total_Emissions", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": { + "to": [ + "CO2", + "CH4", + "CO" + ], + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots" + ] + }, + "data_name": "Total_Biomass_Emissions", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": { + "to": [ + "CO2", + "CH4", + "CO" + ], + "from": [ + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ] + }, + "data_name": "Total_DOM_Emissions", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": { + "to": [ + "CO2" + ], + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots", + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ] + }, + "data_name": "Total_CO2_Emissions", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": { + "to": [ + "CO" + ], + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots", + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ] + }, + "data_name": "Total_CO_Emissions", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": { + "to": [ + "CH4" + ], + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots", + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ] + }, + "data_name": "Total_CH4_Emissions", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": [ + { + "to": [ + "CO2", + "CH4", + "CO" + ], + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots", + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ] + }, + { + "flux_source": "disturbance", + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ], + "to": [ + "Products" + ] + } + ], + "data_name": "Ecosystem_Removals", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": { + "flux_source": "disturbance", + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots" + ], + "to": [ + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ] + }, + "data_name": "Bio_To_DOM_From_Disturbances", + "enabled": true, + "on_notification": "OutputStep" + } + ], + "output_path": "output" + } + }, + "CBMAggregatorLandUnitData": { + "enabled": true, + "order": 12, + "library": "moja.modules.cbm", + "settings": { + "reporting_classifier_set": "reporting_classifiers" + } + }, + "CBMAggregatorSQLiteWriter": { + "enabled": true, + "order": 13, + "library": "moja.modules.cbm", + "settings": { + "databasename": "output/simulation_output.db" + } + } + } +} diff --git a/local/rest_api_skeleton/templates/pools_cbm.json b/local/rest_api_skeleton/templates/pools_cbm.json new file mode 100644 index 00000000..35979d38 --- /dev/null +++ b/local/rest_api_skeleton/templates/pools_cbm.json @@ -0,0 +1,34 @@ +{ + "Pools": { + "AboveGroundFastSoil": 0.0, + "AboveGroundSlowSoil": 0.0, + "AboveGroundVeryFastSoil": 0.0, + "Atmosphere": 0.0, + "BelowGroundFastSoil": 0.0, + "BelowGroundSlowSoil": 0.0, + "BelowGroundVeryFastSoil": 0.0, + "BlackCarbon": 0.0, + "CH4": 0.0, + "CO": 0.0, + "CO2": 0.0, + "DissolvedOrganicCarbon": 0.0, + "HardwoodBranchSnag": 0.0, + "HardwoodCoarseRoots": 0.0, + "HardwoodFineRoots": 0.0, + "HardwoodFoliage": 0.0, + "HardwoodMerch": 0.0, + "HardwoodOther": 0.0, + "HardwoodStemSnag": 0.0, + "MediumSoil": 0.0, + "NO2": 0.0, + "Peat": 0.0, + "Products": 0.0, + "SoftwoodBranchSnag": 0.0, + "SoftwoodCoarseRoots": 0.0, + "SoftwoodFineRoots": 0.0, + "SoftwoodFoliage": 0.0, + "SoftwoodMerch": 0.0, + "SoftwoodOther": 0.0, + "SoftwoodStemSnag": 0.0 + } +} \ No newline at end of file diff --git a/local/rest_api_skeleton/templates/provider_config.json b/local/rest_api_skeleton/templates/provider_config.json new file mode 100644 index 00000000..0888f9fc --- /dev/null +++ b/local/rest_api_skeleton/templates/provider_config.json @@ -0,0 +1,55 @@ +{ + "Providers": { + "SQLite": { + "path": "../input_database/gcbm_input.db", + "type": "SQLite" + }, + "RasterTiled": { + "layers": [ + { + "name": "disturbances_2011", + "layer_path": "disturbances_2011_moja.tiff", + "layer_prefix": "disturbances_2011_moja" + }, + { + "name": "disturbances_2012", + "layer_path": "disturbances_2012_moja.tiff", + "layer_prefix": "disturbances_2012_moja" + }, + { + "name": "disturbances_2013", + "layer_path": "disturbances_2013_moja.tiff", + "layer_prefix": "disturbances_2013_moja" + }, + { + "name": "disturbances_2014", + "layer_path": "disturbances_2014_moja.tiff", + "layer_prefix": "disturbances_2014_moja" + }, + { + "name": "disturbances_2015", + "layer_path": "disturbances_2015_moja.tiff", + "layer_prefix": "disturbances_2015_moja" + }, + { + "name": "disturbances_2016", + "layer_path": "disturbances_2016_moja.tiff", + "layer_prefix": "disturbances_2016_moja" + }, + { + "name": "disturbances_2018", + "layer_path": "disturbances_2018_moja.tiff", + "layer_prefix": "disturbances_2018_moja" + } + ], + "blockLonSize": 0.1, + "tileLatSize": 0.1, + "tileLonSize": 1.0, + "cellLatSize": 0.00025, + "cellLonSize": 0.00025, + "blockLatSize": 0.1, + "type": "RasterTiledGDAL", + "library": "moja.modules.gdal" + } + } +} \ No newline at end of file diff --git a/local/rest_api_skeleton/templates/spinup.json b/local/rest_api_skeleton/templates/spinup.json new file mode 100644 index 00000000..1365b141 --- /dev/null +++ b/local/rest_api_skeleton/templates/spinup.json @@ -0,0 +1,56 @@ +{ + "Spinup": { + "enabled": true, + "sequencer_library": "moja.modules.cbm", + "simulateLandUnit": "simulateLandUnit", + "landUnitBuildSuccess": "landUnitBuildSuccess", + "sequencer": "CBMSpinupSequencer" + }, + "SpinupVariables": { + "delay": 0, + "minimum_rotation": 10, + "run_delay": false + }, + "Variables": { + "spinup_parameters": { + "transform": { + "queryString": "SELECT s.return_interval AS return_interval, s.max_rotations AS max_rotations, dt.name AS historic_disturbance_type, dt.name AS last_pass_disturbance_type, s.mean_annual_temperature AS mean_annual_temperature, 0 as delay FROM spinup_parameter s INNER JOIN disturbance_type dt ON s.historic_disturbance_type_id = dt.id INNER JOIN spatial_unit spu ON spu.spinup_parameter_id = s.id INNER JOIN admin_boundary a ON spu.admin_boundary_id = a.id INNER JOIN eco_boundary e ON spu.eco_boundary_id = e.id WHERE a.name = {var:admin_boundary} AND e.name = {var:eco_boundary}", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + } + }, + "SpinupModules": { + "CBMSpinupSequencer": { + "create_new": true, + "library": "moja.modules.cbm", + "order": 1 + }, + "CBMBuildLandUnitModule": { + "create_new": true, + "library": "moja.modules.cbm", + "order": 2 + }, + "CBMGrowthModule": { + "create_new": true, + "library": "moja.modules.cbm", + "order": 3 + }, + "CBMDecayModule": { + "create_new": true, + "library": "moja.modules.cbm", + "order": 4 + }, + "TransactionManagerAfterSubmitModule": { + "create_new": true, + "library": "internal.flint", + "order": 5 + }, + "CBMSpinupDisturbanceModule": { + "create_new": true, + "library": "moja.modules.cbm", + "order": 6 + } + } +} \ No newline at end of file diff --git a/local/rest_api_skeleton/templates/variables.json b/local/rest_api_skeleton/templates/variables.json new file mode 100644 index 00000000..f2d8bc2f --- /dev/null +++ b/local/rest_api_skeleton/templates/variables.json @@ -0,0 +1,257 @@ +{ + "Variables": { + "enable_peatland": false, + "enable_moss": false, + "admin_boundary": "British Columbia", + "eco_boundary": "Taiga Plains", + "initial_age": { + "transform": { + "library": "internal.flint", + "type": "LocationIdxFromFlintDataTransform", + "provider": "RasterTiled", + "data_id": "initial_age" + } + }, + "initial_historic_land_class": "FL", + "initial_current_land_class": "FL", + "age_class_range": 20, + "age_maximum": 300, + "slow_ag_to_bg_mixing_rate": 0.006, + "disturbance_matrices": { + "transform": { + "queryString": "SELECT dm.id AS disturbance_matrix_id, source_pool.name as source_pool_name, dest_pool.name as dest_pool_name, dv.proportion FROM disturbance_matrix dm INNER JOIN disturbance_matrix_value dv ON dm.id = dv.disturbance_matrix_id INNER JOIN pool source_pool ON dv.source_pool_id = source_pool.id INNER JOIN pool dest_pool ON dv.sink_pool_id = dest_pool.id", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "softwood_yield_table": { + "transform": { + "queryString": "SELECT gcv.age AS age, SUM(gcv.merchantable_volume) AS merchantable_volume FROM (SELECT CASE WHEN gc.id IS NOT NULL THEN gc.id ELSE -1 END AS growth_curve_component_id FROM growth_curve_component gc INNER JOIN species s ON s.id = gc.species_id INNER JOIN forest_type ft ON ft.id = s.forest_type_id WHERE gc.growth_curve_id = {var:growth_curve_id} AND LOWER(ft.name) LIKE LOWER('Softwood')) AS gc INNER JOIN growth_curve_component_value gcv ON gc.growth_curve_component_id = gcv.growth_curve_component_id GROUP BY gcv.age", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "reporting_classifiers": { + "transform": { + "allow_nulls": true, + "type": "CompositeTransform", + "library": "internal.flint", + "vars": [ + "classifier_set" + ] + } + }, + "land_class_transitions": { + "transform": { + "queryString": "SELECT dt.name AS disturbance_type, lc.code AS land_class_transition, lc.is_forest, lc.years_to_permanent FROM disturbance_type dt INNER JOIN land_class lc ON dt.transition_land_class_id = lc.id", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "transition_rules": { + "transform": { + "queryString": "SELECT t.id AS id, age, regen_delay, description, tt.name AS reset_type FROM transition t INNER JOIN transition_type tt ON t.transition_type_id = tt.id", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "transition_rule_matches": { + "transform": { + "classifier_set_var": "classifier_set", + "type": "TransitionRuleTransform", + "library": "moja.modules.cbm", + "provider": "SQLite" + } + }, + "spatial_unit_id": { + "transform": { + "queryString": "SELECT spu.id FROM spatial_unit spu INNER JOIN admin_boundary a ON spu.admin_boundary_id = a.id INNER JOIN eco_boundary e ON spu.eco_boundary_id = e.id WHERE a.name = {var:admin_boundary} AND e.name = {var:eco_boundary}", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "hardwood_yield_table": { + "transform": { + "queryString": "SELECT gcv.age AS age, SUM(gcv.merchantable_volume) AS merchantable_volume FROM (SELECT CASE WHEN gc.id IS NOT NULL THEN gc.id ELSE -1 END AS growth_curve_component_id FROM growth_curve_component gc INNER JOIN species s ON s.id = gc.species_id INNER JOIN forest_type ft ON ft.id = s.forest_type_id WHERE gc.growth_curve_id = {var:growth_curve_id} AND LOWER(ft.name) LIKE LOWER('Hardwood')) AS gc INNER JOIN growth_curve_component_value gcv ON gc.growth_curve_component_id = gcv.growth_curve_component_id GROUP BY gcv.age", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "turnover_rates": { + "transform": { + "queryString": "SELECT COALESCE(sw_turnover.foliage, 0) AS sw_foliage_turnover, COALESCE(hw_turnover.foliage, 0) AS hw_foliage_turnover, COALESCE(sw_turnover.stem, 0) AS sw_stem_turnover, COALESCE(hw_turnover.stem, 0) AS hw_stem_turnover, COALESCE(sw_turnover.branch, 0) AS sw_branch_turnover, COALESCE(hw_turnover.branch, 0) AS hw_branch_turnover, COALESCE(sw_turnover.branch_snag_split, 0) AS sw_other_to_branch_snag_split, COALESCE(hw_turnover.branch_snag_split, 0) AS hw_other_to_branch_snag_split, COALESCE(sw_turnover.stem_snag, 0) AS sw_stem_snag_turnover, COALESCE(hw_turnover.stem_snag, 0) AS hw_stem_snag_turnover, COALESCE(sw_turnover.branch_snag, 0) AS sw_branch_snag_turnover, COALESCE(hw_turnover.branch_snag, 0) AS hw_branch_snag_turnover, COALESCE(sw_turnover.coarse_ag_split, 0) AS sw_coarse_root_split, COALESCE(hw_turnover.coarse_ag_split, 0) AS hw_coarse_root_split, COALESCE(sw_turnover.coarse_root, 0) AS sw_coarse_root_turnover, COALESCE(hw_turnover.coarse_root, 0) AS hw_coarse_root_turnover, COALESCE(sw_turnover.fine_ag_split, 0) AS sw_fine_root_ag_split, COALESCE(hw_turnover.fine_ag_split, 0) AS hw_fine_root_ag_split, COALESCE(sw_turnover.fine_root, 0) AS sw_fine_root_turnover, COALESCE(hw_turnover.fine_root, 0) AS hw_fine_root_turnover FROM growth_curve gc LEFT JOIN ( SELECT growth_curve_id, foliage, stem, branch, branch_snag_split, stem_snag, branch_snag, coarse_ag_split, coarse_root, fine_ag_split, fine_root FROM turnover_parameter_association tpa INNER JOIN eco_boundary e ON tpa.eco_boundary_id = e.id INNER JOIN genus g ON tpa.genus_id = g.id INNER JOIN species s ON s.genus_id = g.id INNER JOIN forest_type f ON s.forest_type_id = f.id INNER JOIN growth_curve_component gcc ON gcc.species_id = s.id INNER JOIN turnover_parameter t ON tpa.turnover_parameter_id = t.id WHERE gcc.growth_curve_id = {var:growth_curve_id} AND e.name = {var:eco_boundary} AND f.name = 'Softwood' ORDER BY gcc.id LIMIT 1 ) AS sw_turnover ON gc.id = sw_turnover.growth_curve_id LEFT JOIN ( SELECT growth_curve_id, foliage, stem, branch, branch_snag_split, stem_snag, branch_snag, coarse_ag_split, coarse_root, fine_ag_split, fine_root FROM turnover_parameter_association tpa INNER JOIN eco_boundary e ON tpa.eco_boundary_id = e.id INNER JOIN genus g ON tpa.genus_id = g.id INNER JOIN species s ON s.genus_id = g.id INNER JOIN forest_type f ON s.forest_type_id = f.id INNER JOIN growth_curve_component gcc ON gcc.species_id = s.id INNER JOIN turnover_parameter t ON tpa.turnover_parameter_id = t.id WHERE gcc.growth_curve_id = {var:growth_curve_id} AND e.name = {var:eco_boundary} AND f.name = 'Hardwood' ORDER BY gcc.id LIMIT 1 ) AS hw_turnover ON gc.id = hw_turnover.growth_curve_id WHERE gc.id = {var:growth_curve_id}", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "disturbance_type_codes": { + "transform": { + "queryString": "SELECT dt.name AS disturbance_type, dt.code AS disturbance_type_code FROM disturbance_type dt", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "transition_rule_classifiers": { + "transform": { + "queryString": "SELECT t.id, c.name AS classifier_name, cv.value AS classifier_value FROM transition t INNER JOIN transition_classifier_value tcv ON t.id = tcv.transition_id INNER JOIN classifier_value cv ON tcv.classifier_value_id = cv.id INNER JOIN classifier c ON cv.classifier_id = c.id", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "initial_classifier_set": { + "transform": { + "type": "CompositeTransform", + "library": "internal.flint", + "vars": [ + "Classifier2", + "Classifier1" + ] + } + }, + "disturbance_matrix_associations": { + "transform": { + "queryString": "SELECT dt.name AS disturbance_type, dma.spatial_unit_id, dma.disturbance_matrix_id FROM disturbance_matrix_association dma INNER JOIN disturbance_type dt ON dma.disturbance_type_id = dt.id", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "other_to_branch_snag_split": { + "transform": { + "queryString": "SELECT t.branch_snag_split AS slow_mixing_rate FROM eco_boundary e INNER JOIN turnover_parameter t ON e.turnover_parameter_id = t.id WHERE e.name LIKE {var:eco_boundary}", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "growth_curve_id": { + "transform": { + "classifier_set_var": "classifier_set", + "type": "GrowthCurveTransform", + "library": "moja.modules.cbm", + "provider": "SQLite" + } + }, + "volume_to_biomass_parameters": { + "transform": { + "queryString": "SELECT ft.name AS forest_type, f.a as a, f.b as b, f.a_nonmerch as a_non_merch, f.b_nonmerch as b_non_merch, f.k_nonmerch as k_non_merch, f.cap_nonmerch as cap_non_merch, f.a_sap as a_sap, f.b_sap as b_sap, f.k_sap as k_sap, f.cap_sap as cap_sap, f.a1 as a1, f.a2 as a2, f.a3 as a3, f.b1 as b1, f.b2 as b2, f.b3 as b3, f.c1 as c1, f.c2 as c2, f.c3 as c3, f.min_volume as min_volume, f.max_volume as max_volume, f.low_stemwood_prop as low_stemwood_prop, f.high_stemwood_prop as high_stemwood_prop, f.low_stembark_prop as low_stembark_prop, f.high_stembark_prop as high_stembark_prop, f.low_branches_prop AS low_branches_prop, f.high_branches_prop as high_branches_prop, f.low_foliage_prop AS low_foliage_prop, f.high_foliage_prop AS high_foliage_prop, sp.sw_top_proportion AS softwood_top_prop, sp.sw_stump_proportion AS softwood_stump_prop, sp.hw_top_proportion AS hardwood_top_prop, sp.hw_stump_proportion AS hardwood_stump_prop, rp.hw_a AS hw_a, rp.hw_b AS hw_b, rp.sw_a AS sw_a, rp.frp_a AS frp_a, rp.frp_b AS frp_b, rp.frp_c AS frp_c FROM vol_to_bio_factor_association fa INNER JOIN vol_to_bio_factor f ON f.id = fa.vol_to_bio_factor_id INNER JOIN species s ON fa.species_id = s.id INNER JOIN growth_curve_component gcc ON s.id = gcc.species_id INNER JOIN forest_type ft ON s.forest_type_id = ft.id INNER JOIN spatial_unit spu ON fa.spatial_unit_id = spu.id INNER JOIN admin_boundary a ON spu.admin_boundary_id = a.id INNER JOIN stump_parameter sp ON a.stump_parameter_id = sp.id INNER JOIN root_parameter rp ON rp.id = fa.root_parameter_id WHERE gcc.growth_curve_id = {var:growth_curve_id} AND spu.id = {var:spatial_unit_id} ORDER BY gcc.id DESC", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "spu": { + "transform": { + "queryString": "select s.id AS spu_id from spatial_unit s inner join admin_boundary a on s.admin_boundary_id = a.id inner join eco_boundary e on s.eco_boundary_id = e.id where a.name like {var:admin_boundary} and e.name like {var:eco_boundary}", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "land_class_data": { + "transform": { + "queryString": "SELECT code AS land_class, is_forest, years_to_permanent FROM land_class lc", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "mean_annual_temperature": { + "transform": { + "library": "internal.flint", + "type": "LocationIdxFromFlintDataTransform", + "provider": "RasterTiled", + "data_id": "mean_annual_temperature" + } + }, + "decay_parameters": { + "transform": { + "queryString": "SELECT p.name AS pool, dp.base_decay_rate AS organic_matter_decay_rate, dp.prop_to_atmosphere AS prop_to_atmosphere, dp.q10 AS q10, dp.reference_temp AS reference_temp, dp.max_rate AS max_decay_rate_soft FROM decay_parameter dp INNER JOIN dom_pool dom ON dp.dom_pool_id = dom.id INNER JOIN pool p ON p.id = dom.pool_id", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "disturbances_2012": { + "transform": { + "library": "internal.flint", + "type": "LocationIdxFromFlintDataTransform", + "provider": "RasterTiled", + "data_id": "disturbances_2012" + } + }, + "disturbances_2011": { + "transform": { + "library": "internal.flint", + "type": "LocationIdxFromFlintDataTransform", + "provider": "RasterTiled", + "data_id": "disturbances_2011" + } + }, + "disturbances_2015": { + "transform": { + "library": "internal.flint", + "type": "LocationIdxFromFlintDataTransform", + "provider": "RasterTiled", + "data_id": "disturbances_2015" + } + }, + "disturbances_2014": { + "transform": { + "library": "internal.flint", + "type": "LocationIdxFromFlintDataTransform", + "provider": "RasterTiled", + "data_id": "disturbances_2014" + } + }, + "disturbances_2018": { + "transform": { + "library": "internal.flint", + "type": "LocationIdxFromFlintDataTransform", + "provider": "RasterTiled", + "data_id": "disturbances_2018" + } + }, + "disturbances_2016": { + "transform": { + "library": "internal.flint", + "type": "LocationIdxFromFlintDataTransform", + "provider": "RasterTiled", + "data_id": "disturbances_2016" + } + }, + "Classifier2": { + "transform": { + "library": "internal.flint", + "type": "LocationIdxFromFlintDataTransform", + "provider": "RasterTiled", + "data_id": "Classifier2" + } + }, + "Classifier1": { + "transform": { + "library": "internal.flint", + "type": "LocationIdxFromFlintDataTransform", + "provider": "RasterTiled", + "data_id": "Classifier1" + } + }, + "disturbances_2013": { + "transform": { + "library": "internal.flint", + "type": "LocationIdxFromFlintDataTransform", + "provider": "RasterTiled", + "data_id": "disturbances_2013" + } + } + } +} \ No newline at end of file From 8c24f84590743b7e757af5fc8af2a38263f5de15 Mon Sep 17 00:00:00 2001 From: Freeman Date: Fri, 28 Oct 2022 00:41:12 +0100 Subject: [PATCH 11/21] GCBM Refactor idea Draft Signed-off-by: Freeman --- .../Endpoints/gcbm_endpoints.py | 62 ++++++++++++++----- 1 file changed, 46 insertions(+), 16 deletions(-) diff --git a/local/rest_api_skeleton/Endpoints/gcbm_endpoints.py b/local/rest_api_skeleton/Endpoints/gcbm_endpoints.py index 46b865a0..a5582bca 100644 --- a/local/rest_api_skeleton/Endpoints/gcbm_endpoints.py +++ b/local/rest_api_skeleton/Endpoints/gcbm_endpoints.py @@ -1,13 +1,24 @@ -from flask_restful import Resource +from flask_restful import Resource import os from threading import Thread -from Helpers.preprocess import DisturbanceConfig, launch_run, ClassifierConfig, MiscellaneousConfig -from Helpers.for_requests import title_query, miscellaneous_query, disturbance_query, classifier_query +from Helpers.preprocess import ( + DisturbanceConfig, + launch_run, + ClassifierConfig, + MiscellaneousConfig, +) +from Helpers.for_requests import ( + title_query, + miscellaneous_query, + disturbance_query, + classifier_query, +) + class disturbance(Resource): def post(self): title = title_query().get("title") or "simulation" - input_dir = f"{os.getcwd()}/input/{title}" + input_dir = f"{os.getcwd()}/input/{title}" if not os.path.exists(f"{input_dir}"): os.makedirs(f"{input_dir}") if not os.path.exists(f"{input_dir}/disturbances"): @@ -20,12 +31,17 @@ def post(self): for file in disturbances.get("disturbances"): file.save(f"{input_dir}/disturbances/{file.filename}") try: - disturb = DisturbanceConfig(input_dir, file.filename, disturbances.get("attributes")) + disturb = DisturbanceConfig( + input_dir, file.filename, disturbances.get("attributes") + ) disturb() - except Exception as e: + except Exception as e: return e - disturb.flatten_directory("disturbances") - return {"data": "Disturbances file uploaded succesfully. Proceed to the next step."} + disturb.flatten_directory("disturbances") + return { + "data": "Disturbances file uploaded succesfully. Proceed to the next step." + } + class classifier(Resource): def post(self): @@ -44,22 +60,28 @@ def post(self): # store disturbances file in a new folder classifiers = classifier_query() if not classifiers: - return {"error": "Missing classifiers file"}, 400 + return {"error": "Missing classifiers file"}, 400 for file in classifiers.get("classifiers"): file.save(f"{input_dir}/classifiers/{file.filename}") try: - classify = ClassifierConfig(input_dir, file.filename, classifiers.get("attributes")) + classify = ClassifierConfig( + input_dir, file.filename, classifiers.get("attributes") + ) classify() except Exception as e: return e classify.flatten_directory("classifiers") - return {"data": "Classifiers file uploaded succesfully. Proceed to the next step."} + return { + "data": "Classifiers file uploaded succesfully. Proceed to the next step." + } + class Database(Resource): def post(self): pass + class miscellaneous(Resource): def post(self): # Get the title from the payload @@ -77,17 +99,22 @@ def post(self): # store miscellaneous file in a new folder mis = miscellaneous_query() if not mis: - return {"error": "Missing classifiers file"}, 400 + return {"error": "Missing classifiers file"}, 400 for file in mis.get("miscellaneous"): file.save(f"{input_dir}/miscellaneous/{file.filename}") try: - miscel = MiscellaneousConfig(input_dir, file.filename, mis.get("attributes")) + miscel = MiscellaneousConfig( + input_dir, file.filename, mis.get("attributes") + ) miscel() except Exception as e: return e miscel.flatten_directory("miscellaneous") - return {"data": "Classifiers file uploaded succesfully. Proceed to the next step."} + return { + "data": "Classifiers file uploaded succesfully. Proceed to the next step." + } + class title(Resource): def post(self): @@ -105,9 +132,11 @@ def post(self): return {"data": message}, 200 + class Run(Resource): """THIS ENDPOINT WILL BE ABLE TO RUN A SIMULATION GOAL IS TO EQUIP IT TO BE ABLE TO RUN MORE THAN ONE SIMULATIONS AT A TIME""" + def post(self): title = title_query().get("title") or "simulation" @@ -115,11 +144,12 @@ def post(self): title = "".join(c for c in title if c.isalnum()) input_dir = f"{os.getcwd()}/input/{title}" - if not os.path.exists(f"{input_dir}"): os.makedirs(f"{input_dir}") - thread = Thread(target=launch_run, kwargs={"title": title, "input_dir": input_dir}) + thread = Thread( + target=launch_run, kwargs={"title": title, "input_dir": input_dir} + ) thread.start() return {"status": "Run started"}, 200 From 9a9a9a1a48952454e947b51b586c99273c35e41b Mon Sep 17 00:00:00 2001 From: Freeman Date: Fri, 28 Oct 2022 00:43:25 +0100 Subject: [PATCH 12/21] GCBM Refactor idea Draft Signed-off-by: Freeman --- .../rest_api_skeleton/Helpers/for_requests.py | 47 +++++-- local/rest_api_skeleton/Helpers/preprocess.py | 117 +++++++++++------- local/rest_api_skeleton/app.py | 6 +- 3 files changed, 109 insertions(+), 61 deletions(-) diff --git a/local/rest_api_skeleton/Helpers/for_requests.py b/local/rest_api_skeleton/Helpers/for_requests.py index db53093d..9851264f 100644 --- a/local/rest_api_skeleton/Helpers/for_requests.py +++ b/local/rest_api_skeleton/Helpers/for_requests.py @@ -1,26 +1,47 @@ from flask_restful import reqparse -from werkzeug.datastructures import FileStorage +from werkzeug.datastructures import FileStorage + def title_query(): query = reqparse.RequestParser() - query.add_argument("title", required=True, location="form" ) - return query.parse_args() + query.add_argument("title", required=True, location="form") + return query.parse_args() + def classifier_query(): query = reqparse.RequestParser() - query.add_argument("classifiers",type=FileStorage, required=True, action="append", location="files" ) - query.add_argument("attributes", location= "form") - return query.parse_args() + query.add_argument( + "classifiers", + type=FileStorage, + required=True, + action="append", + location="files", + ) + query.add_argument("attributes", location="form") + return query.parse_args() + def disturbance_query(): query = reqparse.RequestParser() - query.add_argument("disturbances",type=FileStorage, required=True, action="append", location="files" ) - query.add_argument("attributes", location= "form") - return query.parse_args() + query.add_argument( + "disturbances", + type=FileStorage, + required=True, + action="append", + location="files", + ) + query.add_argument("attributes", location="form") + return query.parse_args() + def miscellaneous_query(): query = reqparse.RequestParser() - query.add_argument("miscellaneous",type=FileStorage, required=True, action="append", location="files" ) - query.add_argument("attributes", location= "form") - return query.parse_args() - + query.add_argument( + "miscellaneous", + type=FileStorage, + required=True, + action="append", + location="files", + ) + query.add_argument("attributes", location="form") + return query.parse_args() diff --git a/local/rest_api_skeleton/Helpers/preprocess.py b/local/rest_api_skeleton/Helpers/preprocess.py index 157a0081..a496b64a 100644 --- a/local/rest_api_skeleton/Helpers/preprocess.py +++ b/local/rest_api_skeleton/Helpers/preprocess.py @@ -3,7 +3,8 @@ import json import rasterio as rst -def launch_run(title, input_dir): + +def launch_run(title, input_dir): s = time.time() with open(f"{input_dir}/gcbm_logs.csv", "w+") as f: res = subprocess.Popen( @@ -36,32 +37,43 @@ def launch_run(title, input_dir): "response": "Operation executed successfully. Downloadable links for input and output are attached in the response. Alternatively, you may also download this simulation input and output results by making a request at gcbm/download with the title in the body.", } -class Configs: +class Configs: def __init__(self, input_dir) -> None: self.input_dir = input_dir self.Rasters = [] - self.Rastersm = [] + self.Rastersm = [] self.nodatam = [] self.nodata = [] self.cellLatSize = [] self.cellLonSize = [] self.paths = [] self.lst = [] - self.provider_config = open(f"{os.getcwd()}/templates/provider_config.json", "r+") + self.provider_config = open( + f"{os.getcwd()}/templates/provider_config.json", "r+" + ) def get_config_templates(self): if not os.path.exists(f"{self.input_dir}/templates"): shutil.copytree( - f"{os.getcwd()}/templates", f"{self.input_dir}/templates", dirs_exist_ok=False + f"{os.getcwd()}/templates", + f"{self.input_dir}/templates", + dirs_exist_ok=False, + ) + self.provider_config = open( + f"{self.input_dir}/templates/provider_config.json", "r+" ) - self.provider_config = open(f"{self.input_dir}/templates/provider_config.json", "r+") def get_modules_cbm_config(self): - with open(f"{self.input_dir}/templates/modules_cbm.json", "r+") as modules_cbm_config: + with open( + f"{self.input_dir}/templates/modules_cbm.json", "r+" + ) as modules_cbm_config: data = json.load(modules_cbm_config) - disturbances = [file.split(".")[0][:-5] for file in os.listdir(f"{self.input_dir}/disturbances/")] # drop `_moja` to match modules_cbm.json template - modules_cbm_config.seek(0) + disturbances = [ + file.split(".")[0][:-5] + for file in os.listdir(f"{self.input_dir}/disturbances/") + ] # drop `_moja` to match modules_cbm.json template + modules_cbm_config.seek(0) data["Modules"]["CBMDisturbanceListener"]["settings"]["vars"] = disturbances json.dump(data, modules_cbm_config, indent=4) modules_cbm_config.truncate() @@ -70,11 +82,10 @@ def get_modules_cbm_config(self): def database_writes(self): data = json.load(self.provider_config) for file in os.listdir(f"{self.input_dir}/db/"): - data["Providers"]["SQLite"] = {"type": "SQLite", "path": file } + data["Providers"]["SQLite"] = {"type": "SQLite", "path": file} self.provider_config.seek(0) - - def write_configs(self, config_type : str): + def write_configs(self, config_type: str): data = json.load(self.provider_config) for file in os.listdir(f"{self.input_dir}/{config_type}/"): d = dict() @@ -82,7 +93,7 @@ def write_configs(self, config_type : str): d["layer_path"] = file d["layer_prefix"] = file[:-5] self.lst.append(d) - if config_type == "disturbances" or config_type == "classifiers": + if config_type == "disturbances" or config_type == "classifiers": for root, _, files in os.walk( os.path.abspath(f"{self.input_dir}/{config_type}/") ): @@ -91,15 +102,15 @@ def write_configs(self, config_type : str): self.Rasters.append(fp) self.paths.append(fp) for self.nd in self.Rasters: - img = rst.open(self.nd) - t = img.transform + img = rst.open(self.nd) + t = img.transform x = t[0] y = -t[4] - n = img.nodata + n = img.nodata self.cellLatSize.append(x) self.cellLonSize.append(y) self.nodata.append(n) - result = all(element == self.cellLatSize[0] for element in self.cellLatSize) + result = all(element == self.cellLatSize[0] for element in self.cellLatSize) if result: cellLat = x cellLon = y @@ -112,8 +123,15 @@ def write_configs(self, config_type : str): print("Corrupt files") self.provider_config.seek(0) - new_values = {"cellLonSize": cellLon,"cellLatSize": cellLat, "blockLonSize": blockLon, "blockLatSize": blockLat, "tileLatSize": tileLat, "tileLonSize": tileLon} - data["Providers"]["RasterTiled"]["layers"] = self.lst + new_values = { + "cellLonSize": cellLon, + "cellLatSize": cellLat, + "blockLonSize": blockLon, + "blockLatSize": blockLat, + "tileLatSize": tileLat, + "tileLonSize": tileLon, + } + data["Providers"]["RasterTiled"]["layers"] = self.lst data["Providers"]["RasterTiled"].update(new_values) json.dump(data, self.provider_config, indent=4) @@ -132,21 +150,23 @@ def write_configs(self, config_type : str): } self.study_area = { - "tile_size": tileLat, - "block_size": blockLat, - "tiles": [ - { - "x": int(t[2]), - "y": int(t[5]), - "index": 12674, - } - ], - "pixel_size": cellLat, - "layers": [], - } + "tile_size": tileLat, + "block_size": blockLat, + "tiles": [ + { + "x": int(t[2]), + "y": int(t[5]), + "index": 12674, + } + ], + "pixel_size": cellLat, + "layers": [], + } def add_file_to_path(self, config_type): - for root, _, files in os.walk(os.path.abspath(f"{self.input_dir}/{config_type}/")): + for root, _, files in os.walk( + os.path.abspath(f"{self.input_dir}/{config_type}/") + ): for file in files: fp = os.path.join(root, file) self.paths.append(fp) @@ -157,15 +177,18 @@ def copy_directory(self): def flatten_directory(self, config_type): shutil.rmtree((f"{self.input_dir}/{config_type}/")) - + + class DisturbanceConfig(Configs): - def __init__(self, input_dir, config_file : str, attribute : dict = None ) -> None: + def __init__(self, input_dir, config_file: str, attribute: dict = None) -> None: super().__init__(input_dir) self.config_file = config_file self.attribute = attribute def disturbances_special(self): - with open(f"{self.input_dir}/{self.config_file}", "w", encoding="utf8") as json_file: + with open( + f"{self.input_dir}/{self.config_file}", "w", encoding="utf8" + ) as json_file: self.dictionary["attributes"] = self.attribute json.dump(self.dictionary, json_file, indent=4) with open( @@ -194,14 +217,17 @@ def __call__(self): self.add_file_to_path("disturbances") self.copy_directory() + class ClassifierConfig(Configs): - def __init__(self, input_dir, config_file : str, attribute : dict ) -> None: + def __init__(self, input_dir, config_file: str, attribute: dict) -> None: super().__init__(input_dir) self.config_file = config_file self.attribute = attribute def classifier_special(self): - with open(f"{self.input_dir}/{self.config_file}", "w", encoding="utf8") as json_file: + with open( + f"{self.input_dir}/{self.config_file}", "w", encoding="utf8" + ) as json_file: self.dictionary["attributes"] = self.attribute json.dump(self.dictionary, json_file, indent=4) @@ -225,16 +251,17 @@ def __call__(self): self.add_file_to_path("classifiers") self.copy_directory() + class MiscellaneousConfig(Configs): - def __init__(self, input_dir, config_file : str, attribute : dict ) -> None: + def __init__(self, input_dir, config_file: str, attribute: dict) -> None: super().__init__(input_dir) self.config_file = config_file self.attribute = attribute - - def miscellaneous_special(self): + + def miscellaneous_special(self): for root, _, files in os.walk( os.path.abspath(f"{self.input_dir}/miscellaneous/") -): + ): for file in files: fp2 = os.path.join(root, file) self.Rastersm.append(fp2) @@ -244,9 +271,11 @@ def miscellaneous_special(self): d = img.nodata self.nodatam.append(d) """this is an experimental thing""" - with open(f"{self.input_dir}/{self.config_file}", "w", encoding="utf8") as json_file: + with open( + f"{self.input_dir}/{self.config_file}", "w", encoding="utf8" + ) as json_file: self.dictionary["layer_type"] = "GridLayer" - if self.config_file == "mean_annual_temperature_moja.json": + if self.config_file == "mean_annual_temperature_moja.json": self.dictionary["layer_data"] = "Float32" else: self.dictionary["layer_data"] = "Int16" @@ -273,5 +302,3 @@ def __call__(self): self.add_file_to_path("miscellaneous") self.copy_directory() self.flatten_directory("miscellaneous") - - diff --git a/local/rest_api_skeleton/app.py b/local/rest_api_skeleton/app.py index c5502bc0..cd256cc9 100644 --- a/local/rest_api_skeleton/app.py +++ b/local/rest_api_skeleton/app.py @@ -2,8 +2,8 @@ from flask_restful import Api from Endpoints.gcbm_endpoints import disturbance, Run, title, classifier, miscellaneous -app = Flask(__name__) -api = Api() +app = Flask(__name__) +api = Api() api.add_resource(disturbance, "/gcbm/upload/disturbances") api.add_resource(title, "/gcbm/create") @@ -12,5 +12,5 @@ api.add_resource(Run, "/gcbm/run") -if __name__ == "__main__": +if __name__ == "__main__": app.run(debug=True) From bbdab43cee26b5597c5f232adf60a7b6353e80c4 Mon Sep 17 00:00:00 2001 From: temitayo Date: Tue, 18 Oct 2022 14:11:20 +0100 Subject: [PATCH 13/21] fix: add a check to see if simulation output exists (#199) Signed-off-by: olalekan temitayo Signed-off-by: olalekan temitayo Signed-off-by: Freeman --- local/rest_api_gcbm/app.py | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/local/rest_api_gcbm/app.py b/local/rest_api_gcbm/app.py index f19a03ad..1feed739 100644 --- a/local/rest_api_gcbm/app.py +++ b/local/rest_api_gcbm/app.py @@ -462,9 +462,24 @@ def gcbm_download(): title = request.form.get("title") or "simulation" # Sanitize title title = "".join(c for c in title if c.isalnum()) + + output_dir = f"{os.getcwd()}/output/{title}.zip" + input_dir = f"{os.getcwd()}/input/{title}" + + # if the title has an input simulation and there is no output simulation then they should check the status. + if not os.path.exists(f"{output_dir}") and os.path.exists(f"{input_dir}"): + return { + "message": "You simulation is currently running, check the status via /gcbm/status" + } + + # if there is no input simulation and no output simulation then the simulation does not exist. + elif not os.path.exists(f"{output_dir}") and not os.path.exists(f"{input_dir}"): + return { + "message": "You don't have a simulation with this title kindly check the title and try again" + } + return send_file( - f"{os.getcwd()}/output/{title}.zip", - attachment_filename="{title}.zip", + f"{os.getcwd()}/output/{title}.zip", attachment_filename="{title}.zip", ) @@ -484,10 +499,13 @@ def gcbm_list_simulations(): for file in os.listdir(f"{os.getcwd()}/input"): list.append(file) - return { - "data": list, - "message": "To create a new simulation, create a request at gcbm/new. To access the results of the existing simulations, create a request at gcbm/download.", - }, 200 + return ( + { + "data": list, + "message": "To create a new simulation, create a request at gcbm/new. To access the results of the existing simulations, create a request at gcbm/download.", + }, + 200, + ) @app.route("/gcbm/status", methods=["POST"]) From 1a6da289e5f7cb0762331077347979db97efb084 Mon Sep 17 00:00:00 2001 From: Clinton Mekwunye Date: Sun, 23 Oct 2022 16:58:04 +0100 Subject: [PATCH 14/21] docs: document the `/gcbm/getConfig` endpoint (#201) Signed-off-by: Clinton Mekwunye Signed-off-by: Clinton Mekwunye Signed-off-by: Freeman --- local/rest_api_gcbm/curl.md | 84 +++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/local/rest_api_gcbm/curl.md b/local/rest_api_gcbm/curl.md index e8b32653..1ef0532c 100644 --- a/local/rest_api_gcbm/curl.md +++ b/local/rest_api_gcbm/curl.md @@ -258,6 +258,90 @@ A file named `output.zip` will be obtained. This file contains the outputs gener curl -d "title=run4" -X POST http://localhost:8080/gcbm/download -L -o run4.zip ``` +### Get Config mean annual temperature moja file + +| **Method** | **Endpoint** | **Response code** | +| ---------- | ----------------- | ----------------- | +| POST | `/gcbm/getConfig` | 200 OK (Success) | + +``` +curl -F file_name="mean_annual_temperature_moja" \ + -F title="run4" \ + http://localhost:8080/gcbm/getConfig + +``` + +Expected response (Success): + +```json +{"data":{ + "blockLatSize":0.1,"blockLonSize":0.1,"cellLatSize":0.00025, + "cellLonSize":0.00025,"layer_data":"Float32","layer_type":"GridLayer", + "nodata":32767.0,"tileLatSize":1.0,"tileLonSize":1.0 + } +} +``` + +### Get Config internal_variables file + +| **Method** | **Endpoint** | **Response code** | +| ---------- | ----------------- | ----------------- | +| POST | `/gcbm/getConfig` | 200 OK (Success) | + +``` +curl -F file_name="internal_variables" \ + -F title="run4" \ + http://localhost:8080/gcbm/getConfig + +``` + +Expected response (Success): + +```json +{"data":{ + "Variables":{ + "LandUnitId":-1,"LocalDomainId":1,"age":0,"age_class":0,"blockIndex":0, + "cellIndex":0,"classifier_set":{},"current_land_class":"FL","historic_land_class":"FL", + "is_decaying":true,"is_forest":true,"landUnitArea":0,"landUnitBuildSuccess":true, + "localDomainId":0,"peatlandId":-1,"regen_delay":0,"run_delay":false,"run_moss":false, + "run_peatland":false,"simulateLandUnit":true, + "spatialLocationInfo":{ + "flintdata":{"library":"internal.flint","settings":{},"type":"SpatialLocationInfo"} + }, + "spinup_moss_only":false,"tileIndex":0,"unfccc_land_class":"UNFCCC_FL_R_FL" + } + } +} +``` + + +### Get Config localdomain file + +| **Method** | **Endpoint** | **Response code** | +| ---------- | ----------------- | ----------------- | +| POST | `/gcbm/getConfig` | 200 OK (Success) | + +``` +curl -F file_name="localdomain" \ + -F title="run4" \ + http://localhost:8080/gcbm/getConfig + +``` + +Expected response (Success): + +```json +{"data":{"Libraries":{"moja.modules.cbm":"external","moja.modules.gdal":"external"}, + "LocalDomain":{"end_date":"2021/01/01","landUnitBuildSuccess":"landUnitBuildSuccess", + "landscape":{"num_threads":4,"provider":"RasterTiled", + "tile_size_x":1.0,"tile_size_y":1.0, + "tiles":[{"index":12674,"x":-106,"y":55}], + "x_pixels":4000,"y_pixels":4000}, + "sequencer":"CBMSequencer","sequencer_library":"moja.modules.cbm", + "simulateLandUnit":"simulateLandUnit","start_date":"2010/01/01", + "timing":"annual","type":"spatial_tiled"}}} +``` + ### List all of the simulations | **Method** | **Endpoint** | **Response code** | From cce284d9bfa5d13ed430c6ae0241457edb35e971 Mon Sep 17 00:00:00 2001 From: Freeman Date: Fri, 28 Oct 2022 00:13:44 +0100 Subject: [PATCH 15/21] GCBM Refactor idea Draft Signed-off-by: Freeman --- .../Endpoints/gcbm_endpoints.py | 62 +++------- .../rest_api_skeleton/Helpers/for_requests.py | 47 ++----- local/rest_api_skeleton/Helpers/preprocess.py | 117 +++++++----------- local/rest_api_skeleton/app.py | 6 +- 4 files changed, 77 insertions(+), 155 deletions(-) diff --git a/local/rest_api_skeleton/Endpoints/gcbm_endpoints.py b/local/rest_api_skeleton/Endpoints/gcbm_endpoints.py index a5582bca..46b865a0 100644 --- a/local/rest_api_skeleton/Endpoints/gcbm_endpoints.py +++ b/local/rest_api_skeleton/Endpoints/gcbm_endpoints.py @@ -1,24 +1,13 @@ -from flask_restful import Resource +from flask_restful import Resource import os from threading import Thread -from Helpers.preprocess import ( - DisturbanceConfig, - launch_run, - ClassifierConfig, - MiscellaneousConfig, -) -from Helpers.for_requests import ( - title_query, - miscellaneous_query, - disturbance_query, - classifier_query, -) - +from Helpers.preprocess import DisturbanceConfig, launch_run, ClassifierConfig, MiscellaneousConfig +from Helpers.for_requests import title_query, miscellaneous_query, disturbance_query, classifier_query class disturbance(Resource): def post(self): title = title_query().get("title") or "simulation" - input_dir = f"{os.getcwd()}/input/{title}" + input_dir = f"{os.getcwd()}/input/{title}" if not os.path.exists(f"{input_dir}"): os.makedirs(f"{input_dir}") if not os.path.exists(f"{input_dir}/disturbances"): @@ -31,17 +20,12 @@ def post(self): for file in disturbances.get("disturbances"): file.save(f"{input_dir}/disturbances/{file.filename}") try: - disturb = DisturbanceConfig( - input_dir, file.filename, disturbances.get("attributes") - ) + disturb = DisturbanceConfig(input_dir, file.filename, disturbances.get("attributes")) disturb() - except Exception as e: + except Exception as e: return e - disturb.flatten_directory("disturbances") - return { - "data": "Disturbances file uploaded succesfully. Proceed to the next step." - } - + disturb.flatten_directory("disturbances") + return {"data": "Disturbances file uploaded succesfully. Proceed to the next step."} class classifier(Resource): def post(self): @@ -60,28 +44,22 @@ def post(self): # store disturbances file in a new folder classifiers = classifier_query() if not classifiers: - return {"error": "Missing classifiers file"}, 400 + return {"error": "Missing classifiers file"}, 400 for file in classifiers.get("classifiers"): file.save(f"{input_dir}/classifiers/{file.filename}") try: - classify = ClassifierConfig( - input_dir, file.filename, classifiers.get("attributes") - ) + classify = ClassifierConfig(input_dir, file.filename, classifiers.get("attributes")) classify() except Exception as e: return e classify.flatten_directory("classifiers") - return { - "data": "Classifiers file uploaded succesfully. Proceed to the next step." - } - + return {"data": "Classifiers file uploaded succesfully. Proceed to the next step."} class Database(Resource): def post(self): pass - class miscellaneous(Resource): def post(self): # Get the title from the payload @@ -99,22 +77,17 @@ def post(self): # store miscellaneous file in a new folder mis = miscellaneous_query() if not mis: - return {"error": "Missing classifiers file"}, 400 + return {"error": "Missing classifiers file"}, 400 for file in mis.get("miscellaneous"): file.save(f"{input_dir}/miscellaneous/{file.filename}") try: - miscel = MiscellaneousConfig( - input_dir, file.filename, mis.get("attributes") - ) + miscel = MiscellaneousConfig(input_dir, file.filename, mis.get("attributes")) miscel() except Exception as e: return e miscel.flatten_directory("miscellaneous") - return { - "data": "Classifiers file uploaded succesfully. Proceed to the next step." - } - + return {"data": "Classifiers file uploaded succesfully. Proceed to the next step."} class title(Resource): def post(self): @@ -132,11 +105,9 @@ def post(self): return {"data": message}, 200 - class Run(Resource): """THIS ENDPOINT WILL BE ABLE TO RUN A SIMULATION GOAL IS TO EQUIP IT TO BE ABLE TO RUN MORE THAN ONE SIMULATIONS AT A TIME""" - def post(self): title = title_query().get("title") or "simulation" @@ -144,12 +115,11 @@ def post(self): title = "".join(c for c in title if c.isalnum()) input_dir = f"{os.getcwd()}/input/{title}" + if not os.path.exists(f"{input_dir}"): os.makedirs(f"{input_dir}") - thread = Thread( - target=launch_run, kwargs={"title": title, "input_dir": input_dir} - ) + thread = Thread(target=launch_run, kwargs={"title": title, "input_dir": input_dir}) thread.start() return {"status": "Run started"}, 200 diff --git a/local/rest_api_skeleton/Helpers/for_requests.py b/local/rest_api_skeleton/Helpers/for_requests.py index 9851264f..db53093d 100644 --- a/local/rest_api_skeleton/Helpers/for_requests.py +++ b/local/rest_api_skeleton/Helpers/for_requests.py @@ -1,47 +1,26 @@ from flask_restful import reqparse -from werkzeug.datastructures import FileStorage - +from werkzeug.datastructures import FileStorage def title_query(): query = reqparse.RequestParser() - query.add_argument("title", required=True, location="form") - return query.parse_args() - + query.add_argument("title", required=True, location="form" ) + return query.parse_args() def classifier_query(): query = reqparse.RequestParser() - query.add_argument( - "classifiers", - type=FileStorage, - required=True, - action="append", - location="files", - ) - query.add_argument("attributes", location="form") - return query.parse_args() - + query.add_argument("classifiers",type=FileStorage, required=True, action="append", location="files" ) + query.add_argument("attributes", location= "form") + return query.parse_args() def disturbance_query(): query = reqparse.RequestParser() - query.add_argument( - "disturbances", - type=FileStorage, - required=True, - action="append", - location="files", - ) - query.add_argument("attributes", location="form") - return query.parse_args() - + query.add_argument("disturbances",type=FileStorage, required=True, action="append", location="files" ) + query.add_argument("attributes", location= "form") + return query.parse_args() def miscellaneous_query(): query = reqparse.RequestParser() - query.add_argument( - "miscellaneous", - type=FileStorage, - required=True, - action="append", - location="files", - ) - query.add_argument("attributes", location="form") - return query.parse_args() + query.add_argument("miscellaneous",type=FileStorage, required=True, action="append", location="files" ) + query.add_argument("attributes", location= "form") + return query.parse_args() + diff --git a/local/rest_api_skeleton/Helpers/preprocess.py b/local/rest_api_skeleton/Helpers/preprocess.py index a496b64a..157a0081 100644 --- a/local/rest_api_skeleton/Helpers/preprocess.py +++ b/local/rest_api_skeleton/Helpers/preprocess.py @@ -3,8 +3,7 @@ import json import rasterio as rst - -def launch_run(title, input_dir): +def launch_run(title, input_dir): s = time.time() with open(f"{input_dir}/gcbm_logs.csv", "w+") as f: res = subprocess.Popen( @@ -37,43 +36,32 @@ def launch_run(title, input_dir): "response": "Operation executed successfully. Downloadable links for input and output are attached in the response. Alternatively, you may also download this simulation input and output results by making a request at gcbm/download with the title in the body.", } - class Configs: + def __init__(self, input_dir) -> None: self.input_dir = input_dir self.Rasters = [] - self.Rastersm = [] + self.Rastersm = [] self.nodatam = [] self.nodata = [] self.cellLatSize = [] self.cellLonSize = [] self.paths = [] self.lst = [] - self.provider_config = open( - f"{os.getcwd()}/templates/provider_config.json", "r+" - ) + self.provider_config = open(f"{os.getcwd()}/templates/provider_config.json", "r+") def get_config_templates(self): if not os.path.exists(f"{self.input_dir}/templates"): shutil.copytree( - f"{os.getcwd()}/templates", - f"{self.input_dir}/templates", - dirs_exist_ok=False, - ) - self.provider_config = open( - f"{self.input_dir}/templates/provider_config.json", "r+" + f"{os.getcwd()}/templates", f"{self.input_dir}/templates", dirs_exist_ok=False ) + self.provider_config = open(f"{self.input_dir}/templates/provider_config.json", "r+") def get_modules_cbm_config(self): - with open( - f"{self.input_dir}/templates/modules_cbm.json", "r+" - ) as modules_cbm_config: + with open(f"{self.input_dir}/templates/modules_cbm.json", "r+") as modules_cbm_config: data = json.load(modules_cbm_config) - disturbances = [ - file.split(".")[0][:-5] - for file in os.listdir(f"{self.input_dir}/disturbances/") - ] # drop `_moja` to match modules_cbm.json template - modules_cbm_config.seek(0) + disturbances = [file.split(".")[0][:-5] for file in os.listdir(f"{self.input_dir}/disturbances/")] # drop `_moja` to match modules_cbm.json template + modules_cbm_config.seek(0) data["Modules"]["CBMDisturbanceListener"]["settings"]["vars"] = disturbances json.dump(data, modules_cbm_config, indent=4) modules_cbm_config.truncate() @@ -82,10 +70,11 @@ def get_modules_cbm_config(self): def database_writes(self): data = json.load(self.provider_config) for file in os.listdir(f"{self.input_dir}/db/"): - data["Providers"]["SQLite"] = {"type": "SQLite", "path": file} + data["Providers"]["SQLite"] = {"type": "SQLite", "path": file } self.provider_config.seek(0) - def write_configs(self, config_type: str): + + def write_configs(self, config_type : str): data = json.load(self.provider_config) for file in os.listdir(f"{self.input_dir}/{config_type}/"): d = dict() @@ -93,7 +82,7 @@ def write_configs(self, config_type: str): d["layer_path"] = file d["layer_prefix"] = file[:-5] self.lst.append(d) - if config_type == "disturbances" or config_type == "classifiers": + if config_type == "disturbances" or config_type == "classifiers": for root, _, files in os.walk( os.path.abspath(f"{self.input_dir}/{config_type}/") ): @@ -102,15 +91,15 @@ def write_configs(self, config_type: str): self.Rasters.append(fp) self.paths.append(fp) for self.nd in self.Rasters: - img = rst.open(self.nd) - t = img.transform + img = rst.open(self.nd) + t = img.transform x = t[0] y = -t[4] - n = img.nodata + n = img.nodata self.cellLatSize.append(x) self.cellLonSize.append(y) self.nodata.append(n) - result = all(element == self.cellLatSize[0] for element in self.cellLatSize) + result = all(element == self.cellLatSize[0] for element in self.cellLatSize) if result: cellLat = x cellLon = y @@ -123,15 +112,8 @@ def write_configs(self, config_type: str): print("Corrupt files") self.provider_config.seek(0) - new_values = { - "cellLonSize": cellLon, - "cellLatSize": cellLat, - "blockLonSize": blockLon, - "blockLatSize": blockLat, - "tileLatSize": tileLat, - "tileLonSize": tileLon, - } - data["Providers"]["RasterTiled"]["layers"] = self.lst + new_values = {"cellLonSize": cellLon,"cellLatSize": cellLat, "blockLonSize": blockLon, "blockLatSize": blockLat, "tileLatSize": tileLat, "tileLonSize": tileLon} + data["Providers"]["RasterTiled"]["layers"] = self.lst data["Providers"]["RasterTiled"].update(new_values) json.dump(data, self.provider_config, indent=4) @@ -150,23 +132,21 @@ def write_configs(self, config_type: str): } self.study_area = { - "tile_size": tileLat, - "block_size": blockLat, - "tiles": [ - { - "x": int(t[2]), - "y": int(t[5]), - "index": 12674, - } - ], - "pixel_size": cellLat, - "layers": [], - } + "tile_size": tileLat, + "block_size": blockLat, + "tiles": [ + { + "x": int(t[2]), + "y": int(t[5]), + "index": 12674, + } + ], + "pixel_size": cellLat, + "layers": [], + } def add_file_to_path(self, config_type): - for root, _, files in os.walk( - os.path.abspath(f"{self.input_dir}/{config_type}/") - ): + for root, _, files in os.walk(os.path.abspath(f"{self.input_dir}/{config_type}/")): for file in files: fp = os.path.join(root, file) self.paths.append(fp) @@ -177,18 +157,15 @@ def copy_directory(self): def flatten_directory(self, config_type): shutil.rmtree((f"{self.input_dir}/{config_type}/")) - - + class DisturbanceConfig(Configs): - def __init__(self, input_dir, config_file: str, attribute: dict = None) -> None: + def __init__(self, input_dir, config_file : str, attribute : dict = None ) -> None: super().__init__(input_dir) self.config_file = config_file self.attribute = attribute def disturbances_special(self): - with open( - f"{self.input_dir}/{self.config_file}", "w", encoding="utf8" - ) as json_file: + with open(f"{self.input_dir}/{self.config_file}", "w", encoding="utf8") as json_file: self.dictionary["attributes"] = self.attribute json.dump(self.dictionary, json_file, indent=4) with open( @@ -217,17 +194,14 @@ def __call__(self): self.add_file_to_path("disturbances") self.copy_directory() - class ClassifierConfig(Configs): - def __init__(self, input_dir, config_file: str, attribute: dict) -> None: + def __init__(self, input_dir, config_file : str, attribute : dict ) -> None: super().__init__(input_dir) self.config_file = config_file self.attribute = attribute def classifier_special(self): - with open( - f"{self.input_dir}/{self.config_file}", "w", encoding="utf8" - ) as json_file: + with open(f"{self.input_dir}/{self.config_file}", "w", encoding="utf8") as json_file: self.dictionary["attributes"] = self.attribute json.dump(self.dictionary, json_file, indent=4) @@ -251,17 +225,16 @@ def __call__(self): self.add_file_to_path("classifiers") self.copy_directory() - class MiscellaneousConfig(Configs): - def __init__(self, input_dir, config_file: str, attribute: dict) -> None: + def __init__(self, input_dir, config_file : str, attribute : dict ) -> None: super().__init__(input_dir) self.config_file = config_file self.attribute = attribute - - def miscellaneous_special(self): + + def miscellaneous_special(self): for root, _, files in os.walk( os.path.abspath(f"{self.input_dir}/miscellaneous/") - ): +): for file in files: fp2 = os.path.join(root, file) self.Rastersm.append(fp2) @@ -271,11 +244,9 @@ def miscellaneous_special(self): d = img.nodata self.nodatam.append(d) """this is an experimental thing""" - with open( - f"{self.input_dir}/{self.config_file}", "w", encoding="utf8" - ) as json_file: + with open(f"{self.input_dir}/{self.config_file}", "w", encoding="utf8") as json_file: self.dictionary["layer_type"] = "GridLayer" - if self.config_file == "mean_annual_temperature_moja.json": + if self.config_file == "mean_annual_temperature_moja.json": self.dictionary["layer_data"] = "Float32" else: self.dictionary["layer_data"] = "Int16" @@ -302,3 +273,5 @@ def __call__(self): self.add_file_to_path("miscellaneous") self.copy_directory() self.flatten_directory("miscellaneous") + + diff --git a/local/rest_api_skeleton/app.py b/local/rest_api_skeleton/app.py index cd256cc9..c5502bc0 100644 --- a/local/rest_api_skeleton/app.py +++ b/local/rest_api_skeleton/app.py @@ -2,8 +2,8 @@ from flask_restful import Api from Endpoints.gcbm_endpoints import disturbance, Run, title, classifier, miscellaneous -app = Flask(__name__) -api = Api() +app = Flask(__name__) +api = Api() api.add_resource(disturbance, "/gcbm/upload/disturbances") api.add_resource(title, "/gcbm/create") @@ -12,5 +12,5 @@ api.add_resource(Run, "/gcbm/run") -if __name__ == "__main__": +if __name__ == "__main__": app.run(debug=True) From b93e51800a94ad453ae92676a38a6fe57208bf2b Mon Sep 17 00:00:00 2001 From: Freeman Date: Fri, 28 Oct 2022 00:41:12 +0100 Subject: [PATCH 16/21] GCBM Refactor idea Draft Signed-off-by: Freeman --- .../Endpoints/gcbm_endpoints.py | 62 ++++++++++++++----- 1 file changed, 46 insertions(+), 16 deletions(-) diff --git a/local/rest_api_skeleton/Endpoints/gcbm_endpoints.py b/local/rest_api_skeleton/Endpoints/gcbm_endpoints.py index 46b865a0..a5582bca 100644 --- a/local/rest_api_skeleton/Endpoints/gcbm_endpoints.py +++ b/local/rest_api_skeleton/Endpoints/gcbm_endpoints.py @@ -1,13 +1,24 @@ -from flask_restful import Resource +from flask_restful import Resource import os from threading import Thread -from Helpers.preprocess import DisturbanceConfig, launch_run, ClassifierConfig, MiscellaneousConfig -from Helpers.for_requests import title_query, miscellaneous_query, disturbance_query, classifier_query +from Helpers.preprocess import ( + DisturbanceConfig, + launch_run, + ClassifierConfig, + MiscellaneousConfig, +) +from Helpers.for_requests import ( + title_query, + miscellaneous_query, + disturbance_query, + classifier_query, +) + class disturbance(Resource): def post(self): title = title_query().get("title") or "simulation" - input_dir = f"{os.getcwd()}/input/{title}" + input_dir = f"{os.getcwd()}/input/{title}" if not os.path.exists(f"{input_dir}"): os.makedirs(f"{input_dir}") if not os.path.exists(f"{input_dir}/disturbances"): @@ -20,12 +31,17 @@ def post(self): for file in disturbances.get("disturbances"): file.save(f"{input_dir}/disturbances/{file.filename}") try: - disturb = DisturbanceConfig(input_dir, file.filename, disturbances.get("attributes")) + disturb = DisturbanceConfig( + input_dir, file.filename, disturbances.get("attributes") + ) disturb() - except Exception as e: + except Exception as e: return e - disturb.flatten_directory("disturbances") - return {"data": "Disturbances file uploaded succesfully. Proceed to the next step."} + disturb.flatten_directory("disturbances") + return { + "data": "Disturbances file uploaded succesfully. Proceed to the next step." + } + class classifier(Resource): def post(self): @@ -44,22 +60,28 @@ def post(self): # store disturbances file in a new folder classifiers = classifier_query() if not classifiers: - return {"error": "Missing classifiers file"}, 400 + return {"error": "Missing classifiers file"}, 400 for file in classifiers.get("classifiers"): file.save(f"{input_dir}/classifiers/{file.filename}") try: - classify = ClassifierConfig(input_dir, file.filename, classifiers.get("attributes")) + classify = ClassifierConfig( + input_dir, file.filename, classifiers.get("attributes") + ) classify() except Exception as e: return e classify.flatten_directory("classifiers") - return {"data": "Classifiers file uploaded succesfully. Proceed to the next step."} + return { + "data": "Classifiers file uploaded succesfully. Proceed to the next step." + } + class Database(Resource): def post(self): pass + class miscellaneous(Resource): def post(self): # Get the title from the payload @@ -77,17 +99,22 @@ def post(self): # store miscellaneous file in a new folder mis = miscellaneous_query() if not mis: - return {"error": "Missing classifiers file"}, 400 + return {"error": "Missing classifiers file"}, 400 for file in mis.get("miscellaneous"): file.save(f"{input_dir}/miscellaneous/{file.filename}") try: - miscel = MiscellaneousConfig(input_dir, file.filename, mis.get("attributes")) + miscel = MiscellaneousConfig( + input_dir, file.filename, mis.get("attributes") + ) miscel() except Exception as e: return e miscel.flatten_directory("miscellaneous") - return {"data": "Classifiers file uploaded succesfully. Proceed to the next step."} + return { + "data": "Classifiers file uploaded succesfully. Proceed to the next step." + } + class title(Resource): def post(self): @@ -105,9 +132,11 @@ def post(self): return {"data": message}, 200 + class Run(Resource): """THIS ENDPOINT WILL BE ABLE TO RUN A SIMULATION GOAL IS TO EQUIP IT TO BE ABLE TO RUN MORE THAN ONE SIMULATIONS AT A TIME""" + def post(self): title = title_query().get("title") or "simulation" @@ -115,11 +144,12 @@ def post(self): title = "".join(c for c in title if c.isalnum()) input_dir = f"{os.getcwd()}/input/{title}" - if not os.path.exists(f"{input_dir}"): os.makedirs(f"{input_dir}") - thread = Thread(target=launch_run, kwargs={"title": title, "input_dir": input_dir}) + thread = Thread( + target=launch_run, kwargs={"title": title, "input_dir": input_dir} + ) thread.start() return {"status": "Run started"}, 200 From 0c6092c0f05c87da1781163e11974305a49d70f7 Mon Sep 17 00:00:00 2001 From: Freeman Date: Fri, 28 Oct 2022 00:43:25 +0100 Subject: [PATCH 17/21] GCBM Refactor idea Draft Signed-off-by: Freeman --- .../rest_api_skeleton/Helpers/for_requests.py | 47 +++++-- local/rest_api_skeleton/Helpers/preprocess.py | 117 +++++++++++------- local/rest_api_skeleton/app.py | 6 +- 3 files changed, 109 insertions(+), 61 deletions(-) diff --git a/local/rest_api_skeleton/Helpers/for_requests.py b/local/rest_api_skeleton/Helpers/for_requests.py index db53093d..9851264f 100644 --- a/local/rest_api_skeleton/Helpers/for_requests.py +++ b/local/rest_api_skeleton/Helpers/for_requests.py @@ -1,26 +1,47 @@ from flask_restful import reqparse -from werkzeug.datastructures import FileStorage +from werkzeug.datastructures import FileStorage + def title_query(): query = reqparse.RequestParser() - query.add_argument("title", required=True, location="form" ) - return query.parse_args() + query.add_argument("title", required=True, location="form") + return query.parse_args() + def classifier_query(): query = reqparse.RequestParser() - query.add_argument("classifiers",type=FileStorage, required=True, action="append", location="files" ) - query.add_argument("attributes", location= "form") - return query.parse_args() + query.add_argument( + "classifiers", + type=FileStorage, + required=True, + action="append", + location="files", + ) + query.add_argument("attributes", location="form") + return query.parse_args() + def disturbance_query(): query = reqparse.RequestParser() - query.add_argument("disturbances",type=FileStorage, required=True, action="append", location="files" ) - query.add_argument("attributes", location= "form") - return query.parse_args() + query.add_argument( + "disturbances", + type=FileStorage, + required=True, + action="append", + location="files", + ) + query.add_argument("attributes", location="form") + return query.parse_args() + def miscellaneous_query(): query = reqparse.RequestParser() - query.add_argument("miscellaneous",type=FileStorage, required=True, action="append", location="files" ) - query.add_argument("attributes", location= "form") - return query.parse_args() - + query.add_argument( + "miscellaneous", + type=FileStorage, + required=True, + action="append", + location="files", + ) + query.add_argument("attributes", location="form") + return query.parse_args() diff --git a/local/rest_api_skeleton/Helpers/preprocess.py b/local/rest_api_skeleton/Helpers/preprocess.py index 157a0081..a496b64a 100644 --- a/local/rest_api_skeleton/Helpers/preprocess.py +++ b/local/rest_api_skeleton/Helpers/preprocess.py @@ -3,7 +3,8 @@ import json import rasterio as rst -def launch_run(title, input_dir): + +def launch_run(title, input_dir): s = time.time() with open(f"{input_dir}/gcbm_logs.csv", "w+") as f: res = subprocess.Popen( @@ -36,32 +37,43 @@ def launch_run(title, input_dir): "response": "Operation executed successfully. Downloadable links for input and output are attached in the response. Alternatively, you may also download this simulation input and output results by making a request at gcbm/download with the title in the body.", } -class Configs: +class Configs: def __init__(self, input_dir) -> None: self.input_dir = input_dir self.Rasters = [] - self.Rastersm = [] + self.Rastersm = [] self.nodatam = [] self.nodata = [] self.cellLatSize = [] self.cellLonSize = [] self.paths = [] self.lst = [] - self.provider_config = open(f"{os.getcwd()}/templates/provider_config.json", "r+") + self.provider_config = open( + f"{os.getcwd()}/templates/provider_config.json", "r+" + ) def get_config_templates(self): if not os.path.exists(f"{self.input_dir}/templates"): shutil.copytree( - f"{os.getcwd()}/templates", f"{self.input_dir}/templates", dirs_exist_ok=False + f"{os.getcwd()}/templates", + f"{self.input_dir}/templates", + dirs_exist_ok=False, + ) + self.provider_config = open( + f"{self.input_dir}/templates/provider_config.json", "r+" ) - self.provider_config = open(f"{self.input_dir}/templates/provider_config.json", "r+") def get_modules_cbm_config(self): - with open(f"{self.input_dir}/templates/modules_cbm.json", "r+") as modules_cbm_config: + with open( + f"{self.input_dir}/templates/modules_cbm.json", "r+" + ) as modules_cbm_config: data = json.load(modules_cbm_config) - disturbances = [file.split(".")[0][:-5] for file in os.listdir(f"{self.input_dir}/disturbances/")] # drop `_moja` to match modules_cbm.json template - modules_cbm_config.seek(0) + disturbances = [ + file.split(".")[0][:-5] + for file in os.listdir(f"{self.input_dir}/disturbances/") + ] # drop `_moja` to match modules_cbm.json template + modules_cbm_config.seek(0) data["Modules"]["CBMDisturbanceListener"]["settings"]["vars"] = disturbances json.dump(data, modules_cbm_config, indent=4) modules_cbm_config.truncate() @@ -70,11 +82,10 @@ def get_modules_cbm_config(self): def database_writes(self): data = json.load(self.provider_config) for file in os.listdir(f"{self.input_dir}/db/"): - data["Providers"]["SQLite"] = {"type": "SQLite", "path": file } + data["Providers"]["SQLite"] = {"type": "SQLite", "path": file} self.provider_config.seek(0) - - def write_configs(self, config_type : str): + def write_configs(self, config_type: str): data = json.load(self.provider_config) for file in os.listdir(f"{self.input_dir}/{config_type}/"): d = dict() @@ -82,7 +93,7 @@ def write_configs(self, config_type : str): d["layer_path"] = file d["layer_prefix"] = file[:-5] self.lst.append(d) - if config_type == "disturbances" or config_type == "classifiers": + if config_type == "disturbances" or config_type == "classifiers": for root, _, files in os.walk( os.path.abspath(f"{self.input_dir}/{config_type}/") ): @@ -91,15 +102,15 @@ def write_configs(self, config_type : str): self.Rasters.append(fp) self.paths.append(fp) for self.nd in self.Rasters: - img = rst.open(self.nd) - t = img.transform + img = rst.open(self.nd) + t = img.transform x = t[0] y = -t[4] - n = img.nodata + n = img.nodata self.cellLatSize.append(x) self.cellLonSize.append(y) self.nodata.append(n) - result = all(element == self.cellLatSize[0] for element in self.cellLatSize) + result = all(element == self.cellLatSize[0] for element in self.cellLatSize) if result: cellLat = x cellLon = y @@ -112,8 +123,15 @@ def write_configs(self, config_type : str): print("Corrupt files") self.provider_config.seek(0) - new_values = {"cellLonSize": cellLon,"cellLatSize": cellLat, "blockLonSize": blockLon, "blockLatSize": blockLat, "tileLatSize": tileLat, "tileLonSize": tileLon} - data["Providers"]["RasterTiled"]["layers"] = self.lst + new_values = { + "cellLonSize": cellLon, + "cellLatSize": cellLat, + "blockLonSize": blockLon, + "blockLatSize": blockLat, + "tileLatSize": tileLat, + "tileLonSize": tileLon, + } + data["Providers"]["RasterTiled"]["layers"] = self.lst data["Providers"]["RasterTiled"].update(new_values) json.dump(data, self.provider_config, indent=4) @@ -132,21 +150,23 @@ def write_configs(self, config_type : str): } self.study_area = { - "tile_size": tileLat, - "block_size": blockLat, - "tiles": [ - { - "x": int(t[2]), - "y": int(t[5]), - "index": 12674, - } - ], - "pixel_size": cellLat, - "layers": [], - } + "tile_size": tileLat, + "block_size": blockLat, + "tiles": [ + { + "x": int(t[2]), + "y": int(t[5]), + "index": 12674, + } + ], + "pixel_size": cellLat, + "layers": [], + } def add_file_to_path(self, config_type): - for root, _, files in os.walk(os.path.abspath(f"{self.input_dir}/{config_type}/")): + for root, _, files in os.walk( + os.path.abspath(f"{self.input_dir}/{config_type}/") + ): for file in files: fp = os.path.join(root, file) self.paths.append(fp) @@ -157,15 +177,18 @@ def copy_directory(self): def flatten_directory(self, config_type): shutil.rmtree((f"{self.input_dir}/{config_type}/")) - + + class DisturbanceConfig(Configs): - def __init__(self, input_dir, config_file : str, attribute : dict = None ) -> None: + def __init__(self, input_dir, config_file: str, attribute: dict = None) -> None: super().__init__(input_dir) self.config_file = config_file self.attribute = attribute def disturbances_special(self): - with open(f"{self.input_dir}/{self.config_file}", "w", encoding="utf8") as json_file: + with open( + f"{self.input_dir}/{self.config_file}", "w", encoding="utf8" + ) as json_file: self.dictionary["attributes"] = self.attribute json.dump(self.dictionary, json_file, indent=4) with open( @@ -194,14 +217,17 @@ def __call__(self): self.add_file_to_path("disturbances") self.copy_directory() + class ClassifierConfig(Configs): - def __init__(self, input_dir, config_file : str, attribute : dict ) -> None: + def __init__(self, input_dir, config_file: str, attribute: dict) -> None: super().__init__(input_dir) self.config_file = config_file self.attribute = attribute def classifier_special(self): - with open(f"{self.input_dir}/{self.config_file}", "w", encoding="utf8") as json_file: + with open( + f"{self.input_dir}/{self.config_file}", "w", encoding="utf8" + ) as json_file: self.dictionary["attributes"] = self.attribute json.dump(self.dictionary, json_file, indent=4) @@ -225,16 +251,17 @@ def __call__(self): self.add_file_to_path("classifiers") self.copy_directory() + class MiscellaneousConfig(Configs): - def __init__(self, input_dir, config_file : str, attribute : dict ) -> None: + def __init__(self, input_dir, config_file: str, attribute: dict) -> None: super().__init__(input_dir) self.config_file = config_file self.attribute = attribute - - def miscellaneous_special(self): + + def miscellaneous_special(self): for root, _, files in os.walk( os.path.abspath(f"{self.input_dir}/miscellaneous/") -): + ): for file in files: fp2 = os.path.join(root, file) self.Rastersm.append(fp2) @@ -244,9 +271,11 @@ def miscellaneous_special(self): d = img.nodata self.nodatam.append(d) """this is an experimental thing""" - with open(f"{self.input_dir}/{self.config_file}", "w", encoding="utf8") as json_file: + with open( + f"{self.input_dir}/{self.config_file}", "w", encoding="utf8" + ) as json_file: self.dictionary["layer_type"] = "GridLayer" - if self.config_file == "mean_annual_temperature_moja.json": + if self.config_file == "mean_annual_temperature_moja.json": self.dictionary["layer_data"] = "Float32" else: self.dictionary["layer_data"] = "Int16" @@ -273,5 +302,3 @@ def __call__(self): self.add_file_to_path("miscellaneous") self.copy_directory() self.flatten_directory("miscellaneous") - - diff --git a/local/rest_api_skeleton/app.py b/local/rest_api_skeleton/app.py index c5502bc0..cd256cc9 100644 --- a/local/rest_api_skeleton/app.py +++ b/local/rest_api_skeleton/app.py @@ -2,8 +2,8 @@ from flask_restful import Api from Endpoints.gcbm_endpoints import disturbance, Run, title, classifier, miscellaneous -app = Flask(__name__) -api = Api() +app = Flask(__name__) +api = Api() api.add_resource(disturbance, "/gcbm/upload/disturbances") api.add_resource(title, "/gcbm/create") @@ -12,5 +12,5 @@ api.add_resource(Run, "/gcbm/run") -if __name__ == "__main__": +if __name__ == "__main__": app.run(debug=True) From 0f6000c5236cb0c7bec9c172ccbe3a0abaa20c28 Mon Sep 17 00:00:00 2001 From: Freeman Date: Fri, 28 Oct 2022 00:13:44 +0100 Subject: [PATCH 18/21] GCBM Refactor idea Draft --- .../Endpoints/gcbm_endpoints.py | 62 +++------- .../rest_api_skeleton/Helpers/for_requests.py | 47 ++----- local/rest_api_skeleton/Helpers/preprocess.py | 117 +++++++----------- local/rest_api_skeleton/app.py | 6 +- 4 files changed, 77 insertions(+), 155 deletions(-) diff --git a/local/rest_api_skeleton/Endpoints/gcbm_endpoints.py b/local/rest_api_skeleton/Endpoints/gcbm_endpoints.py index a5582bca..46b865a0 100644 --- a/local/rest_api_skeleton/Endpoints/gcbm_endpoints.py +++ b/local/rest_api_skeleton/Endpoints/gcbm_endpoints.py @@ -1,24 +1,13 @@ -from flask_restful import Resource +from flask_restful import Resource import os from threading import Thread -from Helpers.preprocess import ( - DisturbanceConfig, - launch_run, - ClassifierConfig, - MiscellaneousConfig, -) -from Helpers.for_requests import ( - title_query, - miscellaneous_query, - disturbance_query, - classifier_query, -) - +from Helpers.preprocess import DisturbanceConfig, launch_run, ClassifierConfig, MiscellaneousConfig +from Helpers.for_requests import title_query, miscellaneous_query, disturbance_query, classifier_query class disturbance(Resource): def post(self): title = title_query().get("title") or "simulation" - input_dir = f"{os.getcwd()}/input/{title}" + input_dir = f"{os.getcwd()}/input/{title}" if not os.path.exists(f"{input_dir}"): os.makedirs(f"{input_dir}") if not os.path.exists(f"{input_dir}/disturbances"): @@ -31,17 +20,12 @@ def post(self): for file in disturbances.get("disturbances"): file.save(f"{input_dir}/disturbances/{file.filename}") try: - disturb = DisturbanceConfig( - input_dir, file.filename, disturbances.get("attributes") - ) + disturb = DisturbanceConfig(input_dir, file.filename, disturbances.get("attributes")) disturb() - except Exception as e: + except Exception as e: return e - disturb.flatten_directory("disturbances") - return { - "data": "Disturbances file uploaded succesfully. Proceed to the next step." - } - + disturb.flatten_directory("disturbances") + return {"data": "Disturbances file uploaded succesfully. Proceed to the next step."} class classifier(Resource): def post(self): @@ -60,28 +44,22 @@ def post(self): # store disturbances file in a new folder classifiers = classifier_query() if not classifiers: - return {"error": "Missing classifiers file"}, 400 + return {"error": "Missing classifiers file"}, 400 for file in classifiers.get("classifiers"): file.save(f"{input_dir}/classifiers/{file.filename}") try: - classify = ClassifierConfig( - input_dir, file.filename, classifiers.get("attributes") - ) + classify = ClassifierConfig(input_dir, file.filename, classifiers.get("attributes")) classify() except Exception as e: return e classify.flatten_directory("classifiers") - return { - "data": "Classifiers file uploaded succesfully. Proceed to the next step." - } - + return {"data": "Classifiers file uploaded succesfully. Proceed to the next step."} class Database(Resource): def post(self): pass - class miscellaneous(Resource): def post(self): # Get the title from the payload @@ -99,22 +77,17 @@ def post(self): # store miscellaneous file in a new folder mis = miscellaneous_query() if not mis: - return {"error": "Missing classifiers file"}, 400 + return {"error": "Missing classifiers file"}, 400 for file in mis.get("miscellaneous"): file.save(f"{input_dir}/miscellaneous/{file.filename}") try: - miscel = MiscellaneousConfig( - input_dir, file.filename, mis.get("attributes") - ) + miscel = MiscellaneousConfig(input_dir, file.filename, mis.get("attributes")) miscel() except Exception as e: return e miscel.flatten_directory("miscellaneous") - return { - "data": "Classifiers file uploaded succesfully. Proceed to the next step." - } - + return {"data": "Classifiers file uploaded succesfully. Proceed to the next step."} class title(Resource): def post(self): @@ -132,11 +105,9 @@ def post(self): return {"data": message}, 200 - class Run(Resource): """THIS ENDPOINT WILL BE ABLE TO RUN A SIMULATION GOAL IS TO EQUIP IT TO BE ABLE TO RUN MORE THAN ONE SIMULATIONS AT A TIME""" - def post(self): title = title_query().get("title") or "simulation" @@ -144,12 +115,11 @@ def post(self): title = "".join(c for c in title if c.isalnum()) input_dir = f"{os.getcwd()}/input/{title}" + if not os.path.exists(f"{input_dir}"): os.makedirs(f"{input_dir}") - thread = Thread( - target=launch_run, kwargs={"title": title, "input_dir": input_dir} - ) + thread = Thread(target=launch_run, kwargs={"title": title, "input_dir": input_dir}) thread.start() return {"status": "Run started"}, 200 diff --git a/local/rest_api_skeleton/Helpers/for_requests.py b/local/rest_api_skeleton/Helpers/for_requests.py index 9851264f..db53093d 100644 --- a/local/rest_api_skeleton/Helpers/for_requests.py +++ b/local/rest_api_skeleton/Helpers/for_requests.py @@ -1,47 +1,26 @@ from flask_restful import reqparse -from werkzeug.datastructures import FileStorage - +from werkzeug.datastructures import FileStorage def title_query(): query = reqparse.RequestParser() - query.add_argument("title", required=True, location="form") - return query.parse_args() - + query.add_argument("title", required=True, location="form" ) + return query.parse_args() def classifier_query(): query = reqparse.RequestParser() - query.add_argument( - "classifiers", - type=FileStorage, - required=True, - action="append", - location="files", - ) - query.add_argument("attributes", location="form") - return query.parse_args() - + query.add_argument("classifiers",type=FileStorage, required=True, action="append", location="files" ) + query.add_argument("attributes", location= "form") + return query.parse_args() def disturbance_query(): query = reqparse.RequestParser() - query.add_argument( - "disturbances", - type=FileStorage, - required=True, - action="append", - location="files", - ) - query.add_argument("attributes", location="form") - return query.parse_args() - + query.add_argument("disturbances",type=FileStorage, required=True, action="append", location="files" ) + query.add_argument("attributes", location= "form") + return query.parse_args() def miscellaneous_query(): query = reqparse.RequestParser() - query.add_argument( - "miscellaneous", - type=FileStorage, - required=True, - action="append", - location="files", - ) - query.add_argument("attributes", location="form") - return query.parse_args() + query.add_argument("miscellaneous",type=FileStorage, required=True, action="append", location="files" ) + query.add_argument("attributes", location= "form") + return query.parse_args() + diff --git a/local/rest_api_skeleton/Helpers/preprocess.py b/local/rest_api_skeleton/Helpers/preprocess.py index a496b64a..157a0081 100644 --- a/local/rest_api_skeleton/Helpers/preprocess.py +++ b/local/rest_api_skeleton/Helpers/preprocess.py @@ -3,8 +3,7 @@ import json import rasterio as rst - -def launch_run(title, input_dir): +def launch_run(title, input_dir): s = time.time() with open(f"{input_dir}/gcbm_logs.csv", "w+") as f: res = subprocess.Popen( @@ -37,43 +36,32 @@ def launch_run(title, input_dir): "response": "Operation executed successfully. Downloadable links for input and output are attached in the response. Alternatively, you may also download this simulation input and output results by making a request at gcbm/download with the title in the body.", } - class Configs: + def __init__(self, input_dir) -> None: self.input_dir = input_dir self.Rasters = [] - self.Rastersm = [] + self.Rastersm = [] self.nodatam = [] self.nodata = [] self.cellLatSize = [] self.cellLonSize = [] self.paths = [] self.lst = [] - self.provider_config = open( - f"{os.getcwd()}/templates/provider_config.json", "r+" - ) + self.provider_config = open(f"{os.getcwd()}/templates/provider_config.json", "r+") def get_config_templates(self): if not os.path.exists(f"{self.input_dir}/templates"): shutil.copytree( - f"{os.getcwd()}/templates", - f"{self.input_dir}/templates", - dirs_exist_ok=False, - ) - self.provider_config = open( - f"{self.input_dir}/templates/provider_config.json", "r+" + f"{os.getcwd()}/templates", f"{self.input_dir}/templates", dirs_exist_ok=False ) + self.provider_config = open(f"{self.input_dir}/templates/provider_config.json", "r+") def get_modules_cbm_config(self): - with open( - f"{self.input_dir}/templates/modules_cbm.json", "r+" - ) as modules_cbm_config: + with open(f"{self.input_dir}/templates/modules_cbm.json", "r+") as modules_cbm_config: data = json.load(modules_cbm_config) - disturbances = [ - file.split(".")[0][:-5] - for file in os.listdir(f"{self.input_dir}/disturbances/") - ] # drop `_moja` to match modules_cbm.json template - modules_cbm_config.seek(0) + disturbances = [file.split(".")[0][:-5] for file in os.listdir(f"{self.input_dir}/disturbances/")] # drop `_moja` to match modules_cbm.json template + modules_cbm_config.seek(0) data["Modules"]["CBMDisturbanceListener"]["settings"]["vars"] = disturbances json.dump(data, modules_cbm_config, indent=4) modules_cbm_config.truncate() @@ -82,10 +70,11 @@ def get_modules_cbm_config(self): def database_writes(self): data = json.load(self.provider_config) for file in os.listdir(f"{self.input_dir}/db/"): - data["Providers"]["SQLite"] = {"type": "SQLite", "path": file} + data["Providers"]["SQLite"] = {"type": "SQLite", "path": file } self.provider_config.seek(0) - def write_configs(self, config_type: str): + + def write_configs(self, config_type : str): data = json.load(self.provider_config) for file in os.listdir(f"{self.input_dir}/{config_type}/"): d = dict() @@ -93,7 +82,7 @@ def write_configs(self, config_type: str): d["layer_path"] = file d["layer_prefix"] = file[:-5] self.lst.append(d) - if config_type == "disturbances" or config_type == "classifiers": + if config_type == "disturbances" or config_type == "classifiers": for root, _, files in os.walk( os.path.abspath(f"{self.input_dir}/{config_type}/") ): @@ -102,15 +91,15 @@ def write_configs(self, config_type: str): self.Rasters.append(fp) self.paths.append(fp) for self.nd in self.Rasters: - img = rst.open(self.nd) - t = img.transform + img = rst.open(self.nd) + t = img.transform x = t[0] y = -t[4] - n = img.nodata + n = img.nodata self.cellLatSize.append(x) self.cellLonSize.append(y) self.nodata.append(n) - result = all(element == self.cellLatSize[0] for element in self.cellLatSize) + result = all(element == self.cellLatSize[0] for element in self.cellLatSize) if result: cellLat = x cellLon = y @@ -123,15 +112,8 @@ def write_configs(self, config_type: str): print("Corrupt files") self.provider_config.seek(0) - new_values = { - "cellLonSize": cellLon, - "cellLatSize": cellLat, - "blockLonSize": blockLon, - "blockLatSize": blockLat, - "tileLatSize": tileLat, - "tileLonSize": tileLon, - } - data["Providers"]["RasterTiled"]["layers"] = self.lst + new_values = {"cellLonSize": cellLon,"cellLatSize": cellLat, "blockLonSize": blockLon, "blockLatSize": blockLat, "tileLatSize": tileLat, "tileLonSize": tileLon} + data["Providers"]["RasterTiled"]["layers"] = self.lst data["Providers"]["RasterTiled"].update(new_values) json.dump(data, self.provider_config, indent=4) @@ -150,23 +132,21 @@ def write_configs(self, config_type: str): } self.study_area = { - "tile_size": tileLat, - "block_size": blockLat, - "tiles": [ - { - "x": int(t[2]), - "y": int(t[5]), - "index": 12674, - } - ], - "pixel_size": cellLat, - "layers": [], - } + "tile_size": tileLat, + "block_size": blockLat, + "tiles": [ + { + "x": int(t[2]), + "y": int(t[5]), + "index": 12674, + } + ], + "pixel_size": cellLat, + "layers": [], + } def add_file_to_path(self, config_type): - for root, _, files in os.walk( - os.path.abspath(f"{self.input_dir}/{config_type}/") - ): + for root, _, files in os.walk(os.path.abspath(f"{self.input_dir}/{config_type}/")): for file in files: fp = os.path.join(root, file) self.paths.append(fp) @@ -177,18 +157,15 @@ def copy_directory(self): def flatten_directory(self, config_type): shutil.rmtree((f"{self.input_dir}/{config_type}/")) - - + class DisturbanceConfig(Configs): - def __init__(self, input_dir, config_file: str, attribute: dict = None) -> None: + def __init__(self, input_dir, config_file : str, attribute : dict = None ) -> None: super().__init__(input_dir) self.config_file = config_file self.attribute = attribute def disturbances_special(self): - with open( - f"{self.input_dir}/{self.config_file}", "w", encoding="utf8" - ) as json_file: + with open(f"{self.input_dir}/{self.config_file}", "w", encoding="utf8") as json_file: self.dictionary["attributes"] = self.attribute json.dump(self.dictionary, json_file, indent=4) with open( @@ -217,17 +194,14 @@ def __call__(self): self.add_file_to_path("disturbances") self.copy_directory() - class ClassifierConfig(Configs): - def __init__(self, input_dir, config_file: str, attribute: dict) -> None: + def __init__(self, input_dir, config_file : str, attribute : dict ) -> None: super().__init__(input_dir) self.config_file = config_file self.attribute = attribute def classifier_special(self): - with open( - f"{self.input_dir}/{self.config_file}", "w", encoding="utf8" - ) as json_file: + with open(f"{self.input_dir}/{self.config_file}", "w", encoding="utf8") as json_file: self.dictionary["attributes"] = self.attribute json.dump(self.dictionary, json_file, indent=4) @@ -251,17 +225,16 @@ def __call__(self): self.add_file_to_path("classifiers") self.copy_directory() - class MiscellaneousConfig(Configs): - def __init__(self, input_dir, config_file: str, attribute: dict) -> None: + def __init__(self, input_dir, config_file : str, attribute : dict ) -> None: super().__init__(input_dir) self.config_file = config_file self.attribute = attribute - - def miscellaneous_special(self): + + def miscellaneous_special(self): for root, _, files in os.walk( os.path.abspath(f"{self.input_dir}/miscellaneous/") - ): +): for file in files: fp2 = os.path.join(root, file) self.Rastersm.append(fp2) @@ -271,11 +244,9 @@ def miscellaneous_special(self): d = img.nodata self.nodatam.append(d) """this is an experimental thing""" - with open( - f"{self.input_dir}/{self.config_file}", "w", encoding="utf8" - ) as json_file: + with open(f"{self.input_dir}/{self.config_file}", "w", encoding="utf8") as json_file: self.dictionary["layer_type"] = "GridLayer" - if self.config_file == "mean_annual_temperature_moja.json": + if self.config_file == "mean_annual_temperature_moja.json": self.dictionary["layer_data"] = "Float32" else: self.dictionary["layer_data"] = "Int16" @@ -302,3 +273,5 @@ def __call__(self): self.add_file_to_path("miscellaneous") self.copy_directory() self.flatten_directory("miscellaneous") + + diff --git a/local/rest_api_skeleton/app.py b/local/rest_api_skeleton/app.py index cd256cc9..c5502bc0 100644 --- a/local/rest_api_skeleton/app.py +++ b/local/rest_api_skeleton/app.py @@ -2,8 +2,8 @@ from flask_restful import Api from Endpoints.gcbm_endpoints import disturbance, Run, title, classifier, miscellaneous -app = Flask(__name__) -api = Api() +app = Flask(__name__) +api = Api() api.add_resource(disturbance, "/gcbm/upload/disturbances") api.add_resource(title, "/gcbm/create") @@ -12,5 +12,5 @@ api.add_resource(Run, "/gcbm/run") -if __name__ == "__main__": +if __name__ == "__main__": app.run(debug=True) From 674b919762dca571a09b15597a4f181c7291ddc8 Mon Sep 17 00:00:00 2001 From: Freeman Date: Fri, 28 Oct 2022 00:41:12 +0100 Subject: [PATCH 19/21] GCBM Refactor idea Draft Signed-off-by: Freeman --- .../Endpoints/gcbm_endpoints.py | 62 ++++++++++++++----- 1 file changed, 46 insertions(+), 16 deletions(-) diff --git a/local/rest_api_skeleton/Endpoints/gcbm_endpoints.py b/local/rest_api_skeleton/Endpoints/gcbm_endpoints.py index 46b865a0..a5582bca 100644 --- a/local/rest_api_skeleton/Endpoints/gcbm_endpoints.py +++ b/local/rest_api_skeleton/Endpoints/gcbm_endpoints.py @@ -1,13 +1,24 @@ -from flask_restful import Resource +from flask_restful import Resource import os from threading import Thread -from Helpers.preprocess import DisturbanceConfig, launch_run, ClassifierConfig, MiscellaneousConfig -from Helpers.for_requests import title_query, miscellaneous_query, disturbance_query, classifier_query +from Helpers.preprocess import ( + DisturbanceConfig, + launch_run, + ClassifierConfig, + MiscellaneousConfig, +) +from Helpers.for_requests import ( + title_query, + miscellaneous_query, + disturbance_query, + classifier_query, +) + class disturbance(Resource): def post(self): title = title_query().get("title") or "simulation" - input_dir = f"{os.getcwd()}/input/{title}" + input_dir = f"{os.getcwd()}/input/{title}" if not os.path.exists(f"{input_dir}"): os.makedirs(f"{input_dir}") if not os.path.exists(f"{input_dir}/disturbances"): @@ -20,12 +31,17 @@ def post(self): for file in disturbances.get("disturbances"): file.save(f"{input_dir}/disturbances/{file.filename}") try: - disturb = DisturbanceConfig(input_dir, file.filename, disturbances.get("attributes")) + disturb = DisturbanceConfig( + input_dir, file.filename, disturbances.get("attributes") + ) disturb() - except Exception as e: + except Exception as e: return e - disturb.flatten_directory("disturbances") - return {"data": "Disturbances file uploaded succesfully. Proceed to the next step."} + disturb.flatten_directory("disturbances") + return { + "data": "Disturbances file uploaded succesfully. Proceed to the next step." + } + class classifier(Resource): def post(self): @@ -44,22 +60,28 @@ def post(self): # store disturbances file in a new folder classifiers = classifier_query() if not classifiers: - return {"error": "Missing classifiers file"}, 400 + return {"error": "Missing classifiers file"}, 400 for file in classifiers.get("classifiers"): file.save(f"{input_dir}/classifiers/{file.filename}") try: - classify = ClassifierConfig(input_dir, file.filename, classifiers.get("attributes")) + classify = ClassifierConfig( + input_dir, file.filename, classifiers.get("attributes") + ) classify() except Exception as e: return e classify.flatten_directory("classifiers") - return {"data": "Classifiers file uploaded succesfully. Proceed to the next step."} + return { + "data": "Classifiers file uploaded succesfully. Proceed to the next step." + } + class Database(Resource): def post(self): pass + class miscellaneous(Resource): def post(self): # Get the title from the payload @@ -77,17 +99,22 @@ def post(self): # store miscellaneous file in a new folder mis = miscellaneous_query() if not mis: - return {"error": "Missing classifiers file"}, 400 + return {"error": "Missing classifiers file"}, 400 for file in mis.get("miscellaneous"): file.save(f"{input_dir}/miscellaneous/{file.filename}") try: - miscel = MiscellaneousConfig(input_dir, file.filename, mis.get("attributes")) + miscel = MiscellaneousConfig( + input_dir, file.filename, mis.get("attributes") + ) miscel() except Exception as e: return e miscel.flatten_directory("miscellaneous") - return {"data": "Classifiers file uploaded succesfully. Proceed to the next step."} + return { + "data": "Classifiers file uploaded succesfully. Proceed to the next step." + } + class title(Resource): def post(self): @@ -105,9 +132,11 @@ def post(self): return {"data": message}, 200 + class Run(Resource): """THIS ENDPOINT WILL BE ABLE TO RUN A SIMULATION GOAL IS TO EQUIP IT TO BE ABLE TO RUN MORE THAN ONE SIMULATIONS AT A TIME""" + def post(self): title = title_query().get("title") or "simulation" @@ -115,11 +144,12 @@ def post(self): title = "".join(c for c in title if c.isalnum()) input_dir = f"{os.getcwd()}/input/{title}" - if not os.path.exists(f"{input_dir}"): os.makedirs(f"{input_dir}") - thread = Thread(target=launch_run, kwargs={"title": title, "input_dir": input_dir}) + thread = Thread( + target=launch_run, kwargs={"title": title, "input_dir": input_dir} + ) thread.start() return {"status": "Run started"}, 200 From 3354a5cf0d7fcb5559859f7dd4ead8b6894bf9ac Mon Sep 17 00:00:00 2001 From: Freeman Date: Fri, 28 Oct 2022 00:43:25 +0100 Subject: [PATCH 20/21] GCBM Refactor idea Draft Signed-off-by: Freeman --- .../rest_api_skeleton/Helpers/for_requests.py | 47 +++++-- local/rest_api_skeleton/Helpers/preprocess.py | 117 +++++++++++------- local/rest_api_skeleton/app.py | 6 +- 3 files changed, 109 insertions(+), 61 deletions(-) diff --git a/local/rest_api_skeleton/Helpers/for_requests.py b/local/rest_api_skeleton/Helpers/for_requests.py index db53093d..9851264f 100644 --- a/local/rest_api_skeleton/Helpers/for_requests.py +++ b/local/rest_api_skeleton/Helpers/for_requests.py @@ -1,26 +1,47 @@ from flask_restful import reqparse -from werkzeug.datastructures import FileStorage +from werkzeug.datastructures import FileStorage + def title_query(): query = reqparse.RequestParser() - query.add_argument("title", required=True, location="form" ) - return query.parse_args() + query.add_argument("title", required=True, location="form") + return query.parse_args() + def classifier_query(): query = reqparse.RequestParser() - query.add_argument("classifiers",type=FileStorage, required=True, action="append", location="files" ) - query.add_argument("attributes", location= "form") - return query.parse_args() + query.add_argument( + "classifiers", + type=FileStorage, + required=True, + action="append", + location="files", + ) + query.add_argument("attributes", location="form") + return query.parse_args() + def disturbance_query(): query = reqparse.RequestParser() - query.add_argument("disturbances",type=FileStorage, required=True, action="append", location="files" ) - query.add_argument("attributes", location= "form") - return query.parse_args() + query.add_argument( + "disturbances", + type=FileStorage, + required=True, + action="append", + location="files", + ) + query.add_argument("attributes", location="form") + return query.parse_args() + def miscellaneous_query(): query = reqparse.RequestParser() - query.add_argument("miscellaneous",type=FileStorage, required=True, action="append", location="files" ) - query.add_argument("attributes", location= "form") - return query.parse_args() - + query.add_argument( + "miscellaneous", + type=FileStorage, + required=True, + action="append", + location="files", + ) + query.add_argument("attributes", location="form") + return query.parse_args() diff --git a/local/rest_api_skeleton/Helpers/preprocess.py b/local/rest_api_skeleton/Helpers/preprocess.py index 157a0081..a496b64a 100644 --- a/local/rest_api_skeleton/Helpers/preprocess.py +++ b/local/rest_api_skeleton/Helpers/preprocess.py @@ -3,7 +3,8 @@ import json import rasterio as rst -def launch_run(title, input_dir): + +def launch_run(title, input_dir): s = time.time() with open(f"{input_dir}/gcbm_logs.csv", "w+") as f: res = subprocess.Popen( @@ -36,32 +37,43 @@ def launch_run(title, input_dir): "response": "Operation executed successfully. Downloadable links for input and output are attached in the response. Alternatively, you may also download this simulation input and output results by making a request at gcbm/download with the title in the body.", } -class Configs: +class Configs: def __init__(self, input_dir) -> None: self.input_dir = input_dir self.Rasters = [] - self.Rastersm = [] + self.Rastersm = [] self.nodatam = [] self.nodata = [] self.cellLatSize = [] self.cellLonSize = [] self.paths = [] self.lst = [] - self.provider_config = open(f"{os.getcwd()}/templates/provider_config.json", "r+") + self.provider_config = open( + f"{os.getcwd()}/templates/provider_config.json", "r+" + ) def get_config_templates(self): if not os.path.exists(f"{self.input_dir}/templates"): shutil.copytree( - f"{os.getcwd()}/templates", f"{self.input_dir}/templates", dirs_exist_ok=False + f"{os.getcwd()}/templates", + f"{self.input_dir}/templates", + dirs_exist_ok=False, + ) + self.provider_config = open( + f"{self.input_dir}/templates/provider_config.json", "r+" ) - self.provider_config = open(f"{self.input_dir}/templates/provider_config.json", "r+") def get_modules_cbm_config(self): - with open(f"{self.input_dir}/templates/modules_cbm.json", "r+") as modules_cbm_config: + with open( + f"{self.input_dir}/templates/modules_cbm.json", "r+" + ) as modules_cbm_config: data = json.load(modules_cbm_config) - disturbances = [file.split(".")[0][:-5] for file in os.listdir(f"{self.input_dir}/disturbances/")] # drop `_moja` to match modules_cbm.json template - modules_cbm_config.seek(0) + disturbances = [ + file.split(".")[0][:-5] + for file in os.listdir(f"{self.input_dir}/disturbances/") + ] # drop `_moja` to match modules_cbm.json template + modules_cbm_config.seek(0) data["Modules"]["CBMDisturbanceListener"]["settings"]["vars"] = disturbances json.dump(data, modules_cbm_config, indent=4) modules_cbm_config.truncate() @@ -70,11 +82,10 @@ def get_modules_cbm_config(self): def database_writes(self): data = json.load(self.provider_config) for file in os.listdir(f"{self.input_dir}/db/"): - data["Providers"]["SQLite"] = {"type": "SQLite", "path": file } + data["Providers"]["SQLite"] = {"type": "SQLite", "path": file} self.provider_config.seek(0) - - def write_configs(self, config_type : str): + def write_configs(self, config_type: str): data = json.load(self.provider_config) for file in os.listdir(f"{self.input_dir}/{config_type}/"): d = dict() @@ -82,7 +93,7 @@ def write_configs(self, config_type : str): d["layer_path"] = file d["layer_prefix"] = file[:-5] self.lst.append(d) - if config_type == "disturbances" or config_type == "classifiers": + if config_type == "disturbances" or config_type == "classifiers": for root, _, files in os.walk( os.path.abspath(f"{self.input_dir}/{config_type}/") ): @@ -91,15 +102,15 @@ def write_configs(self, config_type : str): self.Rasters.append(fp) self.paths.append(fp) for self.nd in self.Rasters: - img = rst.open(self.nd) - t = img.transform + img = rst.open(self.nd) + t = img.transform x = t[0] y = -t[4] - n = img.nodata + n = img.nodata self.cellLatSize.append(x) self.cellLonSize.append(y) self.nodata.append(n) - result = all(element == self.cellLatSize[0] for element in self.cellLatSize) + result = all(element == self.cellLatSize[0] for element in self.cellLatSize) if result: cellLat = x cellLon = y @@ -112,8 +123,15 @@ def write_configs(self, config_type : str): print("Corrupt files") self.provider_config.seek(0) - new_values = {"cellLonSize": cellLon,"cellLatSize": cellLat, "blockLonSize": blockLon, "blockLatSize": blockLat, "tileLatSize": tileLat, "tileLonSize": tileLon} - data["Providers"]["RasterTiled"]["layers"] = self.lst + new_values = { + "cellLonSize": cellLon, + "cellLatSize": cellLat, + "blockLonSize": blockLon, + "blockLatSize": blockLat, + "tileLatSize": tileLat, + "tileLonSize": tileLon, + } + data["Providers"]["RasterTiled"]["layers"] = self.lst data["Providers"]["RasterTiled"].update(new_values) json.dump(data, self.provider_config, indent=4) @@ -132,21 +150,23 @@ def write_configs(self, config_type : str): } self.study_area = { - "tile_size": tileLat, - "block_size": blockLat, - "tiles": [ - { - "x": int(t[2]), - "y": int(t[5]), - "index": 12674, - } - ], - "pixel_size": cellLat, - "layers": [], - } + "tile_size": tileLat, + "block_size": blockLat, + "tiles": [ + { + "x": int(t[2]), + "y": int(t[5]), + "index": 12674, + } + ], + "pixel_size": cellLat, + "layers": [], + } def add_file_to_path(self, config_type): - for root, _, files in os.walk(os.path.abspath(f"{self.input_dir}/{config_type}/")): + for root, _, files in os.walk( + os.path.abspath(f"{self.input_dir}/{config_type}/") + ): for file in files: fp = os.path.join(root, file) self.paths.append(fp) @@ -157,15 +177,18 @@ def copy_directory(self): def flatten_directory(self, config_type): shutil.rmtree((f"{self.input_dir}/{config_type}/")) - + + class DisturbanceConfig(Configs): - def __init__(self, input_dir, config_file : str, attribute : dict = None ) -> None: + def __init__(self, input_dir, config_file: str, attribute: dict = None) -> None: super().__init__(input_dir) self.config_file = config_file self.attribute = attribute def disturbances_special(self): - with open(f"{self.input_dir}/{self.config_file}", "w", encoding="utf8") as json_file: + with open( + f"{self.input_dir}/{self.config_file}", "w", encoding="utf8" + ) as json_file: self.dictionary["attributes"] = self.attribute json.dump(self.dictionary, json_file, indent=4) with open( @@ -194,14 +217,17 @@ def __call__(self): self.add_file_to_path("disturbances") self.copy_directory() + class ClassifierConfig(Configs): - def __init__(self, input_dir, config_file : str, attribute : dict ) -> None: + def __init__(self, input_dir, config_file: str, attribute: dict) -> None: super().__init__(input_dir) self.config_file = config_file self.attribute = attribute def classifier_special(self): - with open(f"{self.input_dir}/{self.config_file}", "w", encoding="utf8") as json_file: + with open( + f"{self.input_dir}/{self.config_file}", "w", encoding="utf8" + ) as json_file: self.dictionary["attributes"] = self.attribute json.dump(self.dictionary, json_file, indent=4) @@ -225,16 +251,17 @@ def __call__(self): self.add_file_to_path("classifiers") self.copy_directory() + class MiscellaneousConfig(Configs): - def __init__(self, input_dir, config_file : str, attribute : dict ) -> None: + def __init__(self, input_dir, config_file: str, attribute: dict) -> None: super().__init__(input_dir) self.config_file = config_file self.attribute = attribute - - def miscellaneous_special(self): + + def miscellaneous_special(self): for root, _, files in os.walk( os.path.abspath(f"{self.input_dir}/miscellaneous/") -): + ): for file in files: fp2 = os.path.join(root, file) self.Rastersm.append(fp2) @@ -244,9 +271,11 @@ def miscellaneous_special(self): d = img.nodata self.nodatam.append(d) """this is an experimental thing""" - with open(f"{self.input_dir}/{self.config_file}", "w", encoding="utf8") as json_file: + with open( + f"{self.input_dir}/{self.config_file}", "w", encoding="utf8" + ) as json_file: self.dictionary["layer_type"] = "GridLayer" - if self.config_file == "mean_annual_temperature_moja.json": + if self.config_file == "mean_annual_temperature_moja.json": self.dictionary["layer_data"] = "Float32" else: self.dictionary["layer_data"] = "Int16" @@ -273,5 +302,3 @@ def __call__(self): self.add_file_to_path("miscellaneous") self.copy_directory() self.flatten_directory("miscellaneous") - - diff --git a/local/rest_api_skeleton/app.py b/local/rest_api_skeleton/app.py index c5502bc0..cd256cc9 100644 --- a/local/rest_api_skeleton/app.py +++ b/local/rest_api_skeleton/app.py @@ -2,8 +2,8 @@ from flask_restful import Api from Endpoints.gcbm_endpoints import disturbance, Run, title, classifier, miscellaneous -app = Flask(__name__) -api = Api() +app = Flask(__name__) +api = Api() api.add_resource(disturbance, "/gcbm/upload/disturbances") api.add_resource(title, "/gcbm/create") @@ -12,5 +12,5 @@ api.add_resource(Run, "/gcbm/run") -if __name__ == "__main__": +if __name__ == "__main__": app.run(debug=True) From 7a561fe54824dbf2c9fa5f9b0a5079a3def63221 Mon Sep 17 00:00:00 2001 From: Freeman Date: Sun, 27 Nov 2022 14:14:38 +0100 Subject: [PATCH 21/21] added MVC architecture --- local/rest_api_skeleton/Helpers/routes.py | 8 ++++++++ local/rest_api_skeleton/app.py | 9 +++------ 2 files changed, 11 insertions(+), 6 deletions(-) create mode 100644 local/rest_api_skeleton/Helpers/routes.py diff --git a/local/rest_api_skeleton/Helpers/routes.py b/local/rest_api_skeleton/Helpers/routes.py new file mode 100644 index 00000000..0a002789 --- /dev/null +++ b/local/rest_api_skeleton/Helpers/routes.py @@ -0,0 +1,8 @@ +from Endpoints.gcbm_endpoints import disturbance, Run, title, classifier, miscellaneous + +def endpoints(api): + api.add_resource(disturbance, "/gcbm/upload/disturbances") + api.add_resource(title, "/gcbm/create") + api.add_resource(classifier, "/gcbm/upload/classifies") + api.add_resource(miscellaneous, "/gcbm/upload/miscellaneous") + api.add_resource(Run, "/gcbm/run") diff --git a/local/rest_api_skeleton/app.py b/local/rest_api_skeleton/app.py index cd256cc9..bc201d42 100644 --- a/local/rest_api_skeleton/app.py +++ b/local/rest_api_skeleton/app.py @@ -1,16 +1,13 @@ from flask import Flask from flask_restful import Api -from Endpoints.gcbm_endpoints import disturbance, Run, title, classifier, miscellaneous +from Helpers.routes import endpoints + app = Flask(__name__) api = Api() -api.add_resource(disturbance, "/gcbm/upload/disturbances") -api.add_resource(title, "/gcbm/create") -api.add_resource(classifier, "/gcbm/upload/classifies") -api.add_resource(miscellaneous, "/gcbm/upload/miscellaneous") -api.add_resource(Run, "/gcbm/run") +endpoints(api=api) if __name__ == "__main__": app.run(debug=True)