add minio auth plugin --skip-tests

Author: OriHoch

cwm_worker_operator/cli.py

@@ -178,5 +178,14 @@ def send_agg_metrics(worker_id, minutes_json):
     CwmApiManager().send_agg_metrics(worker_id, json.loads(minutes_json))
 
 
+@main.command(short_help="Start Minio Auth Server for development")
+def start_minio_auth_server_devel():
+    """
+    Start Minio Auth Server for development
+    """
+    import uvicorn
+    uvicorn.run('cwm_worker_operator.minio_auth_plugin.app:app', host='0.0.0.0', port=5000, reload=True)
+
+
 if __name__ == '__main__':
     main()

cwm_worker_operator/minio_auth_plugin/__init__.py

@@ -0,0 +1,10 @@
+from fastapi import FastAPI, Request
+
+app = FastAPI()
+
+
+@app.post("/")
+async def root(request: Request):
+    body = await request.json()
+    print(body)
+    return {"result": True}

cwm_worker_operator/minio_auth_plugin/app.py

@@ -0,0 +1,68 @@
+# copied from https://github.com/tiangolo/uvicorn-gunicorn-docker/blob/master/docker-images/gunicorn_conf.py August 24, 2021
+import json
+import multiprocessing
+import os
+
+workers_per_core_str = os.getenv("WORKERS_PER_CORE", "1")
+max_workers_str = os.getenv("MAX_WORKERS")
+use_max_workers = None
+if max_workers_str:
+    use_max_workers = int(max_workers_str)
+web_concurrency_str = os.getenv("WEB_CONCURRENCY", None)
+
+host = os.getenv("HOST", "0.0.0.0")
+port = os.getenv("PORT", "80")
+bind_env = os.getenv("BIND", None)
+use_loglevel = os.getenv("LOG_LEVEL", "info")
+if bind_env:
+    use_bind = bind_env
+else:
+    use_bind = f"{host}:{port}"
+
+cores = multiprocessing.cpu_count()
+workers_per_core = float(workers_per_core_str)
+default_web_concurrency = workers_per_core * cores
+if web_concurrency_str:
+    web_concurrency = int(web_concurrency_str)
+    assert web_concurrency > 0
+else:
+    web_concurrency = max(int(default_web_concurrency), 2)
+    if use_max_workers:
+        web_concurrency = min(web_concurrency, use_max_workers)
+accesslog_var = os.getenv("ACCESS_LOG", "-")
+use_accesslog = accesslog_var or None
+errorlog_var = os.getenv("ERROR_LOG", "-")
+use_errorlog = errorlog_var or None
+graceful_timeout_str = os.getenv("GRACEFUL_TIMEOUT", "120")
+timeout_str = os.getenv("TIMEOUT", "120")
+keepalive_str = os.getenv("KEEP_ALIVE", "5")
+
+# Gunicorn config variables
+loglevel = use_loglevel
+workers = web_concurrency
+bind = use_bind
+errorlog = use_errorlog
+worker_tmp_dir = "/dev/shm"
+accesslog = use_accesslog
+graceful_timeout = int(graceful_timeout_str)
+timeout = int(timeout_str)
+keepalive = int(keepalive_str)
+
+
+# For debugging and testing
+log_data = {
+    "loglevel": loglevel,
+    "workers": workers,
+    "bind": bind,
+    "graceful_timeout": graceful_timeout,
+    "timeout": timeout,
+    "keepalive": keepalive,
+    "errorlog": errorlog,
+    "accesslog": accesslog,
+    # Additional, non-gunicorn variables
+    "workers_per_core": workers_per_core,
+    "use_max_workers": use_max_workers,
+    "host": host,
+    "port": port,
+}
+print(json.dumps(log_data))

cwm_worker_operator/minio_auth_plugin/gunicorn_conf.py

@@ -4,3 +4,5 @@ prometheus-client==0.8.0
 pytz==2021.1
 boto3==1.17.49
 click==7.1.2
+uvicorn[standard]==0.27.0.post1
+fastapi==0.109.1