From b89ce4ff320c6a1c9ba98afebb13271a5ace477a Mon Sep 17 00:00:00 2001 From: M Kunde Date: Tue, 12 Nov 2024 20:07:24 +0100 Subject: [PATCH 1/9] handle access denied error from boto --- giftless/error_handling.py | 11 +++++++++++ giftless/storage/amazon_s3.py | 12 ++++++++++-- giftless/storage/exc.py | 4 ++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/giftless/error_handling.py b/giftless/error_handling.py index 969cd08..3ec25c4 100644 --- a/giftless/error_handling.py +++ b/giftless/error_handling.py @@ -5,6 +5,7 @@ from werkzeug.exceptions import default_exceptions from .representation import output_git_lfs_json +from .storage.exc import AccessDenied class ApiErrorHandler: @@ -17,11 +18,21 @@ def init_app(self, app): for code in default_exceptions: app.errorhandler(code)(self.error_as_json) + # Specifically handle AccessDenied with a JSON response and 403 status + app.errorhandler(AccessDenied)(self.access_denied_as_json) + @classmethod def error_as_json(cls, ex): """Handle errors by returning a JSON response + only maps HTTP-based exceptions from werkzeug.exceptions (like NotFound or Forbidden) """ code = ex.code if hasattr(ex, 'code') else 500 data = {"message": str(ex)} return output_git_lfs_json(data=data, code=code) + + @classmethod + def access_denied_as_json(cls, ex): + """Handle AccessDenied by returning a JSON response with 403 status""" + data = ex.as_dict() + return output_git_lfs_json(data=data, code=403) diff --git a/giftless/storage/amazon_s3.py b/giftless/storage/amazon_s3.py index c9eb343..fe131cc 100644 --- a/giftless/storage/amazon_s3.py +++ b/giftless/storage/amazon_s3.py @@ -22,7 +22,13 @@ def __init__(self, bucket_name: str, path_prefix: Optional[str] = None, **_): def get(self, prefix: str, oid: str) -> Iterable[bytes]: if not self.exists(prefix, oid): raise ObjectNotFound() - result: Iterable[bytes] = self._s3_object(prefix, oid).get()['Body'] + try: + result: Iterable[bytes] = self._s3_object(prefix, oid).get()['Body'] + except botocore.exceptions.ClientError as e: + if e.response['Error']['Code'] in ("403", "AccessDenied"): + raise AccessDenied() + else: + raise e return result def put(self, prefix: str, oid: str, data_stream: BinaryIO) -> int: @@ -38,7 +44,7 @@ def upload_callback(size): def exists(self, prefix: str, oid: str) -> bool: try: self.get_size(prefix, oid) - except ObjectNotFound: + except (ObjectNotFound, AccessDenied): return False return True @@ -48,6 +54,8 @@ def get_size(self, prefix: str, oid: str) -> int: except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == "404": raise ObjectNotFound() + elif e.response['Error']['Code'] in ("403", "AccessDenied"): + raise AccessDenied() else: raise e return result diff --git a/giftless/storage/exc.py b/giftless/storage/exc.py index c85f92b..dde47bd 100644 --- a/giftless/storage/exc.py +++ b/giftless/storage/exc.py @@ -19,3 +19,7 @@ class ObjectNotFound(StorageError): class InvalidObject(StorageError): code = 422 + + +class AccessDenied(StorageError): + code = 403 From 2016189cae42ee41810a6705ea256a3cdef5341f Mon Sep 17 00:00:00 2001 From: M Kunde Date: Tue, 12 Nov 2024 21:19:52 +0100 Subject: [PATCH 2/9] missing import --- giftless/storage/amazon_s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/giftless/storage/amazon_s3.py b/giftless/storage/amazon_s3.py index fe131cc..ddd1d84 100644 --- a/giftless/storage/amazon_s3.py +++ b/giftless/storage/amazon_s3.py @@ -5,7 +5,7 @@ import botocore # type: ignore from giftless.storage import ExternalStorage, StreamingStorage -from giftless.storage.exc import ObjectNotFound +from giftless.storage.exc import ObjectNotFound, AccessDenied from giftless.util import safe_filename From 73581fc6e091521e5522430df2782c516353da36 Mon Sep 17 00:00:00 2001 From: M Kunde Date: Tue, 12 Nov 2024 22:45:45 +0100 Subject: [PATCH 3/9] useful information for debugging --- giftless/error_handling.py | 1 + 1 file changed, 1 insertion(+) diff --git a/giftless/error_handling.py b/giftless/error_handling.py index 3ec25c4..b330941 100644 --- a/giftless/error_handling.py +++ b/giftless/error_handling.py @@ -28,6 +28,7 @@ def error_as_json(cls, ex): """ code = ex.code if hasattr(ex, 'code') else 500 data = {"message": str(ex)} + logging.debug(f"Returning error response: {data} with status {code}") return output_git_lfs_json(data=data, code=code) From eb3a342a27173dc9b684b3c329a41a573066e25d Mon Sep 17 00:00:00 2001 From: M Kunde Date: Wed, 13 Nov 2024 13:05:17 +0100 Subject: [PATCH 4/9] initialise logging --- giftless/error_handling.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/giftless/error_handling.py b/giftless/error_handling.py index b330941..ab68f5a 100644 --- a/giftless/error_handling.py +++ b/giftless/error_handling.py @@ -2,6 +2,7 @@ See https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md#response-errors """ +import logging from werkzeug.exceptions import default_exceptions from .representation import output_git_lfs_json @@ -26,14 +27,16 @@ def error_as_json(cls, ex): """Handle errors by returning a JSON response only maps HTTP-based exceptions from werkzeug.exceptions (like NotFound or Forbidden) """ + log = logging.getLogger(__name__) code = ex.code if hasattr(ex, 'code') else 500 data = {"message": str(ex)} - logging.debug(f"Returning error response: {data} with status {code}") + log.debug(f"Returning error response: {data} with status {code}") return output_git_lfs_json(data=data, code=code) @classmethod def access_denied_as_json(cls, ex): """Handle AccessDenied by returning a JSON response with 403 status""" + data = ex.as_dict() return output_git_lfs_json(data=data, code=403) From 4c5ff800d024fc4401553cec1cbd95f0fe96d03f Mon Sep 17 00:00:00 2001 From: M Kunde Date: Wed, 13 Nov 2024 13:59:31 +0100 Subject: [PATCH 5/9] formatting --- giftless/error_handling.py | 1 + giftless/storage/amazon_s3.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/giftless/error_handling.py b/giftless/error_handling.py index ab68f5a..eb4e574 100644 --- a/giftless/error_handling.py +++ b/giftless/error_handling.py @@ -3,6 +3,7 @@ See https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md#response-errors """ import logging + from werkzeug.exceptions import default_exceptions from .representation import output_git_lfs_json diff --git a/giftless/storage/amazon_s3.py b/giftless/storage/amazon_s3.py index ddd1d84..a49c428 100644 --- a/giftless/storage/amazon_s3.py +++ b/giftless/storage/amazon_s3.py @@ -5,7 +5,7 @@ import botocore # type: ignore from giftless.storage import ExternalStorage, StreamingStorage -from giftless.storage.exc import ObjectNotFound, AccessDenied +from giftless.storage.exc import AccessDenied, ObjectNotFound from giftless.util import safe_filename From 1efa2daa3306052dba63f6ffbddecb8a96ccf884 Mon Sep 17 00:00:00 2001 From: M Kunde Date: Wed, 13 Nov 2024 14:06:31 +0100 Subject: [PATCH 6/9] fixed syntax error --- giftless/storage/amazon_s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/giftless/storage/amazon_s3.py b/giftless/storage/amazon_s3.py index a49c428..b2d9344 100644 --- a/giftless/storage/amazon_s3.py +++ b/giftless/storage/amazon_s3.py @@ -55,7 +55,7 @@ def get_size(self, prefix: str, oid: str) -> int: if e.response['Error']['Code'] == "404": raise ObjectNotFound() elif e.response['Error']['Code'] in ("403", "AccessDenied"): - raise AccessDenied() + raise AccessDenied() else: raise e return result From 9dac7efa52ab6f3bb416e86688bc768b4f033fcb Mon Sep 17 00:00:00 2001 From: M Kunde Date: Wed, 13 Nov 2024 14:17:21 +0100 Subject: [PATCH 7/9] print log for returning 403 --- giftless/error_handling.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/giftless/error_handling.py b/giftless/error_handling.py index eb4e574..ba3fbda 100644 --- a/giftless/error_handling.py +++ b/giftless/error_handling.py @@ -38,6 +38,7 @@ def error_as_json(cls, ex): @classmethod def access_denied_as_json(cls, ex): """Handle AccessDenied by returning a JSON response with 403 status""" - + log = logging.getLogger(__name__) + log.debug(f"Returning error response of AccessDenied with status 403") data = ex.as_dict() return output_git_lfs_json(data=data, code=403) From 6496d4f9fae376f5255319c38ad6e0df793816d2 Mon Sep 17 00:00:00 2001 From: M Kunde Date: Wed, 13 Nov 2024 14:24:58 +0100 Subject: [PATCH 8/9] What The F541 --- giftless/error_handling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/giftless/error_handling.py b/giftless/error_handling.py index ba3fbda..04442f7 100644 --- a/giftless/error_handling.py +++ b/giftless/error_handling.py @@ -39,6 +39,6 @@ def error_as_json(cls, ex): def access_denied_as_json(cls, ex): """Handle AccessDenied by returning a JSON response with 403 status""" log = logging.getLogger(__name__) - log.debug(f"Returning error response of AccessDenied with status 403") + log.debug("Returning error response of AccessDenied with status 403") data = ex.as_dict() return output_git_lfs_json(data=data, code=403) From e54ae45f3889322af0a40e2cc79f3304855e9c52 Mon Sep 17 00:00:00 2001 From: M Kunde Date: Wed, 13 Nov 2024 18:03:57 +0100 Subject: [PATCH 9/9] removed references to python 3.7 which is no longer supported, incl broke dockerfile --- .github/workflows/test.yaml | 2 +- .travis.yml | 1 - Dockerfile | 64 ------------------------------------- 3 files changed, 1 insertion(+), 66 deletions(-) delete mode 100644 Dockerfile diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index db211d4..7ddb62c 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: [ 3.7, 3.8 ] + python-version: [ 3.8 ] steps: - uses: actions/checkout@v2 - name: Install Python 3 diff --git a/.travis.yml b/.travis.yml index d180ce5..a77137a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,6 @@ language: python python: - - 3.7 - 3.8 env: diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 710918d..0000000 --- a/Dockerfile +++ /dev/null @@ -1,64 +0,0 @@ -# Dockerfile for uWSGI wrapped Giftless Git LFS Server - -### --- Build Depdendencies --- - -FROM python:3.7 as builder -MAINTAINER "Shahar Evron " - -# Build wheels for uWSGI and all requirements -RUN DEBIAN_FRONTEND=noninteractive apt-get update \ - && apt-get install -y build-essential libpcre3 libpcre3-dev git -RUN pip install -U pip -RUN mkdir /wheels - -ARG UWSGI_VERSION=2.0.18 -RUN pip wheel -w /wheels uwsgi==$UWSGI_VERSION - -COPY requirements.txt / -RUN pip wheel -w /wheels -r /requirements.txt - -### --- Build Final Image --- - -FROM python:3.7-slim - -RUN DEBIAN_FRONTEND=noninteractive apt-get update \ - && apt-get install -y libpcre3 libxml2 tini \ - && apt-get clean \ - && apt -y autoremove - -RUN mkdir /app - -# Install dependencies -COPY --from=builder /wheels /wheels -RUN pip install /wheels/*.whl - -# Copy project code -COPY . /app -RUN pip install -e /app - -ARG USER_NAME=giftless -ARG STORAGE_DIR=/lfs-storage -ENV GIFTLESS_TRANSFER_ADAPTERS_basic_options_storage_options_path $STORAGE_DIR - -RUN useradd -d /app $USER_NAME -RUN mkdir $STORAGE_DIR -RUN chown $USER_NAME $STORAGE_DIR - -# Pip-install some common WSGI middleware modules -# These are not required in every Giftless installation but are common enough -ARG EXTRA_PACKAGES="wsgi_cors_middleware" -RUN pip install ${EXTRA_PACKAGES} - -USER $USER_NAME - -WORKDIR /app - -ENV UWSGI_MODULE "giftless.wsgi_entrypoint" - -ARG PORT=5000 -EXPOSE $PORT - -ENTRYPOINT ["tini", "uwsgi", "--"] - -CMD ["-s", "127.0.0.1:${PORT}", "-M", "-T", "--threads", "2", "-p", "2", \ - "--manage-script-name", "--callable", "app"]