process PR feedback

Author: pi-sigma

.github/workflows/ci.yml

@@ -23,6 +23,16 @@ jobs:
 
     name: Run the test suite (Python ${{ matrix.python }}, Django ${{ matrix.django }})
 
+    services:
+      postgres:
+        image: postgres:14
+        env:
+          POSTGRES_HOST_AUTH_METHOD: trust
+        ports:
+          - 5432:5432
+        # needed because the postgres container does not provide a healthcheck
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4

README.rst

@@ -115,8 +115,9 @@ To use this with your project you need to follow these steps:
         res = requests.get("https://httpbin.org/json")
         print(res.json())
 
-#.  Check stdout for the printable output, and navigate to ``/admin/log_outgoing_requests/outgoingrequestslog/`` to see 
-    the saved log records. The settings for saving logs can by overridden under ``/admin/log_outgoing_requests/outgoingrequestslogconfig/``.
+#.  Check stdout for the printable output, and navigate to ``Admin > Miscellaneous > Outgoing Requests Logs``
+    to see the saved log records. In order to override the settings for saving logs, navigate to
+    ``Admin > Miscellaneous > Outgoing Requests Log Configuration``.
 
 
 Local development

docs/quickstart.rst

@@ -60,6 +60,13 @@ Installation
         LOG_OUTGOING_REQUESTS_DB_SAVE = True # save logs enabled/disabled based on the boolean value
         LOG_OUTGOING_REQUESTS_SAVE_BODY = True # save request/response body
         LOG_OUTGOING_REQUESTS_LOG_BODY_TO_STDOUT = True # log request/response body to STDOUT
+        LOG_OUTGOING_REQUESTS_CONTENT_TYPES = [
+                "text/*",
+                "application/json",
+                "application/xml",
+        ] # save request/response bodies with matching content type
+        LOG_OUTGOING_REQUESTS_MAX_CONTENT_LENGTH = 32000 # maximal size (in bytes) for the request/response body
+        LOG_OUTGOING_REQUESTS_REQUIRE_CONTENT_LENGTH = True # if True, the content size of request/response bodies is not checked
 
 
 #.  Run ``python manage.py migrate`` to create the necessary database tables.

log_outgoing_requests/admin.py

@@ -1,3 +1,4 @@
+from django import forms
 from django.contrib import admin
 from django.utils.translation import gettext as _
 
@@ -62,13 +63,27 @@ class Media:
         }
 
 
+class ConfigAdminForm(forms.ModelForm):
+    class Meta:
+        model = OutgoingRequestsLogConfig
+        fields = ("allowed_content_types",)
+        widgets = {"allowed_content_types": forms.CheckboxSelectMultiple}
+
+
 @admin.register(OutgoingRequestsLogConfig)
 class OutgoingRequestsLogConfigAdmin(SingletonModelAdmin):
+    form = ConfigAdminForm
     fields = (
         "save_to_db",
         "save_body",
+        "allowed_content_types",
+        "require_content_length",
+        "max_content_length",
     )
     list_display = (
         "save_to_db",
         "save_body",
+        "allowed_content_types",
+        "require_content_length",
+        "max_content_length",
     )

log_outgoing_requests/formatters.py

@@ -8,32 +8,40 @@ class HttpFormatter(logging.Formatter):
     def _formatHeaders(self, d):
         return "\n".join(f"{k}: {v}" for k, v in d.items())
 
-    def _formatBody(self, content: dict, request_or_response: str) -> str:
+    def _formatBody(self, content: str, request_or_response: str) -> str:
         if settings.LOG_OUTGOING_REQUESTS_LOG_BODY_TO_STDOUT:
             return f"\n{request_or_response} body:\n{content}"
         return ""
 
     def formatMessage(self, record):
-        result = super().formatMessage(record)
-        if record.name == "requests":
-            result += textwrap.dedent(
-                """
-                ---------------- request ----------------
-                {req.method} {req.url}
-                {reqhdrs} {request_body}
-
-                ---------------- response ----------------
-                {res.status_code} {res.reason} {res.url}
-                {reshdrs} {response_body}
+        if record.name != "requests":
+            return
 
+        result = super().formatMessage(record)
+        result += textwrap.dedent(
             """
-            ).format(
-                req=record.req,
-                res=record.res,
-                reqhdrs=self._formatHeaders(record.req.headers),
-                reshdrs=self._formatHeaders(record.res.headers),
-                request_body=self._formatBody(record.req.body, "Request"),
-                response_body=self._formatBody(record.res.json(), "Response"),
+            ---------------- request ----------------
+            {req.method} {req.url}
+            {reqhdrs} {request_body}
+
+            ---------------- response ----------------
+            {res.status_code} {res.reason} {res.url}
+            {reshdrs} {response_body}
+
+        """
+        ).format(
+            req=record.req,
+            res=record.res,
+            reqhdrs=self._formatHeaders(record.req.headers),
+            reshdrs=self._formatHeaders(record.res.headers),
+            request_body=self._formatBody(record.req.body, "Request")
+            if settings.LOG_OUTGOING_REQUESTS_LOG_BODY_TO_STDOUT is True
+            else "",
+            response_body=self._formatBody(
+                record.res.content.decode("utf-8"), "Response"
             )
+            if settings.LOG_OUTGOING_REQUESTS_LOG_BODY_TO_STDOUT is True
+            else "",
+        )
 
         return result

log_outgoing_requests/handlers.py

@@ -2,16 +2,7 @@
 import traceback
 from urllib.parse import urlparse
 
-from django.conf import settings
-
-ALLOWED_CONTENT_TYPES = [
-    "application/json",
-    "multipart/form-data",
-    "text/html",
-    "text/plain",
-    "",
-    None,
-]
+from .validators import validate_content
 
 
 class DatabaseOutgoingRequestsHandler(logging.Handler):
@@ -20,29 +11,19 @@ def emit(self, record):
 
         config = OutgoingRequestsLogConfig.get_solo()
 
-        if config.save_to_db or settings.LOG_OUTGOING_REQUESTS_DB_SAVE:
+        if config.save_logs_enabled:
             from .models import OutgoingRequestsLog
 
-            trace = None
+            trace = ""
 
             # skip requests not coming from the library requests
             if not record or not record.getMessage() == "Outgoing request":
                 return
 
-            # skip requests with non-allowed content
-            request_content_type = record.req.headers.get("Content-Type", "")
-            response_content_type = record.res.headers.get("Content-Type", "")
-
-            if not (
-                request_content_type in ALLOWED_CONTENT_TYPES
-                and response_content_type in ALLOWED_CONTENT_TYPES
-            ):
-                return
-
-            safe_req_headers = record.req.headers.copy()
+            scrubbed_req_headers = record.req.headers.copy()
 
-            if "Authorization" in safe_req_headers:
-                safe_req_headers["Authorization"] = "***hidden***"
+            if "Authorization" in scrubbed_req_headers:
+                scrubbed_req_headers["Authorization"] = "***hidden***"
 
             if record.exc_info:
                 trace = traceback.format_exc()
@@ -54,18 +35,31 @@ def emit(self, record):
                 "params": parsed_url.params,
                 "status_code": record.res.status_code,
                 "method": record.req.method,
-                "req_content_type": record.req.headers.get("Content-Type", ""),
-                "res_content_type": record.res.headers.get("Content-Type", ""),
+                "req_content_type": (
+                    request_content_type := record.req.headers.get("Content-Type", "")
+                ),
+                "res_content_type": (
+                    response_content_type := record.res.headers.get("Content-Type", "")
+                ),
                 "timestamp": record.requested_at,
                 "response_ms": int(record.res.elapsed.total_seconds() * 1000),
-                "req_headers": self.format_headers(safe_req_headers),
+                "req_headers": self.format_headers(scrubbed_req_headers),
                 "res_headers": self.format_headers(record.res.headers),
                 "trace": trace,
             }
 
-            if config.save_body or settings.LOG_OUTGOING_REQUESTS_SAVE_BODY:
-                kwargs["req_body"] = (record.req.body,)
-                kwargs["res_body"] = (record.res.json(),)
+            if config.save_body_enabled:
+                request_content_length = record.req.headers.get("Content-Length")
+                if validate_content(
+                    request_content_length, request_content_type, config
+                ):
+                    kwargs["req_body"] = record.req.body or ""
+
+                response_content_length = record.res.headers.get("Content-Length")
+                if validate_content(
+                    response_content_length, response_content_type, config
+                ):
+                    kwargs["res_body"] = record.res.content.decode("utf-8") or ""
 
             OutgoingRequestsLog.objects.create(**kwargs)