Optimize getLogRecords()

Author: rogerdahl

gmn/src/d1_gmn/app/auth.py

@@ -196,23 +196,6 @@ def is_allowed(request, level, pid):
     ).exists()
 
 
-# TODO: Change "trusted" / "fully trusted" to something like admin / superuser.
-def is_redacted_log(request, pid):
-    """Determine if ``ipAddress`` and ``subject`` fields must be redacted on
-    any ``LogEntry``s for the SciObj that are returned to the client.
-
-    If access to ``LogEntry``s for the SciObj is denied for the client,
-    this function will return True, indicating that ``LogEntry``s must
-    be redacted. However, the result is irrelevant since no
-    ``LogEntry``s for the SciObj will be returned to the caller.
-    """
-    if is_trusted_subject(request):
-        return False
-    if is_allowed(request, WRITE_LEVEL, pid):
-        return False
-    return True
-
-
 def has_create_update_delete_permission(request):
     whitelisted_subject_set = get_whitelisted_subject_set()
     logging.debug('Whitelisted subjects: {}'.format(', '.join(whitelisted_subject_set)))

gmn/src/d1_gmn/app/db_filter.py

@@ -35,21 +35,51 @@
 return: QuerySet
 """
 
+import d1_gmn.app.auth
 import d1_gmn.app.did
 import d1_gmn.app.models
 import d1_gmn.app.revision
 import d1_gmn.app.views.assert_db
 import d1_gmn.app.views.util
 
+import django.db.models
 
 def add_access_policy_filter(request, query, column_name):
+    """Filter records that do not have ``read`` or better access for one or
+    more of the active subjects.
+
+    Since ``read`` is the lowest access level that a subject can have,
+    this method only has to filter on the presence of the subject.
+    """
     q = d1_gmn.app.models.Subject.objects.filter(
         subject__in=request.all_subjects_set
     ).values('permission__sciobj')
     filter_arg = '{}__in'.format(column_name)
     return query.filter(**{filter_arg: q})
 
 
+def add_redact_annotation(request, query):
+    """Flag LogEntry records that require ``ipAddress`` and ``subject`` fields
+    to be redacted before being returned to the client.
+
+    Only trusted subjects and subjects with ``write`` or ``changePermission`` on a
+    SciObj receive unredacted ``ipAddress`` and ``subject`` in LogEntry records for the
+    associated SciObj.
+
+    Subjects with only ``read`` access receive redacted records.
+    """
+    return query.annotate(
+        redact=django.db.models.Exists(
+            d1_gmn.app.models.Permission.objects.filter(
+                sciobj=django.db.models.OuterRef('sciobj'),
+                subject__subject__in=request.all_subjects_set,
+                level__gte=d1_gmn.app.auth.WRITE_LEVEL,
+            ),
+            negated=True,
+        )
+    )
+
+
 def add_replica_filter(request, query):
     param_name = 'replicaStatus'
     bool_val = request.GET.get(param_name, True)

gmn/src/d1_gmn/app/middleware/response_handler.py

@@ -180,9 +180,15 @@ def _generate_log_records(self, request, db_query, start, total):
             logEntry = d1_gmn.app.views.util.dataoneTypes(request).LogEntry()
             logEntry.entryId = str(row.id)
             logEntry.identifier = row.sciobj.pid.did
-            logEntry.ipAddress = row.ip_address.ip_address
+            # Redact ipAddress and subject on records for which client has only "read"
+            # access.
+            if getattr(row, 'redact', False):
+                logEntry.ipAddress = '<NotAuthorized>'
+                logEntry.subject = '<NotAuthorized>'
+            else:
+                logEntry.ipAddress = row.ip_address.ip_address
+                logEntry.subject = row.subject.subject
             logEntry.userAgent = row.user_agent.user_agent
-            logEntry.subject = row.subject.subject
             logEntry.event = row.event.event
             logEntry.dateLogged = d1_common.date_time.normalize_datetime_to_utc(
                 row.timestamp

gmn/src/d1_gmn/app/models.py

@@ -356,9 +356,8 @@ class Event(django.db.models.Model):
 
 
 def event(event_str):
-    # In v2.0, events are no longer restricted to this set. However, GMN still
-    # only records these types of events, so we'll leave it in while that
-    # remains the case.
+    # In v2.0, events are no longer restricted to this set. However, GMN still only
+    # records these types of events, so we'll leave it in while that remains the case.
     assert event_str in [
         'create',
         'read',

gmn/src/d1_gmn/app/views/external.py

@@ -136,16 +136,12 @@ def get_log(request):
 
     Sorted by timestamp, id.
     """
-    # TODO: Check if select_related() gives better performance
-    query = (
-        d1_gmn.app.models.EventLog.objects.all()
-        .select_related()
-        .order_by('timestamp', 'id')
-    )
+    query = d1_gmn.app.models.EventLog.objects.all().order_by('timestamp', 'id')
     if not d1_gmn.app.auth.is_trusted_subject(request):
         query = d1_gmn.app.db_filter.add_access_policy_filter(
             request, query, 'sciobj__id'
         )
+        query = d1_gmn.app.db_filter.add_redact_annotation(request, query)
     query = d1_gmn.app.db_filter.add_datetime_filter(
         request, query, 'timestamp', 'fromDate', 'gte'
     )

gmn/src/d1_gmn/app/views/util.py

@@ -146,7 +146,6 @@ def http_response_with_boolean_true_type():
 def query_object_list(request, type_name):
     query = (
         d1_gmn.app.models.ScienceObject.objects.all()
-        .select_related()
         .annotate(timestamp=django.db.models.F('modified_timestamp'))
         .order_by('modified_timestamp', 'id')
     )