Merge pull request #256 from editorsnotes/action-api

Improve project serializer and API for getting action histories

editorsnotes/api/serializers/__init__.py

@@ -2,3 +2,4 @@
 from topics import TopicSerializer
 from notes import NoteSerializer
 from activity import ActivitySerializer
+from projects import ProjectSerializer

editorsnotes/api/serializers/activity.py

@@ -21,6 +21,9 @@ class Meta:
         model = LogActivity
         fields = ('user', 'project', 'time', 'type', 'url', 'title', 'action',)
     def get_object_url(self, obj):
-        return None if obj.action == DELETION else obj.content_object.get_absolute_url()
+        if obj.action == DELETION or obj.content_object is None:
+            return None
+        else:
+            return obj.content_object.get_absolute_url()
     def get_action_repr(self, version_obj):
         return VERSION_ACTIONS[version_obj.action]

editorsnotes/api/serializers/base.py

@@ -51,7 +51,7 @@ def __init__(self, *args, **kwargs):
     def get_attribute(self, obj):
         return obj.get_affiliation()
     def to_representation(self, value):
-        url = reverse('api:api-project-detail', args=(value.slug,),
+        url = reverse('api:api-projects-detail', args=(value.slug,),
                       request=self.context['request'])
         return { 'name': value.name, 'url': url }
 

editorsnotes/api/serializers/documents.py

@@ -100,7 +100,7 @@ class CitationSerializer(serializers.ModelSerializer):
     document = HyperlinkedProjectItemField(view_name='api:api-documents-detail',
                                            queryset=Document.objects,
                                            required=True)
-    document_description = serializers.SerializerMethodField('get_document_description')
+    document_description = serializers.SerializerMethodField()
     class Meta:
         model = Citation
         fields = ('id', 'url', 'ordering', 'document', 'document_description', 'notes')

editorsnotes/api/serializers/projects.py

@@ -2,24 +2,35 @@
 from rest_framework.reverse import reverse
 
 from editorsnotes.main.models import Project
+from editorsnotes.search import get_index
+
+from .base import URLField
+
+def count_for(project, doc_type):
+    index = get_index('main')
+    count = index.es.count(
+        { 'query': { 'term': { 'serialized.project.name': project.name }}},
+        index=index.name,
+        doc_type=doc_type)
+    return count['count']
+
 
 class ProjectSerializer(serializers.ModelSerializer):
-    url = serializers.SerializerMethodField('get_api_url')
-    notes = serializers.SerializerMethodField('get_notes_url')
-    topics = serializers.SerializerMethodField('get_topics_url')
-    documents = serializers.SerializerMethodField('get_documents_url')
+    url = URLField('api:api-projects-detail', ('slug',))
+    notes = serializers.SerializerMethodField()
+    notes_url = URLField('api:api-notes-list', ('slug',))
+    topics = serializers.SerializerMethodField()
+    topics_url = URLField('api:api-topics-list', ('slug',))
+    documents = serializers.SerializerMethodField()
+    documents_url = URLField('api:api-documents-list', ('slug',))
+    activity_url = URLField('api:api-projects-activity', ('slug',))
     class Meta:
         model = Project
-        fields = ('slug', 'name', 'url', 'notes', 'topics', 'documents',)
-    def get_api_url(self, obj):
-        return reverse('api:api-project-detail', args=(obj.slug,),
-                       request=self.context['request'])
-    def get_notes_url(self, obj):
-        return reverse('api:api-notes-list', args=(obj.slug,),
-                       request=self.context['request'])
-    def get_topics_url(self, obj):
-        return reverse('api:api-topics-list', args=(obj.slug,),
-                       request=self.context['request'])
-    def get_documents_url(self, obj):
-        return reverse('api:api-documents-list', args=(obj.slug,),
-                       request=self.context['request'])
+        fields = ('slug', 'url', 'name', 'description', 'notes', 'notes_url',
+                  'topics', 'topics_url', 'documents', 'documents_url', 'activity_url')
+    def get_notes(self, obj):
+        return count_for(obj, 'note')
+    def get_topics(self, obj):
+        return count_for(obj, 'topic')
+    def get_documents(self, obj):
+        return count_for(obj, 'document')

editorsnotes/api/serializers/topics.py

@@ -33,7 +33,7 @@ def get_alternate_forms(self, obj):
     def get_project_value(self, obj):
         return [OrderedDict((
             ('project_name', topic.project.name),
-            ('project_url', reverse('api:api-project-detail',
+            ('project_url', reverse('api:api-projects-detail',
                             args=(topic.project.slug,),
                             request=self.context['request'])),
             ('preferred_name', topic.preferred_name),

editorsnotes/api/tests.py

@@ -138,7 +138,7 @@ def test_topic_api_create(self):
         self.assertEqual(Revision.objects.count(), 1)
 
         # Make sure an entry was added to the activity index
-        activity_response = self.client.get(reverse('api:api-project-activity',
+        activity_response = self.client.get(reverse('api:api-projects-activity',
                                                          args=[self.project.slug]))
         self.assertEqual(activity_response.status_code, 200)
         self.assertEqual(len(activity_response.data['activity']), 1)
@@ -259,7 +259,7 @@ def test_topic_api_update(self):
         self.assertEqual(Revision.objects.count(), 1)
 
         # Make sure an entry was added to the activity index
-        activity_response = self.client.get(reverse('api:api-project-activity',
+        activity_response = self.client.get(reverse('api:api-projects-activity',
                                                          args=[self.project.slug]))
         self.assertEqual(activity_response.status_code, 200)
         activity_data = activity_response.data['activity'][0]
@@ -325,7 +325,7 @@ def test_topic_api_delete(self):
         self.assertEqual(main_models.TopicNode.objects.count(), 1)
 
         # Make sure an entry was added to the activity index
-        activity_response = self.client.get(reverse('api:api-project-activity',
+        activity_response = self.client.get(reverse('api:api-projects-activity',
                                                          args=[self.project.slug]))
         self.assertEqual(activity_response.status_code, 200)
         activity_data = activity_response.data['activity'][0]

editorsnotes/api/urls.py

@@ -5,8 +5,8 @@
 
 project_specific_patterns = patterns('',
     ### Project (general) ###
-    url(r'^$', views.ProjectDetail.as_view(), name='api-project-detail'),
-    url(r'^activity/$', views.ActivityView.as_view(), name='api-project-activity'),
+    url(r'^$', views.ProjectDetail.as_view(), name='api-projects-detail'),
+    url(r'^activity/$', views.ActivityView.as_view(), name='api-projects-activity'),
 
     ### Topics ###
     url(r'^topics/$', views.TopicList.as_view(), name='api-topics-list'),

editorsnotes/api/views/people.py

@@ -11,29 +11,55 @@
 __all__ = ['ActivityView', 'ProjectList', 'ProjectDetail']
 
 class ProjectList(ListAPIView):
-    model = Project
+    queryset = Project.objects.all()
     serializer_class = ProjectSerializer
 
 class ProjectDetail(RetrieveAPIView):
-    model = Project
+    queryset = Project.objects.all()
     serializer_class = ProjectSerializer
     def get_object(self):
         qs = self.get_queryset()
         project = get_object_or_404(qs, slug=self.kwargs['project_slug'])
         return project
 
+def parse_int(val, default=25, maximum=100):
+    if not isinstance(val, int):
+        try:
+            val = int(val)
+        except ValueError:
+            val = default
+    return val if val <= maximum else maximum
+
 class ActivityView(GenericAPIView):
     """
     Recent activity for a user or project.
 
     Takes the following arguments:
 
-    * number (int)
+    * count (int)
     * start (int)
     * type ("note", "topic", "document", "transcript", "footnote")
     * action ("add", "change", "delete")
     * order ('asc' or 'desc')
     """
+    TYPES = ['note', 'topic', 'document', 'transcript', 'footnote']
+    ACTIONS = ['added', 'changed', 'deleted']
+    def get_es_query(self):
+        q = {'query': {'filtered': {'filter': {'bool': { 'must': []}}}}}
+        params = self.request.QUERY_PARAMS
+        if 'count' in params:
+            q['size'] = parse_int(params['count'])
+        if 'start' in params:
+            q['from'] = parse_int(params['start'])
+        if 'type' in params and params['type'] in self.TYPES:
+            q['query']['filtered']['filter']['bool']['must'].append({
+                'term': { 'data.type': params['type'] }
+            })
+        if 'action' in params and params['action'] in self.ACTIONS:
+            q['query']['filtered']['filter']['bool']['must'].append({
+                'term': { 'data.action': params['action'] }
+            })
+        return q
     def get_object(self, username=None, project_slug=None):
         if username is not None:
             obj = get_object_or_404(User, username=username)
@@ -44,7 +70,6 @@ def get_object(self, username=None, project_slug=None):
         return obj
     def get(self, request, format=None, **kwargs):
         obj = self.get_object(**kwargs)
-        data = get_index('activity').get_activity_for(obj)
+        es_query = self.get_es_query()
+        data = get_index('activity').get_activity_for(obj, es_query)
         return Response({ 'activity': data })
-
-

editorsnotes/search/index.py

@@ -158,25 +158,36 @@ def search(self, query, highlight=False, **kwargs):
 class ActivityIndex(ElasticSearchIndex):
     def get_name(self):
         return settings.ELASTICSEARCH_PREFIX + '-activitylog'
-    def get_activity_for(self, entity, size=25, **kwargs):
-        query = {
-            'query': {},
-            'sort': {'data.time': { 'order': 'desc', 'ignore_unmapped': True }}
-        }
+    def get_activity_for(self, entity, es_query=None, size=25):
+        query = es_query or {}
+        if not 'size' in query:
+            query['size'] = size
+        if not 'sort' in query:
+            query['sort'] = {
+                'data.time': { 'order': 'desc', 'ignore_unmapped': True }
+            }
+        if not 'query' in query:
+            query['query'] = {'filtered': {'filter': {'bool': { 'must': []}}}}
+
         if isinstance(entity, User):
-            query['query']['match'] = { 'data.user': entity.username }
+            query['query']['filtered']['filter']['bool']['must'].append({
+                'term': { 'data.user': entity.username }
+            })
         elif isinstance(entity, Project):
-            query['query']['match'] = { 'data.project': entity.slug }
+            query['query']['filtered']['filter']['bool']['must'].append({
+                'term': { 'data.project': entity.slug }
+            })
         else:
             raise ValueError('Must pass either project or user')
-        query.update(kwargs)
-        search = self.es.search(query, index=self.name, size=size)
+
+        search = self.es.search(query, index=self.name)
         return [ hit['_source']['data'] for hit in search['hits']['hits'] ]
 
-    def handle_edit(self, instance):
+    def handle_edit(self, instance, refresh=True):
         serializer = ActivitySerializer(instance)
         data = json.loads(JSONRenderer().render(serializer.data),
                           object_pairs_hook=OrderedDict)
 
-        self.es.index(self.name, 'activity',{ 'id': instance.id, 'data': data },
-                      id='id', refresh=True)
+        self.es.index(self.name, 'activity',
+                      { 'id': instance.id, 'data': data },
+                      refresh=refresh)

editorsnotes/search/management/commands/rebuild_activity_index.py

@@ -1,41 +1,26 @@
 from django.core.management.base import BaseCommand
 from django.db import models
 
-from reversion.models import Version
+from editorsnotes.main.models.auth import LogActivity
 
-from editorsnotes.main.models.base import Administered
-
-from ... import activity_index
+from ... import get_index
 
 class Command(BaseCommand):
     help = 'Rebuild elasticsearch activity index'
     def handle(self, *args, **kwargs):
+        activity_index = get_index('activity')
+
         activity_index.delete()
         activity_index.create()
 
-        administered_models = list(
-            m._meta.model_name for m in models.get_models()
-            if issubclass(m, Administered))
-
-        qs = Version.objects\
-                .select_related('revision__user',
-                                'revision__revisionproject__project',
-                                'content_type')\
-                .filter(content_type__model__in=administered_models)\
-                .order_by('-revision__date_created')
+        qs = LogActivity.objects\
+                .select_related('project', 'user', 'content_type')
 
-
-        self.stdout.write('Indexing {} actions'.format(qs.count()))
+        ct = qs.count()
+        self.stdout.write('Indexing {} actions'.format(ct))
 
         i = 0
-        CHUNK_SIZE = 1000
-
-        while True:
-            chunk = qs[i:i + CHUNK_SIZE]
-            if not chunk:
-                break
-            data = (activity_index.data_from_reversion_version(version)
-                    for version in chunk)
-            activity_index.es.bulk_index(activity_index.name, 'activity', data)
-            i += CHUNK_SIZE
-
+        for activity in qs:
+            activity_index.handle_edit(activity, refresh=False)
+            i += 1
+            if (i % 100 == 0): self.stdout.write('{}/{}'.format(i, ct))