From 870600482bd7d2b77542c169f78f125c17cec677 Mon Sep 17 00:00:00 2001 From: Patrick Golden Date: Wed, 21 Jan 2015 14:06:45 -0500 Subject: [PATCH 1/7] Fix bug in action serializer Don't try to get URL of items that have been deleted --- editorsnotes/api/serializers/activity.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/editorsnotes/api/serializers/activity.py b/editorsnotes/api/serializers/activity.py index 1a10f224..7cfc84b1 100644 --- a/editorsnotes/api/serializers/activity.py +++ b/editorsnotes/api/serializers/activity.py @@ -21,6 +21,9 @@ class Meta: model = LogActivity fields = ('user', 'project', 'time', 'type', 'url', 'title', 'action',) def get_object_url(self, obj): - return None if obj.action == DELETION else obj.content_object.get_absolute_url() + if obj.action == DELETION or obj.content_object is None: + return None + else: + return obj.content_object.get_absolute_url() def get_action_repr(self, version_obj): return VERSION_ACTIONS[version_obj.action] From 593906ef04f5dda175091c5de5f344b09b4a9cb5 Mon Sep 17 00:00:00 2001 From: Patrick Golden Date: Wed, 21 Jan 2015 14:08:25 -0500 Subject: [PATCH 2/7] Fix project API view for DRF3 --- editorsnotes/api/views/people.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/editorsnotes/api/views/people.py b/editorsnotes/api/views/people.py index a14e446e..328304bd 100644 --- a/editorsnotes/api/views/people.py +++ b/editorsnotes/api/views/people.py @@ -11,11 +11,11 @@ __all__ = ['ActivityView', 'ProjectList', 'ProjectDetail'] class ProjectList(ListAPIView): - model = Project + queryset = Project.objects.all() serializer_class = ProjectSerializer class ProjectDetail(RetrieveAPIView): - model = Project + queryset = Project.objects.all() serializer_class = ProjectSerializer def get_object(self): qs = self.get_queryset() From 2dd3d37b328505a35ad7f1cd4cdad2a375dc1a03 Mon Sep 17 00:00:00 2001 From: Patrick Golden Date: Wed, 21 Jan 2015 14:09:50 -0500 Subject: [PATCH 3/7] Improve retrieving activity from ES index Adds ability to extend ES query --- editorsnotes/api/views/people.py | 33 ++++++++++++++++++++++++++++---- editorsnotes/search/index.py | 28 ++++++++++++++++++--------- 2 files changed, 48 insertions(+), 13 deletions(-) diff --git a/editorsnotes/api/views/people.py b/editorsnotes/api/views/people.py index 328304bd..b7e30359 100644 --- a/editorsnotes/api/views/people.py +++ b/editorsnotes/api/views/people.py @@ -22,18 +22,44 @@ def get_object(self): project = get_object_or_404(qs, slug=self.kwargs['project_slug']) return project +def parse_int(val, default=25, maximum=100): + if not isinstance(val, int): + try: + val = int(val) + except ValueError: + val = default + return val if val <= maximum else maximum + class ActivityView(GenericAPIView): """ Recent activity for a user or project. Takes the following arguments: - * number (int) + * count (int) * start (int) * type ("note", "topic", "document", "transcript", "footnote") * action ("add", "change", "delete") * order ('asc' or 'desc') """ + TYPES = ['note', 'topic', 'document', 'transcript', 'footnote'] + ACTIONS = ['added', 'changed', 'deleted'] + def get_es_query(self): + q = {'query': {'filtered': {'filter': {'bool': { 'must': []}}}}} + params = self.request.QUERY_PARAMS + if 'count' in params: + q['size'] = parse_int(params['count']) + if 'start' in params: + q['from'] = parse_int(params['start']) + if 'type' in params and params['type'] in self.TYPES: + q['query']['filtered']['filter']['bool']['must'].append({ + 'term': { 'data.type': params['type'] } + }) + if 'action' in params and params['action'] in self.ACTIONS: + q['query']['filtered']['filter']['bool']['must'].append({ + 'term': { 'data.action': params['action'] } + }) + return q def get_object(self, username=None, project_slug=None): if username is not None: obj = get_object_or_404(User, username=username) @@ -44,7 +70,6 @@ def get_object(self, username=None, project_slug=None): return obj def get(self, request, format=None, **kwargs): obj = self.get_object(**kwargs) - data = get_index('activity').get_activity_for(obj) + es_query = self.get_es_query() + data = get_index('activity').get_activity_for(obj, es_query) return Response({ 'activity': data }) - - diff --git a/editorsnotes/search/index.py b/editorsnotes/search/index.py index 15ce43cf..ba98f9ed 100644 --- a/editorsnotes/search/index.py +++ b/editorsnotes/search/index.py @@ -158,19 +158,29 @@ def search(self, query, highlight=False, **kwargs): class ActivityIndex(ElasticSearchIndex): def get_name(self): return settings.ELASTICSEARCH_PREFIX + '-activitylog' - def get_activity_for(self, entity, size=25, **kwargs): - query = { - 'query': {}, - 'sort': {'data.time': { 'order': 'desc', 'ignore_unmapped': True }} - } + def get_activity_for(self, entity, es_query=None, size=25): + query = es_query or {} + if not 'size' in query: + query['size'] = size + if not 'sort' in query: + query['sort'] = { + 'data.time': { 'order': 'desc', 'ignore_unmapped': True } + } + if not 'query' in query: + query['query'] = {'filtered': {'filter': {'bool': { 'must': []}}}} + if isinstance(entity, User): - query['query']['match'] = { 'data.user': entity.username } + query['query']['filtered']['filter']['bool']['must'].append({ + 'term': { 'data.user': entity.username } + }) elif isinstance(entity, Project): - query['query']['match'] = { 'data.project': entity.slug } + query['query']['filtered']['filter']['bool']['must'].append({ + 'term': { 'data.project': entity.slug } + }) else: raise ValueError('Must pass either project or user') - query.update(kwargs) - search = self.es.search(query, index=self.name, size=size) + + search = self.es.search(query, index=self.name) return [ hit['_source']['data'] for hit in search['hits']['hits'] ] def handle_edit(self, instance): From 35f211694b2dd847c137f17625b84140709d8354 Mon Sep 17 00:00:00 2001 From: Patrick Golden Date: Wed, 21 Jan 2015 14:10:32 -0500 Subject: [PATCH 4/7] Fix activity index I had never updated it to take into account the new LogActivity model --- editorsnotes/search/index.py | 7 ++-- .../commands/rebuild_activity_index.py | 39 ++++++------------- 2 files changed, 16 insertions(+), 30 deletions(-) diff --git a/editorsnotes/search/index.py b/editorsnotes/search/index.py index ba98f9ed..75db5a91 100644 --- a/editorsnotes/search/index.py +++ b/editorsnotes/search/index.py @@ -183,10 +183,11 @@ def get_activity_for(self, entity, es_query=None, size=25): search = self.es.search(query, index=self.name) return [ hit['_source']['data'] for hit in search['hits']['hits'] ] - def handle_edit(self, instance): + def handle_edit(self, instance, refresh=True): serializer = ActivitySerializer(instance) data = json.loads(JSONRenderer().render(serializer.data), object_pairs_hook=OrderedDict) - self.es.index(self.name, 'activity',{ 'id': instance.id, 'data': data }, - id='id', refresh=True) + self.es.index(self.name, 'activity', + { 'id': instance.id, 'data': data }, + refresh=refresh) diff --git a/editorsnotes/search/management/commands/rebuild_activity_index.py b/editorsnotes/search/management/commands/rebuild_activity_index.py index 72db7ec3..ce4c1af4 100644 --- a/editorsnotes/search/management/commands/rebuild_activity_index.py +++ b/editorsnotes/search/management/commands/rebuild_activity_index.py @@ -1,41 +1,26 @@ from django.core.management.base import BaseCommand from django.db import models -from reversion.models import Version +from editorsnotes.main.models.auth import LogActivity -from editorsnotes.main.models.base import Administered - -from ... import activity_index +from ... import get_index class Command(BaseCommand): help = 'Rebuild elasticsearch activity index' def handle(self, *args, **kwargs): + activity_index = get_index('activity') + activity_index.delete() activity_index.create() - administered_models = list( - m._meta.model_name for m in models.get_models() - if issubclass(m, Administered)) - - qs = Version.objects\ - .select_related('revision__user', - 'revision__revisionproject__project', - 'content_type')\ - .filter(content_type__model__in=administered_models)\ - .order_by('-revision__date_created') + qs = LogActivity.objects\ + .select_related('project', 'user', 'content_type') - - self.stdout.write('Indexing {} actions'.format(qs.count())) + ct = qs.count() + self.stdout.write('Indexing {} actions'.format(ct)) i = 0 - CHUNK_SIZE = 1000 - - while True: - chunk = qs[i:i + CHUNK_SIZE] - if not chunk: - break - data = (activity_index.data_from_reversion_version(version) - for version in chunk) - activity_index.es.bulk_index(activity_index.name, 'activity', data) - i += CHUNK_SIZE - + for activity in qs: + activity_index.handle_edit(activity, refresh=False) + i += 1 + if (i % 100 == 0): self.stdout.write('{}/{}'.format(i, ct)) From 4e4aef5d2bcca1410f4d34b802a957c799a999c4 Mon Sep 17 00:00:00 2001 From: Patrick Golden Date: Thu, 29 Jan 2015 00:05:41 -0500 Subject: [PATCH 5/7] Tweak project serializer Add item counts and description; change field names around --- editorsnotes/api/serializers/__init__.py | 1 + editorsnotes/api/serializers/base.py | 2 +- editorsnotes/api/serializers/projects.py | 45 +++++++++++++++--------- editorsnotes/api/serializers/topics.py | 2 +- editorsnotes/api/urls.py | 4 +-- 5 files changed, 33 insertions(+), 21 deletions(-) diff --git a/editorsnotes/api/serializers/__init__.py b/editorsnotes/api/serializers/__init__.py index 571c63ce..b01e14ad 100644 --- a/editorsnotes/api/serializers/__init__.py +++ b/editorsnotes/api/serializers/__init__.py @@ -2,3 +2,4 @@ from topics import TopicSerializer from notes import NoteSerializer from activity import ActivitySerializer +from projects import ProjectSerializer diff --git a/editorsnotes/api/serializers/base.py b/editorsnotes/api/serializers/base.py index ec44b497..af4e510a 100644 --- a/editorsnotes/api/serializers/base.py +++ b/editorsnotes/api/serializers/base.py @@ -51,7 +51,7 @@ def __init__(self, *args, **kwargs): def get_attribute(self, obj): return obj.get_affiliation() def to_representation(self, value): - url = reverse('api:api-project-detail', args=(value.slug,), + url = reverse('api:api-projects-detail', args=(value.slug,), request=self.context['request']) return { 'name': value.name, 'url': url } diff --git a/editorsnotes/api/serializers/projects.py b/editorsnotes/api/serializers/projects.py index 1f4fd4b3..d76d72a6 100644 --- a/editorsnotes/api/serializers/projects.py +++ b/editorsnotes/api/serializers/projects.py @@ -2,24 +2,35 @@ from rest_framework.reverse import reverse from editorsnotes.main.models import Project +from editorsnotes.search import get_index + +from .base import URLField + +def count_for(project, doc_type): + index = get_index('main') + count = index.es.count( + { 'query': { 'term': { 'serialized.project.name': project.name }}}, + index=index.name, + doc_type=doc_type) + return count['count'] + class ProjectSerializer(serializers.ModelSerializer): - url = serializers.SerializerMethodField('get_api_url') - notes = serializers.SerializerMethodField('get_notes_url') - topics = serializers.SerializerMethodField('get_topics_url') - documents = serializers.SerializerMethodField('get_documents_url') + url = URLField('api:api-projects-detail', ('slug',)) + notes = serializers.SerializerMethodField() + notes_url = URLField('api:api-notes-list', ('slug',)) + topics = serializers.SerializerMethodField() + topics_url = URLField('api:api-topics-list', ('slug',)) + documents = serializers.SerializerMethodField() + documents_url = URLField('api:api-documents-list', ('slug',)) + activity_url = URLField('api:api-projects-activity', ('slug',)) class Meta: model = Project - fields = ('slug', 'name', 'url', 'notes', 'topics', 'documents',) - def get_api_url(self, obj): - return reverse('api:api-project-detail', args=(obj.slug,), - request=self.context['request']) - def get_notes_url(self, obj): - return reverse('api:api-notes-list', args=(obj.slug,), - request=self.context['request']) - def get_topics_url(self, obj): - return reverse('api:api-topics-list', args=(obj.slug,), - request=self.context['request']) - def get_documents_url(self, obj): - return reverse('api:api-documents-list', args=(obj.slug,), - request=self.context['request']) + fields = ('slug', 'url', 'name', 'description', 'notes', 'notes_url', + 'topics', 'topics_url', 'documents', 'documents_url', 'activity_url') + def get_notes(self, obj): + return count_for(obj, 'note') + def get_topics(self, obj): + return count_for(obj, 'topic') + def get_documents(self, obj): + return count_for(obj, 'document') diff --git a/editorsnotes/api/serializers/topics.py b/editorsnotes/api/serializers/topics.py index a6fc424a..b2974f98 100644 --- a/editorsnotes/api/serializers/topics.py +++ b/editorsnotes/api/serializers/topics.py @@ -33,7 +33,7 @@ def get_alternate_forms(self, obj): def get_project_value(self, obj): return [OrderedDict(( ('project_name', topic.project.name), - ('project_url', reverse('api:api-project-detail', + ('project_url', reverse('api:api-projects-detail', args=(topic.project.slug,), request=self.context['request'])), ('preferred_name', topic.preferred_name), diff --git a/editorsnotes/api/urls.py b/editorsnotes/api/urls.py index 3626c9ad..3ab43c42 100644 --- a/editorsnotes/api/urls.py +++ b/editorsnotes/api/urls.py @@ -5,8 +5,8 @@ project_specific_patterns = patterns('', ### Project (general) ### - url(r'^$', views.ProjectDetail.as_view(), name='api-project-detail'), - url(r'^activity/$', views.ActivityView.as_view(), name='api-project-activity'), + url(r'^$', views.ProjectDetail.as_view(), name='api-projects-detail'), + url(r'^activity/$', views.ActivityView.as_view(), name='api-projects-activity'), ### Topics ### url(r'^topics/$', views.TopicList.as_view(), name='api-topics-list'), From 6aa202e00b24c8a2055ab373894637f0d8a0fc58 Mon Sep 17 00:00:00 2001 From: Patrick Golden Date: Thu, 29 Jan 2015 00:06:13 -0500 Subject: [PATCH 6/7] Fix serializer field syntax in CitationSerializer --- editorsnotes/api/serializers/documents.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/editorsnotes/api/serializers/documents.py b/editorsnotes/api/serializers/documents.py index ecee2c01..e8f35865 100644 --- a/editorsnotes/api/serializers/documents.py +++ b/editorsnotes/api/serializers/documents.py @@ -100,7 +100,7 @@ class CitationSerializer(serializers.ModelSerializer): document = HyperlinkedProjectItemField(view_name='api:api-documents-detail', queryset=Document.objects, required=True) - document_description = serializers.SerializerMethodField('get_document_description') + document_description = serializers.SerializerMethodField() class Meta: model = Citation fields = ('id', 'url', 'ordering', 'document', 'document_description', 'notes') From 312322efa0e006da23b83988b896784c848d599b Mon Sep 17 00:00:00 2001 From: Patrick Golden Date: Thu, 29 Jan 2015 00:26:04 -0500 Subject: [PATCH 7/7] Update changed view name in API test --- editorsnotes/api/tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/editorsnotes/api/tests.py b/editorsnotes/api/tests.py index 6931824e..28bed0dc 100644 --- a/editorsnotes/api/tests.py +++ b/editorsnotes/api/tests.py @@ -137,7 +137,7 @@ def test_topic_api_create(self): self.assertEqual(Revision.objects.count(), 1) # Make sure an entry was added to the activity index - activity_response = self.client.get(reverse('api:api-project-activity', + activity_response = self.client.get(reverse('api:api-projects-activity', args=[self.project.slug])) self.assertEqual(activity_response.status_code, 200) self.assertEqual(len(activity_response.data['activity']), 1) @@ -258,7 +258,7 @@ def test_topic_api_update(self): self.assertEqual(Revision.objects.count(), 1) # Make sure an entry was added to the activity index - activity_response = self.client.get(reverse('api:api-project-activity', + activity_response = self.client.get(reverse('api:api-projects-activity', args=[self.project.slug])) self.assertEqual(activity_response.status_code, 200) activity_data = activity_response.data['activity'][0] @@ -324,7 +324,7 @@ def test_topic_api_delete(self): self.assertEqual(main_models.TopicNode.objects.count(), 1) # Make sure an entry was added to the activity index - activity_response = self.client.get(reverse('api:api-project-activity', + activity_response = self.client.get(reverse('api:api-projects-activity', args=[self.project.slug])) self.assertEqual(activity_response.status_code, 200) activity_data = activity_response.data['activity'][0]