diff --git a/editorsnotes/api/serializers/__init__.py b/editorsnotes/api/serializers/__init__.py index 571c63ce..b01e14ad 100644 --- a/editorsnotes/api/serializers/__init__.py +++ b/editorsnotes/api/serializers/__init__.py @@ -2,3 +2,4 @@ from topics import TopicSerializer from notes import NoteSerializer from activity import ActivitySerializer +from projects import ProjectSerializer diff --git a/editorsnotes/api/serializers/activity.py b/editorsnotes/api/serializers/activity.py index 1a10f224..7cfc84b1 100644 --- a/editorsnotes/api/serializers/activity.py +++ b/editorsnotes/api/serializers/activity.py @@ -21,6 +21,9 @@ class Meta: model = LogActivity fields = ('user', 'project', 'time', 'type', 'url', 'title', 'action',) def get_object_url(self, obj): - return None if obj.action == DELETION else obj.content_object.get_absolute_url() + if obj.action == DELETION or obj.content_object is None: + return None + else: + return obj.content_object.get_absolute_url() def get_action_repr(self, version_obj): return VERSION_ACTIONS[version_obj.action] diff --git a/editorsnotes/api/serializers/base.py b/editorsnotes/api/serializers/base.py index ec44b497..af4e510a 100644 --- a/editorsnotes/api/serializers/base.py +++ b/editorsnotes/api/serializers/base.py @@ -51,7 +51,7 @@ def __init__(self, *args, **kwargs): def get_attribute(self, obj): return obj.get_affiliation() def to_representation(self, value): - url = reverse('api:api-project-detail', args=(value.slug,), + url = reverse('api:api-projects-detail', args=(value.slug,), request=self.context['request']) return { 'name': value.name, 'url': url } diff --git a/editorsnotes/api/serializers/documents.py b/editorsnotes/api/serializers/documents.py index ecee2c01..e8f35865 100644 --- a/editorsnotes/api/serializers/documents.py +++ b/editorsnotes/api/serializers/documents.py @@ -100,7 +100,7 @@ class CitationSerializer(serializers.ModelSerializer): document = HyperlinkedProjectItemField(view_name='api:api-documents-detail', queryset=Document.objects, required=True) - document_description = serializers.SerializerMethodField('get_document_description') + document_description = serializers.SerializerMethodField() class Meta: model = Citation fields = ('id', 'url', 'ordering', 'document', 'document_description', 'notes') diff --git a/editorsnotes/api/serializers/projects.py b/editorsnotes/api/serializers/projects.py index 1f4fd4b3..d76d72a6 100644 --- a/editorsnotes/api/serializers/projects.py +++ b/editorsnotes/api/serializers/projects.py @@ -2,24 +2,35 @@ from rest_framework.reverse import reverse from editorsnotes.main.models import Project +from editorsnotes.search import get_index + +from .base import URLField + +def count_for(project, doc_type): + index = get_index('main') + count = index.es.count( + { 'query': { 'term': { 'serialized.project.name': project.name }}}, + index=index.name, + doc_type=doc_type) + return count['count'] + class ProjectSerializer(serializers.ModelSerializer): - url = serializers.SerializerMethodField('get_api_url') - notes = serializers.SerializerMethodField('get_notes_url') - topics = serializers.SerializerMethodField('get_topics_url') - documents = serializers.SerializerMethodField('get_documents_url') + url = URLField('api:api-projects-detail', ('slug',)) + notes = serializers.SerializerMethodField() + notes_url = URLField('api:api-notes-list', ('slug',)) + topics = serializers.SerializerMethodField() + topics_url = URLField('api:api-topics-list', ('slug',)) + documents = serializers.SerializerMethodField() + documents_url = URLField('api:api-documents-list', ('slug',)) + activity_url = URLField('api:api-projects-activity', ('slug',)) class Meta: model = Project - fields = ('slug', 'name', 'url', 'notes', 'topics', 'documents',) - def get_api_url(self, obj): - return reverse('api:api-project-detail', args=(obj.slug,), - request=self.context['request']) - def get_notes_url(self, obj): - return reverse('api:api-notes-list', args=(obj.slug,), - request=self.context['request']) - def get_topics_url(self, obj): - return reverse('api:api-topics-list', args=(obj.slug,), - request=self.context['request']) - def get_documents_url(self, obj): - return reverse('api:api-documents-list', args=(obj.slug,), - request=self.context['request']) + fields = ('slug', 'url', 'name', 'description', 'notes', 'notes_url', + 'topics', 'topics_url', 'documents', 'documents_url', 'activity_url') + def get_notes(self, obj): + return count_for(obj, 'note') + def get_topics(self, obj): + return count_for(obj, 'topic') + def get_documents(self, obj): + return count_for(obj, 'document') diff --git a/editorsnotes/api/serializers/topics.py b/editorsnotes/api/serializers/topics.py index a6fc424a..b2974f98 100644 --- a/editorsnotes/api/serializers/topics.py +++ b/editorsnotes/api/serializers/topics.py @@ -33,7 +33,7 @@ def get_alternate_forms(self, obj): def get_project_value(self, obj): return [OrderedDict(( ('project_name', topic.project.name), - ('project_url', reverse('api:api-project-detail', + ('project_url', reverse('api:api-projects-detail', args=(topic.project.slug,), request=self.context['request'])), ('preferred_name', topic.preferred_name), diff --git a/editorsnotes/api/tests.py b/editorsnotes/api/tests.py index f9a44f4f..1a5d0ad3 100644 --- a/editorsnotes/api/tests.py +++ b/editorsnotes/api/tests.py @@ -138,7 +138,7 @@ def test_topic_api_create(self): self.assertEqual(Revision.objects.count(), 1) # Make sure an entry was added to the activity index - activity_response = self.client.get(reverse('api:api-project-activity', + activity_response = self.client.get(reverse('api:api-projects-activity', args=[self.project.slug])) self.assertEqual(activity_response.status_code, 200) self.assertEqual(len(activity_response.data['activity']), 1) @@ -259,7 +259,7 @@ def test_topic_api_update(self): self.assertEqual(Revision.objects.count(), 1) # Make sure an entry was added to the activity index - activity_response = self.client.get(reverse('api:api-project-activity', + activity_response = self.client.get(reverse('api:api-projects-activity', args=[self.project.slug])) self.assertEqual(activity_response.status_code, 200) activity_data = activity_response.data['activity'][0] @@ -325,7 +325,7 @@ def test_topic_api_delete(self): self.assertEqual(main_models.TopicNode.objects.count(), 1) # Make sure an entry was added to the activity index - activity_response = self.client.get(reverse('api:api-project-activity', + activity_response = self.client.get(reverse('api:api-projects-activity', args=[self.project.slug])) self.assertEqual(activity_response.status_code, 200) activity_data = activity_response.data['activity'][0] diff --git a/editorsnotes/api/urls.py b/editorsnotes/api/urls.py index 46c6a82f..bbe1d3b4 100644 --- a/editorsnotes/api/urls.py +++ b/editorsnotes/api/urls.py @@ -5,8 +5,8 @@ project_specific_patterns = patterns('', ### Project (general) ### - url(r'^$', views.ProjectDetail.as_view(), name='api-project-detail'), - url(r'^activity/$', views.ActivityView.as_view(), name='api-project-activity'), + url(r'^$', views.ProjectDetail.as_view(), name='api-projects-detail'), + url(r'^activity/$', views.ActivityView.as_view(), name='api-projects-activity'), ### Topics ### url(r'^topics/$', views.TopicList.as_view(), name='api-topics-list'), diff --git a/editorsnotes/api/views/people.py b/editorsnotes/api/views/people.py index a14e446e..b7e30359 100644 --- a/editorsnotes/api/views/people.py +++ b/editorsnotes/api/views/people.py @@ -11,29 +11,55 @@ __all__ = ['ActivityView', 'ProjectList', 'ProjectDetail'] class ProjectList(ListAPIView): - model = Project + queryset = Project.objects.all() serializer_class = ProjectSerializer class ProjectDetail(RetrieveAPIView): - model = Project + queryset = Project.objects.all() serializer_class = ProjectSerializer def get_object(self): qs = self.get_queryset() project = get_object_or_404(qs, slug=self.kwargs['project_slug']) return project +def parse_int(val, default=25, maximum=100): + if not isinstance(val, int): + try: + val = int(val) + except ValueError: + val = default + return val if val <= maximum else maximum + class ActivityView(GenericAPIView): """ Recent activity for a user or project. Takes the following arguments: - * number (int) + * count (int) * start (int) * type ("note", "topic", "document", "transcript", "footnote") * action ("add", "change", "delete") * order ('asc' or 'desc') """ + TYPES = ['note', 'topic', 'document', 'transcript', 'footnote'] + ACTIONS = ['added', 'changed', 'deleted'] + def get_es_query(self): + q = {'query': {'filtered': {'filter': {'bool': { 'must': []}}}}} + params = self.request.QUERY_PARAMS + if 'count' in params: + q['size'] = parse_int(params['count']) + if 'start' in params: + q['from'] = parse_int(params['start']) + if 'type' in params and params['type'] in self.TYPES: + q['query']['filtered']['filter']['bool']['must'].append({ + 'term': { 'data.type': params['type'] } + }) + if 'action' in params and params['action'] in self.ACTIONS: + q['query']['filtered']['filter']['bool']['must'].append({ + 'term': { 'data.action': params['action'] } + }) + return q def get_object(self, username=None, project_slug=None): if username is not None: obj = get_object_or_404(User, username=username) @@ -44,7 +70,6 @@ def get_object(self, username=None, project_slug=None): return obj def get(self, request, format=None, **kwargs): obj = self.get_object(**kwargs) - data = get_index('activity').get_activity_for(obj) + es_query = self.get_es_query() + data = get_index('activity').get_activity_for(obj, es_query) return Response({ 'activity': data }) - - diff --git a/editorsnotes/search/index.py b/editorsnotes/search/index.py index 15ce43cf..75db5a91 100644 --- a/editorsnotes/search/index.py +++ b/editorsnotes/search/index.py @@ -158,25 +158,36 @@ def search(self, query, highlight=False, **kwargs): class ActivityIndex(ElasticSearchIndex): def get_name(self): return settings.ELASTICSEARCH_PREFIX + '-activitylog' - def get_activity_for(self, entity, size=25, **kwargs): - query = { - 'query': {}, - 'sort': {'data.time': { 'order': 'desc', 'ignore_unmapped': True }} - } + def get_activity_for(self, entity, es_query=None, size=25): + query = es_query or {} + if not 'size' in query: + query['size'] = size + if not 'sort' in query: + query['sort'] = { + 'data.time': { 'order': 'desc', 'ignore_unmapped': True } + } + if not 'query' in query: + query['query'] = {'filtered': {'filter': {'bool': { 'must': []}}}} + if isinstance(entity, User): - query['query']['match'] = { 'data.user': entity.username } + query['query']['filtered']['filter']['bool']['must'].append({ + 'term': { 'data.user': entity.username } + }) elif isinstance(entity, Project): - query['query']['match'] = { 'data.project': entity.slug } + query['query']['filtered']['filter']['bool']['must'].append({ + 'term': { 'data.project': entity.slug } + }) else: raise ValueError('Must pass either project or user') - query.update(kwargs) - search = self.es.search(query, index=self.name, size=size) + + search = self.es.search(query, index=self.name) return [ hit['_source']['data'] for hit in search['hits']['hits'] ] - def handle_edit(self, instance): + def handle_edit(self, instance, refresh=True): serializer = ActivitySerializer(instance) data = json.loads(JSONRenderer().render(serializer.data), object_pairs_hook=OrderedDict) - self.es.index(self.name, 'activity',{ 'id': instance.id, 'data': data }, - id='id', refresh=True) + self.es.index(self.name, 'activity', + { 'id': instance.id, 'data': data }, + refresh=refresh) diff --git a/editorsnotes/search/management/commands/rebuild_activity_index.py b/editorsnotes/search/management/commands/rebuild_activity_index.py index 72db7ec3..ce4c1af4 100644 --- a/editorsnotes/search/management/commands/rebuild_activity_index.py +++ b/editorsnotes/search/management/commands/rebuild_activity_index.py @@ -1,41 +1,26 @@ from django.core.management.base import BaseCommand from django.db import models -from reversion.models import Version +from editorsnotes.main.models.auth import LogActivity -from editorsnotes.main.models.base import Administered - -from ... import activity_index +from ... import get_index class Command(BaseCommand): help = 'Rebuild elasticsearch activity index' def handle(self, *args, **kwargs): + activity_index = get_index('activity') + activity_index.delete() activity_index.create() - administered_models = list( - m._meta.model_name for m in models.get_models() - if issubclass(m, Administered)) - - qs = Version.objects\ - .select_related('revision__user', - 'revision__revisionproject__project', - 'content_type')\ - .filter(content_type__model__in=administered_models)\ - .order_by('-revision__date_created') + qs = LogActivity.objects\ + .select_related('project', 'user', 'content_type') - - self.stdout.write('Indexing {} actions'.format(qs.count())) + ct = qs.count() + self.stdout.write('Indexing {} actions'.format(ct)) i = 0 - CHUNK_SIZE = 1000 - - while True: - chunk = qs[i:i + CHUNK_SIZE] - if not chunk: - break - data = (activity_index.data_from_reversion_version(version) - for version in chunk) - activity_index.es.bulk_index(activity_index.name, 'activity', data) - i += CHUNK_SIZE - + for activity in qs: + activity_index.handle_edit(activity, refresh=False) + i += 1 + if (i % 100 == 0): self.stdout.write('{}/{}'.format(i, ct))