Implemented some more Accessions

Author: hotdogee

data/admin.py

@@ -28,7 +28,7 @@ class Media:
 admin.site.register(File, FileAdmin)
 
 class ItemAdmin(admin.ModelAdmin):
-    list_display = ('key', 'id', 'file')
+    list_display = ('key', 'id', 'file', 'attributes')
     search_fields = ('attributes',)
     actions_on_top = True
     actions_on_bottom = True

data/api.py

@@ -0,0 +1,51 @@
+from rest_framework import renderers
+from rest_framework.renderers import JSONRenderer, BrowsableAPIRenderer
+from rest_framework.decorators import action, link
+from rest_framework.response import Response
+from rest_framework.views import APIView
+from rest_framework import viewsets
+from django.contrib.auth.models import User
+from django.http import Http404
+from .models import File, FileRelationship, Item, ItemRelationship
+from .serializers import FileSerializer, ItemSerializer
+
+class FileViewSet(viewsets.ReadOnlyModelViewSet):
+    """
+    Retrieve organisms.
+    """
+    queryset = File.objects.all()
+    serializer_class = FileSerializer
+    lookup_field = 'id'
+
+class ItemViewSet(viewsets.ReadOnlyModelViewSet):
+    """
+    Retrieve sequence types.
+    """
+    queryset = Item.objects.all()
+    serializer_class = ItemSerializer
+    lookup_field = 'key'
+
+class AccessionList(APIView):
+    """
+    List all snippets, or create a new snippet.
+    """
+    def get(self, request, format=None):
+        files = File.objects.all()
+        serializer = FileSerializer(files, many=True)
+        return Response(serializer.data)
+
+class AccessionDetail(APIView):
+    """
+    Retrieve, update or delete a snippet instance.
+    """
+    def get_file(self, id):
+        try:
+            return File.objects.get(id=id)
+        except File.DoesNotExist:
+            raise Http404
+
+    def get(self, request, accession, format=None):
+        ids = accession.split('/')
+        file = self.get_file(ids[0])
+        serializer = FileSerializer(file)
+        return Response(serializer.data)

data/file_types/gff3.py

@@ -1046,7 +1046,7 @@ def __repr__(self, level=0):
         return sorted(list(root_set), key=lambda x: x.value)
 
 try:
-    from ordereddict import OrderedDict
+    from collections import OrderedDict
 except ImportError:
     # Backport of OrderedDict() class that runs on Python 2.4, 2.5, 2.6, 2.7 and pypy.
     # Passes Python2.7's test suite and incorporates all the latest updates.

data/models.py

@@ -1,8 +1,11 @@
 from django.db import models
 from django.contrib.postgres.fields import HStoreField
 from filebrowser.fields import FileBrowseField
-from file_types.gff3 import Gff3
+from file_types.gff3 import Gff3, fasta_file_to_dict
 import os
+import six
+import numbers
+from collections import OrderedDict
 
 class File(models.Model):
     '''A File is a self-contained List, Tree, or Graph of Items.
@@ -13,7 +16,7 @@ class File(models.Model):
     hash = models.TextField(blank=True)
     processed_hash = models.TextField(blank=True)
     processed_date = models.DateTimeField(auto_now_add=True)
-    attributes = HStoreField()
+    attributes = HStoreField(blank=True)
     related = models.ManyToManyField('self', through='FileRelationship', through_fields=('source', 'target'), symmetrical=False)
 
     def process_file(self):
@@ -22,9 +25,63 @@ def process_file(self):
         from sys import platform
         from subprocess import Popen, PIPE
         if self.type == 'gff3':
-            gff = Gff3(str(self.name.path_full))
+            try:
+                # remove existing items
+                self.item_set.all().delete()
+                # parse file
+                gff = Gff3(str(self.name.path_full))
+                #seq_count = 0
+                item_list = []
+                item_relationship_list = []
+
+                # assign every line to an Item
+                for idx, ld in enumerate(gff.lines):
+                    attributes = {}
+                    for k, v in ld.items():
+                        if v:
+                            if isinstance(v, six.string_types):
+                                attributes[k] = v
+                            elif isinstance(v, numbers.Number):
+                                attributes[k] = str(v)
+                    item_list.append(Item(file=self, id=idx+1, text=ld['line_raw'], attributes=attributes))
+
+                # assign Accessions with dfs tranversal
+                def dfs(root_line, path=OrderedDict()):
+                    pass
+                # get a list of root nodes
+                root_lines = [line_data for line_data in gff.lines if line_data['line_type'] == 'feature' and not line_data['parents']]
+                for idx, root_line in enumerate(root_lines):
+                    pass
+
+                # build ItemRelationships
+
+                if len(item_list) > 0:
+                    Item.objects.bulk_create(item_list)
+                if len(item_relationship_list) > 0:
+                    ItemRelationship.objects.bulk_create(item_relationship_list)
+                return 0, '', '%d items added.' % len(item_list)
+            except Exception as e:
+                return 1, str(e), ''
             #return p.returncode, error, output
-            return 1, 'Gff3 lines = {}'.format(len(gff.lines)), ''
+            #return 1, 'Gff3 lines = {}'.format(len(gff.lines)), ''
+        elif self.type == 'fasta':
+            try:
+                # remove existing items
+                self.item_set.all().delete()
+                # parse file
+                fasta, count = fasta_file_to_dict(str(self.name.path_full))
+                #seq_count = 0
+                item_list = []
+                item_relationship_list = []
+                for idx, id in enumerate(fasta):
+                    item_list.append(Item(file=self, id=idx+1, text='{}\n{}'.format(fasta[id]['header'], fasta[id]['seq']), attributes={'header': fasta[id]['header'], 'id': fasta[id]['id'], 'seq': fasta[id]['seq']}))
+                if len(item_list) > 0:
+                    Item.objects.bulk_create(item_list)
+                if len(item_relationship_list) > 0:
+                    ItemRelationship.objects.bulk_create(item_relationship_list)
+                return 0, '', '%d items added.' % len(item_list)
+            except Exception as e:
+                return 1, str(e), ''
         else:
             return 1, 'File type not supported', ''
 
@@ -42,8 +99,9 @@ class Item(models.Model):
     '''Accessions are assigned to Items.
     '''
     key = models.AutoField(primary_key=True) # auto
-    id = models.PositiveIntegerField() # 
+    accession = models.TextField(blank=True)
     file = models.ForeignKey(File)
+    text = models.TextField(blank=True)
     attributes = HStoreField()
     related = models.ManyToManyField('self', through='ItemRelationship', through_fields=('source', 'target'), symmetrical=False)
 
@@ -55,4 +113,13 @@ class ItemRelationship(models.Model):
     '''
     source = models.ForeignKey(Item, related_name='+')
     type = models.CharField(max_length=50, help_text='Relationship Type')
-    target = models.ForeignKey(Item, related_name='+')
+    target = models.ForeignKey(Item, related_name='+')
+
+class Accession(models.Model):
+    '''Accessions are assigned to Items.
+    '''
+    accession = models.TextField(blank=True)
+    item = models.ForeignKey(Item)
+    
+    def __unicode__(self):
+        return str(self.accession)

data/serializers.py

@@ -0,0 +1,17 @@
+from rest_framework import serializers
+from rest_framework.pagination import PaginationSerializer
+from django.contrib.auth.models import User
+from .models import File, FileRelationship, Item, ItemRelationship
+
+class FileSerializer(serializers.HyperlinkedModelSerializer):
+    url = serializers.HyperlinkedIdentityField(view_name='file-detail', lookup_field='id')
+
+    class Meta:
+        model = File
+
+class ItemSerializer(serializers.HyperlinkedModelSerializer):
+    url = serializers.HyperlinkedIdentityField(view_name='item-detail', lookup_field='key')
+    file = serializers.HyperlinkedRelatedField(view_name='file-detail', lookup_field='id')
+
+    class Meta:
+        model = Item

data/urls.py

@@ -0,0 +1,16 @@
+from django.conf.urls import patterns, url, include
+from blast import views
+from .api import FileViewSet, ItemViewSet, AccessionDetail
+from rest_framework.routers import DefaultRouter
+from rest_framework.urlpatterns import format_suffix_patterns
+
+router = DefaultRouter()
+router.register(r'file', FileViewSet)
+router.register(r'item', ItemViewSet)
+
+urlpatterns = format_suffix_patterns('',
+    url(r'^api/', include(router.urls)),
+    url(r'^api/accessions/(?P<accession>.+)/$', AccessionDetail.as_view()),
+    url(r'^api-auth/', include('rest_framework.urls', namespace='rest_framework')),
+    url(r'^api-docs/', include('rest_framework_swagger.urls')),
+)

i5k.pyproj

@@ -75,12 +75,15 @@
     <Compile Include="dashboard\views.py" />
     <Compile Include="dashboard\__init__.py" />
     <Compile Include="data\admin.py" />
+    <Compile Include="data\api.py" />
     <Compile Include="data\file_types\gff3.py" />
     <Compile Include="data\file_types\__init__.py" />
     <Compile Include="data\migrations\0001_initial.py" />
     <Compile Include="data\migrations\__init__.py" />
     <Compile Include="data\models.py" />
+    <Compile Include="data\serializers.py" />
     <Compile Include="data\tests.py" />
+    <Compile Include="data\urls.py" />
     <Compile Include="data\views.py" />
     <Compile Include="data\__init__.py" />
     <Compile Include="i5k\settings_prod.py" />

i5k/settings.py

@@ -271,6 +271,10 @@
             {'model': 'speciespassword', 'label': '[Read Only] Passwords'},
             {'model': 'registration', 'label': '[Read Only] Registrations'},
         )},
+        {'app': 'data', 'label': 'Data', 'icon':'icon-leaf', 'models': (
+            {'model': 'file'},
+            {'model': 'item'},
+        )},
         # auth and axes
         {'label': 'Auth', 'icon':'icon-lock', 'models': (
             {'model': 'auth.user'},

i5k/urls.py

@@ -116,6 +116,8 @@
 
     # BLAST
     url(r'^blast/', include('blast.urls', namespace='blast')),
+    # BLAST
+    url(r'^data/', include('data.urls', namespace='data')),
 )
 from django.conf import settings
 if settings.DEBUG:

web.config

@@ -13,7 +13,7 @@
     </trace>
   </system.diagnostics>
   <appSettings>
-    <add key="WSGI_HANDLER" value="django.core.handlers.wsgi.WSGIHandler()" />
+    <add key="WSGI_HANDLER" value="django.core.wsgi.get_wsgi_application()" />
     <add key="PYTHONPATH" value="%ROOTDIR%" />
     <add key="DJANGO_SETTINGS_MODULE" value="i5k.settings" />
   </appSettings>

web.debug.config

@@ -7,7 +7,7 @@
 <configuration xmlns:xdt="http://schemas.microsoft.com/XML-Document-Transform">
   <appSettings>
     <!-- The following setting is meant to protect the debugging endpoint against inadvertent access, and should be treated as a password. -->
-    <add key="WSGI_PTVSD_SECRET" value="dkryuwojxpn" xdt:Transform="Insert" />
+    <add key="WSGI_PTVSD_SECRET" value="gasgkhy43j1" xdt:Transform="Insert" />
   </appSettings>
   <system.web>
     <!-- Required for websockets. -->