From d71d9b835f1d09a0c8d8ce17f82222f89ffb3c42 Mon Sep 17 00:00:00 2001 From: Andrew Davison Date: Wed, 8 Apr 2020 11:01:03 +0200 Subject: [PATCH 1/4] Add activity id to result data, to help debugging inconsistent behaviour --- model_validation_api/serializer/serializer_kg.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/model_validation_api/serializer/serializer_kg.py b/model_validation_api/serializer/serializer_kg.py index 8fa9709ab..9ba9f7043 100644 --- a/model_validation_api/serializer/serializer_kg.py +++ b/model_validation_api/serializer/serializer_kg.py @@ -666,7 +666,8 @@ def serialize(self, obj): "normalized_score": obj.normalized_score, # the following are temporary. Ideally the client should do a lookup using the IDs above "model_version": ScientificModelInstanceKGSerializer(validation_activity.model_instance, self.client).data, - "test_code": ValidationTestCodeKGSerializer(validation_activity.test_script, self.client).data + "test_code": ValidationTestCodeKGSerializer(validation_activity.test_script, self.client).data, + "activity_uuid": validation_activity.uuid } return data From 05087d9dc4f9263fad72ba42a91eba0b77450e4e Mon Sep 17 00:00:00 2001 From: Andrew Davison Date: Wed, 15 Apr 2020 16:54:19 +0200 Subject: [PATCH 2/4] Add support for REST endpoints for licenses and implementation status options. --- model_validation_api/admin.py | 17 ++++++++++++- model_validation_api/models.py | 20 ++++++++++----- model_validation_api/serializer/serializer.py | 2 ++ .../serializer/simple_serializer.py | 13 ++++++++++ model_validation_api/views.py | 25 ++++++++++++++++++- .../validation_service/settings.py | 14 +++++------ 6 files changed, 76 insertions(+), 15 deletions(-) diff --git a/model_validation_api/admin.py b/model_validation_api/admin.py index ac824695b..0ff960f48 100644 --- a/model_validation_api/admin.py +++ b/model_validation_api/admin.py @@ -4,7 +4,7 @@ CollabParameters, Param_DataModalities, Param_TestType, Param_Species, Param_BrainRegion, Param_CellType, Param_ModelScope, Param_AbstractionLevel, Param_ScoreType,Param_organizations, - Tickets, Comments) + Param_License, Param_ImplementationStatus, Tickets, Comments) admin.site.site_header = "HBP Validation Service administration" @@ -101,6 +101,21 @@ class Param_organizationsAdmin(admin.ModelAdmin): list_display = ('id', 'authorized_value') search_fields = ('id', 'authorized_value') + +@admin.register(Param_ImplementationStatus) +class Param_LicenseAdmin(admin.ModelAdmin): + list_display = ('id', 'authorized_value') + search_fields = ('id', 'authorized_value') + +@admin.register(Param_License) +class Param_LicenseAdmin(admin.ModelAdmin): + list_display = ('id', 'authorized_value') + search_fields = ('id', 'authorized_value') + + + + + @admin.register(Comments) class CommentsAdmin(admin.ModelAdmin): list_display = ('id', 'Ticket', 'author', 'text', 'creation_date') diff --git a/model_validation_api/models.py b/model_validation_api/models.py index 6d9e1d905..0aeb6bd94 100644 --- a/model_validation_api/models.py +++ b/model_validation_api/models.py @@ -8,9 +8,9 @@ from django.db import models from django.utils.encoding import python_2_unicode_compatible -@python_2_unicode_compatible +@python_2_unicode_compatible class CollabParameters(models.Model): - # id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False, ) + # id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False, ) id = models.CharField(primary_key=True, max_length=100 , default="") app_type = models.CharField(max_length=100 ,blank=True, help_text="type of application: model_catalog or validation_app") data_modalities = models.CharField(max_length=500 ,blank=True, help_text="data modalities") @@ -40,13 +40,13 @@ class ValidationTestDefinition(models.Model): data_type = models.CharField(max_length=100, help_text="type of comparison data (number, histogram, time series, etc.)") # S data_modality = models.CharField(max_length=100, default='', blank=True, help_text="recording modality for comparison data (ephys, fMRI, 2-photon, etc)") # J, K - test_type = models.CharField(max_length=100, + test_type = models.CharField(max_length=100, help_text="single cell activity, network structure, network activity, subcellular") # B, C protocol = models.TextField(blank=True, help_text="Description of the experimental protocol") # R (sort of) author = models.CharField(max_length=100, help_text="Author of this test") # H publication = models.CharField(max_length=1000, null=True, blank=True, help_text="Publication in which the validation data set was reported") # E score_type = models.CharField(help_text="Type of score: p-value, r square ..", max_length=20) - alias = models.CharField(max_length=200, unique=True, null=True, blank=True, default=None, help_text="alias of the test") + alias = models.CharField(max_length=200, unique=True, null=True, blank=True, default=None, help_text="alias of the test") creation_date = models.DateTimeField(auto_now_add=True, help_text="creation date of the test") status = models.CharField(max_length=100,blank=True, default='',help_text="status fo the test: Proposal, Published or on Development") # missing fields wrt Lungsi's spreadsheet @@ -181,7 +181,7 @@ class ValidationTestResult(models.Model): project = models.CharField(help_text="Project with which this test run is associated(optional)", max_length=200, blank=True) # project==collab_id for HBP ??rename o collab_id? - normalized_score = models.FloatField(help_text="A normalized numerical measure of the difference between model and experiment") + normalized_score = models.FloatField(help_text="A normalized numerical measure of the difference between model and experiment") hash = models.CharField(max_length=100, blank=True, null=True, default=None, help_text = "hash of the result") runtime = models.CharField(max_length=100, blank=True, null=True, default=None, help_text = "runtime of the simulation") @@ -206,7 +206,7 @@ class Tickets(models.Model): creation_date = models.DateTimeField(auto_now_add=True) -class Comments(models.Model): +class Comments(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False, ) Ticket = models.ForeignKey(Tickets, on_delete=models.CASCADE, default=None) author = models.CharField(max_length=200, default="") @@ -272,3 +272,11 @@ class Param_AbstractionLevel (models.Model): class Param_ScoreType (models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False, ) authorized_value = models.CharField(max_length=200, unique=True, default="") + +class Param_License(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False, ) + authorized_value = models.CharField(max_length=200, unique=True, default="") + +class Param_ImplementationStatus(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False, ) + authorized_value = models.CharField(max_length=200, unique=True, default="") diff --git a/model_validation_api/serializer/serializer.py b/model_validation_api/serializer/serializer.py index 94427e308..2021f52b1 100644 --- a/model_validation_api/serializer/serializer.py +++ b/model_validation_api/serializer/serializer.py @@ -48,6 +48,8 @@ Param_AbstractionLevelSerializer, Param_ScoreTypeSerializer, Param_OrganizationsSerializer, + Param_ImplementationStatusSerializer, + Param_LicenseSerializer ) #### rest framework serializers #### diff --git a/model_validation_api/serializer/simple_serializer.py b/model_validation_api/serializer/simple_serializer.py index bad0f7beb..38a06e723 100644 --- a/model_validation_api/serializer/simple_serializer.py +++ b/model_validation_api/serializer/simple_serializer.py @@ -20,6 +20,8 @@ Param_AbstractionLevel, Param_ScoreType, Param_organizations, + Param_License, + Param_ImplementationStatus, Persons ) @@ -190,6 +192,17 @@ class Meta: model = Param_organizations fields = ('id', 'authorized_value') +class Param_LicenseSerializer(serializers.HyperlinkedModelSerializer): + class Meta: + model = Param_License + fields = ('id', 'authorized_value') + +class Param_ImplementationStatusSerializer(serializers.HyperlinkedModelSerializer): + class Meta: + model = Param_ImplementationStatus + fields = ('id', 'authorized_value') + + class PersonSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Persons diff --git a/model_validation_api/views.py b/model_validation_api/views.py index 2546b0384..0258d5598 100644 --- a/model_validation_api/views.py +++ b/model_validation_api/views.py @@ -54,6 +54,8 @@ CollabParameters, Param_ScoreType, Param_organizations, + Param_License, + Param_ImplementationStatus ) @@ -86,7 +88,9 @@ Param_ModelScopeSerializer, Param_AbstractionLevelSerializer, Param_ScoreTypeSerializer, - Param_OrganizationsSerializer + Param_OrganizationsSerializer, + Param_ImplementationStatusSerializer, + Param_LicenseSerializer ) @@ -227,6 +231,12 @@ def get(self, request, format=None, **kwargs): organization = Param_organizations.objects.all().order_by('authorized_value') organization_serializer = Param_OrganizationsSerializer(organization, context=serializer_context, many=True) + licenses = Param_License.objects.all().order_by('authorized_value') + licenses_serializer = Param_LicenseSerializer(licenses, context=serializer_context, many=True) + + implementation_status_values = Param_ImplementationStatus.objects.all().order_by('authorized_value') + status_serializer = Param_ImplementationStatusSerializer(implementation_status_values, context=serializer_context, many=True) + ##for python client #'python_client=True' 'parameters= list()' if python_client == 'true': params_asked = request.GET.getlist('parameters') @@ -237,6 +247,7 @@ def get(self, request, format=None, **kwargs): # print "if((params_asked[0]=='all') or (len(params_asked) ==0)):" res = { 'data_modalities': data_modalities.values_list('authorized_value', flat=True), + 'data_modality': data_modalities.values_list('authorized_value', flat=True), 'test_type' : test_type.values_list('authorized_value', flat=True), 'species' : species.values_list('authorized_value', flat=True), 'brain_region' : brain_region.values_list('authorized_value', flat=True), @@ -245,6 +256,8 @@ def get(self, request, format=None, **kwargs): 'abstraction_level' : abstraction_level.values_list('authorized_value', flat=True), 'score_type': score_type.values_list('authorized_value', flat=True), 'organization': organization.values_list('authorized_value', flat=True), + 'license': licenses.values_list('authorized_value', flat=True), + 'status': implementation_status_values.values_list('authorized_value', flat=True), } return Response(res) else: @@ -256,6 +269,8 @@ def get(self, request, format=None, **kwargs): res['species']= species.values_list('authorized_value', flat=True) if (param == 'data_modalities'): res['data_modalities'] = data_modalities.values_list('authorized_value', flat=True) + if (param == 'data_modality'): + res['data_modality'] = data_modalities.values_list('authorized_value', flat=True) if (param == 'test_type'): res['test_type'] = test_type.values_list('authorized_value', flat=True) if (param == 'brain_region'): @@ -270,6 +285,10 @@ def get(self, request, format=None, **kwargs): res['score_type'] = score_type.values_list('authorized_value', flat=True) if (param == 'organization'): res['organization'] = organization.values_list('authorized_value', flat=True) + if (param == 'license'): + res['license'] = licenses.values_list('authorized_value', flat=True) + if (param == 'status'): + res['status'] = implementation_status_values.values_list('authorized_value', flat=True) # print "final res" # print res @@ -278,6 +297,7 @@ def get(self, request, format=None, **kwargs): return Response({ 'data_modalities': data_modalities_serializer.data, + 'data_modality': data_modalities_serializer.data, 'test_type' : test_type_serializer.data, 'species' : species_serializer.data, 'brain_region' : brain_region_serializer.data, @@ -286,8 +306,11 @@ def get(self, request, format=None, **kwargs): 'abstraction_level' : abstraction_level_serializer.data, 'score_type': score_type_serializer.data, 'organization': organization_serializer.data, + 'license': licenses_serializer.data, + 'status': status_serializer.data }) + class CollabIDRest(APIView): """ Get collab ID diff --git a/validation_service/validation_service/settings.py b/validation_service/validation_service/settings.py index b35197af9..d6cab4112 100644 --- a/validation_service/validation_service/settings.py +++ b/validation_service/validation_service/settings.py @@ -257,13 +257,13 @@ ) CORS_ALLOW_HEADERS = ( -'x-requested-with', -'content-type', -'accept', -'origin', -'authorization', -'X-CSRFToken', -'access-control-allow-origin' + 'x-requested-with', + 'content-type', + 'accept', + 'origin', + 'authorization', + 'X-CSRFToken', + 'access-control-allow-origin', ) REST_FRAMEWORK = { From 1d3a6bdfa6a6cc747c9df2475b8a417de6091f1c Mon Sep 17 00:00:00 2001 From: Andrew Davison Date: Tue, 24 Mar 2020 17:48:24 +0100 Subject: [PATCH 3/4] hack for testing v2 collab apps with v1 web service (cherry picked from commit 61b579754a3f7f2bdc0a400958194fb58db78223) --- .../user_auth_functions.py | 1 + model_validation_api/views_kg.py | 29 ++++++++++--------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/model_validation_api/validation_framework_toolbox/user_auth_functions.py b/model_validation_api/validation_framework_toolbox/user_auth_functions.py index 37c105e92..c7a8fb127 100644 --- a/model_validation_api/validation_framework_toolbox/user_auth_functions.py +++ b/model_validation_api/validation_framework_toolbox/user_auth_functions.py @@ -20,6 +20,7 @@ logger.addHandler(stream_handler) admin_id='13947' +ADMIN_USERS = ("adavison", "shailesh") # temporary hack for development of v2 collab apps def get_authorization_header(request): diff --git a/model_validation_api/views_kg.py b/model_validation_api/views_kg.py index 3e54e77b7..47a15e601 100644 --- a/model_validation_api/views_kg.py +++ b/model_validation_api/views_kg.py @@ -37,7 +37,8 @@ get_storage_file_by_id, get_authorization_header, get_user_from_token, - get_kg_token + get_kg_token, + ADMIN_USERS ) from .validation_framework_toolbox.validation_framework_functions import ( @@ -300,10 +301,10 @@ def get(self, request, format=None, **kwargs): logger.debug("{} total models".format(len(as_list(models)))) authorized_collabs = [] for collab_id in set(model.collab_id for model in as_list(models) if model.private): - if is_authorised_or_admin(request, collab_id): + if self.user.get("username") in ADMIN_USERS or is_authorised_or_admin(request, collab_id): authorized_collabs.append(collab_id) - logger.debug("Authorized collabs for user '{}': {}".format(self.user["username"], authorized_collabs)) + logger.debug("Authorized collabs for user '{}': {}".format(self.user.get("username"), authorized_collabs)) authorized_models = [model for model in as_list(models) if (not model.private) or (model.collab_id in authorized_collabs)] logger.debug("{} authorized models".format(len(authorized_models))) @@ -343,7 +344,7 @@ def get(self, request, format=None, **kwargs): #check if private if model.private: #if private check if collab member - if not is_authorised_or_admin(request, model.collab_id): + if not (self.user.get("username") in ADMIN_USERS or is_authorised_or_admin(request, model.collab_id)): response["errors"]["unauthorized"].append(model_id) continue model_serializer = ScientificModelKGSerializer(model, self.client) @@ -382,7 +383,7 @@ def post(self, request, format=None): else: return Response("You need to specify the app_id argument", status=status.HTTP_400_BAD_REQUEST) - if not is_authorised_or_admin(request, collab_id): + if not (self.user.get("username") in ADMIN_USERS or is_authorised_or_admin(request, collab_id)): return HttpResponse('Unauthorized for collab {}'.format(collab_id), status=401) data = _reformat_request_data(request.data)[0] @@ -465,7 +466,7 @@ def put(self, request, format=None): return Response('We cannot update the model. Please give the id or the alias of the model.', status=status.HTTP_400_BAD_REQUEST ) # security - if not is_authorised_or_admin(request, model.collab_id): + if not (self.user.get("username") in ADMIN_USERS or is_authorised_or_admin(request, model.collab_id)): return HttpResponse('Access to original model collab ({}) unauthorized'.format(model.collab_id), status=401) if ("collab_id" in model_data and model_data["collab_id"] != model.collab_id @@ -499,7 +500,7 @@ def delete(self, request, format=None): list_ids = request.GET.getlist('id') logger.debug("Request to delete {}".format(list_ids)) - if not is_authorised_or_admin(request, settings.ADMIN_COLLAB_ID): + if not (self.user.get("username") in ADMIN_USERS or is_authorised_or_admin(request, settings.ADMIN_COLLAB_ID)): return HttpResponseForbidden() elements_to_delete = [ @@ -668,7 +669,7 @@ def post(self, request, format=None): # security collab_id = model_project.collab_id - if not is_authorised_or_admin(request, collab_id): + if not (self.user.get("username") in ADMIN_USERS or is_authorised_or_admin(request, collab_id)): return HttpResponseForbidden() # check if versions are unique @@ -779,7 +780,7 @@ def put(self, request, format=None): #security collab_id = model_project.collab_id - if not is_authorised_or_admin(request, collab_id): + if not (self.user.get("username") in ADMIN_USERS or is_authorised_or_admin(request, collab_id)): return HttpResponseForbidden() # #check if version is editable - TODO reimplement once ValidationResult migrated to KG @@ -812,7 +813,7 @@ def delete(self, request, format=None): """ - if not is_authorised_or_admin(request, settings.ADMIN_COLLAB_ID): + if not (self.user.get("username") in ADMIN_USERS or is_authorised_or_admin(request, settings.ADMIN_COLLAB_ID)): return HttpResponseForbidden() list_ids = request.GET.getlist('id') @@ -1075,7 +1076,7 @@ def put(self, request, format=None): def delete(self, request, format=None): - if not is_authorised_or_admin(request, settings.ADMIN_COLLAB_ID): + if not (self.user.get("username") in ADMIN_USERS or is_authorised_or_admin(request, settings.ADMIN_COLLAB_ID)): return HttpResponseForbidden() list_ids = request.GET.getlist('id') @@ -1337,7 +1338,7 @@ def put(self, request, format=None): return Response("To edit a test instance, you need to give an id, or a test_definition_id with a version, or a test_definition_alias with a version ", status=status.HTTP_400_BAD_REQUEST) #check if version is editable - if not is_authorised_or_admin(request,settings.ADMIN_COLLAB_ID): + if not (self.user.get("username") in ADMIN_USERS or is_authorised_or_admin(request,settings.ADMIN_COLLAB_ID)): if not _are_test_code_editable_kg(test_code, self.client): return Response("This version is no longer editable as there is at least one result associated with it", status=status.HTTP_400_BAD_REQUEST) @@ -1359,7 +1360,7 @@ def put(self, request, format=None): def delete(self, request, format=None): - if not is_authorised_or_admin(request, settings.ADMIN_COLLAB_ID): + if not (self.user.get("username") in ADMIN_USERS or is_authorised_or_admin(request, settings.ADMIN_COLLAB_ID)): return HttpResponseForbidden() list_ids = request.GET.getlist('id') @@ -1610,7 +1611,7 @@ def delete(self, request, format=None): """ - if not is_authorised_or_admin(request, settings.ADMIN_COLLAB_ID): + if not (self.user.get("username") in ADMIN_USERS or is_authorised_or_admin(request, settings.ADMIN_COLLAB_ID)): return HttpResponseForbidden() list_ids = request.GET.getlist('id') From d70ae7cb7793a3cc4874aba1a0ad0c6074104be3 Mon Sep 17 00:00:00 2001 From: appukuttan-shailesh Date: Fri, 5 Jun 2020 11:36:32 +0200 Subject: [PATCH 4/4] update link to launch morphology viewer --- validation_service/app/js/service.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validation_service/app/js/service.js b/validation_service/app/js/service.js index 7a480989b..37aa0df71 100644 --- a/validation_service/app/js/service.js +++ b/validation_service/app/js/service.js @@ -181,7 +181,7 @@ ContextServices.service('Context', ['$rootScope', '$location', 'AppIDRest', 'Col var newTab_goToMorphologyViewer = function(model_instance, model_id) { _getCollabIDAndAppIDFromUrl("model_catalog").then(function(ids) { var url_collab = encodeURIComponent("https://collab.humanbrainproject.eu/#/collab/" + ids.collab_id + "/nav/" + ids.app_id + "?state=model." + model_id + ",external") - var url = "https://neuroinformatics.nl/HBP/morphology-viewer-dev/?hbp-model-instance-id=" + model_instance.id + "&referrer=" + url_collab; + var url = "https://neuroinformatics.nl/HBP/morphology-viewer-dev/?url=" + model_instance.morphology + "&referrer=" + url_collab; window.open(url, '_blank') }) }