From 0fcde754f484b48be3ba006c1d9183daf691c6d1 Mon Sep 17 00:00:00 2001 From: susanodd Date: Thu, 6 Feb 2025 08:32:15 +0100 Subject: [PATCH] #1463, #1461, #1484, #1368, #1468, #1484: Fixes to gloss update retrieval of original value, normalisation of Boolean values. Omit non-updated values from TO DO in pre-update method. --- signbank/csv_interface.py | 15 +++ signbank/dictionary/adminviews.py | 4 +- signbank/dictionary/models.py | 2 +- signbank/dictionary/tests.py | 2 +- signbank/gloss_update.py | 100 ++++++++++++------- signbank/query_parameters.py | 2 +- signbank/settings/server_specific/default.py | 2 + 7 files changed, 88 insertions(+), 39 deletions(-) diff --git a/signbank/csv_interface.py b/signbank/csv_interface.py index 497ae6b60..11f822eeb 100644 --- a/signbank/csv_interface.py +++ b/signbank/csv_interface.py @@ -1141,6 +1141,21 @@ def csv_focusgloss_to_minimalpairs(focusgloss, dataset, language_code, csv_rows) return csv_rows +def normalize_boolean(gloss_field, new_boolean, original_boolean): + # return 'True', 'False', 'None' depending on field type + print('normalize boolean: ', gloss_field, new_boolean, original_boolean) + if gloss_field.name in ['weakdrop', 'weakprop'] and new_boolean.lower() in ['neutral']: + return 'None' + elif new_boolean.lower() in ['true', 'ja', 'yes']: + return 'True' + elif new_boolean.lower() in ['false', 'nee', 'no']: + return 'False' + else: + if settings.DEBUG_API: + print('normalize_boolean NO MATCH: ', gloss_field, new_boolean) + return original_boolean + + def normalize_field_choice(field_choice): # add spaces around > and + diff --git a/signbank/dictionary/adminviews.py b/signbank/dictionary/adminviews.py index dbd184ae4..f4f9e318f 100755 --- a/signbank/dictionary/adminviews.py +++ b/signbank/dictionary/adminviews.py @@ -5868,10 +5868,10 @@ def glosslist_ajax_complete(request, gloss_id): this_gloss = Gloss.objects.get(id=gloss_id, archived=False) default_language = this_gloss.lemma.dataset.default_language.language_code_2char - + activate(default_language) # TO DO could use the following method to generate the column values # but the domain of that method is the column header - # gloss_data = this_gloss.get_fields_dict(display_fields) + # gloss_data = this_gloss.get_fields_dict(display_fields, default_language) # list_gloss_data = list(gloss_data) # print(list_gloss_data) diff --git a/signbank/dictionary/models.py b/signbank/dictionary/models.py index 13af16eb6..9975f23b7 100755 --- a/signbank/dictionary/models.py +++ b/signbank/dictionary/models.py @@ -1399,7 +1399,7 @@ def get_hasothermedia_display(self): other_media_paths.append(other_media_filename) return ", ".join(other_media_paths) - def get_fields_dict(self, fieldnames, language_code='en'): + def get_fields_dict(self, fieldnames, language_code): from django.utils import translation translation.activate(language_code) diff --git a/signbank/dictionary/tests.py b/signbank/dictionary/tests.py index 7e3545c80..4e5e769f2 100644 --- a/signbank/dictionary/tests.py +++ b/signbank/dictionary/tests.py @@ -340,7 +340,7 @@ def test_package_function(self): # this calculates the data retrieved by get_gloss_data for packages # it shows the format/display of the returned gloss fields # note that the value of the phonology fields are numerical rather than human readable - result = changed_gloss.get_fields_dict(settings.API_FIELDS) + result = changed_gloss.get_fields_dict(settings.API_FIELDS, 'en') print('test_package_function: settings.API_FIELDS: ', settings.API_FIELDS) print('test_package_function: get_fields_dict: ', result) diff --git a/signbank/gloss_update.py b/signbank/gloss_update.py index 262f633c1..6ce416f7a 100755 --- a/signbank/gloss_update.py +++ b/signbank/gloss_update.py @@ -3,6 +3,7 @@ from signbank.attachments.views import upload_file from signbank.dictionary.models import * +from django.db.models.fields import BooleanField from signbank.video.models import GlossVideoDescription, GlossVideoNME from signbank.tools import get_default_annotationidglosstranslation from django.db.transaction import atomic @@ -11,7 +12,7 @@ from django.http import JsonResponse from guardian.shortcuts import get_objects_for_user from signbank.tools import get_interface_language_and_default_language_codes -from signbank.csv_interface import normalize_field_choice +from signbank.csv_interface import normalize_field_choice, normalize_boolean from signbank.api_token import put_api_user_in_request import datetime as DT from signbank.tools import get_default_annotationidglosstranslation @@ -303,7 +304,10 @@ def detect_type_related_problems_for_gloss_update(changes, dataset, language_cod language_fields = internal_language_fields(dataset, language_code) errors = dict() + # in the following loop, field is either a language field as described above + # or a field of model Gloss, not a string for field, (original_value, new_value) in changes.items(): + print('field: ', field, type(field)) if not new_value: continue if field in language_fields: @@ -319,27 +323,30 @@ def detect_type_related_problems_for_gloss_update(changes, dataset, language_cod fieldchoice = FieldChoice.objects.get(field=field_choice_category, name__iexact=normalised_choice) continue except (ObjectDoesNotExist, MultipleObjectsReturned): - errors[field.verbose_name.title()] = gettext('NOT FOUND: ') + new_value + errors[field.verbose_name.title()] = gettext('Incorrect value') continue if isinstance(field, models.ForeignKey) and field.related_model == Handshape: try: handshape = Handshape.objects.get(name__iexact=new_value) continue except (ObjectDoesNotExist, MultipleObjectsReturned): - errors[field.verbose_name.title()] = gettext('NOT FOUND: ') + new_value + errors[field.verbose_name.title()] = gettext('Incorrect value') continue if field.__class__.__name__ == 'BooleanField': - normalised_boolean_string = new_value.lower() - if normalised_boolean_string not in ['true', 'false', 'neutral', 'ja', 'nee']: - errors[field.verbose_name.title()] = gettext('NOT FOUND: ') + new_value + if field.name in ['weakdrop', 'weakprop']: + if new_value not in ['True', 'False', 'None']: + errors[field.verbose_name.title()] = gettext('Incorrect value') + else: + if new_value not in ['True', 'False']: + errors[field.verbose_name.title()] = gettext('Incorrect value') elif field.name == 'semField': type_check = type_check_multiselect('SemField', new_value, language_code) if not type_check: - errors[field.verbose_name.title()] = gettext('NOT FOUND: ') + new_value + errors[field.verbose_name.title()] = gettext('Incorrect value') elif field.name == 'derivHist': type_check = type_check_multiselect('derivHist', new_value, language_code) if not type_check: - errors[field.verbose_name.title()] = gettext('NOT FOUND: ') + new_value + errors[field.verbose_name.title()] = gettext('Incorrect value') return errors @@ -461,25 +468,22 @@ def gloss_update_do_changes(user, gloss, changes, language_code): setattr(gloss, field.name, fieldchoice) changes_done.append((field.name, original_value, new_value)) elif field.__class__.__name__ == 'BooleanField': + # the new value is a string, do not use a coercion because we want + # to store None as a value if new_value in ['true', 'True', 'TRUE']: - new_value = True + boolean_value = True elif new_value in ['false', 'False', 'FALSE']: - new_value = False + boolean_value = False elif new_value == 'None' or new_value == 'Neutral': - new_value = None + boolean_value = None else: - new_value = False - setattr(gloss, field.name, new_value) + boolean_value = False + setattr(gloss, field.name, boolean_value) changes_done.append((field.name, original_value, new_value)) elif isinstance(field, models.ForeignKey) and field.related_model == Handshape: handshape = Handshape.objects.get(name__iexact=new_value) setattr(gloss, field.name, handshape) changes_done.append((field.name, original_value, new_value)) - elif field in ["senses", "Senses"]: - original_senses = collect_revision_history_for_senses(gloss) - update_senses(gloss, new_value) - new_senses = collect_revision_history_for_senses(gloss) - changes_done.append((field, original_senses, new_senses)) elif field.name == 'semField': update_semantic_field(gloss, new_value, language_code) changes_done.append((field.name, original_value, new_value)) @@ -520,40 +524,62 @@ def get_original_senses_value(gloss): def gloss_pre_update(gloss, input_fields_dict, language_code): + # this returns a dictionary mapping internal Gloss fields to tuples of original and new values + # for fields of Gloss, this is the internal model field + # for language fields, which are relations, this is an encoding of the field used in many interfaces + # of the templates and the methods dataset = gloss.lemma.dataset language_fields, gloss_fields = names_of_update_gloss_fields(dataset, language_code) human_readable_to_internal, human_readable_to_json = update_gloss_columns_to_value_dict_keys(dataset, language_code) # initial value of fields is put in combined_fields variable combined_fields = gloss_fields for language_field in language_fields: - gloss_dict_language_field = human_readable_to_json[language_field] - combined_fields.append(gloss_dict_language_field) + package_gloss_language_field = human_readable_to_json[language_field] + combined_fields.append(package_gloss_language_field) # retrieve what the packages knows for these fields gloss_package_data_dict = gloss.get_fields_dict(combined_fields, language_code) fields_to_update = dict() for human_readable_field, new_field_value in input_fields_dict.items(): if not new_field_value: continue - if human_readable_field not in gloss_package_data_dict.keys(): - # ignore the field if it's not visible to the API user - continue - original_value = gloss_package_data_dict[human_readable_field] - if human_readable_field in language_fields: - gloss_language_field = human_readable_to_json[human_readable_field] + package_gloss_language_field = human_readable_to_json[human_readable_field] + original_value = gloss_package_data_dict[package_gloss_language_field] + if original_value == new_field_value: + continue gloss_field = human_readable_to_internal[human_readable_field] fields_to_update[gloss_field] = (original_value, new_field_value) continue if human_readable_field not in human_readable_to_internal.keys(): - # if this field is not available, ignore it - continue - if original_value in ['False'] and new_field_value in ['', 'False']: - # treat empty as False + if settings.DEBUG_API: + print('gloss_pre_update: json field not found in Gloss fields: ', human_readable_field) continue + internal_field = human_readable_to_internal[human_readable_field] + original_internal_value = getattr(gloss, internal_field.name) + if original_internal_value is None and internal_field.name in ['weakdrop', 'weakprop']: + original_value = 'Neutral' + elif isinstance(original_internal_value, bool): + original_value = str(original_internal_value) + elif isinstance(internal_field, FieldChoiceForeignKey): + original_value = original_internal_value.name if original_internal_value else '-' + elif internal_field.name in ['domhndsh', 'subhndsh', + 'semField', 'derivHist', 'dialect', 'signlanguage']: + display_method = 'get_'+internal_field.name+'_display' + original_value = getattr(gloss, display_method)() + else: + # value is a string + original_value = original_internal_value if new_field_value == original_value: continue - gloss_field = human_readable_to_internal[human_readable_field] - fields_to_update[gloss_field] = (original_value, new_field_value) + if internal_field.__class__.__name__ == 'BooleanField': + # normalise if this is a Boolean and compare again + normalised_new_boolean = normalize_boolean(internal_field, new_field_value, original_value) + if original_value == normalised_new_boolean: + continue + fields_to_update[internal_field] = (original_value, normalised_new_boolean) + else: + fields_to_update[internal_field] = (original_value, new_field_value) + # return the dictionary of input fields that differ from stored values return fields_to_update @@ -622,7 +648,8 @@ def api_update_gloss(request, datasetid, glossid): return JsonResponse(results, status=400) value_dict = get_gloss_update_human_readable_value_dict(request) - + if settings.DEBUG_API: + print('api_update_gloss ', glossid, ' input value_dict: ', value_dict) errors = check_fields_can_be_updated(value_dict, dataset, interface_language_code) if errors: # "Field name not found in fields that can be updated." @@ -645,10 +672,15 @@ def api_update_gloss(request, datasetid, glossid): results['updatestatus'] = "Failed" return JsonResponse(results, status=400) + if settings.DEBUG_API: + # now we are ready to do the update, save the intended update fields to the log + print('api_update_gloss ', glossid, ' fields_to_update: ', fields_to_update) + try: gloss_update_do_changes(request.user, gloss, fields_to_update, interface_language_code) except (TransactionManagementError, ValueError, IntegrityError): - errors['Exception'] = "Transaction management error." + errors = dict() + errors[gettext('Exception')] = gettext("Transaction management error.") results['errors'] = errors results['updatestatus'] = "Failed" return JsonResponse(results, status=400) diff --git a/signbank/query_parameters.py b/signbank/query_parameters.py index ebfcff5d6..37d2c5747 100644 --- a/signbank/query_parameters.py +++ b/signbank/query_parameters.py @@ -25,7 +25,7 @@ from django.core.exceptions import ObjectDoesNotExist, EmptyResultSet from django.db import OperationalError, ProgrammingError from django.db.models import Q, Count, CharField, TextField, Value as V -from django.db.models.fields import BooleanField, BooleanField +from django.db.models.fields import BooleanField from django.urls import reverse from tagging.models import TaggedItem, Tag diff --git a/signbank/settings/server_specific/default.py b/signbank/settings/server_specific/default.py index de8d3540b..3051c69f2 100644 --- a/signbank/settings/server_specific/default.py +++ b/signbank/settings/server_specific/default.py @@ -317,6 +317,8 @@ DEBUG_VIDEOS = False +DEBUG_API = False + # Set this to True to avoid deleting glosses that are in relations with other glosses GUARDED_GLOSS_DELETE = False