From d46b63ee35a71559c0d5ea75757b0747b298c5d4 Mon Sep 17 00:00:00 2001 From: devowit Date: Mon, 3 May 2021 20:35:13 +0300 Subject: [PATCH] Optimization (#455) * timeoutSuggest + remove property Project.load * make get_partial_csv a separate endpoint * ammendment to previous commit: until we fix frontend to not expect partial csv, need to return some default * display all the option for role * add a cache within wikidata provider * fix bug onSelectionChange- selectedAnnotationBlock can be undefine * get partial csv * add more activating css and striping css * add an await so we get the spinner for suggest * onSelectionChange fix bug change area * switch to a different endpoint * stop using a database for entities * remove database from test client * limit partialCsv to 150 rows for now (will be changed once statement generation limits are changed) * fix stupid bug that fetches mapping and partial csv three times... * get partial csv when submitting annotations, and add a spinner when submitting annotations Co-authored-by: ChanaChelem --- backend/app_config.py | 5 +- backend/application.py | 15 ++ backend/calc_params.py | 19 +- backend/database_provider.py | 32 ++- backend/migrations/versions/0a923cbc1283_.py | 32 +++ backend/t2wml_web.py | 30 +-- .../files_for_tests/aid/project_results.json | 14 +- .../tests/files_for_tests/aid/results.json | 14 +- .../files_for_tests/empty_cells/project.t2wml | 4 +- backend/tests/utils.py | 11 - backend/web_dict_provider.py | 56 ++++ backend/wikidata_models.py | 2 +- backend/wikidata_utils.py | 44 ++-- electron/src/renderer/common/dtos.ts | 8 +- electron/src/renderer/common/service.ts | 75 +++--- electron/src/renderer/project/project.tsx | 49 ++-- .../sidebar/file-tree/entities-tree.tsx | 7 + .../project/sidebar/file-tree/file-tree.tsx | 9 +- .../src/renderer/project/sidebar/sidebar.tsx | 7 + .../annotation-table/annotation-form.tsx | 25 +- .../annotation-table/annotation-menu.tsx | 19 +- .../annotation-table/annotation-table.tsx | 242 ++++++++++-------- .../project/table/table-component.css | 59 ++++- .../project/table/table-container.tsx | 61 +++-- .../table/wikify-table/wikify-menu.tsx | 7 + .../renderer/project/wikifier/wikifier.tsx | 21 ++ 26 files changed, 569 insertions(+), 298 deletions(-) create mode 100644 backend/migrations/versions/0a923cbc1283_.py create mode 100644 backend/web_dict_provider.py diff --git a/backend/app_config.py b/backend/app_config.py index fa5b349c6..c37f97698 100644 --- a/backend/app_config.py +++ b/backend/app_config.py @@ -51,7 +51,7 @@ class AppConfig: #############SQL STUFF - +""" AUTO_MIGRATE = "sqlite" in AppConfig.SQLALCHEMY_DATABASE_URI # only set to true if database is sqlite @@ -73,6 +73,7 @@ def auto_constraint_name(constraint, table): "pk": "pk_%(table_name)s" } + metadata = MetaData(naming_convention=convention) db = SQLAlchemy(app, metadata=metadata) @@ -83,4 +84,4 @@ def auto_constraint_name(constraint, table): if AUTO_MIGRATE: with app.app_context(): - upgrade(directory=os.path.join(BASEDIR, 'migrations')) + upgrade(directory=os.path.join(BASEDIR, 'migrations')) """ diff --git a/backend/application.py b/backend/application.py index f085a4b8b..1ec4dd5dd 100644 --- a/backend/application.py +++ b/backend/application.py @@ -156,6 +156,21 @@ def get_data(): response.update(calc_response) return response, code +@app.route('/api/partialcsv', methods=['GET']) +@json_response +def partial_csv(): + project = get_project() + calc_params = get_calc_params(project) + response=dict() + try: + response["partialCsv"]=get_partial_csv(calc_params) + except Exception as e: + print(e) + response["partialCsv"]=dict(dims=[1,3], + firstRowIndex=0, + cells=[["subject", "property", "value"]]) + return response, 200 + @app.route('/api/project', methods=['POST']) @json_response diff --git a/backend/calc_params.py b/backend/calc_params.py index bae0921ab..f6ff94365 100644 --- a/backend/calc_params.py +++ b/backend/calc_params.py @@ -7,8 +7,10 @@ class CalcParams: def __init__(self, project, data_path, sheet_name, yaml_path=None, annotation_path=None): self.project_path = project.directory + self.project = Project.load(self.project_path) self.data_path = Path(project.directory) / data_path self.sheet_name = sheet_name + self.sheet = Sheet(self.data_path, self.sheet_name) self.yaml_path = None if yaml_path: self.yaml_path = Path(project.directory) / yaml_path @@ -17,13 +19,13 @@ def __init__(self, project, data_path, sheet_name, yaml_path=None, annotation_pa self.annotation_path= Path(project.directory) / annotation_path - @property - def project(self): - return Project.load(self.project_path) + # @property + # def project(self): + # return Project.load(self.project_path) - @property - def sheet(self): - return Sheet(self.data_path, self.sheet_name) + # @property + # def sheet(self): + # return Sheet(self.data_path, self.sheet_name) @property def cache(self): @@ -53,5 +55,6 @@ def sheet_names(self): @property def sparql_endpoint(self): - p = Project.load(self.project_path) - return p.sparql_endpoint + # p = Project.load(self.project_path) + return self.project.sparql_endpoint + # return p.sparql_endpoint diff --git a/backend/database_provider.py b/backend/database_provider.py index 7f89abd18..fa922014e 100644 --- a/backend/database_provider.py +++ b/backend/database_provider.py @@ -7,6 +7,7 @@ def __init__(self): super().__init__() self.cache_id=None self.project=None + self.property_cache={} def change_project(self, project): self.project = project @@ -26,29 +27,36 @@ def save_entry(self, wd_id, data_type, from_file=False, **kwargs): return WikidataEntity.add_or_update(wd_id, data_type, do_session_commit=False, cache_id=cache_id, **kwargs) def get_entity(self, wikidata_property, *args, **kwargs): - #check for project-specific first - prop = WikidataEntity.query.filter_by(wd_id=wikidata_property, cache_id=self.cache_id).first() - #check for generic wikidata entry + prop=self.property_cache.get(wikidata_property, None) if not prop: - prop = WikidataEntity.query.filter_by(wd_id=wikidata_property, cache_id=self.sparql_endpoint).first() - if not prop: - raise ValueError("Not found") + #check for project-specific first + prop = WikidataEntity.query.filter_by(wd_id=wikidata_property, cache_id=self.cache_id).first() + #check for generic wikidata entry + if not prop: + prop = WikidataEntity.query.filter_by(wd_id=wikidata_property, cache_id=self.sparql_endpoint).first() + if not prop: + raise ValueError("Not found") + self.property_cache[wikidata_property]=prop return prop.__dict__ def try_get_property_type(self, wikidata_property, *args, **kwargs): - #check for project-specific first - prop = WikidataEntity.query.filter_by(wd_id=wikidata_property, cache_id=self.cache_id).first() - #check for generic wikidata entry - if not prop or prop.data_type is None or prop.data_type == "Property Not Found": - prop = WikidataEntity.query.filter_by(wd_id=wikidata_property, cache_id=self.sparql_endpoint).first() + prop=self.property_cache.get(wikidata_property, None) if not prop: - raise ValueError("Not found") + #check for project-specific first + prop = WikidataEntity.query.filter_by(wd_id=wikidata_property, cache_id=self.cache_id).first() + #check for generic wikidata entry + if not prop or prop.data_type is None or prop.data_type == "Property Not Found": + prop = WikidataEntity.query.filter_by(wd_id=wikidata_property, cache_id=self.sparql_endpoint).first() + if not prop: + raise ValueError("Not found") if prop.data_type == "Property Not Found": return prop.data_type if prop.data_type is None: raise ValueError("No datatype defined for that ID") + self.property_cache[wikidata_property]=prop return prop.data_type def __exit__(self, exc_type, exc_value, exc_traceback): WikidataEntity.do_commit() + diff --git a/backend/migrations/versions/0a923cbc1283_.py b/backend/migrations/versions/0a923cbc1283_.py new file mode 100644 index 000000000..56330e990 --- /dev/null +++ b/backend/migrations/versions/0a923cbc1283_.py @@ -0,0 +1,32 @@ +"""add index to cache_id + +Revision ID: 0a923cbc1283 +Revises: 28a3625f6dd6 +Create Date: 2021-05-02 13:27:58.664291 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '0a923cbc1283' +down_revision = '28a3625f6dd6' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('wikidata_entity', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_wikidata_entity_cache_id'), ['cache_id'], unique=False) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('wikidata_entity', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_wikidata_entity_cache_id')) + + # ### end Alembic commands ### diff --git a/backend/t2wml_web.py b/backend/t2wml_web.py index d5204c7bc..b84cc1adc 100644 --- a/backend/t2wml_web.py +++ b/backend/t2wml_web.py @@ -1,10 +1,8 @@ -from collections import defaultdict import os import json import numpy as np import pandas as pd from pathlib import Path -from numpy.core.numeric import full from t2wml.api import add_entities_from_file as api_add_entities_from_file from t2wml.api import (WikifierService, t2wml_settings, KnowledgeGraph, YamlMapper, AnnotationMapper, kgtk_to_dict, dict_to_kgtk) @@ -12,11 +10,10 @@ from t2wml.input_processing.annotation_parsing import AnnotationNodeGenerator, Annotation from t2wml.input_processing.annotation_suggesting import block_finder from t2wml.mapping.statement_mapper import PartialAnnotationMapper -from t2wml.utils.t2wml_exceptions import T2WMLException from t2wml.spreadsheets.conversions import cell_str_to_tuple from t2wml.api import Project -from app_config import db, CACHE_FOLDER -from database_provider import DatabaseProvider +from app_config import CACHE_FOLDER +from web_dict_provider import WebDictionaryProvider from utils import get_empty_layers from wikidata_utils import get_labels_and_descriptions, get_qnode_url, QNode @@ -47,7 +44,7 @@ def set_web_settings(): if not os.path.isdir(CACHE_FOLDER): os.makedirs(CACHE_FOLDER, exist_ok=True) t2wml_settings.cache_data_files_folder = CACHE_FOLDER - t2wml_settings.wikidata_provider = DatabaseProvider() + t2wml_settings.wikidata_provider = WebDictionaryProvider() def update_t2wml_settings(project): t2wml_settings.update_from_dict(**project.__dict__) @@ -77,8 +74,8 @@ def get_kg(calc_params): ang.preload(calc_params.sheet, wikifier) else: cell_mapper = YamlMapper(calc_params.yaml_path) - kg = KnowledgeGraph.generate(cell_mapper, calc_params.sheet, wikifier) - db.session.commit() # save any queried properties + with t2wml_settings.wikidata_provider as p: + kg = KnowledgeGraph.generate(cell_mapper, calc_params.sheet, wikifier) return kg @@ -119,7 +116,7 @@ def get_qnodes_layer(calc_params): qNode= QNode(id, value, context), indices=[[row, col]]) - labels_and_descriptions = get_labels_and_descriptions(list(ids_to_get), calc_params.sparql_endpoint) + labels_and_descriptions = get_labels_and_descriptions(t2wml_settings.wikidata_provider, list(ids_to_get), calc_params.sparql_endpoint) for id in qnode_entries: if id in labels_and_descriptions: qnode_entries[id]['qNode'].update(**labels_and_descriptions[id]) @@ -245,7 +242,7 @@ def get_yaml_layers(calc_params): cleanedLayer=get_cleaned(kg) - labels = get_labels_and_descriptions(qnodes, calc_params.project.sparql_endpoint) + labels = get_labels_and_descriptions(t2wml_settings.wikidata_provider, qnodes, calc_params.project.sparql_endpoint) qnodes.update(labels) for id in qnodes: if qnodes[id]: @@ -301,14 +298,9 @@ def get_layers(response, calc_params): except Exception as e: response["yamlError"] = str(e) - try: - response["partialCsv"]=get_partial_csv(calc_params) - except Exception as e: - print(e) - response["partialCsv"]=dict(dims=[1,3], - firstRowIndex=0, - cells=[["subject", "property", "value"]]) - + response["partialCsv"]=dict(dims=[1,3], + firstRowIndex=0, + cells=[["subject", "property", "value"]]) def get_annotations(calc_params): annotations_path=calc_params.annotation_path @@ -371,7 +363,7 @@ def get_partial_csv(calc_params): wikifier=calc_params.wikifier annotation= calc_params.annotation_path cell_mapper = PartialAnnotationMapper(calc_params.annotation_path) - kg = KnowledgeGraph.generate(cell_mapper, calc_params.sheet, wikifier) + kg = KnowledgeGraph.generate(cell_mapper, calc_params.sheet, wikifier, start=0, end=150) if not kg.statements: if cell_mapper.annotation.subject_annotations: df=pd.DataFrame([], columns=["subject", "property", "value"]) diff --git a/backend/tests/files_for_tests/aid/project_results.json b/backend/tests/files_for_tests/aid/project_results.json index e4fc502bf..6ef84c79c 100644 --- a/backend/tests/files_for_tests/aid/project_results.json +++ b/backend/tests/files_for_tests/aid/project_results.json @@ -1171,31 +1171,39 @@ "layerType": "statement", "qnodes": { "P17": { + "data_type": "WikibaseItem", "description": "sovereign state of this item (not to be used for human beings)", "id": "P17", "label": "country", "url": "https://www.wikidata.org/wiki/Property:P17" }, "P585": { + "data_type": "Time", "description": "time and date something took place, existed or a statement was true", "id": "P585", "label": "point in time", "url": "https://www.wikidata.org/wiki/Property:P585" }, "Paid-security-002": { - "description": "", + "P31": "Q18616576", + "data_type": "Quantity", + "from_file": true, "id": "Paid-security-002", "label": "UN", "url": "" }, "Paid-security-003": { - "description": "", + "P31": "Q18616576", + "data_type": "Quantity", + "from_file": true, "id": "Paid-security-003", "label": "INGO", "url": "" }, "Paid-security-004": { - "description": "", + "P31": "Q18616576", + "data_type": "Quantity", + "from_file": true, "id": "Paid-security-004", "label": "LNGO/NRCS", "url": "" diff --git a/backend/tests/files_for_tests/aid/results.json b/backend/tests/files_for_tests/aid/results.json index 2fc474bd8..eb6736736 100644 --- a/backend/tests/files_for_tests/aid/results.json +++ b/backend/tests/files_for_tests/aid/results.json @@ -1763,31 +1763,39 @@ "layerType": "statement", "qnodes": { "P17": { + "data_type": "WikibaseItem", "description": "sovereign state of this item (not to be used for human beings)", "id": "P17", "label": "country", "url": "https://www.wikidata.org/wiki/Property:P17" }, "P585": { + "data_type": "Time", "description": "time and date something took place, existed or a statement was true", "id": "P585", "label": "point in time", "url": "https://www.wikidata.org/wiki/Property:P585" }, "Paid-security-002": { - "description": "", + "P31": "Q18616576", + "data_type": "Quantity", + "from_file": true, "id": "Paid-security-002", "label": "UN", "url": "" }, "Paid-security-003": { - "description": "", + "P31": "Q18616576", + "data_type": "Quantity", + "from_file": true, "id": "Paid-security-003", "label": "INGO", "url": "" }, "Paid-security-004": { - "description": "", + "P31": "Q18616576", + "data_type": "Quantity", + "from_file": true, "id": "Paid-security-004", "label": "LNGO/NRCS", "url": "" diff --git a/backend/tests/files_for_tests/empty_cells/project.t2wml b/backend/tests/files_for_tests/empty_cells/project.t2wml index b0600af73..d730c571a 100644 --- a/backend/tests/files_for_tests/empty_cells/project.t2wml +++ b/backend/tests/files_for_tests/empty_cells/project.t2wml @@ -10,7 +10,7 @@ entity_files: - item_definitions.tsv - properties_all.tsv handle_calendar: leave -sparql_endpoint: https://dsbox02.isi.edu:8888/bigdata/namespace/wdq/sparql +sparql_endpoint: https://query.wikidata.org/bigdata/namespace/wdq/sparql title: empty cells url: '' warn_for_empty_cells: true @@ -24,5 +24,3 @@ yaml_sheet_associations: selected: t2wml.yaml val_arr: - t2wml.yaml - - t2wml.yaml - - t2wml.yaml diff --git a/backend/tests/utils.py b/backend/tests/utils.py index a50605d10..0b43ce9ab 100644 --- a/backend/tests/utils.py +++ b/backend/tests/utils.py @@ -11,17 +11,6 @@ @pytest.fixture(scope="session") def client(request): - def fin(): - os.close(db_fd) - os.unlink(name) - app.config['TESTING']=True - db_fd, name = tempfile.mkstemp() - app.config['SQLALCHEMY_DATABASE_URI']='sqlite:///' +name - app.config['USE_CACHE']=False - request.addfinalizer(fin) - with app.app_context(): - upgrade(directory=os.path.join(BACKEND_DIR, 'migrations')) - with app.test_client() as client: yield client diff --git a/backend/web_dict_provider.py b/backend/web_dict_provider.py new file mode 100644 index 000000000..0317dea54 --- /dev/null +++ b/backend/web_dict_provider.py @@ -0,0 +1,56 @@ + +import json +from t2wml.wikification.preloaded_properties import preloaded_properties +from t2wml.wikification.wikidata_provider import FallbackSparql + +class WebDictionaryProvider(FallbackSparql): + def __init__(self): + super().__init__() + self.project=None + self.cache=preloaded_properties + + def change_project(self, project): + self.project = project + self.sparql_endpoint=project.sparql_endpoint + with open(project.entity_file, 'r') as f: + self.cache.update(json.load(f)) + + def save_entry(self, wd_id, data_type, **kwargs): + self.cache[wd_id]=dict(kwargs) + self.cache[wd_id]["data_type"]=data_type + + def try_get_property_type(self, wikidata_property, *args, **kwargs): + property_dict=self.cache.get(wikidata_property, None) + if not property_dict: + raise ValueError("Property not founds") + + data_type= property_dict.get("data_type", None) + if not data_type: + raise ValueError("No datatype defined for that id") + return data_type + + def get_entity(self, wikidata_property, *args, **kwargs): + try: + property_dict=self.cache[wikidata_property] + return property_dict + except KeyError: + raise ValueError(wikidata_property+" not found") + + def save_entry(self, property, data_type, *args, **kwargs): + added=True + if property in self.cache: + added=False + self.cache[property]=dict(kwargs) + if data_type: + self.cache[property]["data_type"]=data_type + return added + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, exc_traceback): + if self.project is not None: + properties=json.dumps(self.cache) + with open(self.project.entity_file, 'w') as f: + f.write(properties) + diff --git a/backend/wikidata_models.py b/backend/wikidata_models.py index 71205f18d..0d44776b1 100644 --- a/backend/wikidata_models.py +++ b/backend/wikidata_models.py @@ -19,7 +19,7 @@ class WikidataEntity(db.Model): label = db.Column(db.String(64)) description = db.Column(db.String(200)) P31 = db.Column(db.String(64)) - cache_id = db.Column(db.String(300), nullable=True) + cache_id = db.Column(db.String(300), nullable=True, index=True) @staticmethod def add_or_update(wd_id, data_type=None, label=None, description=None, P31=None, cache_id=None, diff --git a/backend/wikidata_utils.py b/backend/wikidata_utils.py index ac454ea3e..8490b711c 100644 --- a/backend/wikidata_utils.py +++ b/backend/wikidata_utils.py @@ -1,13 +1,13 @@ from SPARQLWrapper import SPARQLWrapper, JSON -from wikidata_models import WikidataEntity -wikidata_label_query_cache = {} + +wikidata_label_query_cache = {} def query_wikidata_for_label_and_description(items, sparql_endpoint): items = ' wd:'.join(items) items = "wd:" + items - query = """SELECT ?qnode ?qnodeLabel ?qnodeDescription WHERE + query = """SELECT ?qnode ?qnodeLabel ?qnodeDescription WHERE {{ VALUES ?qnode {{{items}}} SERVICE wikibase:label {{ bd:serviceParam wikibase:language "[AUTO_LANGUAGE],en". }} @@ -33,24 +33,15 @@ def query_wikidata_for_label_and_description(items, sparql_endpoint): return response -def get_labels_and_descriptions(items, sparql_endpoint): - response = dict() - missing_items = {} +def get_labels_and_descriptions(provider, items, sparql_endpoint): + response=dict() + missing_items={} for item in items: - wp = WikidataEntity.query.filter_by(wd_id=item).first() - if wp: - label = desc = "" - if wp.label: - label = wp.label - if wp.description: - desc = wp.description - response[item] = dict(label=label, description=desc) - else: - if item not in wikidata_label_query_cache: - missing_items[item]=True - else: - if item not in wikidata_label_query_cache: - missing_items[item]=True + try: + wp=provider.get_entity(item) + response[item]=wp + except: + missing_items[item]=True try: if missing_items: wikidata_label_query_cache.update(missing_items) @@ -58,18 +49,18 @@ def get_labels_and_descriptions(items, sparql_endpoint): additional_items = query_wikidata_for_label_and_description( missing_items, sparql_endpoint) response.update(additional_items) - try: + with provider as p: for item in additional_items: - WikidataEntity.add_or_update(item, do_session_commit=False, **additional_items[item]) - except Exception as e: - print(e) - WikidataEntity.do_commit() - + prop_dict=additional_items[item] + data_type=prop_dict.pop("data_type", None) + p.save_entry(item, data_type, **prop_dict) except: # eg 502 bad gateway error pass return response + + def get_qnode_url(id): url="" first_letter=str(id).upper()[0] @@ -95,4 +86,3 @@ def __init__(self, id, value, context="", label="", description=""): def update(self, label="", description="", **kwargs): self.label=label self.description=description - \ No newline at end of file diff --git a/electron/src/renderer/common/dtos.ts b/electron/src/renderer/common/dtos.ts index 908415a8b..33d0932bf 100644 --- a/electron/src/renderer/common/dtos.ts +++ b/electron/src/renderer/common/dtos.ts @@ -138,7 +138,8 @@ export interface AnnotationBlock{ mainSubject?: string; // the ID of the block with the subject of this block unit?: string; }; - + link?: string; + //upperBound?: string; //lowerBound?: string; @@ -174,10 +175,13 @@ export interface ResponseWithProjectandFileName extends ResponseWithProjectDTO{ filename: string; } +export interface ResponseWithPartialCsvDTO{ + partialCsv: TableDTO; +} + export interface ResponseWithMappingDTO{ project: ProjectDTO; layers: LayersDTO; - partialCsv: TableDTO; yamlContent: string; yamlError?: string; annotations: AnnotationBlock[]; diff --git a/electron/src/renderer/common/service.ts b/electron/src/renderer/common/service.ts index b2fc0deb5..7c34bf071 100644 --- a/electron/src/renderer/common/service.ts +++ b/electron/src/renderer/common/service.ts @@ -5,7 +5,7 @@ import { backendGet, backendPost, backendPut } from './comm'; import { ResponseWithProjectDTO, ResponseWithMappingDTO, ResponseWithTableDTO, ResponseWithQNodeLayerDTO, ResponseCallWikifierServiceDTO, ResponseUploadEntitiesDTO, ResponseWithEverythingDTO, ResponseWithProjectAndMappingDTO, - TableDTO, GlobalSettingsDTO, ResponseEntitiesPropertiesDTO, QNode, ResponseWithProjectandFileName, ResponseWithQNodesDTO, ResponseWithSuggestion + TableDTO, GlobalSettingsDTO, ResponseEntitiesPropertiesDTO, QNode, ResponseWithProjectandFileName, ResponseWithQNodesDTO, ResponseWithSuggestion, ResponseWithPartialCsvDTO } from './dtos'; import { ErrorMessage } from './general'; @@ -38,8 +38,8 @@ class RequestService { } public getDataFileParams(required = true) { - if ( !currentFilesService.currentState.dataFile ) { - if ( required ) { + if (!currentFilesService.currentState.dataFile) { + if (required) { console.error("There is no data file"); //TODO: actual proper error handling? } return this.getProjectFolder(); @@ -47,9 +47,9 @@ class RequestService { return this.getProjectFolder() + `&data_file=${currentFilesService.currentState.dataFile}&sheet_name=${currentFilesService.currentState.sheetName}`; } - public getMappingParams(){ - let url=this.getDataFileParams(); - if ( currentFilesService.currentState.mappingFile ) { + public getMappingParams() { + let url = this.getDataFileParams(); + if (currentFilesService.currentState.mappingFile) { url += `&mapping_file=${currentFilesService.currentState.mappingFile}`; url += `&mapping_type=${currentFilesService.currentState.mappingType}`; } @@ -57,19 +57,19 @@ class RequestService { } @action - public switchProjectState(response: ResponseWithProjectDTO){ + public switchProjectState(response: ResponseWithProjectDTO) { console.debug('switchProjectState called'); wikiStore.project.projectDTO = response.project; currentFilesService.getFiles(response.project); // new project - if ( !Object.keys(response.project.data_files).length ) { + if (!Object.keys(response.project.data_files).length) { this.resetPreProject(); } } @action - public resetPreProject(){ + public resetPreProject() { wikiStore.table.updateTable({} as TableDTO); wikiStore.layers.resetLayers(); wikiStore.yaml.yamlContent = ''; @@ -78,50 +78,57 @@ class RequestService { } @action - public fillMapping(response: ResponseWithMappingDTO){ + public fillMapping(response: ResponseWithMappingDTO) { console.log("mapping", response) + wikiStore.layers.partialCsv = {} as TableDTO; wikiStore.project.projectDTO = response.project; wikiStore.layers.updateFromDTO(response.layers); - wikiStore.layers.partialCsv = response.partialCsv; wikiStore.yaml.yamlContent = response.yamlContent; wikiStore.yaml.yamlError = response.yamlError; wikiStore.annotations.blocks = response.annotations || []; } @action - public fillTable(response: ResponseWithTableDTO){ + public fillTable(response: ResponseWithTableDTO) { wikiStore.table.updateTable(response.table); this.fillMapping(response); } @action - public updateProjectandQnode(response: ResponseWithQNodeLayerDTO){ + public updateProjectandQnode(response: ResponseWithQNodeLayerDTO) { console.log("qnode", response) wikiStore.layers.updateFromDTO(response.layers); wikiStore.project.projectDTO = response.project; } + public async getPartialCsv() { + const updater = currentFilesService.createUpdater(); + const response = await backendGet(`/partialcsv?${this.getMappingParams()}`) as ResponseWithPartialCsvDTO; + updater.update(() => { wikiStore.layers.partialCsv = response.partialCsv; }); + } + + public async getProperties(search: string, type: string) { const url = `/properties?q=${search}&data_type=${type}`; const response = await backendGet(url) as ResponseWithQNodesDTO; wikiStore.annotateProperties.properties = response.qnodes; } - public async getQNodes(search: string, isClass: boolean, instanceOf?: QNode, searchProperties?: boolean, isSubject=false) { + public async getQNodes(search: string, isClass: boolean, instanceOf?: QNode, searchProperties?: boolean, isSubject = false) { let url = `/qnodes?q=${search}`; - if ( searchProperties ) { + if (searchProperties) { url = `/properties?q=${search}`; } - if ( isClass ) { + if (isClass) { url += `&is_class=true`; } - if ( instanceOf ) { + if (instanceOf) { url += `&instance_of=${instanceOf.id}`; } const response = await backendGet(url) as ResponseWithQNodesDTO; - if(isSubject){ + if (isSubject) { wikiStore.subjectQnodes.qnodes = response.qnodes; - } else{ + } else { wikiStore.wikifyQnodes.qnodes = response.qnodes; } @@ -137,7 +144,7 @@ class RequestService { this.updateProjectandQnode(response); } - public async addExistingMapping(data: any){ + public async addExistingMapping(data: any) { const response = await backendPost(`/files/add_mapping?${this.getProjectFolder()}`, data) as ResponseWithProjectandFileName; wikiStore.project.projectDTO = response.project; return response.filename; @@ -155,7 +162,7 @@ class RequestService { wikiStore.project.projectDTO = response.project; } - public async createAnnotation(data: any): Promise{ + public async createAnnotation(data: any): Promise { const response = await backendPost(`/annotation/create?${this.getProjectFolder()}`, data) as ResponseWithProjectandFileName; wikiStore.project.projectDTO = response.project; return response.filename; @@ -164,22 +171,24 @@ class RequestService { public async postAnnotationBlocks(data: any) { const updater = currentFilesService.createUpdater(); const response = await backendPost(`/annotation?${this.getDataFileParams()}`, data) as ResponseWithProjectAndMappingDTO; - updater.update(()=>{wikiStore.project.projectDTO = response.project; - this.fillMapping(response);}, "postAnnotationBlocks") + updater.update(() => { + wikiStore.project.projectDTO = response.project; + this.fillMapping(response); + }, "postAnnotationBlocks") } - public async getAnnotationSuggestions(data: any): Promise { + public async getAnnotationSuggestions(data: any): Promise { const response = await backendPut(`/annotation/suggest?${this.getDataFileParams()}`, data) as ResponseWithSuggestion; //TODO return response } - public async getSuggestedAnnotationBlocks(){ + public async getSuggestedAnnotationBlocks() { const updater = currentFilesService.createUpdater(); const response = await backendGet(`/annotation/guess-blocks?${this.getMappingParams()}`) as ResponseWithMappingDTO; - updater.update(()=>{this.fillMapping(response);}, "getsuggestedAnnotationBlocks") + updater.update(() => { this.fillMapping(response); }, "getsuggestedAnnotationBlocks") } - public async createProject(folder:string, data?: any) { + public async createProject(folder: string, data?: any) { const response = await backendPost(`/project?project_folder=${folder}`, data) as ResponseWithProjectDTO; wikiStore.project.projectDTO = response.project; // not necessary? wikiStore.changeWindowDisplayMode(folder); @@ -195,7 +204,7 @@ class RequestService { public async uploadDataFile(folder: string, data: any) { const updater = currentFilesService.createUpdater(); const response = await backendPost(`/data?project_folder=${folder}`, data) as ResponseWithEverythingDTO; - updater.update(()=>this.fillTable(response), "uploadDataFile"); + updater.update(() => this.fillTable(response), "uploadDataFile"); } public async uploadWikifierOutput(data: any) { @@ -213,13 +222,13 @@ class RequestService { public async getTable() { const updater = currentFilesService.createUpdater(); const response = await backendGet(`/table?${this.getMappingParams()}`) as ResponseWithTableDTO; - updater.update(()=>this.fillTable(response), "getTable"); + updater.update(() => this.fillTable(response), "getTable"); } public async getMappingCalculation() { const updater = currentFilesService.createUpdater(); const response = await backendGet(`/mapping?${this.getMappingParams()}`) as ResponseWithMappingDTO; - updater.update(()=>this.fillMapping(response), "getMappingCalculation"); + updater.update(() => this.fillMapping(response), "getMappingCalculation"); } public async getSettings(folder: string) { @@ -234,12 +243,12 @@ class RequestService { public async getGlobalSettings() { const response = await backendGet(`/project/globalsettings`) as GlobalSettingsDTO; - wikiStore.globalSettings.datamart_api=response.datamart_api; + wikiStore.globalSettings.datamart_api = response.datamart_api; } public async putGlobalSettings(data: any) { const response = await backendPut(`/project/globalsettings`, data) as GlobalSettingsDTO; - wikiStore.globalSettings.datamart_api=response.datamart_api; + wikiStore.globalSettings.datamart_api = response.datamart_api; } public async uploadEntities(data: any) { @@ -282,7 +291,7 @@ class RequestService { public async call( component: React.Component, - func: () => Promise ) { + func: () => Promise) { component.setState({ errorMessage: {} as ErrorMessage }); diff --git a/electron/src/renderer/project/project.tsx b/electron/src/renderer/project/project.tsx index 5854a5242..e2ad7cc94 100644 --- a/electron/src/renderer/project/project.tsx +++ b/electron/src/renderer/project/project.tsx @@ -111,7 +111,6 @@ class Project extends Component { async loadProject() { // before fetching project files wikiStore.table.showSpinner = true; - wikiStore.wikifier.showSpinner = true; wikiStore.yaml.showSpinner = true; wikiStore.output.isDownloadDisabled = true; @@ -123,28 +122,36 @@ class Project extends Component { await this.requestService.call(this, () => this.requestService.getTable()); } - if (wikiStore.project.projectDTO){ - document.title = 't2wml: ' + wikiStore.project.projectDTO.title; - this.setState({ name: wikiStore.project.projectDTO.title }); + if (wikiStore.project.projectDTO) { + document.title = 't2wml: ' + wikiStore.project.projectDTO.title; + this.setState({ name: wikiStore.project.projectDTO.title }); - if (wikiStore.yaml.yamlContent !== null) { - wikiStore.output.isDownloadDisabled = false; - } + if (wikiStore.yaml.yamlContent !== null) { + wikiStore.output.isDownloadDisabled = false; + } - // load settings - if (!wikiStore.project.projectDTO.sparql_endpoint) { - wikiStore.project.projectDTO.sparql_endpoint = Config.defaultSparqlEndpoint; + // load settings + if (!wikiStore.project.projectDTO.sparql_endpoint) { + wikiStore.project.projectDTO.sparql_endpoint = Config.defaultSparqlEndpoint; + } } - } } catch (error) { console.log(error); } finally { wikiStore.table.showSpinner = false; - wikiStore.wikifier.showSpinner = false; wikiStore.yaml.showSpinner = false; } + if (currentFilesService.currentState.dataFile && currentFilesService.currentState.sheetName) { + wikiStore.wikifier.showSpinner = true; + try { + await this.requestService.getPartialCsv(); + } + finally { + wikiStore.wikifier.showSpinner = false; + } + } } @@ -173,17 +180,19 @@ class Project extends Component { }); } - async handleSaveSettings(endpoint: string, warn: boolean, calendar:string, title: string, description: string | undefined, url: string | undefined) { + async handleSaveSettings(endpoint: string, warn: boolean, calendar: string, title: string, description: string | undefined, url: string | undefined) { // update settings this.setState({ showSettings: false }); // notify backend - const data = {"endpoint": endpoint, - "warnEmpty": warn, - "handleCalendar": calendar, - "title": title, - "description": description, - "url": url }; + const data = { + "endpoint": endpoint, + "warnEmpty": warn, + "handleCalendar": calendar, + "title": title, + "description": description, + "url": url + }; try { await this.requestService.call(this, () => this.requestService.putSettings(this.props.path, data)); @@ -199,7 +208,7 @@ class Project extends Component { async handleSaveEntities(file: string, property: string, propertyVals: any) { const data = { entity_file: file, - updated_entries: {[property]: propertyVals}, + updated_entries: { [property]: propertyVals }, } try { await this.requestService.saveEntities(data); diff --git a/electron/src/renderer/project/sidebar/file-tree/entities-tree.tsx b/electron/src/renderer/project/sidebar/file-tree/entities-tree.tsx index 936c9ef40..3a88a8d2b 100644 --- a/electron/src/renderer/project/sidebar/file-tree/entities-tree.tsx +++ b/electron/src/renderer/project/sidebar/file-tree/entities-tree.tsx @@ -73,6 +73,13 @@ class EntitiesTree extends Component { wikiStore.table.showSpinner = false; wikiStore.yaml.showSpinner = false; } + wikiStore.wikifier.showSpinner = true; + try{ + await this.requestService.getPartialCsv(); + } + finally{ + wikiStore.wikifier.showSpinner = false; + } } async changeFile() { diff --git a/electron/src/renderer/project/sidebar/file-tree/file-tree.tsx b/electron/src/renderer/project/sidebar/file-tree/file-tree.tsx index fb0cb8d4d..494c367c3 100644 --- a/electron/src/renderer/project/sidebar/file-tree/file-tree.tsx +++ b/electron/src/renderer/project/sidebar/file-tree/file-tree.tsx @@ -57,11 +57,18 @@ class FileTree extends Component { wikiStore.table.showSpinner = true; wikiStore.yaml.showSpinner = true; try{ - await this.requestService.getTable(); + await this.requestService.getTable(); }finally{ wikiStore.table.showSpinner = false; wikiStore.yaml.showSpinner = false; } + wikiStore.wikifier.showSpinner = true; + try{ + await this.requestService.getPartialCsv(); + } + finally{ + wikiStore.wikifier.showSpinner = false; + } } async changeDataFile(dataFile: string) { diff --git a/electron/src/renderer/project/sidebar/sidebar.tsx b/electron/src/renderer/project/sidebar/sidebar.tsx index 32fe46f09..ad28fbcaa 100644 --- a/electron/src/renderer/project/sidebar/sidebar.tsx +++ b/electron/src/renderer/project/sidebar/sidebar.tsx @@ -92,6 +92,13 @@ class Sidebar extends Component<{}, SidebarState> { wikiStore.table.showSpinner = false; wikiStore.yaml.showSpinner = false; } + wikiStore.wikifier.showSpinner = true; + try{ + await this.requestService.getPartialCsv(); + } + finally{ + wikiStore.wikifier.showSpinner = false; + } } getFilesData() { diff --git a/electron/src/renderer/project/table/annotation-table/annotation-form.tsx b/electron/src/renderer/project/table/annotation-table/annotation-form.tsx index 52b575b4c..1d98ed951 100644 --- a/electron/src/renderer/project/table/annotation-table/annotation-form.tsx +++ b/electron/src/renderer/project/table/annotation-table/annotation-form.tsx @@ -17,7 +17,7 @@ import { columnToLetter } from '../table-utils'; interface AnnotationFormProperties { selection?: CellSelection; - onSelectionChange: (selection: CellSelection) => void; + onSelectionChange: (selection: CellSelection, role?: string) => void; selectedAnnotationBlock?: AnnotationBlock; annotationSuggestions: ResponseWithSuggestion; onChange: any | null; // Use the actual function type: (arg: argType) => returnType @@ -167,7 +167,7 @@ class AnnotationForm extends React.Component { if (this.state.validArea) { - onSelectionChange(selection); + onSelectionChange(selection, fields.role); } }, 500); } else { @@ -334,7 +334,7 @@ class AnnotationForm extends React.Component Type - {selectedOption?.children?.filter(typeOption => annotationSuggestions.types.includes(typeOption.value)).map((type, i) => ( + {selectedOption?.children?.map((type, i) => (