From 6b0a516542d161163b49da31ff2d2c3886f62e3e Mon Sep 17 00:00:00 2001 From: Anne Schumacher Date: Thu, 3 Nov 2022 15:57:13 +0100 Subject: [PATCH 001/175] set new dev version number --- docs/docsify/_coverpage.md | 2 +- sparv/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/docsify/_coverpage.md b/docs/docsify/_coverpage.md index 2a0aa5ee..aff138c4 100644 --- a/docs/docsify/_coverpage.md +++ b/docs/docsify/_coverpage.md @@ -4,7 +4,7 @@ > Språkbanken's text analysis tool -

version 5.1.0

+

version 5.1.1dev0

Sparv on GitHub diff --git a/sparv/__init__.py b/sparv/__init__.py index 966ce6e6..cb9e8c52 100644 --- a/sparv/__init__.py +++ b/sparv/__init__.py @@ -1,3 +1,3 @@ """Main Sparv package.""" -__version__ = "5.1.0" +__version__ = "5.1.1dev0" From c751627fd94b392601e9ff3c266875bdbdde11e6 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Mon, 7 Nov 2022 17:20:17 +0100 Subject: [PATCH 002/175] Add collation to lemgram index table --- sparv/modules/korp/lemgram_index.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/sparv/modules/korp/lemgram_index.py b/sparv/modules/korp/lemgram_index.py index 55711643..1387db76 100644 --- a/sparv/modules/korp/lemgram_index.py +++ b/sparv/modules/korp/lemgram_index.py @@ -65,9 +65,13 @@ def lemgram_sql(corpus: Corpus = Corpus(), MYSQL_TABLE = "lemgram_index" -MYSQL_INDEX = {"columns": [("lemgram", "varchar(64)", "", "NOT NULL"), - ("freq", int, 0, "NOT NULL"), - ("corpus", "varchar(64)", "", "NOT NULL")], - "indexes": ["lemgram corpus freq"], # Can't make this primary due to collation - "default charset": "utf8mb4", - } +MYSQL_INDEX = { + "columns": [ + ("lemgram", "varchar(64)", "", "NOT NULL"), + ("freq", int, 0, "NOT NULL"), + ("corpus", "varchar(64)", "", "NOT NULL"), + ], + "indexes": ["lemgram corpus freq"], # Can't make this primary due to collation + "default charset": "utf8mb4", + "collate": "utf8mb4_bin", +} From 1498165a589a8f7162a95a34bf1e775228143803 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Thu, 10 Nov 2022 13:45:19 +0100 Subject: [PATCH 003/175] Add annotator for inheriting attributes --- sparv/modules/misc/misc.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/sparv/modules/misc/misc.py b/sparv/modules/misc/misc.py index 32d37039..790fa410 100644 --- a/sparv/modules/misc/misc.py +++ b/sparv/modules/misc/misc.py @@ -142,6 +142,21 @@ def struct_to_token(attr: Annotation = Annotation("{struct}:{attr}"), out.write(out_values) +@annotator("Inherit {attr} from {parent}:{attr} to {child}", wildcards=[ + Wildcard("parent", Wildcard.ANNOTATION), + Wildcard("child", Wildcard.ANNOTATION), + Wildcard("attr", Wildcard.ATTRIBUTE) +]) +def inherit(parent: Annotation = Annotation("{parent}:{attr}"), + child: Annotation = Annotation("{child}"), + out: Output = Output("{child}:misc.inherit_{parent}_{attr}")): + """Inherit attribute from a structural parent annotation to a child.""" + child_parents = child.get_parents(parent) + attr_values = list(parent.read()) + out_values = [attr_values[p] if p is not None else "" for p in child_parents] + out.write(out_values) + + # TODO: Do we still need this? struct_to_token probably mostly replaces it def chain(out, annotations, default=None): """Create a functional composition of a list of annotations. From 6a10be516465e7e68ca0f5760d33096f341b0c14 Mon Sep 17 00:00:00 2001 From: Anne Schumacher Date: Thu, 10 Nov 2022 18:11:45 +0100 Subject: [PATCH 004/175] cleanup docstrings --- sparv/modules/dateformat/dateformat.py | 52 +++++-------- sparv/modules/hist/diapivot.py | 7 +- sparv/modules/hist/hist.py | 85 +++++++++++----------- sparv/modules/hist/models.py | 4 +- sparv/modules/hunpos/morphtable.py | 19 ++--- sparv/modules/hunpos/morphtable_hist.py | 12 +-- sparv/modules/korp/lemgram_index.py | 10 +-- sparv/modules/korp/relations.py | 8 +- sparv/modules/korp/timespan.py | 8 +- sparv/modules/misc/misc.py | 2 +- sparv/modules/saldo/saldo.py | 51 ++++++------- sparv/modules/stats_export/sbx_stats.py | 56 ++++++-------- sparv/modules/stats_export/stats_export.py | 21 ++---- 13 files changed, 143 insertions(+), 192 deletions(-) diff --git a/sparv/modules/dateformat/dateformat.py b/sparv/modules/dateformat/dateformat.py index 085d1182..4982b747 100644 --- a/sparv/modules/dateformat/dateformat.py +++ b/sparv/modules/dateformat/dateformat.py @@ -42,24 +42,18 @@ def dateformat(in_from: Annotation = Annotation("[dateformat.datetime_from]"), http://docs.python.org/library/datetime.html#strftime-and-strptime-behavior Args: - in_from (str, optional): Annotation containing from-dates (and times). - Defaults to Annotation("[dateformat.datetime_from]"). - in_to (Optional[str], optional): Annotation containing to-dates. - Defaults to Annotation("[dateformat.datetime_to]"). - out_from (str, optional): Annotation with from-times to be written. - Defaults to Output("[dateformat.out_annotation]:dateformat.datefrom",description="From-dates"). - out_to (Optional[str], optional): Annotation with to-times to be written. - Defaults to Output("[dateformat.out_annotation]:dateformat.dateto",description="To-dates"). - informat (str, optional): Format of the in_from and in_to dates/times. + in_from: Annotation containing from-dates (and times). + in_to: Annotation containing to-dates. + out_from: Annotation with from-times to be written. + out_to: Annotation with to-times to be written. + informat: Format of the in_from and in_to dates/times. Several formats can be specified separated by |. They will be tried in order. - Defaults to Config("dateformat.datetime_informat"). - outformat (str, optional): Desired format of the out_from and out_to dates. + outformat: Desired format of the out_from and out_to dates. Several formats can be specified separated by |. They will be tied to their respective in-format. - Defaults to Config("dateformat.date_outformat", "%Y%m%d"). - splitter (str, optional): One or more characters separating two dates in 'in_from', - treating them as from-date and to-date. Defaults to Config("dateformat.splitter", None). - regex (str, optional): Regular expression with a catching group whose content will be used in the parsing - instead of the whole string. Defaults to Config("dateformat.regex", None). + splitter: One or more characters separating two dates in 'in_from', + treating them as from-date and to-date. + regex: Regular expression with a catching group whose content will be used in the parsing + instead of the whole string. """ _formatter(in_from, in_to, out_from, out_to, informat, outformat, splitter, regex) @@ -95,24 +89,18 @@ def timeformat(in_from: Annotation = Annotation("[dateformat.datetime_from]"), http://docs.python.org/library/datetime.html#strftime-and-strptime-behavior Args: - in_from (str, optional): Annotation containing from-dates (and times). - Defaults to Annotation("[dateformat.datetime_from]"). - in_to (Optional[str], optional): Annotation containing to-dates. - Defaults to Annotation("[dateformat.datetime_to]"). - out_from (str, optional): Annotation with from-times to be written. - Defaults to Output("[dateformat.out_annotation]:dateformat.timefrom",description="From-times"). - out_to (Optional[str], optional): Annotation with to-times to be written. - Defaults to Output("[dateformat.out_annotation]:dateformat.timeto",description="To-times"). - informat (str, optional): Format of the in_from and in_to dates/times. + in_from: Annotation containing from-dates (and times). + in_to: Annotation containing to-dates. + out_from: Annotation with from-times to be written. + out_to: Annotation with to-times to be written. + informat: Format of the in_from and in_to dates/times. Several formats can be specified separated by |. They will be tried in order. - Defaults to Config("dateformat.datetime_informat"). - outformat (str, optional): Desired format of the out_from and out_to times. + outformat: Desired format of the out_from and out_to times. Several formats can be specified separated by |. They will be tied to their respective in-format. - Defaults to Config("dateformat.time_outformat", "%Y%m%d"). - splitter (str, optional): One or more characters separating two dates in 'in_from', - treating them as from-date and to-date. Defaults to Config("dateformat.splitter", None). - regex (str, optional): Regular expression with a catching group whose content will be used in the parsing - instead of the whole string. Defaults to Config("dateformat.regex", None). + splitter: One or more characters separating two dates in 'in_from', + treating them as from-date and to-date. + regex: Regular expression with a catching group whose content will be used in the parsing + instead of the whole string. """ _formatter(in_from, in_to, out_from, out_to, informat, outformat, splitter, regex) diff --git a/sparv/modules/hist/diapivot.py b/sparv/modules/hist/diapivot.py index 426cd8c7..2048bace 100644 --- a/sparv/modules/hist/diapivot.py +++ b/sparv/modules/hist/diapivot.py @@ -18,10 +18,9 @@ def diapivot_annotate(out: Output = Output(":hist.diapivot", cls="token:l """Annotate each lemgram with its corresponding saldo_id according to model. Args: - out (str, optional): Resulting annotation file. - Defaults to Output(":hist.diapivot", description="SALDO IDs corresponding to lemgrams"). - lemgram (str, optional): Existing lemgram annotation. Defaults to Annotation(":saldo.lemgram"). - model (str, optional): Crosslink model. Defaults to Model("hist/diapivot.pickle"). + out: Resulting annotation file. + lemgram: Existing lemgram annotation. Defaults to Annotation(":saldo.lemgram"). + model: Crosslink model. """ lexicon = PivotLexicon(model.path) lemgram_annotation = list(lemgram.read()) diff --git a/sparv/modules/hist/hist.py b/sparv/modules/hist/hist.py index 2ea77bf5..25befbd3 100644 --- a/sparv/modules/hist/hist.py +++ b/sparv/modules/hist/hist.py @@ -86,12 +86,11 @@ def extract_pos(out: Output = Output(":hist.homograph_set", description=" """Extract POS tags from lemgrams. Args: - out (Output): The output annotation. Defaults to Output(":hist.homograph_set"). - lemgrams (Annotation): Input lemgram annotation. Defaults to Annotation(":saldo.lemgram"). - extralemgrams (Optional[Annotation], optional): Additional annotation from which more pos-tags can be extracted. - Defaults to Annotation("[hist.extralemgrams]"). - delimiter (str): Character to put between ambiguous results. Defaults to Config("hist.delimiter"). - affix (str): Character to put before and after sets of results. Defaults to Config("hist.affix"). + out: The output annotation. + lemgrams: Input lemgram annotation. + extralemgrams: Additional annotation from which more pos-tags can be extracted. + delimiter: Character to put between ambiguous results. + affix: Character to put before and after sets of results. """ def oktag(tag): return tag is not None and tag.group(1) not in ["e", "sxc", "mxc"] @@ -123,15 +122,15 @@ def lemgram_fallback( """Lookup lemgrams in models for words that do not already have a lemgram. Args: - out (Output): The output annotation. Defaults to Output(":hist.lemgram"). - word (Annotation): Input annotation with token strings. Defaults to Annotation(""). - msd (Annotation): Input annotation with POS and morphosyntactig desciptions. Defaults to Annotation(""). - lemgram (Annotation): Input annotation with SALDO lemgrams. Defaults to Annotation(":saldo.lemgram"). - key (str): Key to lookup in the models. Defaults to Config("hist.lemgram_key"). - models (List[Model], optional): A list of lexicon models. Defaults to [Model("[hist.dalin_model]"), Model("[hist.swedberg_model]")]. - delimiter (str): Character to put between ambiguous results. Defaults to Config("hist.delimiter"). - affix (str): Character to put before and after sets of results. Defaults to Config("hist.affix"). - models_preloaded (dict, optional): Preloaded models. Defaults to None. + out: The output annotation. + word: Input annotation with token strings. + msd: Input annotation with POS and morphosyntactig desciptions. + lemgram: Input annotation with SALDO lemgrams. + key: Key to lookup in the models. + models: A list of lexicon models. + delimiter: Character to put between ambiguous results. + affix: Character to put before and after sets of results. + models_preloaded: Preloaded models. """ _annotate_fallback(out=out, word=word, msd=msd, main_annotation=lemgram, key=key, models=models, delimiter=delimiter, @@ -156,15 +155,15 @@ def baseform_fallback( """Lookup baseforms in models for words that do not already have a baseform. Args: - out (Output): The output annotation. Defaults to Output(":hist.baseform"). - word (Annotation): Input annotation with token strings. Defaults to Annotation(""). - msd (Annotation): Input annotation with POS and morphosyntactig desciptions. Defaults to Annotation(""). - baseform (Annotation): Input annotation with SALDO baseforms. Defaults to Annotation(":saldo.baseform"). - key (str): Key to lookup in the models. Defaults to Config("hist.baseform_key"). - models (List[Model], optional): A list of lexicon models. Defaults to [Model("[hist.dalin_model]"), Model("[hist.swedberg_model]")]. - delimiter (str): Character to put between ambiguous results. Defaults to Config("hist.delimiter"). - affix (str): Character to put before and after sets of results. Defaults to Config("hist.affix"). - models_preloaded (dict, optional): Preloaded models. Defaults to None. + out: The output annotation. + word: Input annotation with token strings. + msd: Input annotation with POS and morphosyntactig desciptions. + baseform: Input annotation with SALDO baseforms. + key: Key to lookup in the models. + models: A list of lexicon models. + delimiter: Character to put between ambiguous results. + affix: Character to put before and after sets of results. + models_preloaded: Preloaded models. """ _annotate_fallback(out=out, word=word, msd=msd, main_annotation=baseform, key=key, models=models, delimiter=delimiter, @@ -179,11 +178,10 @@ def posset(pos: Annotation = Annotation(""), """Annotate with POS sets by converting a single POS into a set (mostly used to make corpora comparable). Args: - pos (Annotation, optional): Input annotation with part-of-speech tags. Defaults to Annotation(""). - out (Output, optional): Output annotation with sets of part-of-speech tags. - Defaults to Output(":hist.homograph_set"). - delimiter (str): Character to put between ambiguous results. Defaults to Config("hist.delimiter"). - affix (str): Character to put before and after sets of results. Defaults to Config("hist.affix"). + pos: Input annotation with part-of-speech tags. + out: Output annotation with sets of part-of-speech tags. + delimiter: Character to put between ambiguous results. + affix: Character to put before and after sets of results. """ def makeset(_, thepos): """Annotate thepos with separators (dummy function).""" @@ -205,13 +203,13 @@ def spelling_variants(word: Annotation = Annotation(""), """Use a lexicon model and a spelling model to annotate words with their spelling variants. Args: - word (Annotation, optional): Input annotation with token strings. Defaults to Annotation(""). - out (Output, optional): Output annotation with spelling variations. Defaults to Output(""). - spellingmodel (Model): The spelling model. Defaults to Model("[hist.fsv_spelling]") - model (Model): The lexicon model. Defaults to Model("[hist.fsv_model]") - delimiter (str): Character to put between ambiguous results. Defaults to Config("hist.delimiter"). - affix (str): Character to put before and after sets of results. Defaults to Config("hist.affix"). - model_preloaded (dict, optional): Preloaded morphology model. Defaults to None. + word: Input annotation with token strings. + out: Output annotation with spelling variations. + spellingmodel: The spelling model. + model: The lexicon model. + delimiter: Character to put between ambiguous results. + affix: Character to put before and after sets of results. + model_preloaded: Preloaded morphology model. """ # # Load model # model_name = model.path.stem @@ -263,8 +261,8 @@ def all_spelling_variants( # Auxiliaries ################################################################################ -def _annotate_standard(out, input_annotation, annotator, extra_input="", delimiter: str = util.constants.DELIM, - affix: str = util.constants.AFFIX, split=True): +def _annotate_standard(out, input_annotation, annotator, extra_input:str = "", delimiter: str = util.constants.DELIM, + affix: str = util.constants.AFFIX, split:bool = True): """Apply the 'annotator' function to the annotations in 'input_annotation' and write the new output to 'out'. Args: @@ -272,12 +270,11 @@ def _annotate_standard(out, input_annotation, annotator, extra_input="", delimit input_annotation: The input annotation. annotator: function which is to be applied to the input annotation. It should have type :: oldannotations -> newannotations - extra_input (str, optional): An additional input annotation. Defaults to "". - delimiter (str, optional): Delimiter character to put between ambiguous results. Defaults to - util.constants.DELIM. - affix (str, optional): Character to put before and after results. Defaults to util.constants.AFFIX. - split (bool, optional): Defines whether the input annatoation is a set, with elements separated by delimiter. - If so, return a list. Else, return one single element. Defaults to True. + extra_input: An additional input annotation. + delimiter: Delimiter character to put between ambiguous results. + affix: Character to put before and after results. + split: Defines whether the input annatoation is a set, with elements separated by delimiter. + If so, return a list. Else, return one single element. """ # Join input_annotation and extra_input with delimiter annotations = input_annotation.read() diff --git a/sparv/modules/hist/models.py b/sparv/modules/hist/models.py index c8803bc3..01985a24 100644 --- a/sparv/modules/hist/models.py +++ b/sparv/modules/hist/models.py @@ -86,8 +86,8 @@ def read_lmf(xml, annotation_elements=("writtenForm", "lemgram"), verbose=True, verbose (bool, optional): Whether to turn on verbose mode. Defaults to True. skip_multiword (bool, optional): Whether to make special entries for multiword expressions. Set this to False only if the tool used for text annotation cannot handle this at all. Defaults to False. - translate_tags (bool, optional): [description]. Defaults to True. - use_fallback (bool, optional): [description]. Defaults to False. + translate_tags (bool, optional): Whether to translate SALDO tags into SUC tags. Defaults to True. + use_fallback (bool, optional): Whether to get SUC POS tag from POS in lemgram as a fallback. Defaults to False. Returns: A lexicon dict: diff --git a/sparv/modules/hunpos/morphtable.py b/sparv/modules/hunpos/morphtable.py index fa873c40..9269234c 100644 --- a/sparv/modules/hunpos/morphtable.py +++ b/sparv/modules/hunpos/morphtable.py @@ -33,18 +33,13 @@ def saldo_morphtable(out: ModelOutput = ModelOutput("hunpos/saldo_suc-tags.morph is saved. Args: - out (str, optional): Resulting morphtable file to be written. - Defaults to ModelOutput("hunpos/saldo_suc-tags.morphtable"). - saldo_model (str, optional): Path to a pickled SALDO model. - Defaults to Model("saldo/saldo.pickle"). - suc (str, optional): Tab-separated file with wordforms from SUC, containing: frequency, wordform, tag. - Defaults to Model("hunpos/suc3_morphtable.words"). - morphtable_base (str, optional): Existing morphtable file, whose contents will be included in the new one. - Defaults to Model("hunpos/suc.morphtable"). - morphtable_patterns (str, optional): Optional file with regular expressions. - Defaults to Model("hunpos/suc.patterns"). - add_capitalized (bool, optional): Whether or not capitalized word forms should be added. Defaults to True. - add_lowercase (bool, optional): Whether or not lower case word forms should be added. Defaults to False. + out: Resulting morphtable file to be written. + saldo_model: Path to a pickled SALDO model. + suc: Tab-separated file with wordforms from SUC, containing: frequency, wordform, tag. + morphtable_base: Existing morphtable file, whose contents will be included in the new one. + morphtable_patterns: Optional file with regular expressions. + add_capitalized: Whether or not capitalized word forms should be added. + add_lowercase: Whether or not lower case word forms should be added. """ lex = saldo.SaldoLexicon(saldo_model.path) tags = defaultdict(set) diff --git a/sparv/modules/hunpos/morphtable_hist.py b/sparv/modules/hunpos/morphtable_hist.py index a1d2ccd4..21d85351 100644 --- a/sparv/modules/hunpos/morphtable_hist.py +++ b/sparv/modules/hunpos/morphtable_hist.py @@ -19,14 +19,10 @@ def hist_morphtable(out: ModelOutput = ModelOutput("hunpos/hist/dalinm-swedberg_ """Read files and make a morphtable together with the information from SALDO (saldosuc_morphtable). Args: - out (str, optional): Resulting morphtable file to be written. - Defaults to ModelOutput("hunpos/hist/dalinm-swedberg_saldo_suc-tags.morphtable"). - swedberg (str, optional): Wordlist from Swedberg and corresponding SALDO MSD-tags. - Defaults to Model("hunpos/hist/swedberg-gender.hunpos"). - dalin (str, optional): Wordlist from Dalin and corresponding SALDO MSD-tags. - Defaults to Model("hunpos/hist/dalinm.hunpos"). - saldosuc_morphtable (str, optional): SALDO Hunpos morphtable. - Defaults to Model("hunpos/saldo_suc-tags.morphtable"). + out: Resulting morphtable file to be written. + swedberg: Wordlist from Swedberg and corresponding SALDO MSD-tags. + dalin: Wordlist from Dalin and corresponding SALDO MSD-tags. + saldosuc_morphtable: SALDO Hunpos morphtable. """ words = {} _read_saldosuc(words, saldosuc_morphtable.path) diff --git a/sparv/modules/korp/lemgram_index.py b/sparv/modules/korp/lemgram_index.py index 1387db76..ab48cffb 100644 --- a/sparv/modules/korp/lemgram_index.py +++ b/sparv/modules/korp/lemgram_index.py @@ -21,12 +21,10 @@ def install_lemgrams(sqlfile: ExportInput = ExportInput("korp.lemgram_index/lemg """Install lemgram SQL on remote host. Args: - sqlfile (str, optional): SQL file to be installed. - Defaults to ExportInput("korp.lemgram_index/lemgram_index.sql"). - marker (str, optional): Marker file to be written. - Defaults to OutputMarker("korp.install_lemgram_marker"). - db_name (str, optional): Name of the data base. Defaults to Config("korp.mysql_dbname"). - host (str, optional): Remote host to install to. Defaults to Config("korp.remote_host"). + sqlfile: SQL file to be installed. + marker: Marker file to be written. + db_name: Name of the data base. + host: Remote host to install to. """ util.install.install_mysql(host, db_name, sqlfile) marker.write() diff --git a/sparv/modules/korp/relations.py b/sparv/modules/korp/relations.py index f54f37a8..011a8774 100644 --- a/sparv/modules/korp/relations.py +++ b/sparv/modules/korp/relations.py @@ -25,10 +25,10 @@ def install_relations(sqlfile: ExportInput = ExportInput("korp.wordpicture/relat """Install Korp's Word Picture SQL on remote host. Args: - sqlfile (str, optional): SQL file to be installed. Defaults to ExportInput("korp.wordpicture/relations.sql"). - out (str, optional): Marker file to be written. - db_name (str, optional): Name of the data base. Defaults to Config("korp.mysql_dbname"). - host (str, optional): Remote host to install to. Defaults to Config("korp.remote_host"). + sqlfile: SQL file to be installed. + out: Marker file to be written. + db_name: Name of the data base. + host: Remote host to install to. """ util.install.install_mysql(host, db_name, sqlfile) out.write() diff --git a/sparv/modules/korp/timespan.py b/sparv/modules/korp/timespan.py index 08e32759..6c7d9a10 100644 --- a/sparv/modules/korp/timespan.py +++ b/sparv/modules/korp/timespan.py @@ -17,10 +17,10 @@ def install_timespan(sqlfile: ExportInput = ExportInput("korp.timespan/timespan. """Install timespan SQL on remote host. Args: - sqlfile (str, optional): SQL file to be installed. Defaults to ExportInput("korp.timespan/timespan.sql"). - out (str, optional): Marker file to be written. - db_name (str, optional): Name of the data base. Defaults to Config("korp.mysql_dbname"). - host (str, optional): Remote host to install to. Defaults to Config("korp.remote_host"). + sqlfile: SQL file to be installed. + out: Marker file to be written. + db_name: Name of the data base. + host: Remote host to install to. """ util.install.install_mysql(host, db_name, sqlfile) out.write() diff --git a/sparv/modules/misc/misc.py b/sparv/modules/misc/misc.py index 790fa410..ed2b89a8 100644 --- a/sparv/modules/misc/misc.py +++ b/sparv/modules/misc/misc.py @@ -384,7 +384,7 @@ def source(out: Output = Output(":misc.source"), @annotator("Get the first annotation from a cwb set") def first_from_set(out: Output, - chunk: Annotation,): + chunk: Annotation): """"Get the first annotation from a set.""" out_annotation = [] for val in chunk.read(): diff --git a/sparv/modules/saldo/saldo.py b/sparv/modules/saldo/saldo.py index 9eaf5b22..bdba3471 100644 --- a/sparv/modules/saldo/saldo.py +++ b/sparv/modules/saldo/saldo.py @@ -65,36 +65,29 @@ def annotate(token: Annotation = Annotation(""), """Use the Saldo lexicon model to annotate msd-tagged words. Args: - token (Annotation): Input annotation with token spans. Defaults to Annotation(""). - word (Annotation): Input annotation with token strings. Defaults to Annotation(""). - sentence (Annotation): Input annotation with sentence spans. Defaults to Annotation(""). - reference (Annotation): Input annotation with token indices for each sentence. - Defaults to Annotation(""). - out_sense (Output): Output annotation with senses from SALDO. Defaults to Output(":saldo.sense"). - out_lemgram (Output): Output annotation with lemgrams from SALDO. Defaults to Output(":saldo.lemgram"). - out_baseform (Output): Output annotation with baseforms from SALDO. - Defaults to Output(":saldo.baseform"). - models (List[Model]): A list of pickled lexicons, typically the SALDO model (saldo.pickle) - and optional lexicons for older Swedish. Defaults to [Model("[saldo.model]")]. - msd (Annotation, optional): Input annotation with POS and morphological descriptions. - Defaults to Annotation(""). - delimiter (str): Character to put between ambiguous results. Defaults to Config("saldo.delimiter"). - affix (str): Character to put before and after sets of results. Defaults to Config("saldo.affix"). - precision (str): Format string for appending precision to each value (e.g. ':%.3f'). use empty string for no - precision. Defaults to Config("saldo.precision"). - precision_filter (str): Precision filter with values 'max' (only use the annotations that are most probable), + token: Input annotation with token spans. + word: Input annotation with token strings. + sentence: Input annotation with sentence spans. + reference: Input annotation with token indices for each sentence. + out_sense: Output annotation with senses from SALDO. + out_lemgram: Output annotation with lemgrams from SALDO. + out_baseform: Output annotation with baseforms from SALDO. + models: A list of pickled lexicons, typically the SALDO model (saldo.pickle) + and optional lexicons for older Swedish. + msd: Input annotation with POS and morphological descriptions. + delimiter: Character to put between ambiguous results. + affix: Character to put before and after sets of results. + precision: Format string for appending precision to each value (e.g. ':%.3f'). use empty string for no + precision. + precision_filter: Precision filter with values 'max' (only use the annotations that are most probable), 'first' (only use the most probable annotation(s)), 'none' (use all annotations)". - Defaults to Config("saldo.precision_filter"). - min_precision (float): Only use annotations with a probability score higher than this. - Defaults to Config("saldo.min_precision"). - skip_multiword (bool): Whether to disable annotation of multiword expressions. - Defaults to Config("saldo.skip_multiword"). - max_gaps (int): Max amount of gaps allowed within a multiword expression. Defaults to Config("saldo.max_gaps"). - allow_multiword_overlap (bool): Whether all multiword expressions may overlap with each other. If set to False, - some cleanup is done. Defaults to Config("saldo.allow_multiword_overlap"). - word_separator (str): Character used to split the values of 'word' into several word variations. - Defaults to Config("saldo.word_separator"). - models_preloaded (dict, optional): Preloaded models. Defaults to None. + min_precision: Only use annotations with a probability score higher than this. + skip_multiword: Whether to disable annotation of multiword expressions. + max_gaps (int): Max amount of gaps allowed within a multiword expression. + allow_multiword_overlap: Whether all multiword expressions may overlap with each other. If set to False, + some cleanup is done. + word_separator: Character used to split the values of 'word' into several word variations. + models_preloaded: Preloaded models. """ main(token=token, word=word, sentence=sentence, reference=reference, out_sense=out_sense, out_lemgram=out_lemgram, out_baseform=out_baseform, models=models, msd=msd, delimiter=delimiter, affix=affix, precision=precision, diff --git a/sparv/modules/stats_export/sbx_stats.py b/sparv/modules/stats_export/sbx_stats.py index 2d83b7ba..f34e5ef6 100644 --- a/sparv/modules/stats_export/sbx_stats.py +++ b/sparv/modules/stats_export/sbx_stats.py @@ -77,22 +77,17 @@ def sbx_freq_list( """Create a word frequency list for the entire corpus. Args: - source_files (list, optional): The source files belonging to this corpus. Defaults to AllSourceFilenames. - word (str, optional): Word annotations. Defaults to AnnotationAllSourceFiles(""). - token (str, optional): Token span annotations. Defaults to AnnotationAllSourceFiles(""). - msd (str, optional): MSD annotations. Defaults to AnnotationAllSourceFiles(""). - baseform (str, optional): Annotations with first baseform from each set. - Defaults to AnnotationAllSourceFiles(""). - sense (str, optional): Best sense annotations. Defaults to AnnotationAllSourceFiles(""). - lemgram (str, optional): Annotations with first lemgram from each set. - Defaults to AnnotationAllSourceFiles(":saldo.lemgram"). - complemgram (str, optional): Conditional best compound lemgram annotations. - Defaults to AnnotationAllSourceFiles(":saldo.complemgram"). - out (str, optional): The output word frequency file. - Defaults to Export("stats_export.frequency_list_sbx/[metadata.id].csv"). - delimiter (str, optional): Column delimiter to use in the csv. Defaults to Config("stats_export.delimiter"). - cutoff (int, optional): The minimum frequency a word must have in order to be included in the result. - Defaults to Config("stats_export.cutoff"). + source_files: The source files belonging to this corpus. + word: Word annotations. + token: Token span annotations. + msd: MSD annotations. + baseform: Annotations with first baseform from each set. + sense: Best sense annotations. + lemgram: Annotations with first lemgram from each set. + complemgram: Conditional best compound lemgram annotations. + out: The output word frequency file. + delimiter: Column delimiter to use in the csv. + cutoff: The minimum frequency a word must have in order to be included in the result. """ annotations = [(word, "token"), (msd, "POS"), (baseform, "lemma"), (sense, "SALDO sense"), (lemgram, "lemgram"), (complemgram, "compound")] @@ -119,23 +114,18 @@ def sbx_freq_list_date( """Create a word frequency list for the entire corpus. Args: - source_files (list, optional): The source files belonging to this corpus. Defaults to AllSourceFilenames. - word (str, optional): Word annotations. Defaults to AnnotationAllSourceFiles(""). - token (str, optional): Token span annotations. Defaults to AnnotationAllSourceFiles(""). - msd (str, optional): MSD annotations. Defaults to AnnotationAllSourceFiles(""). - baseform (str, optional): Annotations with first baseform from each set. - Defaults to AnnotationAllSourceFiles(""). - sense (str, optional): Best sense annotations. Defaults to AnnotationAllSourceFiles(""). - lemgram (str, optional): Annotations with first lemgram from each set. - Defaults to AnnotationAllSourceFiles(":saldo.lemgram"). - complemgram (str, optional): Conditional best compound lemgram annotations. - Defaults to AnnotationAllSourceFiles(":saldo.complemgram"). - date (str, optional): date annotation - out (str, optional): The output word frequency file. - Defaults to Export("stats_export.frequency_list_sbx_date/[metadata.id].csv"). - delimiter (str, optional): Column delimiter to use in the csv. Defaults to Config("stats_export.delimiter"). - cutoff (int, optional): The minimum frequency a word must have in order to be included in the result. - Defaults to Config("stats_export.cutoff"). + source_files: The source files belonging to this corpus. + word: Word annotations. + token: Token span annotations. + msd: MSD annotations. + baseform: Annotations with first baseform from each set. + sense: Best sense annotations. + lemgram: Annotations with first lemgram from each set. + complemgram: Conditional best compound lemgram annotations. + date: date annotation + out: The output word frequency file. + delimiter: Column delimiter to use in the csv. + cutoff: The minimum frequency a word must have in order to be included in the result. """ annotations = [(word, "token"), (msd, "POS"), (baseform, "lemma"), (sense, "SALDO sense"), (lemgram, "lemgram"), (complemgram, "compound"), (date, "date")] diff --git a/sparv/modules/stats_export/stats_export.py b/sparv/modules/stats_export/stats_export.py index b80b2e1c..7a144b09 100644 --- a/sparv/modules/stats_export/stats_export.py +++ b/sparv/modules/stats_export/stats_export.py @@ -27,22 +27,17 @@ def freq_list(source_files: AllSourceFilenames = AllSourceFilenames(), """Create a word frequency list for the entire corpus. Args: - source_files (list, optional): The source files belonging to this corpus. Defaults to AllSourceFilenames. - word (str, optional): Word annotations. Defaults to AnnotationAllSourceFiles(""). - token (str, optional): Token span annotations. Defaults to AnnotationAllSourceFiles(""). - annotations (str, optional): All automatic annotations to include in the export. Defaults to - ExportAnnotationsAllSourceFiles("stats_export.annotations"). - source_annotations (str, optional): All source annotations to include in the export. If left empty, none will be - included. Defaults to SourceAnnotations("stats_export.source_annotations"). + source_files: The source files belonging to this corpus. + word: Word annotations. + token: Token span annotations. + annotations: All automatic annotations to include in the export. + source_annotations: All source annotations to include in the export. If left empty, none will be included. remove_namespaces: Whether to remove module "namespaces" from element and attribute names. - Disabled by default. sparv_namespace: The namespace to be added to all Sparv annotations. source_namespace: The namespace to be added to all annotations present in the source. - out (str, optional): The output word frequency file. - Defaults to Export("stats_export.frequency_list/[metadata.id].csv"). - delimiter (str, optional): Column delimiter to use in the csv. Defaults to Config("stats_export.delimiter"). - cutoff (int, optional): The minimum frequency a word must have in order to be included in the result. - Defaults to Config("stats_export.cutoff"). + out: The output word frequency file. + delimiter: Column delimiter to use in the csv. + cutoff: The minimum frequency a word must have in order to be included in the result. """ # Add "word" to annotations annotations = [(word, None)] + annotations From 27e4b811fce6a3fe177b63a07c66552530d8ccd1 Mon Sep 17 00:00:00 2001 From: Anne Schumacher Date: Thu, 10 Nov 2022 18:13:17 +0100 Subject: [PATCH 005/175] add stats export for Swedish from the 1800s --- sparv/modules/stats_export/sbx_stats.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/sparv/modules/stats_export/sbx_stats.py b/sparv/modules/stats_export/sbx_stats.py index f34e5ef6..f7a2ad72 100644 --- a/sparv/modules/stats_export/sbx_stats.py +++ b/sparv/modules/stats_export/sbx_stats.py @@ -168,7 +168,30 @@ def sbx_freq_list_simple( out=out, sparv_namespace="", source_namespace="", delimiter=delimiter, cutoff=cutoff) -@exporter("Corpus word frequency list for Old Swedish (without part-of-speech)", language=["swe-fsv"], order=4) +@exporter("Corpus word frequency list for Swedish from the 1800's", language=["swe-1800"], order=4) +def sbx_freq_list_1800( + source_files: AllSourceFilenames = AllSourceFilenames(), + word: AnnotationAllSourceFiles = AnnotationAllSourceFiles(""), + token: AnnotationAllSourceFiles = AnnotationAllSourceFiles(""), + msd: AnnotationAllSourceFiles = AnnotationAllSourceFiles(""), + baseform: AnnotationAllSourceFiles = AnnotationAllSourceFiles(":stats_export.baseform_first"), + sense: AnnotationAllSourceFiles = AnnotationAllSourceFiles(":hist.sense"), + lemgram: AnnotationAllSourceFiles = AnnotationAllSourceFiles(":stats_export.lemgram_first"), + complemgram: AnnotationAllSourceFiles = AnnotationAllSourceFiles( + ":stats_export.complemgram_best_cond"), + out: Export = Export("stats_export.frequency_list_sbx/stats_[metadata.id].csv"), + delimiter: str = Config("stats_export.delimiter"), + cutoff: int = Config("stats_export.cutoff")): + """Create a word frequency list for the entire corpus.""" + + annotations = [(word, "token"), (msd, "POS"), (baseform, "lemma"), (sense, "SALDO sense"), (lemgram, "lemgram"), + (complemgram, "compound")] + + freq_list(source_files=source_files, word=word, token=token, annotations=annotations, source_annotations=[], + out=out, sparv_namespace="", source_namespace="", delimiter=delimiter, cutoff=cutoff) + + +@exporter("Corpus word frequency list for Old Swedish (without part-of-speech)", language=["swe-fsv"], order=5) def sbx_freq_list_fsv( source_files: AllSourceFilenames = AllSourceFilenames(), token: AnnotationAllSourceFiles = AnnotationAllSourceFiles(""), From 784c536f9eb7c75c97567f125ecb2a1d989d01d2 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Thu, 10 Nov 2022 18:35:43 +0100 Subject: [PATCH 006/175] Fix hidden_annotations bug in korp:config --- sparv/modules/korp/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sparv/modules/korp/config.py b/sparv/modules/korp/config.py index 57e1c81d..89e6c926 100644 --- a/sparv/modules/korp/config.py +++ b/sparv/modules/korp/config.py @@ -240,11 +240,12 @@ def config(id: str = Config("metadata.id"), presets = get_presets(remote_host, config_dir) token_annotations = [] struct_annotations = [] + hidden_annotation_names = [a.name for a in hidden_annotations] for annotation in annotation_list: export_name = export_names.get(annotation.name, annotation.name) # Skip certain annotations unless explicitly listed in annotation_definitions - if (annotation.name in hidden_annotations or annotation.attribute_name is None or export_name.split(":", 1)[ + if (annotation.name in hidden_annotation_names or annotation.attribute_name is None or export_name.split(":", 1)[ -1].startswith("_")) and annotation.name not in annotation_definitions and not ( reading_mode and export_name in READING_MODE_ANNOTATIONS ): From 5ec710b2aedabbed857ce7510e17f4d05c0f1b74 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Mon, 14 Nov 2022 14:17:39 +0100 Subject: [PATCH 007/175] Replace old Python 2 unicode literals --- sparv/modules/korp/relations.py | 2 +- sparv/modules/phrase_structure/phrase_structure.py | 2 +- sparv/modules/saldo/compound.py | 6 +++--- sparv/modules/saldo/saldo_model.py | 3 ++- sparv/modules/segment/segment.py | 2 +- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/sparv/modules/korp/relations.py b/sparv/modules/korp/relations.py index 011a8774..919c1a3d 100644 --- a/sparv/modules/korp/relations.py +++ b/sparv/modules/korp/relations.py @@ -328,7 +328,7 @@ def relations_sql(corpus: Corpus = Corpus(), relations_data = relations.read(file) for triple in relations_data.splitlines(): - head, headpos, rel, dep, deppos, extra, sid, refh, refd, bfhead, bfdep, wfhead, wfdep = triple.split(u"\t") + head, headpos, rel, dep, deppos, extra, sid, refh, refd, bfhead, bfdep, wfhead, wfdep = triple.split("\t") bfhead, bfdep, wfhead, wfdep = int(bfhead), int(bfdep), int(wfhead), int(wfdep) if not (head, headpos) in strings: diff --git a/sparv/modules/phrase_structure/phrase_structure.py b/sparv/modules/phrase_structure/phrase_structure.py index 21f31781..5e8b291c 100644 --- a/sparv/modules/phrase_structure/phrase_structure.py +++ b/sparv/modules/phrase_structure/phrase_structure.py @@ -207,7 +207,7 @@ def words(self): out = "" for n in self.tokens: if n.word: - out = out + n.word + u" " + out = out + n.word + " " return out.strip() def is_cyclic(self): diff --git a/sparv/modules/saldo/compound.py b/sparv/modules/saldo/compound.py index f3ffdffe..c6d9bde9 100644 --- a/sparv/modules/saldo/compound.py +++ b/sparv/modules/saldo/compound.py @@ -401,12 +401,12 @@ def split_word(saldo_lexicon, altlexicon, w, msd): def exception(w): """Filter out unwanted suffixes.""" return w.lower() in [ - "il", u"ör", "en", "ens", "ar", "ars", - "or", "ors", "ur", "urs", u"lös", "tik", "bar", + "il", "ör", "en", "ens", "ar", "ars", + "or", "ors", "ur", "urs", "lös", "tik", "bar", "lik", "het", "hets", "lig", "ligt", "te", "tet", "tets", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", - "y", "z", u"ä"] + "y", "z", "ä"] def three_consonant_rule(compound): diff --git a/sparv/modules/saldo/saldo_model.py b/sparv/modules/saldo/saldo_model.py index 7d3c6ea7..03246cb2 100644 --- a/sparv/modules/saldo/saldo_model.py +++ b/sparv/modules/saldo/saldo_model.py @@ -177,7 +177,8 @@ def read_lmf(xml, tagmap, annotation_elements=("gf", "lem", "saldo"), verbose=Tr x_insert = "1" # Only vbm and certain paradigms allow gaps - gap_allowed = (pos == "vbm" or p in (u"abm_x1_var_än", u"knm_x_ju_ju", u"pnm_x1_inte_ett_dugg", u"pnm_x1_vad_än", u"ppm_x1_för_skull")) + gap_allowed = (pos == "vbm" or p in ( + "abm_x1_var_än", "knm_x_ju_ju", "pnm_x1_inte_ett_dugg", "pnm_x1_vad_än", "ppm_x1_för_skull")) table = elem.find("table") multiwords = [] diff --git a/sparv/modules/segment/segment.py b/sparv/modules/segment/segment.py index f7ce3614..ce07c972 100644 --- a/sparv/modules/segment/segment.py +++ b/sparv/modules/segment/segment.py @@ -205,7 +205,7 @@ def train_punkt_segmenter(textfiles, modelfile, encoding=util.constants.UTF8, pr textfiles = textfiles.split() logger.info("Reading files") - text = u"" + text = "" for filename in textfiles: with open(filename, encoding=encoding) as stream: text += stream.read() From 64ca3dced9a47626a25e7e40ee754ecf4e0f53cc Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Mon, 14 Nov 2022 14:39:40 +0100 Subject: [PATCH 008/175] Replace tabs with spaces in word picture data --- sparv/modules/korp/relations.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/sparv/modules/korp/relations.py b/sparv/modules/korp/relations.py index 919c1a3d..5df7f27d 100644 --- a/sparv/modules/korp/relations.py +++ b/sparv/modules/korp/relations.py @@ -188,10 +188,29 @@ def _findrel(head, rel, dep): triples = sorted(set(triples)) - out_data = "\n".join(["\t".join((head, headpos, rel, dep, deppos, extra, sentid, refhead, refdep, str(bfhead), - str(bfdep), str(wfhead), str(wfdep))) for ( - head, headpos, rel, dep, deppos, extra, sentid, refhead, refdep, bfhead, bfdep, wfhead, wfdep) - in triples]) + out_data = "\n".join( + [ + "\t".join( + ( + head.replace("\t", " "), + headpos, + rel, + dep.replace("\t", " "), + deppos, + extra.replace("\t", " "), + sentid, + refhead, + refdep, + str(bfhead), + str(bfdep), + str(wfhead), + str(wfdep) + ) + ) + for (head, headpos, rel, dep, deppos, extra, sentid, refhead, refdep, bfhead, bfdep, wfhead, wfdep) in + triples + ] + ) out.write(out_data) logger.progress() From a8b1702fd4c9c97d85a640591ea825afda0085c6 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Mon, 14 Nov 2022 14:40:13 +0100 Subject: [PATCH 009/175] Fix spelling --- sparv/modules/korp/relations.py | 2 +- sparv/modules/korp/timespan.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sparv/modules/korp/relations.py b/sparv/modules/korp/relations.py index 5df7f27d..8a0911d1 100644 --- a/sparv/modules/korp/relations.py +++ b/sparv/modules/korp/relations.py @@ -27,7 +27,7 @@ def install_relations(sqlfile: ExportInput = ExportInput("korp.wordpicture/relat Args: sqlfile: SQL file to be installed. out: Marker file to be written. - db_name: Name of the data base. + db_name: Name of the database. host: Remote host to install to. """ util.install.install_mysql(host, db_name, sqlfile) diff --git a/sparv/modules/korp/timespan.py b/sparv/modules/korp/timespan.py index 6c7d9a10..118e0562 100644 --- a/sparv/modules/korp/timespan.py +++ b/sparv/modules/korp/timespan.py @@ -19,7 +19,7 @@ def install_timespan(sqlfile: ExportInput = ExportInput("korp.timespan/timespan. Args: sqlfile: SQL file to be installed. out: Marker file to be written. - db_name: Name of the data base. + db_name: Name of the database. host: Remote host to install to. """ util.install.install_mysql(host, db_name, sqlfile) From e37c035dcf8b96670de1a000dc5c1eb463c50e32 Mon Sep 17 00:00:00 2001 From: Anne Schumacher Date: Mon, 14 Nov 2022 14:58:12 +0100 Subject: [PATCH 010/175] korp config: make description optional and combine short and long description --- sparv/modules/korp/config.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/sparv/modules/korp/config.py b/sparv/modules/korp/config.py index 89e6c926..20fd9ab6 100644 --- a/sparv/modules/korp/config.py +++ b/sparv/modules/korp/config.py @@ -64,7 +64,8 @@ ]) def config(id: str = Config("metadata.id"), name: dict = Config("metadata.name"), - description: dict = Config("metadata.description"), + description: Optional[dict] = Config("metadata.description"), + short_description: Optional[dict] = Config("metadata.short_description"), language: str = Config("metadata.language"), modes: list = Config("korp.modes"), protected: bool = Config("korp.protected"), @@ -101,6 +102,7 @@ def config(id: str = Config("metadata.id"), id: Corpus ID. name: Corpus name. description: Corpus description. + short_description: Short corpus description. language: Corpus language. modes: List of modes and folders where the corpus will be available in Korp. protected: Whether the corpus is password protected. @@ -133,11 +135,11 @@ def config(id: str = Config("metadata.id"), config_dict = { "id": id, "title": name, - "description": description, "lang": language, "mode": modes } optional = { + "description": build_description(description, short_description), "limited_access": protected, "custom_attributes": custom_annotations, "morphology": morphology, @@ -326,6 +328,21 @@ def get_presets(remote_host, config_dir): return presets +def build_description(description, short_description): + """Combine description and short_description if they exist.""" + lang_dict = {} + langs = set(list(description.keys()) + list(short_description.keys())) + for lang in langs: + if short_description.get(lang) and description.get(lang): + descr = f"{short_description.get(lang)}

{description.get(lang)}" + elif description.get(lang): + descr = description.get(lang) + elif short_description.get(lang): + descr = short_description.get(lang) + lang_dict[lang] = descr + return lang_dict + + @installer("Install Korp corpus configuration file.") def install_config( remote_host: Optional[str] = Config("korp.remote_host"), From afab739a38378af94dc78143258897d6816a71b0 Mon Sep 17 00:00:00 2001 From: Anne Schumacher Date: Mon, 14 Nov 2022 16:51:04 +0100 Subject: [PATCH 011/175] fix bug with korp config description --- sparv/modules/korp/config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sparv/modules/korp/config.py b/sparv/modules/korp/config.py index 20fd9ab6..2dce31cb 100644 --- a/sparv/modules/korp/config.py +++ b/sparv/modules/korp/config.py @@ -333,13 +333,15 @@ def build_description(description, short_description): lang_dict = {} langs = set(list(description.keys()) + list(short_description.keys())) for lang in langs: + descr = None if short_description.get(lang) and description.get(lang): descr = f"{short_description.get(lang)}

{description.get(lang)}" elif description.get(lang): descr = description.get(lang) elif short_description.get(lang): descr = short_description.get(lang) - lang_dict[lang] = descr + if descr: + lang_dict[lang] = descr return lang_dict From b799efda9e6316ca2706a33184a2a81a026f8ae0 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Mon, 14 Nov 2022 18:17:50 +0100 Subject: [PATCH 012/175] Add korp.wordpicture_no_sentences setting --- CHANGELOG.md | 10 +++++ sparv/modules/korp/relations.py | 65 +++++++++++++++++---------------- 2 files changed, 43 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7cfc1cd6..1cd3c6b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## [5.2.0.dev0] + +### Added + +- Add `korp.wordpicture_no_sentences` setting to disable generation of Word Picture sentences table. + +### Changed + +### Fixed + ## [5.1.0] - 2022-11-03 ### Added diff --git a/sparv/modules/korp/relations.py b/sparv/modules/korp/relations.py index 8a0911d1..601614fb 100644 --- a/sparv/modules/korp/relations.py +++ b/sparv/modules/korp/relations.py @@ -285,10 +285,14 @@ def mi_lex(rel, x_rel_y, x_rel, rel_y): return x_rel_y * math.log((rel * x_rel_y) / (x_rel * rel_y * 1.0), 2) -@exporter("Word Picture SQL for use in Korp", language=["swe"]) +@exporter("Word Picture SQL for use in Korp", language=["swe"], config=[ + Config("korp.wordpicture_no_sentences", default=False, + description="Set to 'true' to skip generating sentences table.") +]) def relations_sql(corpus: Corpus = Corpus(), out: Export = Export("korp.wordpicture/relations.sql"), relations: AnnotationDataAllSourceFiles = AnnotationDataAllSourceFiles("korp.relations"), + no_sentences: bool = Config("korp.wordpicture_no_sentences"), source_files: Optional[AllSourceFilenames] = AllSourceFilenames(), source_files_list: str = "", split: bool = False): @@ -298,6 +302,7 @@ def relations_sql(corpus: Corpus = Corpus(), corpus: the corpus name out: the name for the SQL file which will contain the resulting SQL statements relations: the name of the relations annotation + no_sentences: set to True to skip generating sentences table source_files: a list of source filenames source_files_list: can be used instead of source_files, and should be a file containing the name of source files, one per row @@ -370,7 +375,7 @@ def relations_sql(corpus: Corpus = Corpus(), freq.setdefault(head, {}).setdefault(rel, {}).setdefault(dep, [this_index, 0, [0, 0, 0, 0]]) freq[head][rel][dep][1] += 1 # Frequency - if sentence_count[this_index] < MAX_SENTENCES: + if not no_sentences and sentence_count[this_index] < MAX_SENTENCES: sentences.setdefault(this_index, set()) sentences[this_index].add((sid, refh, refd)) # Sentence ID and "ref" for both head and dep sentence_count[this_index] += 1 @@ -392,25 +397,29 @@ def relations_sql(corpus: Corpus = Corpus(), if split: # Don't print string table until the last file _write_sql({}, sentences, freq, rel_count, head_rel_count, dep_rel_count, out, db_table, split, - first=(file_count == 1)) + first=(file_count == 1), no_sentences=no_sentences) else: # Only save sentences data, save the rest for the last file - _write_sql({}, sentences, {}, {}, {}, {}, out, db_table, split, first=(file_count == 1)) + _write_sql({}, sentences, {}, {}, {}, {}, out, db_table, split, first=(file_count == 1), + no_sentences=no_sentences) logger.progress() # Create the final file, including the string table _write_sql(strings, sentences, freq, rel_count, head_rel_count, dep_rel_count, out, db_table, split, - first=(file_count == 1), last=True) + first=(file_count == 1), last=True, no_sentences=no_sentences) logger.progress() logger.info("Done creating SQL files") def _write_sql(strings, sentences, freq, rel_count, head_rel_count, dep_rel_count, sql_file, db_table, - split=False, first=False, last=False): + split=False, first=False, last=False, no_sentences=False): temp_db_table = "temp_" + db_table + tables = ["", "_strings", "_rel", "_head_rel", "_dep_rel"] + if not no_sentences: + tables.append("_sentences") update_freq = "ON DUPLICATE KEY UPDATE freq = freq + VALUES(freq)" if split else "" mysql = MySQL(output=sql_file, append=True) @@ -426,9 +435,10 @@ def _write_sql(strings, sentences, freq, rel_count, head_rel_count, dep_rel_coun mysql.create_table(temp_db_table + "_rel", drop=True, **MYSQL_REL) mysql.create_table(temp_db_table + "_head_rel", drop=True, **MYSQL_HEAD_REL) mysql.create_table(temp_db_table + "_dep_rel", drop=True, **MYSQL_DEP_REL) - mysql.create_table(temp_db_table + "_sentences", drop=True, **MYSQL_SENTENCES) - mysql.disable_keys(temp_db_table, temp_db_table + "_strings", temp_db_table + "_rel", - temp_db_table + "_head_rel", temp_db_table + "_dep_rel", temp_db_table + "_sentences") + if not no_sentences: + mysql.create_table(temp_db_table + "_sentences", drop=True, **MYSQL_SENTENCES) + + mysql.disable_keys(*[f"{temp_db_table}{t}" for t in tables]) mysql.disable_checks() mysql.set_names() @@ -502,32 +512,23 @@ def _write_sql(strings, sentences, freq, rel_count, head_rel_count, dep_rel_coun mysql.add_row(temp_db_table + "_dep_rel", rows, update_freq) - for index, sentenceset in sorted(sentences.items()): - for sentence in sorted(sentenceset): - srow = { - "id": index, - "sentence": sentence[0], - "start": int(sentence[1]), - "end": int(sentence[2]) - } - sentence_rows.append(srow) + if not no_sentences: + for index, sentenceset in sorted(sentences.items()): + for sentence in sorted(sentenceset): + srow = { + "id": index, + "sentence": sentence[0], + "start": int(sentence[1]), + "end": int(sentence[2]) + } + sentence_rows.append(srow) - mysql.add_row(temp_db_table + "_sentences", sentence_rows) + mysql.add_row(temp_db_table + "_sentences", sentence_rows) if last: - mysql.enable_keys(temp_db_table, temp_db_table + "_strings", temp_db_table + "_rel", - temp_db_table + "_head_rel", temp_db_table + "_dep_rel", temp_db_table + "_sentences") - mysql.drop_table(db_table, db_table + "_strings", db_table + "_rel", db_table + "_head_rel", - db_table + "_dep_rel", db_table + "_sentences") - mysql.rename_table({ - temp_db_table: db_table, - temp_db_table + "_strings": db_table + "_strings", - temp_db_table + "_rel": db_table + "_rel", - temp_db_table + "_head_rel": db_table + "_head_rel", - temp_db_table + "_dep_rel": db_table + "_dep_rel", - temp_db_table + "_sentences": db_table + "_sentences" - }) - + mysql.enable_keys(*[f"{temp_db_table}{t}" for t in tables]) + mysql.drop_table(*[f"{db_table}{t}" for t in tables]) + mysql.rename_table({f"{temp_db_table}{t}": f"{db_table}{t}" for t in tables}) mysql.enable_checks() logger.info("%s written", sql_file) From d5a14db0730e5d910ab9849cfad6f6fc285caa35 Mon Sep 17 00:00:00 2001 From: Anne Schumacher Date: Tue, 15 Nov 2022 11:39:26 +0100 Subject: [PATCH 013/175] download pre-built SALDO models instead of building them on demand --- sparv/modules/saldo/compound.py | 8 +++++++- sparv/modules/saldo/saldo_model.py | 12 +++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/sparv/modules/saldo/compound.py b/sparv/modules/saldo/compound.py index c6d9bde9..db318e87 100644 --- a/sparv/modules/saldo/compound.py +++ b/sparv/modules/saldo/compound.py @@ -142,7 +142,13 @@ def annotate(out_complemgrams: Output = Output(":saldo.complemgram", logger.progress() -@modelbuilder("SALDO compound model", language=["swe"]) +@modelbuilder("SALDO compound model", language=["swe"], order=1) +def download_saldo_comp(out: ModelOutput = ModelOutput("saldo/saldo.compound.pickle")): + """Download SALDO compound model from sparv-models repo.""" + out.download("https://github.com/spraakbanken/sparv-models/raw/master/saldo/saldo.compound.pickle") + + +@modelbuilder("SALDO compound model", language=["swe"], order=2) def build_saldo_comp(out: ModelOutput = ModelOutput("saldo/saldo.compound.pickle"), saldom: Model = Model("saldo/saldom.xml")): """Extract compound info from saldom.xml and save as a pickle file.""" diff --git a/sparv/modules/saldo/saldo_model.py b/sparv/modules/saldo/saldo_model.py index 03246cb2..c98802f2 100644 --- a/sparv/modules/saldo/saldo_model.py +++ b/sparv/modules/saldo/saldo_model.py @@ -19,13 +19,19 @@ @modelbuilder("SALDO morphology XML", language=["swe"]) -def download_saldo(out: ModelOutput = ModelOutput("saldo/saldom.xml")): +def download_saldo_xml(out: ModelOutput = ModelOutput("saldo/saldom.xml")): """Download SALDO morphology XML.""" out.download("https://svn.spraakdata.gu.se/sb-arkiv/pub/lexikon/saldom/saldom.xml") -@modelbuilder("SALDO morphology model", language=["swe"]) -def build_saldo(out: ModelOutput = ModelOutput("saldo/saldo.pickle"), +@modelbuilder("SALDO morphology model", language=["swe"], order=1) +def download_saldo_pickle(out: ModelOutput = ModelOutput("saldo/nst_comp_pos.pickle")): + """Download SALDO morphology model from sparv-models repo.""" + out.download("https://github.com/spraakbanken/sparv-models/raw/master/saldo/saldo.pickle") + + +@modelbuilder("SALDO morphology model", language=["swe"], order=2) +def build_saldo_pickle(out: ModelOutput = ModelOutput("saldo/saldo.pickle"), saldom: Model = Model("saldo/saldom.xml")): """Save SALDO morphology as a pickle file.""" tagmap = tagmappings.mappings["saldo_to_suc"] From d2f7ddd893ff2fc8d543212cb896a96f40009110 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Wed, 16 Nov 2022 14:12:07 +0100 Subject: [PATCH 014/175] Implement changes needed by Strix-plugin --- sparv/modules/korp/config.py | 39 +++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/sparv/modules/korp/config.py b/sparv/modules/korp/config.py index 2dce31cb..8e7a2c56 100644 --- a/sparv/modules/korp/config.py +++ b/sparv/modules/korp/config.py @@ -134,12 +134,12 @@ def config(id: str = Config("metadata.id"), """ config_dict = { "id": id, - "title": name, "lang": language, "mode": modes } optional = { "description": build_description(description, short_description), + "title": name, "limited_access": protected, "custom_attributes": custom_annotations, "morphology": morphology, @@ -240,8 +240,28 @@ def config(id: str = Config("metadata.id"), # Annotations presets = get_presets(remote_host, config_dir) + token_annotations, struct_annotations, _ = build_annotations(annotation_definitions, annotation_list, export_names, + hidden_annotations, presets, reading_mode, token) + + config_dict["struct_attributes"] = struct_annotations + config_dict["pos_attributes"] = token_annotations + + if filters: + config_dict["attribute_filters"] = [] + for a in filters: + config_dict["attribute_filters"].append(cwb_escape(export_names[a].replace(":", "_"))) + + with open(out, "w", encoding="utf-8") as out_yaml: + out_yaml.write("# This file was automatically generated by Sparv. Do not make changes directly to this file as " + "they will get overwritten.\n") + out_yaml.write(dict_to_yaml(config_dict)) + + +def build_annotations(annotation_definitions, annotation_list, export_names, hidden_annotations, presets, reading_mode, + token, text_annotation=None): token_annotations = [] struct_annotations = [] + text_annotations = [] hidden_annotation_names = [a.name for a in hidden_annotations] for annotation in annotation_list: @@ -255,6 +275,7 @@ def config(id: str = Config("metadata.id"), continue export_name_cwb = cwb_escape(export_name.replace(":", "_")) is_token = annotation.annotation_name == token.name + is_text = text_annotation and annotation.annotation_name == text_annotation definition: Union[str, dict] = annotation_definitions.get(annotation.name, export_name_cwb) if isinstance(definition, str): # Referring to a preset @@ -286,21 +307,11 @@ def config(id: str = Config("metadata.id"), if is_token: token_annotations.append({export_name_cwb: definition}) + elif is_text: + text_annotations.append({export_name_cwb: definition}) else: struct_annotations.append({export_name_cwb: definition}) - - config_dict["struct_attributes"] = struct_annotations - config_dict["pos_attributes"] = token_annotations - - if filters: - config_dict["attribute_filters"] = [] - for a in filters: - config_dict["attribute_filters"].append(cwb_escape(export_names[a].replace(":", "_"))) - - with open(out, "w", encoding="utf-8") as out_yaml: - out_yaml.write("# This file was automatically generated by Sparv. Do not make changes directly to this file as " - "they will get overwritten.\n") - out_yaml.write(dict_to_yaml(config_dict)) + return token_annotations, struct_annotations, text_annotations def get_presets(remote_host, config_dir): From 599ec4efae9f04f2b75bcaeed60e91c78c72060c Mon Sep 17 00:00:00 2001 From: Anne Schumacher Date: Wed, 16 Nov 2022 16:37:20 +0100 Subject: [PATCH 015/175] bug fix (download_saldo_pickle downloaded to the wrong file name) --- sparv/modules/saldo/saldo_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sparv/modules/saldo/saldo_model.py b/sparv/modules/saldo/saldo_model.py index c98802f2..b98d62e3 100644 --- a/sparv/modules/saldo/saldo_model.py +++ b/sparv/modules/saldo/saldo_model.py @@ -25,7 +25,7 @@ def download_saldo_xml(out: ModelOutput = ModelOutput("saldo/saldom.xml")): @modelbuilder("SALDO morphology model", language=["swe"], order=1) -def download_saldo_pickle(out: ModelOutput = ModelOutput("saldo/nst_comp_pos.pickle")): +def download_saldo_pickle(out: ModelOutput = ModelOutput("saldo/saldo.pickle")): """Download SALDO morphology model from sparv-models repo.""" out.download("https://github.com/spraakbanken/sparv-models/raw/master/saldo/saldo.pickle") From bfcda6a3b3fb38f55210d650518f0ae131709331 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 18 Nov 2022 13:55:31 +0100 Subject: [PATCH 016/175] Fix bug in korp:config --- sparv/modules/korp/config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sparv/modules/korp/config.py b/sparv/modules/korp/config.py index 8e7a2c56..99542db3 100644 --- a/sparv/modules/korp/config.py +++ b/sparv/modules/korp/config.py @@ -342,6 +342,8 @@ def get_presets(remote_host, config_dir): def build_description(description, short_description): """Combine description and short_description if they exist.""" lang_dict = {} + description = description or {} + short_description = short_description or {} langs = set(list(description.keys()) + list(short_description.keys())) for lang in langs: descr = None From 8430ce63b3a77aede448e93f7bfaa1d955cd43c3 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 18 Nov 2022 14:13:19 +0100 Subject: [PATCH 017/175] Update documentation --- docs/developers-guide/utilities.md | 5 +++-- sparv/api/util/install.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/developers-guide/utilities.md b/docs/developers-guide/utilities.md index 528e77b7..6096ed8a 100644 --- a/docs/developers-guide/utilities.md +++ b/docs/developers-guide/utilities.md @@ -94,11 +94,12 @@ remotely. ### install_path() -Transfer a file or directory to a target destination, optionally on a different host. +Transfer a file or the contents of a directory to a target destination, optionally on a different host. **Arguments:** -- `source_path`: Path to the local file or directory to sync. +- `source_path`: Path to the local file or directory to sync. If a directory is used, its contents are synced, not the + directory itself. - `host` (optional): The remote host to install to. - `target_path`: The name of the target file or directory. diff --git a/sparv/api/util/install.py b/sparv/api/util/install.py index b6e9a519..dcbbbdd8 100644 --- a/sparv/api/util/install.py +++ b/sparv/api/util/install.py @@ -14,7 +14,7 @@ def install_path(source_path: Union[str, Path], host: Optional[str] = None, target_path: Optional[Union[str, Path]] = None) -> None: - """Transfer a file or directory to a target destination, optionally on a different host.""" + """Transfer a file or the contents of a directory to a target destination, optionally on a different host.""" system.rsync(source_path, host, target_path) From dca9f0c081a0c8c058d9553f744d79700d3cd1f2 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Tue, 22 Nov 2022 15:56:01 +0100 Subject: [PATCH 018/175] Make replacing colon with underscore optional in korp:config --- sparv/modules/korp/config.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sparv/modules/korp/config.py b/sparv/modules/korp/config.py index 99542db3..1df7c228 100644 --- a/sparv/modules/korp/config.py +++ b/sparv/modules/korp/config.py @@ -258,7 +258,7 @@ def config(id: str = Config("metadata.id"), def build_annotations(annotation_definitions, annotation_list, export_names, hidden_annotations, presets, reading_mode, - token, text_annotation=None): + token, text_annotation=None, cwb_annotations: bool = True): token_annotations = [] struct_annotations = [] text_annotations = [] @@ -306,11 +306,11 @@ def build_annotations(annotation_definitions, annotation_list, export_names, hid definition.pop("use_as_positional", None) if is_token: - token_annotations.append({export_name_cwb: definition}) + token_annotations.append({export_name_cwb if cwb_annotations else export_name: definition}) elif is_text: - text_annotations.append({export_name_cwb: definition}) + text_annotations.append({export_name_cwb if cwb_annotations else export_name: definition}) else: - struct_annotations.append({export_name_cwb: definition}) + struct_annotations.append({export_name_cwb if cwb_annotations else export_name: definition}) return token_annotations, struct_annotations, text_annotations From ed5e6ec679c61d77341dcba954b6b152700b3064 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Thu, 24 Nov 2022 17:08:46 +0100 Subject: [PATCH 019/175] Update plugin installation documentation --- docs/user-manual/installation-and-setup.md | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/user-manual/installation-and-setup.md b/docs/user-manual/installation-and-setup.md index 79ff94b4..a57bad5c 100644 --- a/docs/user-manual/installation-and-setup.md +++ b/docs/user-manual/installation-and-setup.md @@ -241,25 +241,33 @@ Please follow the installation instructions given in the fast_align repository a ## Plugins -If you have the Sparv Pipeline installed on your machine, you can install plugins by injecting them into the Sparv -Pipeline code using pipx: +Plugins for Sparv can be installed by one of the following commands, depending on whether you installed Sparv using pipx +or pip: ``` pipx inject sparv-pipeline [pointer-to-sparv-plugin] ``` +``` +pip install [pointer-to-sparv-plugin] +``` + The `pointer-to-sparv-plugin` can be a package available on the [Python Package Index (PyPI)](https://pypi.org/), a remote public repository, or a local directory on your machine. For now there are two plugins available for Sparv: [sparv-sbx-freeling](https://github.com/spraakbanken/sparv-sbx-freeling) and -[sparv-sbx-metadata](https://github.com/spraakbanken/sparv-sbx-metadata). Please refer to their GitHub page for more +[sparv-sbx-metadata](https://github.com/spraakbanken/sparv-sbx-metadata). Please refer to their GitHub pages for more information. -Plugins can be uninstalled by running: +Plugins can be uninstalled using one of the following commands: ``` pipx runpip sparv-pipeline uninstall [name-of-sparv-plugin] ``` +``` +pip uninstall [name-of-sparv-plugin] +``` + ## Uninstalling Sparv To uninstall Sparv completely, follow these steps: From b644f31e868e03f4d7626f81daec2ed9f78bf0e6 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 25 Nov 2022 15:28:59 +0100 Subject: [PATCH 020/175] Add all methods from Annotation to AnnotationAllSourceFiles --- docs/developers-guide/sparv-classes.md | 27 +++- sparv/api/classes.py | 169 ++++++++++++++++++------- 2 files changed, 140 insertions(+), 56 deletions(-) diff --git a/docs/developers-guide/sparv-classes.md b/docs/developers-guide/sparv-classes.md index a449af13..90472b69 100644 --- a/docs/developers-guide/sparv-classes.md +++ b/docs/developers-guide/sparv-classes.md @@ -33,6 +33,9 @@ annotation is needed as input for a function, e.g. `Annotation("")`. - `exists()`: Return True if annotation file exists. - `remove()`: Remove annotation file. - `read(allow_newlines: bool = False)`: Yield each line from the annotation. +- `read_spans(decimals=False, with_annotation_name=False)`: Yield the spans of the annotation. +- `read_attributes(annotations: Union[List[BaseAnnotation], Tuple[BaseAnnotation, ...]], with_annotation_name: bool = + False, allow_newlines: bool = False)`: Yield tuples of multiple attributes on the same annotation. - `get_children(child: BaseAnnotation, orphan_alert=False, preserve_parent_annotation_order=False)`: Return two lists. The first one is a list with n (= total number of parents) elements where every element is a list of indices in the child annotation. The second one is a list of orphans, i.e. containing indices in the child annotation that have no @@ -43,9 +46,6 @@ annotation is needed as input for a function, e.g. `Annotation("")`. elements where every element is an index in the parent annotation. Return None when no parent is found. - `read_parents_and_children(parent, child)`: Read parent and child annotations. Reorder them according to span position, but keep original index information. -- `read_attributes(annotations: Union[List[BaseAnnotation], Tuple[BaseAnnotation, ...]], with_annotation_name: bool = - False, allow_newlines: bool = False)`: Yield tuples of multiple attributes on the same annotation. -- `read_spans(decimals=False, with_annotation_name=False)`: Yield the spans of the annotation. - `create_empty_attribute()`: Return a list filled with None of the same size as this annotation. - `get_size()`: Get the number of values. @@ -60,17 +60,30 @@ require the specificed annotation for every source file in the corpus. **Properties:** +- `has_attribute`: Return True if the annotation has an attribute. - `annotation_name`: Get annotation name (excluding name of any attribute). -- `attribute_name`: Get attribute name (excluding name of annotation) +- `attribute_name`: Get attribute name (excluding name of annotation). **Methods:** - `split()`: Split name into annotation name and attribute. -- `read(source_file: str)`: Yield each line from the annotation. -- `read_spans(source_file: str, decimals=False, with_annotation_name=False)`: Yield the spans of the annotation. -- `create_empty_attribute(source_file: str)`: Return a list filled with None of the same size as this annotation. - `exists(source_file: str)`: Return True if annotation file exists. - `remove(source_file: str)`: Remove annotation file. +- `read(source_file: str, allow_newlines: bool = False)`: Yield each line from the annotation. +- `read_spans(source_file: str, decimals=False, with_annotation_name=False)`: Yield the spans of the annotation. +- `read_attributes(source_file: str, annotations: Union[List[BaseAnnotation], Tuple[BaseAnnotation, ...]], with_annotation_name: bool = + False, allow_newlines: bool = False)`: Yield tuples of multiple attributes on the same annotation. +- `get_children(source_file: str, child: BaseAnnotation, orphan_alert=False, preserve_parent_annotation_order=False)`: Return two lists. + The first one is a list with n (= total number of parents) elements where every element is a list of indices in the + child annotation. The second one is a list of orphans, i.e. containing indices in the child annotation that have no + parent. Both parents and children are sorted according to their position in the source file, unless + preserve_parent_annotation_order is set to True, in which case the parents keep the order from the parent + annotation. +- `get_parents(source_file: str, parent: BaseAnnotation, orphan_alert: bool = False)`: Return a list with n (= total number of children) + elements where every element is an index in the parent annotation. Return None when no parent is found. +- `read_parents_and_children(source_file: str, parent, child)`: Read parent and child annotations. Reorder them according to span + position, but keep original index information. +- `create_empty_attribute(source_file: str)`: Return a list filled with None of the same size as this annotation. - `get_size(source_file: str)`: Get the number of values. diff --git a/sparv/api/classes.py b/sparv/api/classes.py index e61f4324..710edbdc 100644 --- a/sparv/api/classes.py +++ b/sparv/api/classes.py @@ -126,25 +126,56 @@ def remove(self, source_file: str): io.remove_annotation(self, source_file) -class Annotation(CommonMixin, BaseAnnotation): - """Regular Annotation tied to one source file.""" +class CommonAnnotationMixin(BaseAnnotation): + """Methods common to Annotation and AnnotationAllSourceFiles.""" - def __init__(self, name: str = "", source_file: Optional[str] = None, is_input: bool = True): - super().__init__(name, source_file=source_file) - self._size = None - self.is_input = is_input + def __init__(self, name: str = "", source_file: Optional[str] = None): + super().__init__(name, source_file) + self._size = {} - def get_size(self): + def _read(self, source_file: str, allow_newlines: bool = False): + """Yield each line from the annotation.""" + return io.read_annotation(source_file, self, allow_newlines=allow_newlines) + + def _read_spans(self, source_file: str, decimals=False, with_annotation_name=False): + """Yield the spans of the annotation.""" + return io.read_annotation_spans(source_file, self, decimals=decimals, with_annotation_name=with_annotation_name) + + @staticmethod + def _read_attributes(source_file: str, annotations: Union[List[BaseAnnotation], Tuple[BaseAnnotation, ...]], + with_annotation_name: bool = False, allow_newlines: bool = False): + """Yield tuples of multiple attributes on the same annotation.""" + return io.read_annotation_attributes(source_file, annotations, with_annotation_name=with_annotation_name, + allow_newlines=allow_newlines) + + def _get_size(self, source_file: str): """Get number of values.""" - if self._size is None: - self._size = io.get_annotation_size(self.source_file, self) - return self._size + if self._size.get(source_file) is None: + self._size[source_file] = io.get_annotation_size(source_file, self) + return self._size[source_file] - def read(self, allow_newlines: bool = False): - """Yield each line from the annotation.""" - return io.read_annotation(self.source_file, self, allow_newlines=allow_newlines) + def _create_empty_attribute(self, source_file: str): + """Return a list filled with None of the same size as this annotation.""" + return [None] * self._get_size(source_file) - def get_children(self, child: BaseAnnotation, orphan_alert=False, preserve_parent_annotation_order=False): + def _read_parents_and_children(self, source_file: str, parent: BaseAnnotation, child: BaseAnnotation): + """Read parent and child annotations. + + Reorder them according to span position, but keep original index information. + """ + parent_spans = sorted(enumerate(io.read_annotation_spans(source_file, parent, decimals=True)), key=lambda x: x[1]) + child_spans = sorted(enumerate(io.read_annotation_spans(source_file, child, decimals=True)), key=lambda x: x[1]) + + # Only use sub-positions if both parent and child have them + if parent_spans and child_spans: + if len(parent_spans[0][1][0]) == 1 or len(child_spans[0][1][0]) == 1: + parent_spans = [(p[0], (p[1][0][0], p[1][1][0])) for p in parent_spans] + child_spans = [(c[0], (c[1][0][0], c[1][1][0])) for c in child_spans] + + return iter(parent_spans), iter(child_spans) + + def _get_children(self, source_file: str, child: BaseAnnotation, orphan_alert=False, + preserve_parent_annotation_order=False): """Return two lists. The first one is a list with n (= total number of parents) elements where every element is a list @@ -154,7 +185,7 @@ def get_children(self, child: BaseAnnotation, orphan_alert=False, preserve_paren preserve_parent_annotation_order is set to True, in which case the parents keep the order from the parent annotation. """ - parent_spans, child_spans = self.read_parents_and_children(self, child) + parent_spans, child_spans = self._read_parents_and_children(source_file, self, child) parent_children = [] orphans = [] previous_parent_i = None @@ -196,12 +227,12 @@ def get_children(self, child: BaseAnnotation, orphan_alert=False, preserve_paren return parent_children, orphans - def get_parents(self, parent: BaseAnnotation, orphan_alert: bool = False): + def _get_parents(self, source_file: str, parent: BaseAnnotation, orphan_alert: bool = False): """Return a list with n (= total number of children) elements where every element is an index in the parent annotation. Return None when no parent is found. """ - parent_spans, child_spans = self.read_parents_and_children(parent, self) + parent_spans, child_spans = self._read_parents_and_children(source_file, parent, self) child_parents = [] previous_parent_i = None try: @@ -231,36 +262,60 @@ def get_parents(self, parent: BaseAnnotation, orphan_alert: bool = False): return child_parents - def read_parents_and_children(self, parent: BaseAnnotation, child: BaseAnnotation): - """Read parent and child annotations. - Reorder them according to span position, but keep original index information. - """ - parent_spans = sorted(enumerate(io.read_annotation_spans(self.source_file, parent, decimals=True)), key=lambda x: x[1]) - child_spans = sorted(enumerate(io.read_annotation_spans(self.source_file, child, decimals=True)), key=lambda x: x[1]) +class Annotation(CommonAnnotationMixin, CommonMixin, BaseAnnotation): + """Regular Annotation tied to one source file.""" - # Only use sub-positions if both parent and child have them - if parent_spans and child_spans: - if len(parent_spans[0][1][0]) == 1 or len(child_spans[0][1][0]) == 1: - parent_spans = [(p[0], (p[1][0][0], p[1][1][0])) for p in parent_spans] - child_spans = [(c[0], (c[1][0][0], c[1][1][0])) for c in child_spans] + def __init__(self, name: str = "", source_file: Optional[str] = None, is_input: bool = True): + super().__init__(name, source_file=source_file) + self.is_input = is_input - return iter(parent_spans), iter(child_spans) + def read(self, allow_newlines: bool = False): + """Yield each line from the annotation.""" + return self._read(self.source_file, allow_newlines=allow_newlines) + + def read_spans(self, decimals=False, with_annotation_name=False): + """Yield the spans of the annotation.""" + return self._read_spans(self.source_file, decimals=decimals, with_annotation_name=with_annotation_name) def read_attributes(self, annotations: Union[List[BaseAnnotation], Tuple[BaseAnnotation, ...]], with_annotation_name: bool = False, allow_newlines: bool = False): """Yield tuples of multiple attributes on the same annotation.""" - return io.read_annotation_attributes(self.source_file, annotations, with_annotation_name=with_annotation_name, - allow_newlines=allow_newlines) + return self._read_attributes(self.source_file, annotations, with_annotation_name, allow_newlines) - def read_spans(self, decimals=False, with_annotation_name=False): - """Yield the spans of the annotation.""" - return io.read_annotation_spans(self.source_file, self, decimals=decimals, - with_annotation_name=with_annotation_name) + def get_children(self, child: BaseAnnotation, orphan_alert=False, preserve_parent_annotation_order=False): + """Return two lists. + + The first one is a list with n (= total number of parents) elements where every element is a list + of indices in the child annotation. + The second one is a list of orphans, i.e. containing indices in the child annotation that have no parent. + Both parents and children are sorted according to their position in the source file, unless + preserve_parent_annotation_order is set to True, in which case the parents keep the order from the parent + annotation. + """ + return self._get_children(self.source_file, child, orphan_alert, preserve_parent_annotation_order) + + def get_parents(self, parent: BaseAnnotation, orphan_alert: bool = False): + """Return a list with n (= total number of children) elements where every element is an index in the parent annotation. + + Return None when no parent is found. + """ + return self._get_parents(self.source_file, parent, orphan_alert) + + def read_parents_and_children(self, parent: BaseAnnotation, child: BaseAnnotation): + """Read parent and child annotations. + + Reorder them according to span position, but keep original index information. + """ + return self._read_parents_and_children(self.source_file, parent, child) def create_empty_attribute(self): """Return a list filled with None of the same size as this annotation.""" - return [None] * self.get_size() + return self._create_empty_attribute(self.source_file) + + def get_size(self): + """Get number of values.""" + return self._get_size(self.source_file) class AnnotationName(BaseAnnotation): @@ -286,7 +341,7 @@ def read(self, source_file: Optional[str] = None): return io.read_data(self.source_file or source_file, self) -class AnnotationAllSourceFiles(CommonAllSourceFilesMixin, BaseAnnotation): +class AnnotationAllSourceFiles(CommonAnnotationMixin, CommonAllSourceFilesMixin, BaseAnnotation): """Regular annotation but source file must be specified for all actions. Use as input to an annotator to require the specificed annotation for every source file in the corpus. @@ -300,28 +355,44 @@ def __init__(self, name: str = ""): def read(self, source_file: str): """Yield each line from the annotation.""" - return io.read_annotation(source_file, self) + return self._read(source_file) def read_spans(self, source_file: str, decimals=False, with_annotation_name=False): """Yield the spans of the annotation.""" - return io.read_annotation_spans(source_file, self, decimals=decimals, with_annotation_name=with_annotation_name) + return self._read_spans(source_file, decimals=decimals, with_annotation_name=with_annotation_name) - @staticmethod - def read_attributes(source_file: str, annotations: Union[List[BaseAnnotation], Tuple[BaseAnnotation, ...]], + def read_attributes(self, source_file: str, annotations: Union[List[BaseAnnotation], Tuple[BaseAnnotation, ...]], with_annotation_name: bool = False, allow_newlines: bool = False): """Yield tuples of multiple attributes on the same annotation.""" - return io.read_annotation_attributes(source_file, annotations, with_annotation_name=with_annotation_name, - allow_newlines=allow_newlines) + return self._read_attributes(source_file, annotations, with_annotation_name=with_annotation_name, + allow_newlines=allow_newlines) - def get_size(self, source_file: str): - """Get number of values.""" - if self._size.get(source_file) is None: - self._size[source_file] = io.get_annotation_size(source_file, self) - return self._size[source_file] + def get_children(self, source_file: str, child: BaseAnnotation, orphan_alert=False, preserve_parent_annotation_order=False): + """Return two lists. + + The first one is a list with n (= total number of parents) elements where every element is a list + of indices in the child annotation. + The second one is a list of orphans, i.e. containing indices in the child annotation that have no parent. + Both parents and children are sorted according to their position in the source file, unless + preserve_parent_annotation_order is set to True, in which case the parents keep the order from the parent + annotation. + """ + return self._get_children(source_file, child, orphan_alert, preserve_parent_annotation_order) + + def get_parents(self, source_file: str, parent: BaseAnnotation, orphan_alert: bool = False): + """Return a list with n (= total number of children) elements where every element is an index in the parent annotation. + + Return None when no parent is found. + """ + return self._get_parents(source_file, parent, orphan_alert) def create_empty_attribute(self, source_file: str): """Return a list filled with None of the same size as this annotation.""" - return [None] * self.get_size(source_file) + return self._create_empty_attribute(source_file) + + def get_size(self, source_file: str): + """Get number of values.""" + return self._get_size(source_file) class AnnotationDataAllSourceFiles(CommonAllSourceFilesMixin, BaseAnnotation): From adcd659fb43f614bee2a3a73facbeca14c8b6967 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 25 Nov 2022 15:35:20 +0100 Subject: [PATCH 021/175] Add short arguments to 'clean' command --- sparv/__main__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sparv/__main__.py b/sparv/__main__.py index 218beaaa..819f1a4f 100644 --- a/sparv/__main__.py +++ b/sparv/__main__.py @@ -110,9 +110,9 @@ def main(): clean_parser = subparsers.add_parser("clean", description="Remove output directories (by default only the " "sparv-workdir directory).") - clean_parser.add_argument("--export", action="store_true", help="Remove export directory") - clean_parser.add_argument("--logs", action="store_true", help="Remove logs directory") - clean_parser.add_argument("--all", action="store_true", help="Remove workdir, export and logs directories") + clean_parser.add_argument("-e", "--export", action="store_true", help="Remove export directory") + clean_parser.add_argument("-l", "--logs", action="store_true", help="Remove logs directory") + clean_parser.add_argument("-a", "--all", action="store_true", help="Remove workdir, export and logs directories") # Inspect config_parser = subparsers.add_parser("config", description="Display the corpus configuration.") From 113a1bcbcf3bc708188e69533cb22022bc470705 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 25 Nov 2022 17:25:33 +0100 Subject: [PATCH 022/175] Add MarkerOptional class --- docs/developers-guide/sparv-classes.md | 5 +++++ docs/developers-guide/sparv-decorators.md | 6 +++-- sparv/api/__init__.py | 1 + sparv/api/classes.py | 27 ++++++++++++++--------- 4 files changed, 26 insertions(+), 13 deletions(-) diff --git a/docs/developers-guide/sparv-classes.md b/docs/developers-guide/sparv-classes.md index 90472b69..f4cf16f2 100644 --- a/docs/developers-guide/sparv-classes.md +++ b/docs/developers-guide/sparv-classes.md @@ -281,6 +281,11 @@ been run. Created by using `OutputMarker`. - `remove()`: Remove marker file. +## MarkerOptional +Same as `Marker`, but if the marker file doesn't exist, it won't be created. This is mainly used to remove markers +from connected (un)installers without triggering the connected (un)installation. + + ## Model An instance of this class holds a path to a model file relative to the Sparv model directory. This class is typically used as input to annotator functions. diff --git a/docs/developers-guide/sparv-decorators.md b/docs/developers-guide/sparv-decorators.md index 6272c812..665ce99a 100644 --- a/docs/developers-guide/sparv-decorators.md +++ b/docs/developers-guide/sparv-decorators.md @@ -120,7 +120,8 @@ the XML output could be copied to a web server, or SQL data could be inserted in Every installer needs to create a marker of the type `OutputMarker` at the end of a successful installation. Simply call the `write()` method on the marker to create the required empty file. -It is recommended that an installer removes any related uninstaller's marker, to enable uninstallation. +It is recommended that an installer removes any related uninstaller's marker, to enable uninstallation. Use the +`MarkerOptional` class to refer to the uninstaller's marker without triggering an unnecessary installation. **Arguments:** @@ -151,7 +152,8 @@ remote location or delete corpus data from a database. Every uninstaller needs to create a marker of the type `OutputMarker` at the end of a successful uninstallation. Simply call the `write()` method on the marker to create the required empty file. -It is recommended that an uninstaller removes any related installer's marker, to enable re-installation. +It is recommended that an uninstaller removes any related installer's marker, to enable re-installation. Use the +`MarkerOptional` class to refer to the installer's marker without triggering an unnecessary installation. **Arguments:** diff --git a/sparv/api/__init__.py b/sparv/api/__init__.py index 9026fb8b..646f2183 100644 --- a/sparv/api/__init__.py +++ b/sparv/api/__init__.py @@ -24,6 +24,7 @@ Headers, Language, Marker, + MarkerOptional, Model, ModelOutput, Namespaces, diff --git a/sparv/api/classes.py b/sparv/api/classes.py index 710edbdc..35f62a26 100644 --- a/sparv/api/classes.py +++ b/sparv/api/classes.py @@ -63,10 +63,11 @@ class BaseAnnotation(Base): common = False is_input = True - def __init__(self, name: str = "", source_file: Optional[str] = None, is_input: bool = True): + def __init__(self, name: str = "", source_file: Optional[str] = None, is_input: Optional[bool] = None): super().__init__(name) self.source_file = source_file - self.is_input = is_input + if is_input is not None: + self.is_input = is_input def expand_variables(self, rule_name: str = "") -> List[str]: """Update name by replacing references with annotation names, and [config] references with config values. @@ -266,10 +267,6 @@ def _get_parents(self, source_file: str, parent: BaseAnnotation, orphan_alert: b class Annotation(CommonAnnotationMixin, CommonMixin, BaseAnnotation): """Regular Annotation tied to one source file.""" - def __init__(self, name: str = "", source_file: Optional[str] = None, is_input: bool = True): - super().__init__(name, source_file=source_file) - self.is_input = is_input - def read(self, allow_newlines: bool = False): """Yield each line from the annotation.""" return self._read(self.source_file, allow_newlines=allow_newlines) @@ -324,8 +321,7 @@ class AnnotationName(BaseAnnotation): To be used when only the name is of interest and not the actual annotation file. """ - def __init__(self, name: str = "", source_file: Optional[str] = None): - super().__init__(name, source_file=source_file, is_input=False) + is_input = False class AnnotationData(CommonMixin, BaseAnnotation): @@ -429,6 +425,12 @@ class Marker(AnnotationCommonData): pass +class MarkerOptional(Marker): + """Same as regular Marker, except if it doesn't exist, it won't be created.""" + + is_input = False + + class BaseOutput(BaseAnnotation): """Base class for all Output classes.""" @@ -789,10 +791,11 @@ class ExportAnnotations(List[Tuple[Annotation, Optional[str]]]): # If is_input = False the annotations won't be added to the rule's input. is_input = True - def __init__(self, config_name: str, items=(), is_input: bool = True): + def __init__(self, config_name: str, items=(), is_input: Optional[bool] = None): list.__init__(self, items) self.config_name = config_name - self.is_input = is_input + if is_input is not None: + self.is_input = is_input class ExportAnnotationNames(ExportAnnotations): @@ -801,8 +804,10 @@ class ExportAnnotationNames(ExportAnnotations): To be used when only the annotation names are of interest and not the actual annotation files. """ + is_input = False + def __init__(self, config_name: str, items=()): - super().__init__(config_name, items=items, is_input=False) + super().__init__(config_name, items=items) class ExportAnnotationsAllSourceFiles(List[Tuple[AnnotationAllSourceFiles, Optional[str]]]): From 6e15c75d5251d4efe1ac43d776c6f730cc309290 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 25 Nov 2022 17:36:55 +0100 Subject: [PATCH 023/175] Add korp:uninstall_config --- sparv/modules/korp/config.py | 41 ++++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/sparv/modules/korp/config.py b/sparv/modules/korp/config.py index 1df7c228..cb8d2dd4 100644 --- a/sparv/modules/korp/config.py +++ b/sparv/modules/korp/config.py @@ -6,8 +6,22 @@ import yaml -from sparv.api import (AllSourceFilenames, AnnotationName, Config, Export, ExportAnnotationNames, ExportInput, - OutputMarker, SourceAnnotationsAllSourceFiles, exporter, get_logger, installer, util) +from sparv.api import ( + AllSourceFilenames, + AnnotationName, + Config, + Export, + ExportAnnotationNames, + ExportInput, + MarkerOptional, + OutputMarker, + SourceAnnotationsAllSourceFiles, + exporter, + get_logger, + installer, + uninstaller, + util +) from sparv.core.io import split_annotation from sparv.modules.cwb.cwb import cwb_escape @@ -358,17 +372,36 @@ def build_description(description, short_description): return lang_dict -@installer("Install Korp corpus configuration file.") +@installer("Install Korp corpus configuration file.", uninstaller="korp:uninstall_config") def install_config( remote_host: Optional[str] = Config("korp.remote_host"), config_dir: str = Config("korp.config_dir"), config_file: ExportInput = ExportInput("korp.config/[metadata.id].yaml"), - marker: OutputMarker = OutputMarker("korp.install_config_marker") + marker: OutputMarker = OutputMarker("korp.install_config_marker"), + uninstall_marker: MarkerOptional = MarkerOptional("korp.uninstall_config_marker") ): """Install Korp corpus configuration file.""" corpus_dir = Path(config_dir) / "corpora" logger.info(f"Installing Korp corpus configuration file to {remote_host}:{corpus_dir}") util.install.install_path(config_file, remote_host, corpus_dir) + uninstall_marker.remove() + marker.write() + + +@uninstaller("Uninstall Korp corpus configuration file.") +def uninstall_config( + remote_host: Optional[str] = Config("korp.remote_host"), + config_dir: str = Config("korp.config_dir"), + corpus_id: str = Config("metadata.id"), + marker: OutputMarker = OutputMarker("korp.uninstall_config_marker"), + install_marker: MarkerOptional = MarkerOptional("korp.install_config_marker") +): + """Uninstall Korp corpus configuration file.""" + corpus_file = Path(config_dir) / "corpora" / f"{corpus_id}.yaml" + logger.info(f"Uninstalling Korp corpus configuration file from {remote_host + ':' if remote_host else ''}" + f"{corpus_file}") + util.install.uninstall_path(corpus_file, host=remote_host) + install_marker.remove() marker.write() From fe2f820508a2b5986d5a09b6ab3380965fab8e16 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Mon, 28 Nov 2022 14:44:03 +0100 Subject: [PATCH 024/175] Update install_mysql to allow local installations as well --- CHANGELOG.md | 2 ++ docs/developers-guide/utilities.md | 8 +++--- sparv/api/util/install.py | 41 +++++++++++++++++++----------- 3 files changed, 32 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1cd3c6b3..e595d329 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ ### Changed +- The util function `install_mysql` can now install locally as well as to a remote server. + ### Fixed ## [5.1.0] - 2022-11-03 diff --git a/docs/developers-guide/utilities.md b/docs/developers-guide/utilities.md index 6096ed8a..d63454b1 100644 --- a/docs/developers-guide/utilities.md +++ b/docs/developers-guide/utilities.md @@ -114,13 +114,13 @@ Remove a file or directory, optionally on a different host. ### install_mysql() -Insert tables and data from local SQL-file to remote MySQL database. +Insert tables and data from one or more SQL-files to local or remote MySQL database. **Arguments:** -- `host`: The remote host to install to. -- `db_name`: Name of the remote database. -- `sqlfile`: Path to a local SQL file, or multiple paths separated by whitespaces. +- `host`: The remote host to install to. Set to None to install locally. +- `db_name`: Name of the database. +- `sqlfile`: Path to a SQL file, or list of paths. ### install_mysql_dump() diff --git a/sparv/api/util/install.py b/sparv/api/util/install.py index dcbbbdd8..5319be02 100644 --- a/sparv/api/util/install.py +++ b/sparv/api/util/install.py @@ -1,9 +1,10 @@ """Util functions for installations on remote servers.""" import os +import shlex import subprocess from pathlib import Path -from typing import Optional, Union +from typing import Optional, Union, List from sparv.api import get_logger from sparv.api.util import system @@ -23,27 +24,37 @@ def uninstall_path(path: Union[str, Path] = None, host: Optional[str] = None) -> system.remove_path(path, host) -def install_mysql(host, db_name, sqlfile): - """Insert tables and data from local SQL-file to remote MySQL database. +def install_mysql(host: Optional[str], db_name: str, sqlfile: Union[str, List[str]]): + """Insert tables and data from SQL-file(s) to local or remote MySQL database. - sqlfile may be a whitespace separated list of SQL files. + Args: + host: The remote host to install to. Set to None to install locally. + db_name: Name of the database. + sqlfile: Path to a SQL file, or list of paths. """ - if not host: - raise Exception("No host provided! Installations aborted.") + local = host is None or host == "localhost" - sqlfiles = sqlfile.split() + if isinstance(sqlfile, str): + sqlfile = [sqlfile] file_count = 0 - file_total = len(sqlfiles) + file_total = len(sqlfile) - for sqlf in sqlfiles: + for f in sqlfile: file_count += 1 - if not os.path.exists(sqlf): - logger.error("Missing SQL file: %s", sqlf) - elif os.path.getsize(sqlf) < 10: - logger.info("Skipping empty file: %s (%d/%d)", sqlf, file_count, file_total) + if not os.path.exists(f): + logger.error("Missing SQL file: %s", f) + elif os.path.getsize(f) < 10: + logger.info("Skipping empty file: %s (%d/%d)", f, file_count, file_total) else: - logger.info("Installing MySQL database: %s, source: %s (%d/%d)", db_name, sqlf, file_count, file_total) - subprocess.check_call('cat %s | ssh %s "mysql %s"' % (sqlf, host, db_name), shell=True) + logger.info(f"Installing MySQL database: {db_name}, source: {f} ({file_count}/{file_total})") + if local: + subprocess.check_call( + f"cat {shlex.quote(f)} | mysql {shlex.quote(db_name)}", shell=True + ) + else: + subprocess.check_call( + f"cat {shlex.quote(f)} | ssh {shlex.quote(host)} {shlex.quote(f'mysql {db_name}')}", shell=True + ) def install_mysql_dump(host, db_name, tables): From 505e5835e110db44a9f7211e96b5b6b318162d50 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Tue, 6 Dec 2022 16:24:55 +0100 Subject: [PATCH 025/175] Add annotation to korp:config reading mode annotations --- sparv/modules/korp/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sparv/modules/korp/config.py b/sparv/modules/korp/config.py index cb8d2dd4..eaecd759 100644 --- a/sparv/modules/korp/config.py +++ b/sparv/modules/korp/config.py @@ -40,6 +40,7 @@ # Annotations needed by reading mode (using export names) READING_MODE_ANNOTATIONS = ( + "text:_id", "sentence:id", "_head", "_tail" From eabb7bc81268848ca678fe04dc6a8465d6e22d57 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 9 Dec 2022 12:29:16 +0100 Subject: [PATCH 026/175] Add execution of SQL queries over SSH in mysql_wrapper --- CHANGELOG.md | 1 + sparv/api/util/mysql_wrapper.py | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e595d329..283d44e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Added - Add `korp.wordpicture_no_sentences` setting to disable generation of Word Picture sentences table. +- `util.mysql_wrapper` can now execute SQL queries remotely over SSH. ### Changed diff --git a/sparv/api/util/mysql_wrapper.py b/sparv/api/util/mysql_wrapper.py index cd5549b6..ee6f3167 100644 --- a/sparv/api/util/mysql_wrapper.py +++ b/sparv/api/util/mysql_wrapper.py @@ -1,6 +1,7 @@ """Util function for creating mysql files.""" import os +import shlex from sparv.api import get_logger from . import system @@ -12,9 +13,11 @@ class MySQL: - binaries = ("mysql", "mysql5") + binaries = ("mysql", "mariadb") - def __init__(self, database=None, username=None, password=None, encoding="UTF-8", output="", append=False): + def __init__( + self, database=None, host=None, username=None, password=None, encoding="UTF-8", output=None, append=False + ): assert database or output, "Either 'database' or 'output' must be used." if database: self.arguments = [database] @@ -22,6 +25,7 @@ def __init__(self, database=None, username=None, password=None, encoding="UTF-8" self.arguments += ["-u", username] if password: self.arguments += ["-p", password] + self.host = host self.encoding = encoding self.output = output self.first_output = True @@ -40,7 +44,12 @@ def execute(self, sql, *args): outfile.write(sql + "\n") else: # Execute SQL statement - out, err = system.call_binary(self.binaries, self.arguments, sql % args, encoding=self.encoding) + if self.host is None: + out, err = system.call_binary(self.binaries, self.arguments, sql % args, encoding=self.encoding) + else: + out, err = system.call_binary( + "ssh", [self.host, " ".join([self.binaries[0], *self.arguments, "-e", shlex.quote(sql % args)])] + ) if out: logger.info("MySQL: %s", out) if err: From 626dc64219adc089c5b77121a3aa5dff51b8be93 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 9 Dec 2022 12:40:54 +0100 Subject: [PATCH 027/175] Don't require indexes when creating table in mysql_wrapper --- sparv/api/util/mysql_wrapper.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/sparv/api/util/mysql_wrapper.py b/sparv/api/util/mysql_wrapper.py index ee6f3167..6fa62da7 100644 --- a/sparv/api/util/mysql_wrapper.py +++ b/sparv/api/util/mysql_wrapper.py @@ -64,10 +64,11 @@ def create_table(self, table, drop, columns, primary=None, indexes=None, constra if isinstance(primary, str): primary = primary.split() sqlcolumns += ["PRIMARY KEY (%s)" % _atomseq(primary)] - for index in indexes: - if isinstance(index, str): - index = index.split() - sqlcolumns += ["INDEX %s (%s)" % (_atom("-".join(index)), _atomseq(index))] + if indexes: + for index in indexes: + if isinstance(index, str): + index = index.split() + sqlcolumns += ["INDEX %s (%s)" % (_atom("-".join(index)), _atomseq(index))] if constraints: for constraint in constraints: sqlcolumns += ["CONSTRAINT %s %s (%s)" % (constraint[0], _atom(constraint[1]), _atomseq(constraint[2]))] From accadcd33ca15af881740766c402a2460985cdc8 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 9 Dec 2022 14:05:24 +0100 Subject: [PATCH 028/175] Add korp:uninstall_lemgrams uninstaller --- sparv/modules/korp/lemgram_index.py | 57 +++++++++++++++++++++++++---- 1 file changed, 49 insertions(+), 8 deletions(-) diff --git a/sparv/modules/korp/lemgram_index.py b/sparv/modules/korp/lemgram_index.py index ab48cffb..f8624c9e 100644 --- a/sparv/modules/korp/lemgram_index.py +++ b/sparv/modules/korp/lemgram_index.py @@ -1,9 +1,23 @@ """Create files needed for the lemgram search in Korp.""" from collections import defaultdict - -from sparv.api import (AllSourceFilenames, AnnotationAllSourceFiles, Config, Corpus, Export, ExportInput, OutputMarker, - exporter, get_logger, installer, util) +from typing import Optional + +from sparv.api import ( + AllSourceFilenames, + AnnotationAllSourceFiles, + Config, + Corpus, + Export, + ExportInput, + MarkerOptional, + OutputMarker, + exporter, + get_logger, + installer, + uninstaller, + util +) from sparv.api.util.mysql_wrapper import MySQL logger = get_logger(__name__) @@ -13,23 +27,49 @@ CWB_SCAN_EXECUTABLE = "cwb-scan-corpus" -@installer("Install lemgram SQL on remote host", language=["swe"]) +@installer("Install lemgram SQL on remote host", language=["swe"], uninstaller="korp:uninstall_lemgrams") def install_lemgrams(sqlfile: ExportInput = ExportInput("korp.lemgram_index/lemgram_index.sql"), marker: OutputMarker = OutputMarker("korp.install_lemgram_marker"), + uninstall_marker: MarkerOptional = MarkerOptional("korp.uninstall_lemgram_marker"), db_name: str = Config("korp.mysql_dbname"), - host: str = Config("korp.remote_host")): + host: Optional[str] = Config("korp.remote_host")): """Install lemgram SQL on remote host. Args: sqlfile: SQL file to be installed. marker: Marker file to be written. - db_name: Name of the data base. + uninstall_marker: Uninstall marker to remove. + db_name: Name of the database. host: Remote host to install to. """ + uninstall_marker.remove() util.install.install_mysql(host, db_name, sqlfile) marker.write() +@uninstaller("Uninstall lemgrams from database", language=["swe"]) +def uninstall_lemgrams( + corpus: Corpus = Corpus(), + marker: OutputMarker = OutputMarker("korp.uninstall_lemgram_marker"), + install_marker: MarkerOptional = MarkerOptional("korp.install_lemgram_marker"), + db_name: str = Config("korp.mysql_dbname"), + host: Optional[str] = Config("korp.remote_host") +): + """Remove lemgram index data from database. + + Args: + corpus: Corpus ID. + marker: Uninstall marker to write. + install_marker: Install marker to remove. + db_name: Name of the database. + host: Remote host. + """ + sql = MySQL(database=db_name, host=host) + sql.delete_rows(MYSQL_TABLE, {"corpus": corpus.upper()}) + install_marker.remove() + marker.write() + + @exporter("Lemgram index SQL file for use in Korp", language=["swe"]) def lemgram_sql(corpus: Corpus = Corpus(), source_files: AllSourceFilenames = AllSourceFilenames(), @@ -69,7 +109,8 @@ def lemgram_sql(corpus: Corpus = Corpus(), ("freq", int, 0, "NOT NULL"), ("corpus", "varchar(64)", "", "NOT NULL"), ], - "indexes": ["lemgram corpus freq"], # Can't make this primary due to collation + "primary": "lemgram corpus freq", + "indexes": ["corpus"], # Used by uninstaller "default charset": "utf8mb4", - "collate": "utf8mb4_bin", + "collate": "utf8mb4_bin" } From cc3ebca5fd793a6974c79d6f15a6bf3b1a8e966c Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 9 Dec 2022 14:35:06 +0100 Subject: [PATCH 029/175] Pass query using stdin instead of command line argument in mysql_wrapper --- sparv/api/util/mysql_wrapper.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sparv/api/util/mysql_wrapper.py b/sparv/api/util/mysql_wrapper.py index 6fa62da7..d7ed1a53 100644 --- a/sparv/api/util/mysql_wrapper.py +++ b/sparv/api/util/mysql_wrapper.py @@ -48,7 +48,10 @@ def execute(self, sql, *args): out, err = system.call_binary(self.binaries, self.arguments, sql % args, encoding=self.encoding) else: out, err = system.call_binary( - "ssh", [self.host, " ".join([self.binaries[0], *self.arguments, "-e", shlex.quote(sql % args)])] + "ssh", + [self.host, " ".join([self.binaries[0], *self.arguments])], + stdin=sql % args, + encoding=self.encoding ) if out: logger.info("MySQL: %s", out) From 74b45763dfa6dfdb5e710e07b8da36fb6b5250ea Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 9 Dec 2022 14:40:04 +0100 Subject: [PATCH 030/175] Add korp:uninstall_timespan uninstaller --- sparv/modules/korp/timespan.py | 65 ++++++++++++++++++++++++++++------ 1 file changed, 55 insertions(+), 10 deletions(-) diff --git a/sparv/modules/korp/timespan.py b/sparv/modules/korp/timespan.py index 118e0562..b8a18e52 100644 --- a/sparv/modules/korp/timespan.py +++ b/sparv/modules/korp/timespan.py @@ -1,29 +1,74 @@ """Create timespan SQL data for use in Korp.""" from collections import defaultdict - -from sparv.api import (AllSourceFilenames, Annotation, AnnotationAllSourceFiles, Config, Corpus, Export, ExportInput, - OutputMarker, annotator, exporter, get_logger, installer, util) +from typing import Optional + +from sparv.api import ( + AllSourceFilenames, + Annotation, + AnnotationAllSourceFiles, + Config, + Corpus, + Export, + ExportInput, + MarkerOptional, + OutputMarker, + annotator, + exporter, + get_logger, + installer, + uninstaller, + util +) from sparv.api.util.mysql_wrapper import MySQL logger = get_logger(__name__) -@installer("Install timespan SQL on remote host") -def install_timespan(sqlfile: ExportInput = ExportInput("korp.timespan/timespan.sql"), - out: OutputMarker = OutputMarker("korp.install_timespan_marker"), - db_name: str = Config("korp.mysql_dbname"), - host: str = Config("korp.remote_host")): +@installer("Install timespan SQL on remote host", uninstaller="korp:uninstall_timespan") +def install_timespan( + sqlfile: ExportInput = ExportInput("korp.timespan/timespan.sql"), + marker: OutputMarker = OutputMarker("korp.install_timespan_marker"), + uninstall_marker: MarkerOptional = MarkerOptional("korp.uninstall_timespan_marker"), + db_name: str = Config("korp.mysql_dbname"), + host: Optional[str] = Config("korp.remote_host"), +): """Install timespan SQL on remote host. Args: sqlfile: SQL file to be installed. - out: Marker file to be written. + marker: Marker file to be written. + uninstall_marker: Uninstall marker to remove. db_name: Name of the database. host: Remote host to install to. """ util.install.install_mysql(host, db_name, sqlfile) - out.write() + uninstall_marker.remove() + marker.write() + + +@uninstaller("Uninstall timespan data from database", language=["swe"]) +def uninstall_timespan( + corpus: Corpus = Corpus(), + marker: OutputMarker = OutputMarker("korp.uninstall_timespan_marker"), + install_marker: MarkerOptional = MarkerOptional("korp.install_timespan_marker"), + db_name: str = Config("korp.mysql_dbname"), + host: Optional[str] = Config("korp.remote_host") +): + """Remove timespan data from database. + + Args: + corpus: Corpus ID. + marker: Uninstall marker to write. + install_marker: Install marker to remove. + db_name: Name of the database. + host: Remote host. + """ + sql = MySQL(database=db_name, host=host) + sql.delete_rows(MYSQL_TABLE, {"corpus": corpus.upper()}) + sql.delete_rows(MYSQL_TABLE_DATE, {"corpus": corpus.upper()}) + install_marker.remove() + marker.write() @exporter("Timespan SQL data for use in Korp", abstract=True) From a8ac8a1528891cdecc6c62b5ec07b61a75f993d5 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 9 Dec 2022 14:43:33 +0100 Subject: [PATCH 031/175] Add korp:uninstall_relations uninstaller --- sparv/modules/korp/relations.py | 69 ++++++++++++++++++++++++++++----- 1 file changed, 60 insertions(+), 9 deletions(-) diff --git a/sparv/modules/korp/relations.py b/sparv/modules/korp/relations.py index 601614fb..11f85e3d 100644 --- a/sparv/modules/korp/relations.py +++ b/sparv/modules/korp/relations.py @@ -5,8 +5,24 @@ from collections import defaultdict from typing import Optional -from sparv.api import (AllSourceFilenames, Annotation, AnnotationDataAllSourceFiles, Config, Corpus, Export, ExportInput, - OutputMarker, OutputData, annotator, exporter, get_logger, installer, util) +from sparv.api import ( + AllSourceFilenames, + Annotation, + AnnotationDataAllSourceFiles, + Config, + Corpus, + Export, + ExportInput, + MarkerOptional, + OutputMarker, + OutputData, + annotator, + exporter, + get_logger, + installer, + uninstaller, + util +) from sparv.api.util.mysql_wrapper import MySQL logger = get_logger(__name__) @@ -17,21 +33,56 @@ MAX_POS_LENGTH = 5 -@installer("Install Korp's Word Picture SQL on remote host", language=["swe"]) -def install_relations(sqlfile: ExportInput = ExportInput("korp.wordpicture/relations.sql"), - out: OutputMarker = OutputMarker("korp.install_relations_marker"), - db_name: str = Config("korp.mysql_dbname"), - host: str = Config("korp.remote_host")): +@installer("Install Korp's Word Picture SQL on remote host", language=["swe"], uninstaller="korp:uninstall_relations") +def install_relations( + sqlfile: ExportInput = ExportInput("korp.wordpicture/relations.sql"), + marker: OutputMarker = OutputMarker("korp.install_relations_marker"), + uninstall_marker: MarkerOptional = MarkerOptional("korp.uninstall_relations_marker"), + db_name: str = Config("korp.mysql_dbname"), + host: Optional[str] = Config("korp.remote_host") +): """Install Korp's Word Picture SQL on remote host. Args: sqlfile: SQL file to be installed. - out: Marker file to be written. + marker: Marker file to be written. + uninstall_marker: Uninstall marker to remove. db_name: Name of the database. host: Remote host to install to. """ util.install.install_mysql(host, db_name, sqlfile) - out.write() + uninstall_marker.remove() + marker.write() + + +@uninstaller("Uninstall lemgrams from database", language=["swe"]) +def uninstall_relations( + corpus: Corpus = Corpus(), + marker: OutputMarker = OutputMarker("korp.uninstall_relations_marker"), + install_marker: MarkerOptional = MarkerOptional("korp.install_relations_marker"), + db_name: str = Config("korp.mysql_dbname"), + host: Optional[str] = Config("korp.remote_host") +): + """Remove lemgram index data from database. + + Args: + corpus: Corpus ID. + marker: Uninstall marker to write. + install_marker: Install marker to remove. + db_name: Name of the database. + host: Remote host. + """ + db_table = MYSQL_TABLE + "_" + corpus.upper() + tables = ["", "_strings", "_rel", "_head_rel", "_dep_rel", "_sentences"] + + sql = MySQL(database=db_name, host=host) + sql.drop_table( + *[db_table + t for t in tables], + *["temp_" + db_table + t for t in tables] + ) + + install_marker.remove() + marker.write() @annotator("Find dependencies for Korp's Word Picture", language=["swe"]) From 526c97f981d7df55f2e20268ee01f1c872a5caad Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 9 Dec 2022 14:54:25 +0100 Subject: [PATCH 032/175] Make path mandatory in uninstall_path function --- sparv/api/util/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sparv/api/util/install.py b/sparv/api/util/install.py index 5319be02..11690f62 100644 --- a/sparv/api/util/install.py +++ b/sparv/api/util/install.py @@ -19,7 +19,7 @@ def install_path(source_path: Union[str, Path], system.rsync(source_path, host, target_path) -def uninstall_path(path: Union[str, Path] = None, host: Optional[str] = None) -> None: +def uninstall_path(path: Union[str, Path], host: Optional[str] = None) -> None: """Remove a file or directory, optionally on a remote host.""" system.remove_path(path, host) From f3c7fc31b552af6f8221274c69be2b9b6695affe Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 9 Dec 2022 14:55:51 +0100 Subject: [PATCH 033/175] Handle pathlib.Path in remove_path function --- sparv/api/util/system.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sparv/api/util/system.py b/sparv/api/util/system.py index c04976bf..63439b4a 100644 --- a/sparv/api/util/system.py +++ b/sparv/api/util/system.py @@ -175,11 +175,11 @@ def rsync(local, host=None, remote=None): subprocess.check_call(["rsync"] + args + [remote]) -def remove_path(path, host=None): +def remove_path(path: Union[str, Path], host: Optional[str] = None): """Remove a file or directory, either locally or remotely.""" assert path, "'path' must not be empty." if host: - subprocess.check_call(["ssh", host, f"rm -rf {shlex.quote(path)}"]) + subprocess.check_call(["ssh", host, f"rm -rf {shlex.quote(str(path))}"]) else: p = Path(path) if p.is_file(): From f2fab92d658e52c3e6ebc451d9d6503dc97a4c73 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 9 Dec 2022 14:57:24 +0100 Subject: [PATCH 034/175] Add cwb:uninstall_corpus uninstaller --- sparv/modules/cwb/install_corpus.py | 66 ++++++++++++++++++++++++----- 1 file changed, 55 insertions(+), 11 deletions(-) diff --git a/sparv/modules/cwb/install_corpus.py b/sparv/modules/cwb/install_corpus.py index 6cbeb40f..bbef6ff7 100644 --- a/sparv/modules/cwb/install_corpus.py +++ b/sparv/modules/cwb/install_corpus.py @@ -1,15 +1,30 @@ """Module for installing cwb binary files on remote host.""" import os +from pathlib import Path from typing import Optional -from sparv.api import Config, Corpus, ExportInput, OutputMarker, SparvErrorMessage, installer, util +from sparv.api import ( + Config, + Corpus, + ExportInput, + MarkerOptional, + OutputMarker, + SparvErrorMessage, + get_logger, + installer, + uninstaller, + util +) +logger = get_logger(__name__) -@installer("Install CWB datafiles on remote host") + +@installer("Install CWB datafiles", uninstaller="korp:uninstall_corpus") def install_corpus( corpus: Corpus = Corpus(), - out: OutputMarker = OutputMarker("cwb.install_corpus_marker"), + marker: OutputMarker = OutputMarker("cwb.install_corpus_marker"), + uninstall_marker: MarkerOptional = MarkerOptional("cwb.uninstall_corpus_marker"), host: Optional[str] = Config("cwb.remote_host"), registry_file: ExportInput = ExportInput("cwb.encoded/registry/[metadata.id]"), info_file: ExportInput = ExportInput("cwb.encoded/data/.info"), @@ -17,15 +32,17 @@ def install_corpus( target_registry_dir: str = Config("cwb.remote_registry_dir"), # The remaining arguments are needed by Snakemake _marker: ExportInput = ExportInput("cwb.encoded/data/.marker")): - """Install CWB datafiles on server, by rsyncing datadir and registry.""" - sync_cwb(corpus=corpus, out=out, host=host, info_file=info_file, registry_file=registry_file, + """Install CWB datafiles, by rsyncing datadir and registry.""" + sync_cwb(corpus=corpus, marker=marker, host=host, info_file=info_file, registry_file=registry_file, target_data_dir=target_data_dir, target_registry_dir=target_registry_dir) + uninstall_marker.remove() -@installer("Install CWB datafiles for a scrambled corpus on remote host") +@installer("Install CWB datafiles for a scrambled corpus", uninstaller="korp:uninstall_corpus") def install_corpus_scrambled( corpus: Corpus = Corpus(), - out: OutputMarker = OutputMarker("cwb.install_corpus_scrambled_marker"), + marker: OutputMarker = OutputMarker("cwb.install_corpus_scrambled_marker"), + uninstall_marker: MarkerOptional = MarkerOptional("cwb.uninstall_corpus_marker"), host: Optional[str] = Config("cwb.remote_host"), registry_file: ExportInput = ExportInput("cwb.encoded_scrambled/registry/[metadata.id]"), info_file: ExportInput = ExportInput("cwb.encoded_scrambled/data/.info"), @@ -33,12 +50,39 @@ def install_corpus_scrambled( target_registry_dir: str = Config("cwb.remote_registry_dir"), # The remaining arguments are needed by Snakemake _scrambled_marker: ExportInput = ExportInput("cwb.encoded_scrambled/data/.scrambled_marker")): - """Install scrambled CWB datafiles on server, by rsyncing datadir and registry.""" - sync_cwb(corpus=corpus, out=out, host=host, info_file=info_file, registry_file=registry_file, + """Install scrambled CWB datafiles, by rsyncing datadir and registry.""" + sync_cwb(corpus=corpus, marker=marker, host=host, info_file=info_file, registry_file=registry_file, target_data_dir=target_data_dir, target_registry_dir=target_registry_dir) + uninstall_marker.remove() + + +@uninstaller("Uninstall CWB datafiles") +def uninstall_corpus( + corpus: Corpus = Corpus(), + marker: OutputMarker = OutputMarker("cwb.uninstall_corpus_marker"), + install_marker: MarkerOptional = MarkerOptional("cwb.install_corpus_marker"), + install_scrambled_marker: MarkerOptional = MarkerOptional("cwb.install_corpus_scrambled_marker"), + host: Optional[str] = Config("cwb.remote_host"), + data_dir: str = Config("cwb.remote_data_dir"), + registry_dir: str = Config("cwb.remote_registry_dir") +): + """Uninstall CWB data.""" + assert corpus and data_dir and registry_dir # Already checked by Sparv, but just to be sure + + registry_file = Path(registry_dir) / corpus + logger.info(f"Removing CWB registry file from {host + ':' if host else ''}{registry_file}") + util.install.uninstall_path(registry_file, host=host) + + corpus_dir = Path(data_dir) / corpus + logger.info(f"Removing CWB data from {host + ':' if host else ''}{corpus_dir}") + util.install.uninstall_path(corpus_dir, host=host) + + install_marker.remove() + install_scrambled_marker.remove() + marker.write() -def sync_cwb(corpus, out, host, info_file, registry_file, target_data_dir, target_registry_dir): +def sync_cwb(corpus, marker, host, info_file, registry_file, target_data_dir, target_registry_dir): """Install CWB datafiles on server, by rsyncing CWB datadir and registry.""" if not corpus: raise SparvErrorMessage("Missing corpus name. Corpus not installed.") @@ -73,4 +117,4 @@ def sync_cwb(corpus, out, host, info_file, registry_file, target_data_dir, targe os.remove(source_registry_file) # Write marker file - out.write() + marker.write() From 53c58a93fb9dc406382d208cdd3cf4f90aafba26 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 9 Dec 2022 15:23:52 +0100 Subject: [PATCH 035/175] Add xml_export:uninstall and xml_export:uninstall_scrambled uninstallers --- sparv/modules/xml_export/pretty.py | 45 ++++++++++++++++++++++----- sparv/modules/xml_export/scrambled.py | 45 +++++++++++++++++++++++---- sparv/modules/xml_export/xml_utils.py | 11 +++++-- 3 files changed, 86 insertions(+), 15 deletions(-) diff --git a/sparv/modules/xml_export/pretty.py b/sparv/modules/xml_export/pretty.py index 9a2a0706..4f51a767 100644 --- a/sparv/modules/xml_export/pretty.py +++ b/sparv/modules/xml_export/pretty.py @@ -2,10 +2,26 @@ import os -from sparv.api import (AllSourceFilenames, Annotation, AnnotationData, Config, Corpus, Export, ExportAnnotations, - ExportInput, Namespaces, OutputMarker, SourceAnnotations, SourceFilename, exporter, - get_logger, installer, util) - +from sparv.api import ( + AllSourceFilenames, + Annotation, + AnnotationData, + Config, + Corpus, + Export, + ExportAnnotations, + ExportInput, + MarkerOptional, + Namespaces, + OutputMarker, + SourceAnnotations, + SourceFilename, + exporter, + get_logger, + installer, + uninstaller, + util +) from . import xml_utils logger = get_logger(__name__) @@ -119,11 +135,26 @@ def compressed(out: Export = Export("xml_export.combined/[xml_export.filename_co @installer("Copy compressed XML to remote host", config=[ Config("xml_export.export_host", "", description="Remote host to copy XML export to."), Config("xml_export.export_path", "", description="Path on remote host to copy XML export to.") -]) +], uninstaller="xml_export:uninstall") def install(corpus: Corpus = Corpus(), bz2file: ExportInput = ExportInput("xml_export.combined/[xml_export.filename_compressed]"), - out: OutputMarker = OutputMarker("xml_export.install_export_pretty_marker"), + marker: OutputMarker = OutputMarker("xml_export.install_export_pretty_marker"), + uninstall_marker: MarkerOptional = MarkerOptional("xml_export.uninstall_export_pretty_marker"), export_path: str = Config("xml_export.export_path"), host: str = Config("xml_export.export_host")): """Copy compressed combined XML to remote host.""" - xml_utils.install_compressed_xml(corpus, bz2file, out, export_path, host) + xml_utils.install_compressed_xml(corpus, bz2file, marker, export_path, host) + uninstall_marker.remove() + + +@uninstaller("Remove compressed XML from remote location") +def uninstall( + corpus: Corpus = Corpus(), + marker: OutputMarker = OutputMarker("xml_export.uninstall_export_pretty_marker"), + install_marker: MarkerOptional = MarkerOptional("xml_export.install_export_pretty_marker"), + export_path: str = Config("xml_export.export_path"), + host: str = Config("xml_export.export_host") +): + """Remove compressed XML from remote location.""" + xml_utils.uninstall_compressed_xml(corpus, marker, export_path, host) + install_marker.remove() diff --git a/sparv/modules/xml_export/scrambled.py b/sparv/modules/xml_export/scrambled.py index 590e87b3..2b2999a0 100644 --- a/sparv/modules/xml_export/scrambled.py +++ b/sparv/modules/xml_export/scrambled.py @@ -2,9 +2,27 @@ import os -from sparv.api import (AllSourceFilenames, Annotation, AnnotationData, Config, Corpus, Export, ExportAnnotations, - ExportInput, Namespaces, OutputMarker, SourceAnnotations, SourceFilename, SparvErrorMessage, - exporter, get_logger, installer, util) +from sparv.api import ( + AllSourceFilenames, + Annotation, + AnnotationData, + Config, + Corpus, + Export, + ExportAnnotations, + ExportInput, + MarkerOptional, + Namespaces, + OutputMarker, + SourceAnnotations, + SourceFilename, + SparvErrorMessage, + exporter, + get_logger, + installer, + uninstaller, + util +) from . import xml_utils logger = get_logger(__name__) @@ -92,12 +110,27 @@ def compressed_scrambled(out: Export = Export("xml_export.combined_scrambled/[me @installer("Copy compressed scrambled XML to remote host", config=[ Config("xml_export.export_scrambled_host", "", description="Remote host to copy scrambled XML export to"), Config("xml_export.export_scrambled_path", "", description="Path on remote host to copy scrambled XML export to") -]) +], uninstaller="xml_export:uninstall_scrambled") def install_scrambled(corpus: Corpus = Corpus(), bz2file: ExportInput = ExportInput( "xml_export.combined_scrambled/[metadata.id]_scrambled.xml.bz2"), - out: OutputMarker = OutputMarker("xml_export.install_export_scrambled_marker"), + marker: OutputMarker = OutputMarker("xml_export.install_export_scrambled_marker"), + uninstall_marker: MarkerOptional = MarkerOptional("xml_export.uninstall_export_scrambled_marker"), export_path: str = Config("xml_export.export_scrambled_path"), host: str = Config("xml_export.export_scrambled_host")): """Copy compressed combined scrambled XML to remote host.""" - xml_utils.install_compressed_xml(corpus, bz2file, out, export_path, host) + xml_utils.install_compressed_xml(corpus, bz2file, marker, export_path, host) + uninstall_marker.remove() + + +@uninstaller("Remove compressed scrambled XML from remote location") +def uninstall_scrambled( + corpus: Corpus = Corpus(), + marker: OutputMarker = OutputMarker("xml_export.uninstall_export_scrambled_marker"), + install_marker: MarkerOptional = MarkerOptional("xml_export.install_export_scrambled_marker"), + export_path: str = Config("xml_export.export_scrambled_path"), + host: str = Config("xml_export.export_scrambled_host") +): + """Remove compressed XML from remote location.""" + xml_utils.uninstall_compressed_xml(corpus, marker, export_path, host) + install_marker.remove() diff --git a/sparv/modules/xml_export/xml_utils.py b/sparv/modules/xml_export/xml_utils.py index dde77e81..cc69be20 100644 --- a/sparv/modules/xml_export/xml_utils.py +++ b/sparv/modules/xml_export/xml_utils.py @@ -175,11 +175,18 @@ def compress(xmlfile, out): copyfileobj(infile, outfile) -def install_compressed_xml(corpus, bz2file, out, export_path, host): +def install_compressed_xml(corpus, bz2file, marker, export_path, host): """Install xml file on remote server.""" if not host: raise Exception("No host provided! Export not installed.") filename = corpus + ".xml.bz2" remote_file_path = os.path.join(export_path, filename) util.install.install_path(bz2file, host, remote_file_path) - out.write() + marker.write() + + +def uninstall_compressed_xml(corpus, marker, export_path, host): + """Uninstall XML file.""" + remote_file_path = os.path.join(export_path, corpus + ".xml.bz2") + util.install.uninstall_path(remote_file_path, host) + marker.write() From 5b12fd624d840aa60238d068a1e497231829109e Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 9 Dec 2022 15:40:49 +0100 Subject: [PATCH 036/175] Allow local installations for xml_export:install and xml_export:install_scrambled --- CHANGELOG.md | 1 + sparv/modules/xml_export/pretty.py | 25 ++++++++++++++----------- sparv/modules/xml_export/scrambled.py | 24 +++++++++++++----------- sparv/modules/xml_export/xml_utils.py | 4 +--- 4 files changed, 29 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 283d44e1..ab34a0d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ ### Changed - The util function `install_mysql` can now install locally as well as to a remote server. +- `xml_export:install` and `xml_export:install_scrambled` can now install locally. ### Fixed diff --git a/sparv/modules/xml_export/pretty.py b/sparv/modules/xml_export/pretty.py index 4f51a767..562f5912 100644 --- a/sparv/modules/xml_export/pretty.py +++ b/sparv/modules/xml_export/pretty.py @@ -1,6 +1,7 @@ """Export annotated corpus data to pretty-printed xml.""" import os +from typing import Optional from sparv.api import ( AllSourceFilenames, @@ -132,17 +133,19 @@ def compressed(out: Export = Export("xml_export.combined/[xml_export.filename_co xml_utils.compress(xmlfile, out) -@installer("Copy compressed XML to remote host", config=[ - Config("xml_export.export_host", "", description="Remote host to copy XML export to."), - Config("xml_export.export_path", "", description="Path on remote host to copy XML export to.") +@installer("Copy compressed XML to a target path, optionally on a remote host", config=[ + Config("xml_export.export_host", "", description="Remote host to copy XML export to"), + Config("xml_export.export_path", "", description="Target path to copy XML export to") ], uninstaller="xml_export:uninstall") -def install(corpus: Corpus = Corpus(), - bz2file: ExportInput = ExportInput("xml_export.combined/[xml_export.filename_compressed]"), - marker: OutputMarker = OutputMarker("xml_export.install_export_pretty_marker"), - uninstall_marker: MarkerOptional = MarkerOptional("xml_export.uninstall_export_pretty_marker"), - export_path: str = Config("xml_export.export_path"), - host: str = Config("xml_export.export_host")): - """Copy compressed combined XML to remote host.""" +def install( + corpus: Corpus = Corpus(), + bz2file: ExportInput = ExportInput("xml_export.combined/[xml_export.filename_compressed]"), + marker: OutputMarker = OutputMarker("xml_export.install_export_pretty_marker"), + uninstall_marker: MarkerOptional = MarkerOptional("xml_export.uninstall_export_pretty_marker"), + export_path: str = Config("xml_export.export_path"), + host: Optional[str] = Config("xml_export.export_host") +): + """Copy compressed XML to a target path, optionally on a remote host.""" xml_utils.install_compressed_xml(corpus, bz2file, marker, export_path, host) uninstall_marker.remove() @@ -153,7 +156,7 @@ def uninstall( marker: OutputMarker = OutputMarker("xml_export.uninstall_export_pretty_marker"), install_marker: MarkerOptional = MarkerOptional("xml_export.install_export_pretty_marker"), export_path: str = Config("xml_export.export_path"), - host: str = Config("xml_export.export_host") + host: Optional[str] = Config("xml_export.export_host") ): """Remove compressed XML from remote location.""" xml_utils.uninstall_compressed_xml(corpus, marker, export_path, host) diff --git a/sparv/modules/xml_export/scrambled.py b/sparv/modules/xml_export/scrambled.py index 2b2999a0..3598e52b 100644 --- a/sparv/modules/xml_export/scrambled.py +++ b/sparv/modules/xml_export/scrambled.py @@ -1,6 +1,7 @@ """Export annotated corpus data to scrambled xml.""" import os +from typing import Optional from sparv.api import ( AllSourceFilenames, @@ -107,18 +108,19 @@ def compressed_scrambled(out: Export = Export("xml_export.combined_scrambled/[me xml_utils.compress(xmlfile, out) -@installer("Copy compressed scrambled XML to remote host", config=[ +@installer("Copy compressed scrambled XML to a target path, optionally on a remote host", config=[ Config("xml_export.export_scrambled_host", "", description="Remote host to copy scrambled XML export to"), - Config("xml_export.export_scrambled_path", "", description="Path on remote host to copy scrambled XML export to") + Config("xml_export.export_scrambled_path", "", description="Target path to copy scrambled XML export to") ], uninstaller="xml_export:uninstall_scrambled") -def install_scrambled(corpus: Corpus = Corpus(), - bz2file: ExportInput = ExportInput( - "xml_export.combined_scrambled/[metadata.id]_scrambled.xml.bz2"), - marker: OutputMarker = OutputMarker("xml_export.install_export_scrambled_marker"), - uninstall_marker: MarkerOptional = MarkerOptional("xml_export.uninstall_export_scrambled_marker"), - export_path: str = Config("xml_export.export_scrambled_path"), - host: str = Config("xml_export.export_scrambled_host")): - """Copy compressed combined scrambled XML to remote host.""" +def install_scrambled( + corpus: Corpus = Corpus(), + bz2file: ExportInput = ExportInput("xml_export.combined_scrambled/[metadata.id]_scrambled.xml.bz2"), + marker: OutputMarker = OutputMarker("xml_export.install_export_scrambled_marker"), + uninstall_marker: MarkerOptional = MarkerOptional("xml_export.uninstall_export_scrambled_marker"), + export_path: str = Config("xml_export.export_scrambled_path"), + host: Optional[str] = Config("xml_export.export_scrambled_host") +): + """Copy compressed combined scrambled XML to a target path, optionally on a remote host.""" xml_utils.install_compressed_xml(corpus, bz2file, marker, export_path, host) uninstall_marker.remove() @@ -129,7 +131,7 @@ def uninstall_scrambled( marker: OutputMarker = OutputMarker("xml_export.uninstall_export_scrambled_marker"), install_marker: MarkerOptional = MarkerOptional("xml_export.install_export_scrambled_marker"), export_path: str = Config("xml_export.export_scrambled_path"), - host: str = Config("xml_export.export_scrambled_host") + host: Optional[str] = Config("xml_export.export_scrambled_host") ): """Remove compressed XML from remote location.""" xml_utils.uninstall_compressed_xml(corpus, marker, export_path, host) diff --git a/sparv/modules/xml_export/xml_utils.py b/sparv/modules/xml_export/xml_utils.py index cc69be20..b8a9061b 100644 --- a/sparv/modules/xml_export/xml_utils.py +++ b/sparv/modules/xml_export/xml_utils.py @@ -176,9 +176,7 @@ def compress(xmlfile, out): def install_compressed_xml(corpus, bz2file, marker, export_path, host): - """Install xml file on remote server.""" - if not host: - raise Exception("No host provided! Export not installed.") + """Install XML file.""" filename = corpus + ".xml.bz2" remote_file_path = os.path.join(export_path, filename) util.install.install_path(bz2file, host, remote_file_path) From 3cac1b4f36d69b4396d0863f2033ef6855e5f1d2 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 9 Dec 2022 15:54:20 +0100 Subject: [PATCH 037/175] Update changelog Should be up to date now. --- CHANGELOG.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ab34a0d9..797365b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,16 +4,31 @@ ### Added -- Add `korp.wordpicture_no_sentences` setting to disable generation of Word Picture sentences table. +- Added new `misc:inherit` annotator for inheriting attributes. +- Added `korp.wordpicture_no_sentences` setting to disable generation of Word Picture sentences table. - `util.mysql_wrapper` can now execute SQL queries remotely over SSH. +- Added several uninstallers: + - `cwb:uninstall_corpus` + - `korp:uninstall_config` + - `korp:uninstall_lemgrams` + - `korp:uninstall_timespan` + - `korp:uninstall_relations` + - `xml_export:uninstall` + - `xml_export:uninstall` +- Added `MarkerOptional` class. +- Added stats export for Swedish from the 1800s. ### Changed +- `AnnotationAllSourceFiles` now have the same methods as `Annotation`. - The util function `install_mysql` can now install locally as well as to a remote server. +- Pre-built SALDO models are now downloaded instead of being built on demand. - `xml_export:install` and `xml_export:install_scrambled` can now install locally. ### Fixed +- Several bugs fixed in `korp:config`. + ## [5.1.0] - 2022-11-03 ### Added From 85c84afd87d9e372fdc8990955b5e35e4717b286 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 16 Dec 2022 14:53:25 +0100 Subject: [PATCH 038/175] Rename korp:relations to korp:wordpicture --- CHANGELOG.md | 5 +- sparv/modules/korp/__init__.py | 6 +- .../korp/{relations.py => wordpicture.py} | 68 ++++++++++--------- 3 files changed, 45 insertions(+), 34 deletions(-) rename sparv/modules/korp/{relations.py => wordpicture.py} (93%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 797365b4..fdcadd8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,11 +12,12 @@ - `korp:uninstall_config` - `korp:uninstall_lemgrams` - `korp:uninstall_timespan` - - `korp:uninstall_relations` + - `korp:uninstall_wordpicture` - `xml_export:uninstall` - `xml_export:uninstall` - Added `MarkerOptional` class. - Added stats export for Swedish from the 1800s. +- `korp:wordpicture` table name is now configurable using `korp.wordpicture_table`. ### Changed @@ -24,6 +25,8 @@ - The util function `install_mysql` can now install locally as well as to a remote server. - Pre-built SALDO models are now downloaded instead of being built on demand. - `xml_export:install` and `xml_export:install_scrambled` can now install locally. +- `korp:relations`, `korp:relations_sql` and `korp:install_relations` has been renamed to `korp:wordpicture`, + `korp:wordpicture_sql` and `korp:install_wordpicture` respectively. ### Fixed diff --git a/sparv/modules/korp/__init__.py b/sparv/modules/korp/__init__.py index 94c5379b..7f5d753d 100644 --- a/sparv/modules/korp/__init__.py +++ b/sparv/modules/korp/__init__.py @@ -1,7 +1,7 @@ """Korp-related annotators, exporters and installers.""" from sparv.api import Config -from . import config, lemgram_index, relations, timespan +from . import config, lemgram_index, wordpicture, timespan __config__ = [ Config("korp.remote_host", description="Remote host to install to. Leave blank to install locally."), @@ -9,5 +9,7 @@ Config("korp.modes", default=[{"name": "default"}], description="The Korp modes in which the corpus will be published"), Config("korp.protected", False, description="Whether this corpus should have limited access or not"), - Config("korp.config_dir", description="Path on remote host where Korp corpus configuration files are stored") + Config("korp.config_dir", description="Path on remote host where Korp corpus configuration files are stored"), + Config("korp.wordpicture_table", default="relations", + description="Prefix used for Word Picture database table names") ] diff --git a/sparv/modules/korp/relations.py b/sparv/modules/korp/wordpicture.py similarity index 93% rename from sparv/modules/korp/relations.py rename to sparv/modules/korp/wordpicture.py index 11f85e3d..6fc77e18 100644 --- a/sparv/modules/korp/relations.py +++ b/sparv/modules/korp/wordpicture.py @@ -33,11 +33,11 @@ MAX_POS_LENGTH = 5 -@installer("Install Korp's Word Picture SQL on remote host", language=["swe"], uninstaller="korp:uninstall_relations") -def install_relations( - sqlfile: ExportInput = ExportInput("korp.wordpicture/relations.sql"), - marker: OutputMarker = OutputMarker("korp.install_relations_marker"), - uninstall_marker: MarkerOptional = MarkerOptional("korp.uninstall_relations_marker"), +@installer("Install Korp's Word Picture SQL on remote host", language=["swe"], uninstaller="korp:uninstall_wordpicture") +def install_wordpicture( + sqlfile: ExportInput = ExportInput("korp.wordpicture/wordpicture.sql"), + marker: OutputMarker = OutputMarker("korp.install_wordpicture_marker"), + uninstall_marker: MarkerOptional = MarkerOptional("korp.uninstall_wordpicture_marker"), db_name: str = Config("korp.mysql_dbname"), host: Optional[str] = Config("korp.remote_host") ): @@ -56,11 +56,12 @@ def install_relations( @uninstaller("Uninstall lemgrams from database", language=["swe"]) -def uninstall_relations( +def uninstall_wordpicture( corpus: Corpus = Corpus(), - marker: OutputMarker = OutputMarker("korp.uninstall_relations_marker"), - install_marker: MarkerOptional = MarkerOptional("korp.install_relations_marker"), + marker: OutputMarker = OutputMarker("korp.uninstall_wordpicture_marker"), + install_marker: MarkerOptional = MarkerOptional("korp.install_wordpicture_marker"), db_name: str = Config("korp.mysql_dbname"), + table_name: str = Config("korp.wordpicture_table"), host: Optional[str] = Config("korp.remote_host") ): """Remove lemgram index data from database. @@ -70,9 +71,10 @@ def uninstall_relations( marker: Uninstall marker to write. install_marker: Install marker to remove. db_name: Name of the database. + table_name: Name of database table. host: Remote host. """ - db_table = MYSQL_TABLE + "_" + corpus.upper() + db_table = table_name + "_" + corpus.upper() tables = ["", "_strings", "_rel", "_head_rel", "_dep_rel", "_sentences"] sql = MySQL(database=db_name, host=host) @@ -86,15 +88,17 @@ def uninstall_relations( @annotator("Find dependencies for Korp's Word Picture", language=["swe"]) -def relations(out: OutputData = OutputData("korp.relations"), - word: Annotation = Annotation(""), - pos: Annotation = Annotation(""), - lemgram: Annotation = Annotation(":saldo.lemgram"), - dephead: Annotation = Annotation(""), - deprel: Annotation = Annotation(""), - sentence_id: Annotation = Annotation(":misc.id"), - ref: Annotation = Annotation(""), - baseform: Annotation = Annotation(":saldo.baseform")): +def wordpicture( + out: OutputData = OutputData("korp.wordpicture"), + word: Annotation = Annotation(""), + pos: Annotation = Annotation(""), + lemgram: Annotation = Annotation(":saldo.lemgram"), + dephead: Annotation = Annotation(""), + deprel: Annotation = Annotation(""), + sentence_id: Annotation = Annotation(":misc.id"), + ref: Annotation = Annotation(""), + baseform: Annotation = Annotation(":saldo.baseform") +): """Find certain dependencies between words, to be used by the Word Picture feature in Korp.""" sentence_ids = sentence_id.read() sentence_tokens, _ = sentence_id.get_children(word) @@ -340,27 +344,31 @@ def mi_lex(rel, x_rel_y, x_rel, rel_y): Config("korp.wordpicture_no_sentences", default=False, description="Set to 'true' to skip generating sentences table.") ]) -def relations_sql(corpus: Corpus = Corpus(), - out: Export = Export("korp.wordpicture/relations.sql"), - relations: AnnotationDataAllSourceFiles = AnnotationDataAllSourceFiles("korp.relations"), - no_sentences: bool = Config("korp.wordpicture_no_sentences"), - source_files: Optional[AllSourceFilenames] = AllSourceFilenames(), - source_files_list: str = "", - split: bool = False): +def wordpicture_sql( + corpus: Corpus = Corpus(), + out: Export = Export("korp.wordpicture/wordpicture.sql"), + wordpicture: AnnotationDataAllSourceFiles = AnnotationDataAllSourceFiles("korp.wordpicture"), + no_sentences: bool = Config("korp.wordpicture_no_sentences"), + source_files: Optional[AllSourceFilenames] = AllSourceFilenames(), + source_files_list: str = "", + table_name: str = Config("korp.wordpicture_table"), + split: bool = False +): """Calculate statistics of the dependencies and saves to SQL files. Args: corpus: the corpus name out: the name for the SQL file which will contain the resulting SQL statements - relations: the name of the relations annotation + wordpicture: the name of the wordpicture annotation no_sentences: set to True to skip generating sentences table source_files: a list of source filenames source_files_list: can be used instead of source_files, and should be a file containing the name of source files, one per row + table_name: Name of database table split: when set to true leads to SQL commands being split into several parts, requiring less memory during creation, but installing the data will take much longer """ - db_table = MYSQL_TABLE + "_" + corpus.upper() + db_table = table_name + "_" + corpus.upper() # Relations that will be grouped together rel_grouping = { @@ -400,9 +408,9 @@ def relations_sql(corpus: Corpus = Corpus(), head_rel_count = defaultdict(int) # Frequency of (head, rel) dep_rel_count = defaultdict(int) # Frequency of (rel, dep) - relations_data = relations.read(file) + wordpicture_data = wordpicture.read(file) - for triple in relations_data.splitlines(): + for triple in wordpicture_data.splitlines(): head, headpos, rel, dep, deppos, extra, sid, refh, refd, bfhead, bfdep, wfhead, wfdep = triple.split("\t") bfhead, bfdep, wfhead, wfdep = int(bfhead), int(bfdep), int(wfhead), int(wfdep) @@ -591,8 +599,6 @@ def _write_sql(strings, sentences, freq, rel_count, head_rel_count, dep_rel_coun RELNAMES = ["SS", "OBJ", "ADV", "AA", "AT", "ET", "PA"] rel_enum = "ENUM(%s)" % ", ".join("'%s'" % r for r in RELNAMES) -MYSQL_TABLE = "relations" - MYSQL_RELATIONS = {"columns": [("id", int, 0, "NOT NULL"), ("head", int, 0, "NOT NULL"), ("rel", rel_enum, RELNAMES[0], "NOT NULL"), From 75a7ea9f709a3028afe445faa7d0de70206819d5 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 16 Dec 2022 15:06:16 +0100 Subject: [PATCH 039/175] Update tests to match changes --- tests/test_corpora/standard-swe/config.yaml | 2 +- .../korp.wordpicture/{relations.sql => wordpicture.sql} | 0 .../gold_sparv-workdir/r\303\244var/korp.wordpicture" | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename tests/test_corpora/standard-swe/gold_export/korp.wordpicture/{relations.sql => wordpicture.sql} (100%) rename "tests/test_corpora/standard-swe/gold_sparv-workdir/r\303\244var/korp.relations" => "tests/test_corpora/standard-swe/gold_sparv-workdir/r\303\244var/korp.wordpicture" (100%) diff --git a/tests/test_corpora/standard-swe/config.yaml b/tests/test_corpora/standard-swe/config.yaml index 29d79ff3..8e6c85c0 100644 --- a/tests/test_corpora/standard-swe/config.yaml +++ b/tests/test_corpora/standard-swe/config.yaml @@ -67,7 +67,7 @@ export: - csv_export:csv - cwb:vrt - cwb:vrt_scrambled - - korp:relations_sql + - korp:wordpicture_sql - korp:timespan_sql - stats_export:freq_list - xml_export:pretty diff --git a/tests/test_corpora/standard-swe/gold_export/korp.wordpicture/relations.sql b/tests/test_corpora/standard-swe/gold_export/korp.wordpicture/wordpicture.sql similarity index 100% rename from tests/test_corpora/standard-swe/gold_export/korp.wordpicture/relations.sql rename to tests/test_corpora/standard-swe/gold_export/korp.wordpicture/wordpicture.sql diff --git "a/tests/test_corpora/standard-swe/gold_sparv-workdir/r\303\244var/korp.relations" "b/tests/test_corpora/standard-swe/gold_sparv-workdir/r\303\244var/korp.wordpicture" similarity index 100% rename from "tests/test_corpora/standard-swe/gold_sparv-workdir/r\303\244var/korp.relations" rename to "tests/test_corpora/standard-swe/gold_sparv-workdir/r\303\244var/korp.wordpicture" From 1758b3fc309c5325cdb1ce67e273d6731bf36028 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Mon, 19 Dec 2022 14:17:37 +0100 Subject: [PATCH 040/175] Fix CWB uninstaller bug --- sparv/modules/cwb/install_corpus.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sparv/modules/cwb/install_corpus.py b/sparv/modules/cwb/install_corpus.py index bbef6ff7..dcdf9b2d 100644 --- a/sparv/modules/cwb/install_corpus.py +++ b/sparv/modules/cwb/install_corpus.py @@ -20,7 +20,7 @@ logger = get_logger(__name__) -@installer("Install CWB datafiles", uninstaller="korp:uninstall_corpus") +@installer("Install CWB datafiles", uninstaller="cwb:uninstall_corpus") def install_corpus( corpus: Corpus = Corpus(), marker: OutputMarker = OutputMarker("cwb.install_corpus_marker"), @@ -38,7 +38,7 @@ def install_corpus( uninstall_marker.remove() -@installer("Install CWB datafiles for a scrambled corpus", uninstaller="korp:uninstall_corpus") +@installer("Install CWB datafiles for a scrambled corpus", uninstaller="cwb:uninstall_corpus") def install_corpus_scrambled( corpus: Corpus = Corpus(), marker: OutputMarker = OutputMarker("cwb.install_corpus_scrambled_marker"), From cdbd605dc49428fe326a9b94ca6fcdcbe1667b0a Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Tue, 20 Dec 2022 18:24:24 +0100 Subject: [PATCH 041/175] Make target mandatory for install_path and rsync functions --- CHANGELOG.md | 1 + docs/developers-guide/utilities.md | 11 +++++------ sparv/api/util/install.py | 8 +++++--- sparv/api/util/system.py | 6 ++---- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fdcadd8c..7b350d58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ - `xml_export:install` and `xml_export:install_scrambled` can now install locally. - `korp:relations`, `korp:relations_sql` and `korp:install_relations` has been renamed to `korp:wordpicture`, `korp:wordpicture_sql` and `korp:install_wordpicture` respectively. +- Target path is no longer optional for the utility functions `install_path` and `rsync`. ### Fixed diff --git a/docs/developers-guide/utilities.md b/docs/developers-guide/utilities.md index d63454b1..78d8cdb7 100644 --- a/docs/developers-guide/utilities.md +++ b/docs/developers-guide/utilities.md @@ -99,9 +99,9 @@ Transfer a file or the contents of a directory to a target destination, optional **Arguments:** - `source_path`: Path to the local file or directory to sync. If a directory is used, its contents are synced, not the - directory itself. -- `host` (optional): The remote host to install to. -- `target_path`: The name of the target file or directory. + directory itself, and any extraneous files in destination directories are deleted. +- `host`: The remote host to install to. Set to `None` to install locally. +- `target_path`: Path to target file or directory. ### uninstall_path() @@ -211,9 +211,8 @@ deleted. **Arguments:** - `local`: Path to a local file or directory. -- `host`: The remote host to rsync to. -- `remote`: Path on the remote host to rsync to. Defaults to `None`. If not provided, the path will be the same as on - the local machine. +- `host`: The remote host to rsync to. Set to `None` to sync locally. +- `remote`: Path to target file or directory. ## Tagsets diff --git a/sparv/api/util/install.py b/sparv/api/util/install.py index 11690f62..b1882b31 100644 --- a/sparv/api/util/install.py +++ b/sparv/api/util/install.py @@ -12,9 +12,11 @@ logger = get_logger(__name__) -def install_path(source_path: Union[str, Path], - host: Optional[str] = None, - target_path: Optional[Union[str, Path]] = None) -> None: +def install_path( + source_path: Union[str, Path], + host: Optional[str], + target_path: Union[str, Path] +) -> None: """Transfer a file or the contents of a directory to a target destination, optionally on a different host.""" system.rsync(source_path, host, target_path) diff --git a/sparv/api/util/system.py b/sparv/api/util/system.py index 63439b4a..030e7593 100644 --- a/sparv/api/util/system.py +++ b/sparv/api/util/system.py @@ -150,14 +150,12 @@ def find_binary(name: Union[str, list], search_paths=(), executable: bool = True return None -def rsync(local, host=None, remote=None): +def rsync(local: Union[str, Path], host: Optional[str], remote: Union[str, Path]): """Transfer files and/or directories using rsync. When syncing directories, extraneous files in destination dirs are deleted. """ - assert host or remote, "Either 'host' or 'remote' must be set." - if remote is None: - remote = local + assert local and remote, "Both 'local' and 'remote' must be set." remote_dir = os.path.dirname(remote) if os.path.isdir(local): From 2f80cadf0e010ab52d4b00b3e445e3d1a45ac824 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Tue, 20 Dec 2022 18:24:47 +0100 Subject: [PATCH 042/175] Simplify code --- sparv/modules/korp/config.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sparv/modules/korp/config.py b/sparv/modules/korp/config.py index eaecd759..a58fd62e 100644 --- a/sparv/modules/korp/config.py +++ b/sparv/modules/korp/config.py @@ -10,6 +10,7 @@ AllSourceFilenames, AnnotationName, Config, + Corpus, Export, ExportAnnotationNames, ExportInput, @@ -77,7 +78,7 @@ Config("korp.hidden_annotations", description="List of annotations not to include in corpus config", default=HIDDEN_ANNOTATIONS) ]) -def config(id: str = Config("metadata.id"), +def config(id: Corpus = Corpus(), name: dict = Config("metadata.name"), description: Optional[dict] = Config("metadata.description"), short_description: Optional[dict] = Config("metadata.short_description"), @@ -393,7 +394,7 @@ def install_config( def uninstall_config( remote_host: Optional[str] = Config("korp.remote_host"), config_dir: str = Config("korp.config_dir"), - corpus_id: str = Config("metadata.id"), + corpus_id: Corpus = Corpus(), marker: OutputMarker = OutputMarker("korp.uninstall_config_marker"), install_marker: MarkerOptional = MarkerOptional("korp.install_config_marker") ): From b378e700e50895f9186f81ab8a857378b943b8be Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Wed, 21 Dec 2022 14:43:27 +0100 Subject: [PATCH 043/175] Remove unnecessary default config values --- docs/developers-guide/sparv-decorators.md | 12 ++++++------ sparv/modules/conll_export/conllu.py | 6 ++---- sparv/modules/cwb/__init__.py | 4 ++-- sparv/modules/cwb/install_corpus.py | 6 ------ sparv/modules/dateformat/dateformat.py | 12 ++++++------ sparv/modules/docx_import/docx_import.py | 6 +++--- sparv/modules/geo/geo.py | 2 +- sparv/modules/hist/__init__.py | 2 +- sparv/modules/hunpos/__init__.py | 4 ++-- sparv/modules/misc/misc.py | 2 +- sparv/modules/odt_import/odt_import.py | 5 +++-- sparv/modules/saldo/saldo.py | 12 +++++------- sparv/modules/stanza/__init__.py | 5 ++--- sparv/modules/stats_export/__init__.py | 4 ++-- sparv/modules/text_import/text_import.py | 5 +++-- sparv/modules/xml_export/pretty.py | 4 ++-- sparv/modules/xml_export/scrambled.py | 4 ++-- sparv/modules/xml_import/xml_import.py | 8 ++++---- 18 files changed, 47 insertions(+), 56 deletions(-) diff --git a/docs/developers-guide/sparv-decorators.md b/docs/developers-guide/sparv-decorators.md index 665ce99a..2d1b0227 100644 --- a/docs/developers-guide/sparv-decorators.md +++ b/docs/developers-guide/sparv-decorators.md @@ -134,14 +134,14 @@ It is recommended that an installer removes any related uninstaller's marker, to **Example:** ```python @installer("Copy compressed XML to remote host", config=[ - Config("xml_export.export_host", "", description="Remote host to copy XML export to."), - Config("xml_export.export_path", "", description="Path on remote host to copy XML export to.") + Config("xml_export.export_host", description="Remote host to copy XML export to."), + Config("xml_export.export_path", description="Path on remote host to copy XML export to.") ]) def install(corpus: Corpus = Corpus(), xmlfile: ExportInput = ExportInput("xml_export.combined/[metadata.id].xml.bz2"), out: OutputMarker = OutputMarker("xml_export.install_export_pretty_marker"), export_path: str = Config("xml_export.export_path"), - host: str = Config("xml_export.export_host")): + host: Optional[str] = Config("xml_export.export_host")): ... ``` @@ -165,14 +165,14 @@ It is recommended that an uninstaller removes any related installer's marker, to **Example:** ```python @uninstaller("Remove compressed XML from remote host", config=[ - Config("xml_export.export_host", "", description="Remote host to remove XML export from."), - Config("xml_export.export_path", "", description="Path on remote host to remove XML export from.") + Config("xml_export.export_host", description="Remote host to remove XML export from."), + Config("xml_export.export_path", description="Path on remote host to remove XML export from.") ]) def uninstall(corpus: Corpus = Corpus(), xmlfile: ExportInput = ExportInput("xml_export.combined/[metadata.id].xml.bz2"), out: OutputMarker = OutputMarker("xml_export.uninstall_export_pretty_marker"), export_path: str = Config("xml_export.export_path"), - host: str = Config("xml_export.export_host")): + host: Optional[str] = Config("xml_export.export_host")): ... ``` diff --git a/sparv/modules/conll_export/conllu.py b/sparv/modules/conll_export/conllu.py index 3d7958cb..b0a803e5 100644 --- a/sparv/modules/conll_export/conllu.py +++ b/sparv/modules/conll_export/conllu.py @@ -26,10 +26,8 @@ description="Annotation in HEAD field of CoNLL-U output"), Config("conll_export.conll_fields.deprel", default="", description="Annotation in DEPREL field of CoNLL-U output"), - Config("conll_export.conll_fields.deps", default=None, - description="Annotation in DEPS field of CoNLL-U output"), - Config("conll_export.conll_fields.misc", default=None, - description="Annotation in MISC field of CoNLL-U output") + Config("conll_export.conll_fields.deps", description="Annotation in DEPS field of CoNLL-U output"), + Config("conll_export.conll_fields.misc", description="Annotation in MISC field of CoNLL-U output") ]) def conllu(source_file: SourceFilename = SourceFilename(), out: Export = Export("conll_export/{file}.conllu"), diff --git a/sparv/modules/cwb/__init__.py b/sparv/modules/cwb/__init__.py index 30ea304e..1faed0b0 100644 --- a/sparv/modules/cwb/__init__.py +++ b/sparv/modules/cwb/__init__.py @@ -5,6 +5,6 @@ __config__ = [ Config("cwb.remote_host", description="Remote host to install CWB files to"), - Config("cwb.remote_registry_dir", "", description="CWB registry path on remote host"), - Config("cwb.remote_data_dir", "", description="CWB datadir path on remote host") + Config("cwb.remote_registry_dir", description="CWB registry path on remote host"), + Config("cwb.remote_data_dir", description="CWB datadir path on remote host") ] diff --git a/sparv/modules/cwb/install_corpus.py b/sparv/modules/cwb/install_corpus.py index dcdf9b2d..e97f5579 100644 --- a/sparv/modules/cwb/install_corpus.py +++ b/sparv/modules/cwb/install_corpus.py @@ -87,12 +87,6 @@ def sync_cwb(corpus, marker, host, info_file, registry_file, target_data_dir, ta if not corpus: raise SparvErrorMessage("Missing corpus name. Corpus not installed.") - if not target_data_dir: - raise SparvErrorMessage("Configuration variable cwb.remote_data_dir not set! Corpus not installed.") - - if not target_registry_dir: - raise SparvErrorMessage("Configuration variable cwb.remote_registry_dir not set! Corpus not installed.") - source_data_dir = os.path.dirname(info_file) source_registry_dir = os.path.dirname(registry_file) diff --git a/sparv/modules/dateformat/dateformat.py b/sparv/modules/dateformat/dateformat.py index 4982b747..a3ada9a5 100644 --- a/sparv/modules/dateformat/dateformat.py +++ b/sparv/modules/dateformat/dateformat.py @@ -35,8 +35,8 @@ def dateformat(in_from: Annotation = Annotation("[dateformat.datetime_from]"), description="To-dates"), informat: str = Config("dateformat.datetime_informat"), outformat: str = Config("dateformat.date_outformat"), - splitter: Optional[str] = Config("dateformat.splitter", None), - regex: Optional[str] = Config("dateformat.regex", None)): + splitter: Optional[str] = Config("dateformat.splitter"), + regex: Optional[str] = Config("dateformat.regex")): """Convert existing dates/times to specified date output format. http://docs.python.org/library/datetime.html#strftime-and-strptime-behavior @@ -63,8 +63,8 @@ def dateformat_pretty(in_date: Annotation = Annotation("[dateformat.datetime_fro out: Output = Output("[dateformat.out_annotation]:dateformat.date_pretty", description="Date without timestamp 'YYYY-MM-DD'"), informat: str = Config("dateformat.datetime_informat"), - splitter: Optional[str] = Config("dateformat.splitter", None), - regex: Optional[str] = Config("dateformat.regex", None)): + splitter: Optional[str] = Config("dateformat.splitter"), + regex: Optional[str] = Config("dateformat.regex")): """Convert existing dates to format YYYY-MM-DD.""" _formatter(in_date, Annotation(), out, Annotation(), informat, "%Y-%m-%d", splitter, regex) @@ -82,8 +82,8 @@ def timeformat(in_from: Annotation = Annotation("[dateformat.datetime_from]"), description="To-times"), informat: str = Config("dateformat.datetime_informat"), outformat: str = Config("dateformat.time_outformat"), - splitter: Optional[str] = Config("dateformat.splitter", None), - regex: Optional[str] = Config("dateformat.regex", None)): + splitter: Optional[str] = Config("dateformat.splitter"), + regex: Optional[str] = Config("dateformat.regex")): """Convert existing dates/times to specified time output format. http://docs.python.org/library/datetime.html#strftime-and-strptime-behavior diff --git a/sparv/modules/docx_import/docx_import.py b/sparv/modules/docx_import/docx_import.py index 59bcefcc..f30a229e 100644 --- a/sparv/modules/docx_import/docx_import.py +++ b/sparv/modules/docx_import/docx_import.py @@ -1,6 +1,7 @@ """Import module for docx source files.""" import unicodedata +from typing import Optional from docx2python import docx2python from docx2python.iterators import iter_at_depth @@ -8,9 +9,8 @@ from sparv.api import Config, SourceFilename, Output, Source, SourceStructure, Text, importer, util - @importer("docx import", file_extension="docx", outputs=["text"], text_annotation="text", config=[ - Config("docx_import.prefix", "", description="Optional prefix to add to annotation names."), + Config("docx_import.prefix", description="Optional prefix to add to annotation names."), Config("docx_import.keep_control_chars", False, description="Set to True if control characters should not be " "removed from the text."), Config("docx_import.normalize", "NFC", description="Normalize input using any of the following forms: " @@ -18,7 +18,7 @@ ]) def parse(source_file: SourceFilename = SourceFilename(), source_dir: Source = Source(), - prefix: str = Config("docx_import.prefix"), + prefix: Optional[str] = Config("docx_import.prefix"), keep_control_chars: bool = Config("docx_import.keep_control_chars"), normalize: str = Config("docx_import.normalize")) -> None: """Parse docx file as input to the Sparv Pipeline. diff --git a/sparv/modules/geo/geo.py b/sparv/modules/geo/geo.py index e19537bd..02b40b30 100644 --- a/sparv/modules/geo/geo.py +++ b/sparv/modules/geo/geo.py @@ -50,7 +50,7 @@ def contextual(out: Output = Output("{chunk}:geo.geo_context", description="Geog @annotator("Annotate {chunk} with location data, based on metadata containing location names", config=[ - Config("geo.metadata_source", default="", description="Source attribute for location metadata"), + Config("geo.metadata_source", description="Source attribute for location metadata"), Config("geo.model", default="geo/geo.pickle", description="Path to model") ], wildcards=[Wildcard("chunk", Wildcard.ANNOTATION)]) def metadata(out: Output = Output("{chunk}:geo.geo_metadata", description="Geographical places with coordinates"), diff --git a/sparv/modules/hist/__init__.py b/sparv/modules/hist/__init__.py index a9b5c656..01de8882 100644 --- a/sparv/modules/hist/__init__.py +++ b/sparv/modules/hist/__init__.py @@ -14,5 +14,5 @@ Config("hist.max_mwe_gaps", default=0, description="Max amount of gaps allowed within a multiword expression"), Config("hist.delimiter", default=util.constants.DELIM, description="Character to put between ambiguous results"), Config("hist.affix", default=util.constants.AFFIX, description="Character to put before and after sets of results"), - Config("hist.extralemgrams", default="", description="Additional lemgram annotation") + Config("hist.extralemgrams", description="Additional lemgram annotation") ] diff --git a/sparv/modules/hunpos/__init__.py b/sparv/modules/hunpos/__init__.py index a3dc85a4..a3f5d064 100644 --- a/sparv/modules/hunpos/__init__.py +++ b/sparv/modules/hunpos/__init__.py @@ -14,13 +14,13 @@ Config("hunpos.morphtable", default="hunpos/saldo_suc-tags.morphtable", description="Path to optional Hunpos morphtable file"), Config("hunpos.patterns", default="hunpos/suc.patterns", description="Path to optional patterns file"), - Config("hunpos.tag_mapping", default=None, description="Optional tag mapping for translating the output tags"), + Config("hunpos.tag_mapping", description="Optional tag mapping for translating the output tags"), # Config for swe-1800 Config("hunpos.model_hist", default="hunpos/suc3_suc-tags_default-setting_utf8.model", description="Path to Hunpos model (older Swedish)"), Config("hunpos.morphtable_hist", default="hunpos/hist/dalinm-swedberg_saldo_suc-tags.morphtable", description="Path to optional Hunpos morphtable file (older Swedish)"), - Config("hunpos.tag_mapping_hist", default=None, + Config("hunpos.tag_mapping_hist", description="Optional tag mapping for translating the output tags (older Swedish)") ] diff --git a/sparv/modules/misc/misc.py b/sparv/modules/misc/misc.py index ed2b89a8..8276e0ad 100644 --- a/sparv/modules/misc/misc.py +++ b/sparv/modules/misc/misc.py @@ -35,7 +35,7 @@ def text_spans(text: Text = Text(), @annotator("Head and tail whitespace characters for tokens", config=[ - Config("misc.head_tail_max_length", default=None, + Config("misc.head_tail_max_length", description="If set to an int misc.head and misc.tail will be truncated to that many characters.")]) def text_headtail(text: Text = Text(), chunk: Annotation = Annotation(""), diff --git a/sparv/modules/odt_import/odt_import.py b/sparv/modules/odt_import/odt_import.py index c614bc07..44eac8d7 100644 --- a/sparv/modules/odt_import/odt_import.py +++ b/sparv/modules/odt_import/odt_import.py @@ -3,6 +3,7 @@ import unicodedata import xml.etree.ElementTree as etree import zipfile +from typing import Optional from sparv.api import Config, SourceFilename, Output, Source, SourceStructure, Text, get_logger, importer, util @@ -10,7 +11,7 @@ @importer("odt import", file_extension="odt", outputs=["text"], text_annotation="text", config=[ - Config("odt_import.prefix", "", description="Optional prefix to add to annotation names."), + Config("odt_import.prefix", description="Optional prefix to add to annotation names."), Config("odt_import.keep_control_chars", False, description="Set to True if control characters should not be " "removed from the text."), Config("odt_import.normalize", "NFC", description="Normalize input using any of the following forms: " @@ -18,7 +19,7 @@ ]) def parse(source_file: SourceFilename = SourceFilename(), source_dir: Source = Source(), - prefix: str = Config("odt_import.prefix"), + prefix: Optional[str] = Config("odt_import.prefix"), keep_control_chars: bool = Config("odt_import.keep_control_chars"), normalize: str = Config("odt_import.normalize")) -> None: """Parse odt file as input to the Sparv Pipeline. diff --git a/sparv/modules/saldo/saldo.py b/sparv/modules/saldo/saldo.py index bdba3471..97dac8a4 100644 --- a/sparv/modules/saldo/saldo.py +++ b/sparv/modules/saldo/saldo.py @@ -25,8 +25,7 @@ def preloader(models): Config("saldo.model", default="saldo/saldo.pickle", description="Path to SALDO model"), Config("saldo.delimiter", default=util.constants.DELIM, description="Character to put between ambiguous results"), Config("saldo.affix", default=util.constants.AFFIX, description="Character to put before and after sets of results"), - Config("saldo.precision", "", - description="Format string for appending precision to each value (e.g. ':%.3f')"), + Config("saldo.precision", description="Format string for appending precision to each value (e.g. ':%.3f')"), Config("saldo.precision_filter", default="max", description="Precision filter with possible values 'max' (only use the most probable annotations), " "'first' (only use the single most probable annotation), 'none' (use all annotations)"), @@ -39,7 +38,7 @@ def preloader(models): Config("saldo.allow_multiword_overlap", default=False, description="Whether all multiword expressions may overlap with each other. " "If set to False, some cleanup is done."), - Config("saldo.word_separator", default="", + Config("saldo.word_separator", description="Character used to split the values of 'word' into several word variations"), ], preloader=preloader, preloader_params=["models"], preloader_target="models_preloaded") def annotate(token: Annotation = Annotation(""), @@ -54,13 +53,13 @@ def annotate(token: Annotation = Annotation(""), msd: Optional[Annotation] = Annotation(""), delimiter: str = Config("saldo.delimiter"), affix: str = Config("saldo.affix"), - precision: str = Config("saldo.precision"), + precision: Optional[str] = Config("saldo.precision"), precision_filter: str = Config("saldo.precision_filter"), min_precision: float = Config("saldo.min_precision"), skip_multiword: bool = Config("saldo.skip_multiword"), max_gaps: int = Config("saldo.max_mwe_gaps"), allow_multiword_overlap: bool = Config("saldo.allow_multiword_overlap"), - word_separator: str = Config("saldo.word_separator"), + word_separator: Optional[str] = Config("saldo.word_separator"), models_preloaded: Optional[dict] = None): """Use the Saldo lexicon model to annotate msd-tagged words. @@ -77,8 +76,7 @@ def annotate(token: Annotation = Annotation(""), msd: Input annotation with POS and morphological descriptions. delimiter: Character to put between ambiguous results. affix: Character to put before and after sets of results. - precision: Format string for appending precision to each value (e.g. ':%.3f'). use empty string for no - precision. + precision: Optional format string for appending precision to each value (e.g. ':%.3f'). precision_filter: Precision filter with values 'max' (only use the annotations that are most probable), 'first' (only use the most probable annotation(s)), 'none' (use all annotations)". min_precision: Only use annotations with a probability score higher than this. diff --git a/sparv/modules/stanza/__init__.py b/sparv/modules/stanza/__init__.py index 6f2903df..3501fee9 100644 --- a/sparv/modules/stanza/__init__.py +++ b/sparv/modules/stanza/__init__.py @@ -22,10 +22,9 @@ "be excluded from analysis. Disabled by default."), Config("stanza.sentence_chunk", default="", description="Text chunk (annotation) to use as input when segmenting sentences (not used for Swedish)"), - Config("stanza.sentence_annotation", default="", + Config("stanza.sentence_annotation", description="Optional existing sentence segmentation annotation (not used for Swedish)"), - Config("stanza.token_annotation", default="", - description="Optional existing token annotation (not used for Swedish)"), + Config("stanza.token_annotation", description="Optional existing token annotation (not used for Swedish)"), # Config for Swedish Config("stanza.swe_lem_model", default="stanza/swe/lem/sv_suc_lemmatizer.pt", diff --git a/sparv/modules/stats_export/__init__.py b/sparv/modules/stats_export/__init__.py index 88ba44df..70a584ab 100644 --- a/sparv/modules/stats_export/__init__.py +++ b/sparv/modules/stats_export/__init__.py @@ -12,6 +12,6 @@ Config("stats_export.delimiter", default="\t", description="Delimiter separating columns"), Config("stats_export.cutoff", default=1, description="The minimum frequency a word must have in order to be included in the result"), - Config("stats_export.remote_host", "", description="Remote host to install to"), - Config("stats_export.remote_dir", "", description="Path on remote host to install to") + Config("stats_export.remote_host", description="Remote host to install to"), + Config("stats_export.remote_dir", description="Path on remote host to install to") ] diff --git a/sparv/modules/text_import/text_import.py b/sparv/modules/text_import/text_import.py index 68a827e1..6cbd0321 100644 --- a/sparv/modules/text_import/text_import.py +++ b/sparv/modules/text_import/text_import.py @@ -1,12 +1,13 @@ """Import module for plain text source files.""" import unicodedata +from typing import Optional from sparv.api import Config, SourceFilename, Output, Source, SourceStructure, Text, importer, util @importer("TXT import", file_extension="txt", outputs=["text"], text_annotation="text", config=[ - Config("text_import.prefix", "", description="Optional prefix to add to annotation names."), + Config("text_import.prefix", description="Optional prefix to add to annotation names."), Config("text_import.encoding", util.constants.UTF8, description="Encoding of source file. Defaults to UTF-8."), Config("text_import.keep_control_chars", False, description="Set to True if control characters should not be " "removed from the text."), @@ -15,7 +16,7 @@ ]) def parse(source_file: SourceFilename = SourceFilename(), source_dir: Source = Source(), - prefix: str = Config("text_import.prefix"), + prefix: Optional[str] = Config("text_import.prefix"), encoding: str = Config("text_import.encoding"), keep_control_chars: bool = Config("text_import.keep_control_chars"), normalize: str = Config("text_import.normalize")) -> None: diff --git a/sparv/modules/xml_export/pretty.py b/sparv/modules/xml_export/pretty.py index 562f5912..638bf573 100644 --- a/sparv/modules/xml_export/pretty.py +++ b/sparv/modules/xml_export/pretty.py @@ -134,8 +134,8 @@ def compressed(out: Export = Export("xml_export.combined/[xml_export.filename_co @installer("Copy compressed XML to a target path, optionally on a remote host", config=[ - Config("xml_export.export_host", "", description="Remote host to copy XML export to"), - Config("xml_export.export_path", "", description="Target path to copy XML export to") + Config("xml_export.export_host", description="Remote host to copy XML export to"), + Config("xml_export.export_path", description="Target path to copy XML export to") ], uninstaller="xml_export:uninstall") def install( corpus: Corpus = Corpus(), diff --git a/sparv/modules/xml_export/scrambled.py b/sparv/modules/xml_export/scrambled.py index 3598e52b..e4f09da7 100644 --- a/sparv/modules/xml_export/scrambled.py +++ b/sparv/modules/xml_export/scrambled.py @@ -109,8 +109,8 @@ def compressed_scrambled(out: Export = Export("xml_export.combined_scrambled/[me @installer("Copy compressed scrambled XML to a target path, optionally on a remote host", config=[ - Config("xml_export.export_scrambled_host", "", description="Remote host to copy scrambled XML export to"), - Config("xml_export.export_scrambled_path", "", description="Target path to copy scrambled XML export to") + Config("xml_export.export_scrambled_host", description="Remote host to copy scrambled XML export to"), + Config("xml_export.export_scrambled_path", description="Target path to copy scrambled XML export to") ], uninstaller="xml_export:uninstall_scrambled") def install_scrambled( corpus: Corpus = Corpus(), diff --git a/sparv/modules/xml_import/xml_import.py b/sparv/modules/xml_import/xml_import.py index 754a415d..7d4c0b5f 100644 --- a/sparv/modules/xml_import/xml_import.py +++ b/sparv/modules/xml_import/xml_import.py @@ -5,7 +5,7 @@ import unicodedata import xml.etree.ElementTree as etree from itertools import chain -from typing import List +from typing import List, Optional from sparv.api import (Config, Headers, Namespaces, Output, Source, SourceFilename, SourceStructure, SourceStructureParser, SparvErrorMessage, Text, get_logger, importer, util) @@ -55,7 +55,7 @@ def get_annotations(self, corpus_config: dict) -> List[str]: "included in corpus text."), Config("xml_import.header_data", [], description="List of header elements and attributes from which to extract " "metadata."), - Config("xml_import.prefix", "", description="Optional prefix to add to annotation names."), + Config("xml_import.prefix", description="Optional prefix to add to annotation names."), Config("xml_import.remove_namespaces", False, description="Remove XML namespaces upon import."), Config("xml_import.encoding", util.constants.UTF8, description="Encoding of source file. Defaults to UTF-8."), Config("xml_import.keep_control_chars", False, description="Set to True if control characters should not be " @@ -69,7 +69,7 @@ def parse(filename: SourceFilename = SourceFilename(), skip: list = Config("xml_import.skip"), header_elements: list = Config("xml_import.header_elements"), header_data: list = Config("xml_import.header_data"), - prefix: str = Config("xml_import.prefix"), + prefix: Optional[str] = Config("xml_import.prefix"), remove_namespaces: bool = Config("xml_import.remove_namespaces"), encoding: str = Config("xml_import.encoding"), keep_control_chars: bool = Config("xml_import.keep_control_chars"), @@ -100,7 +100,7 @@ class SparvXMLParser: """XML parser class for parsing XML.""" def __init__(self, elements: list, skip: list, header_elements: list, header_data: list, source_dir: Source, - encoding: str = util.constants.UTF8, prefix: str = "", remove_namespaces: bool = False, + encoding: str = util.constants.UTF8, prefix: Optional[str] = None, remove_namespaces: bool = False, keep_control_chars: bool = True, normalize: str = "NFC"): """Initialize XML parser.""" self.source_dir = source_dir From ebca22cd8fe0a8762879dc082932b63b95dc5479 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Wed, 21 Dec 2022 14:49:58 +0100 Subject: [PATCH 044/175] Add stats_export uninstallers --- CHANGELOG.md | 3 + sparv/modules/stats_export/sbx_stats.py | 84 ++++++++++++++++++---- sparv/modules/stats_export/stats_export.py | 57 ++++++++++++--- 3 files changed, 120 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b350d58..bd048f6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,9 @@ - `korp:uninstall_lemgrams` - `korp:uninstall_timespan` - `korp:uninstall_wordpicture` + - `stats_export:uninstall_freq_list` + - `stats_export:uninstall_sbx_freq_list` + - `stats_export:uninstall_sbx_freq_list_date` - `xml_export:uninstall` - `xml_export:uninstall` - Added `MarkerOptional` class. diff --git a/sparv/modules/stats_export/sbx_stats.py b/sparv/modules/stats_export/sbx_stats.py index f7a2ad72..4689bf84 100644 --- a/sparv/modules/stats_export/sbx_stats.py +++ b/sparv/modules/stats_export/sbx_stats.py @@ -1,8 +1,25 @@ """SBX specific annotation and export functions related to the stats export.""" - -from sparv.api import (AllSourceFilenames, Annotation, AnnotationAllSourceFiles, Config, Export, ExportInput, Output, - OutputMarker, annotator, exporter, get_logger, installer, util) - +import os +from typing import Optional + +from sparv.api import ( + AllSourceFilenames, + Annotation, + AnnotationAllSourceFiles, + Config, + Corpus, + Export, + ExportInput, + MarkerOptional, + Output, + OutputMarker, + annotator, + exporter, + get_logger, + installer, + uninstaller, + util +) from .stats_export import freq_list logger = get_logger(__name__) @@ -208,23 +225,62 @@ def sbx_freq_list_fsv( out=out, sparv_namespace="", source_namespace="", delimiter=delimiter, cutoff=cutoff) -@installer("Install SBX word frequency list on remote host") +@installer("Install SBX word frequency list on remote host", uninstaller="stats_export:uninstall_sbx_freq_list") def install_sbx_freq_list( freq_list: ExportInput = ExportInput("stats_export.frequency_list_sbx/stats_[metadata.id].csv"), - out: OutputMarker = OutputMarker("stats_export.install_sbx_freq_list_marker"), - host: str = Config("stats_export.remote_host"), - target_dir: str = Config("stats_export.remote_dir")): + marker: OutputMarker = OutputMarker("stats_export.install_sbx_freq_list_marker"), + uninstall_marker: MarkerOptional = MarkerOptional("stats_export.uninstall_sbx_freq_list_marker"), + host: Optional[str] = Config("stats_export.remote_host"), + target_dir: str = Config("stats_export.remote_dir") +): """Install frequency list on server by rsyncing.""" util.install.install_path(freq_list, host, target_dir) - out.write() + uninstall_marker.remove() + marker.write() -@installer("Install SBX word frequency list with dates on remote host") +@installer("Install SBX word frequency list with dates on remote host", + uninstaller="stats_export:uninstall_sbx_freq_list_date") def install_sbx_freq_list_date( freq_list: ExportInput = ExportInput("stats_export.frequency_list_sbx_date/stats_[metadata.id].csv"), - out: OutputMarker = OutputMarker("stats_export.install_sbx_freq_list_date_marker"), - host: str = Config("stats_export.remote_host"), - target_dir: str = Config("stats_export.remote_dir")): + marker: OutputMarker = OutputMarker("stats_export.install_sbx_freq_list_date_marker"), + uninstall_marker: MarkerOptional = MarkerOptional("stats_export.uninstall_sbx_freq_list_date_marker"), + host: Optional[str] = Config("stats_export.remote_host"), + target_dir: str = Config("stats_export.remote_dir") +): """Install frequency list on server by rsyncing.""" util.install.install_path(freq_list, host, target_dir) - out.write() + uninstall_marker.remove() + marker.write() + + +@uninstaller("Uninstall SBX word frequency list") +def uninstall_sbx_freq_list( + corpus_id: Corpus = Corpus(), + marker: OutputMarker = OutputMarker("stats_export.uninstall_sbx_freq_list_marker"), + install_marker: MarkerOptional = MarkerOptional("stats_export.install_sbx_freq_list_marker"), + host: Optional[str] = Config("stats_export.remote_host"), + remote_dir: str = Config("stats_export.remote_dir") +): + """Uninstall SBX word frequency list.""" + remote_file = os.path.join(remote_dir, f"stats_{corpus_id}.csv") + logger.info(f"Removing SBX word frequency file {host + ':' if host else ''}{remote_file}") + util.install.uninstall_path(remote_file, host) + install_marker.remove() + marker.write() + + +@uninstaller("Uninstall SBX word frequency list with dates") +def uninstall_sbx_freq_list_date( + corpus_id: Corpus = Corpus(), + marker: OutputMarker = OutputMarker("stats_export.uninstall_sbx_freq_list_date_marker"), + install_marker: MarkerOptional = MarkerOptional("stats_export.install_sbx_freq_list_date_marker"), + host: Optional[str] = Config("stats_export.remote_host"), + remote_dir: str = Config("stats_export.remote_dir") +): + """Uninstall SBX word frequency list with dates.""" + remote_file = os.path.join(remote_dir, f"stats_{corpus_id}.csv") + logger.info(f"Removing SBX word frequency with dates file {host + ':' if host else ''}{remote_file}") + util.install.uninstall_path(remote_file, host) + install_marker.remove() + marker.write() diff --git a/sparv/modules/stats_export/stats_export.py b/sparv/modules/stats_export/stats_export.py index 7a144b09..e5b46343 100644 --- a/sparv/modules/stats_export/stats_export.py +++ b/sparv/modules/stats_export/stats_export.py @@ -1,11 +1,28 @@ """Build word frequency list.""" import csv +import os.path from collections import defaultdict - -from sparv.api import (AllSourceFilenames, Annotation, AnnotationAllSourceFiles, Config, Export, - ExportAnnotationsAllSourceFiles, ExportInput, OutputMarker, SourceAnnotationsAllSourceFiles, - exporter, get_logger, installer, util) +from typing import Optional + +from sparv.api import ( + AllSourceFilenames, + Annotation, + AnnotationAllSourceFiles, + Config, + Corpus, + Export, + ExportAnnotationsAllSourceFiles, + ExportInput, + MarkerOptional, + OutputMarker, + SourceAnnotationsAllSourceFiles, + exporter, + get_logger, + installer, + uninstaller, + util +) logger = get_logger(__name__) @@ -81,14 +98,34 @@ def freq_list(source_files: AllSourceFilenames = AllSourceFilenames(), write_csv(out, column_names, freq_dict, delimiter, cutoff) -@installer("Install word frequency list on remote host") -def install_freq_list(freq_list: ExportInput = ExportInput("stats_export.frequency_list/stats_[metadata.id].csv"), - out: OutputMarker = OutputMarker("stats_export.install_freq_list_marker"), - host: str = Config("stats_export.remote_host"), - target_dir: str = Config("stats_export.remote_dir")): +@installer("Install word frequency list on remote host", uninstaller="stats_export:uninstall_freq_list") +def install_freq_list( + freq_list: ExportInput = ExportInput("stats_export.frequency_list/stats_[metadata.id].csv"), + marker: OutputMarker = OutputMarker("stats_export.install_freq_list_marker"), + uninstall_marker: MarkerOptional = MarkerOptional("stats_export.uninstall_freq_list_marker"), + host: Optional[str] = Config("stats_export.remote_host"), + target_dir: str = Config("stats_export.remote_dir") +): """Install frequency list on server by rsyncing.""" util.install.install_path(freq_list, host, target_dir) - out.write() + uninstall_marker.remove() + marker.write() + + +@uninstaller("Uninstall word frequency list") +def uninstall_freq_list( + corpus_id: Corpus = Corpus(), + marker: OutputMarker = OutputMarker("stats_export.uninstall_freq_list_marker"), + install_marker: MarkerOptional = MarkerOptional("stats_export.install_freq_list_marker"), + host: Optional[str] = Config("stats_export.remote_host"), + remote_dir: str = Config("stats_export.remote_dir") +): + """Uninstall word frequency list.""" + remote_file = os.path.join(remote_dir, f"stats_{corpus_id}.csv") + logger.info(f"Removing word frequency file {host + ':' if host else ''}{remote_file}") + util.install.uninstall_path(remote_file, host) + install_marker.remove() + marker.write() ################################################################################ From 0ebe834cddc66cbe9f3e9537ff4323a287051a37 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Wed, 21 Dec 2022 16:42:30 +0100 Subject: [PATCH 045/175] Fix bug where Sparv would hang if an error occurred in a preloaded annotator --- CHANGELOG.md | 1 + sparv/core/preload.py | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd048f6c..eb582f8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ ### Fixed - Several bugs fixed in `korp:config`. +- Fixed bug where Sparv would hang if an error occurred in a preloaded annotator. ## [5.1.0] - 2022-11-03 diff --git a/sparv/core/preload.py b/sparv/core/preload.py index da0e5247..0c20bc78 100644 --- a/sparv/core/preload.py +++ b/sparv/core/preload.py @@ -179,10 +179,10 @@ def handle(client_sock, annotators: Dict[str, Preloader]): console.print_exception() send_data(client_sock, e) return - - # Clear log handlers - logger = logging.getLogger("sparv") - logger.handlers.clear() + finally: + # Clear log handlers + logger = logging.getLogger("sparv") + logger.handlers.clear() log.info("Done") From eea3f1eddf274529361a957724d11331985aba7d Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Thu, 22 Dec 2022 16:28:15 +0100 Subject: [PATCH 046/175] Mark some arguments as optional --- sparv/modules/hist/hist.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sparv/modules/hist/hist.py b/sparv/modules/hist/hist.py index 25befbd3..d001cf26 100644 --- a/sparv/modules/hist/hist.py +++ b/sparv/modules/hist/hist.py @@ -27,13 +27,13 @@ def annotate_saldo( msd: Optional[Annotation] = Annotation(""), delimiter: str = Config("hist.delimiter"), affix: str = Config("hist.affix"), - precision: str = Config("saldo.precision"), + precision: Optional[str] = Config("saldo.precision"), precision_filter: str = Config("saldo.precision_filter"), min_precision: float = Config("saldo.min_precision"), skip_multiword: bool = Config("saldo.skip_multiword"), max_gaps: int = Config("hist.max_mwe_gaps"), allow_multiword_overlap: bool = Config("saldo.allow_multiword_overlap"), - word_separator: str = Config("saldo.word_separator"), + word_separator: Optional[str] = Config("saldo.word_separator"), models_preloaded: Optional[dict] = None): """Use lexicon models (SALDO, Dalin and Swedberg) to annotate (potentially msd-tagged) words.""" saldo.main(token=token, word=word, sentence=sentence, reference=reference, out_sense=out_sense, @@ -61,7 +61,7 @@ def annotate_saldo_fsv( models: List[Model] = [Model("[hist.fsv_model]")], delimiter: str = Config("hist.delimiter"), affix: str = Config("hist.affix"), - precision: str = Config("saldo.precision"), + precision: Optional[str] = Config("saldo.precision"), precision_filter: str = Config("saldo.precision_filter"), min_precision: float = Config("hist.fsv_min_precision"), skip_multiword: bool = Config("saldo.skip_multiword"), From e06a91a3cb7bb70d8f6ec26b1cb0614cfc9304cf Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Thu, 22 Dec 2022 16:44:27 +0100 Subject: [PATCH 047/175] Remove any existing CWB encoded files before encoding (fixes #169) --- CHANGELOG.md | 1 + sparv/modules/cwb/cwb.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb582f8b..ec782b95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ - Several bugs fixed in `korp:config`. - Fixed bug where Sparv would hang if an error occurred in a preloaded annotator. +- Fixed occasional crash in `cwb:encode` when old CWB export hadn't been removed first. ## [5.1.0] - 2022-11-03 diff --git a/sparv/modules/cwb/cwb.py b/sparv/modules/cwb/cwb.py index 7dd66b81..257f15cf 100644 --- a/sparv/modules/cwb/cwb.py +++ b/sparv/modules/cwb/cwb.py @@ -207,6 +207,11 @@ def cwb_encode(corpus, annotations, source_annotations, source_files, words, vrt data_dir.mkdir(exist_ok=True) registry_dir.mkdir(exist_ok=True) + # Remove any existing files in data dir except for the .info file + for f in data_dir.glob("*"): + if not f.name == ".info": + f.unlink() + encode_args = ["-s", "-p", "-", "-d", data_dir, "-R", registry_file, From 66a7c5b4ab2ef986a4193d445d5a56c0cb74471a Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Wed, 4 Jan 2023 14:35:31 +0100 Subject: [PATCH 048/175] Fix small bug in korp:config --- sparv/modules/korp/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sparv/modules/korp/config.py b/sparv/modules/korp/config.py index a58fd62e..3ebf2589 100644 --- a/sparv/modules/korp/config.py +++ b/sparv/modules/korp/config.py @@ -149,7 +149,7 @@ def config(id: Corpus = Corpus(), out: YAML file to create. """ config_dict = { - "id": id, + "id": str(id), "lang": language, "mode": modes } From 447eeec49281a477bcdc3667fe7e038598381a27 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Thu, 12 Jan 2023 16:59:56 +0100 Subject: [PATCH 049/175] Fix using relative socket path together with --dir (fixes #170) --- CHANGELOG.md | 1 + sparv/__main__.py | 18 ++++++++++++------ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec782b95..d38b2672 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ - Several bugs fixed in `korp:config`. - Fixed bug where Sparv would hang if an error occurred in a preloaded annotator. - Fixed occasional crash in `cwb:encode` when old CWB export hadn't been removed first. +- Fixed bug when using relative socket path while also using `--dir`. ## [5.1.0] - 2022-11-03 diff --git a/sparv/__main__.py b/sparv/__main__.py index 819f1a4f..07be30c5 100644 --- a/sparv/__main__.py +++ b/sparv/__main__.py @@ -300,7 +300,7 @@ def main(): if getattr(args, t): config["types"].append(t) elif args.command == "preload": - config["socket"] = args.socket + config["socket"] = str(Path(args.socket).resolve()) config["preloader"] = True config["processes"] = args.processes config["preload_command"] = args.preload_command @@ -391,18 +391,24 @@ def main(): log_level = args.log or "warning" log_file_level = args.log_to_file or "warning" simple_mode = args.simple + socket = args.socket + + if socket: + # Convert to absolute path, to work together with --dir + socket_path = Path(socket).resolve() + if not socket_path.is_socket(): + print(f"Socket file '{socket}' doesn't exist or isn't a socket.") + sys.exit(1) + socket = str(socket_path) + config.update({"debug": args.debug, "file": vars(args).get("file", []), "log_level": log_level, "log_file_level": log_file_level, - "socket": args.socket, + "socket": socket, "force_preloader": args.force_preloader, "targets": snakemake_args["targets"], "threads": args.cores}) - # If using socket, make sure that socket file exists - if args.socket and not Path(args.socket).is_socket(): - print(f"Socket file '{args.socket}' doesn't exist or isn't a socket.") - sys.exit(1) if simple_target: # Force Snakemake to use threads to prevent unnecessary processes for simple targets From a2f2bb7262138bc8d13bf1607a020b7ea8056f5b Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Mon, 16 Jan 2023 13:28:44 +0100 Subject: [PATCH 050/175] Handle rare logging error --- sparv/core/log_handler.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/sparv/core/log_handler.py b/sparv/core/log_handler.py index 2e8234b5..7bc32e46 100644 --- a/sparv/core/log_handler.py +++ b/sparv/core/log_handler.py @@ -211,12 +211,15 @@ def get_renderables(self): elapsed = str(timedelta(seconds=round(time.time() - task["starttime"]))) if len(elapsed) > elapsed_max_len: elapsed_max_len = len(elapsed) - rows.append(( - task["name"], - f"[dim]{task['file']}[/dim]", - bar_col(self._tasks[task["task"]]) if task["task"] else "", - elapsed - )) + try: + rows.append(( + task["name"], + f"[dim]{task['file']}[/dim]", + bar_col(self._tasks[task["task"]]) if task["task"] else "", + elapsed + )) + except KeyError: # May happen if self._tasks has changed + pass table = Table(show_header=False, box=box.SIMPLE, expand=True) table.add_column("Task", no_wrap=True, min_width=self.task_max_len + 2, ratio=1) From 60188a13d60539fb0c07189ad1f28955502aedfb Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Tue, 17 Jan 2023 16:13:45 +0100 Subject: [PATCH 051/175] Add util.system.gpus() --- CHANGELOG.md | 1 + docs/developers-guide/utilities.md | 6 ++++++ sparv/api/util/system.py | 16 +++++++++++++++- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d38b2672..9d7d0778 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ - Added `MarkerOptional` class. - Added stats export for Swedish from the 1800s. - `korp:wordpicture` table name is now configurable using `korp.wordpicture_table`. +- Added utility function `util.system.gpus()` which returns a list of GPUs, ordered by free memory in descending order. ### Changed diff --git a/docs/developers-guide/utilities.md b/docs/developers-guide/utilities.md index 78d8cdb7..0a0784c7 100644 --- a/docs/developers-guide/utilities.md +++ b/docs/developers-guide/utilities.md @@ -196,6 +196,12 @@ Search for the binary for a program. Returns the path to binary, or `None` if no - `raise_error`: If set to `True` raises error if binary could not be found. Defaults to `False`. +### gpus() +Returns a list of available GPUs, ordered by free memory in descending order. Returns `None` if it fails. +Currently only works for NVIDIA GPUs, and requires the `nvidia-smi` utility to be installed. +Takes no arguments. + + ### kill_process() Kill a process, and ignore the error if it is already dead. diff --git a/sparv/api/util/system.py b/sparv/api/util/system.py index 030e7593..7366c255 100644 --- a/sparv/api/util/system.py +++ b/sparv/api/util/system.py @@ -6,7 +6,7 @@ import shutil import subprocess from pathlib import Path -from typing import Optional, Union +from typing import List, Optional, Union import sparv.core.paths as paths from sparv.api import get_logger, SparvErrorMessage @@ -184,3 +184,17 @@ def remove_path(path: Union[str, Path], host: Optional[str] = None): p.unlink() elif p.is_dir(): shutil.rmtree(p) + + +def gpus() -> Optional[List[int]]: + """Return a list of available GPUs, sorted by free memory in descending order. + + Returns None on failure.""" + + try: + cmd = ["nvidia-smi", "--query-gpu=memory.free", "--format=csv"] + memory_info = subprocess.check_output(cmd).decode().splitlines()[1:] + memory = sorted(((int(free.split()[0]), i) for i, free in enumerate(memory_info)), reverse=True) + return [i[1] for i in memory] + except: + return None From 4bb7d14a9bf7451a599f89ae5c18e048868a1350 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Tue, 17 Jan 2023 16:16:48 +0100 Subject: [PATCH 052/175] Automatically order GPUs in the environment variable CUDA_VISIBLE_DEVICES by amount of free memory --- CHANGELOG.md | 2 ++ sparv/core/Snakefile | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d7d0778..82af2fdd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,8 @@ - Added stats export for Swedish from the 1800s. - `korp:wordpicture` table name is now configurable using `korp.wordpicture_table`. - Added utility function `util.system.gpus()` which returns a list of GPUs, ordered by free memory in descending order. +- Sparv will automatically order the GPUs in the environment variable `CUDA_VISIBLE_DEVICES` by the amount of free + memory that was available when Sparv started. ### Changed diff --git a/sparv/core/Snakefile b/sparv/core/Snakefile index 74e1b822..b57cf6d3 100644 --- a/sparv/core/Snakefile +++ b/sparv/core/Snakefile @@ -1,4 +1,5 @@ """Snakefile used by Snakemake.""" +import os from pathlib import Path from rich import box @@ -199,6 +200,11 @@ if not config_missing and selected_targets and not snake_storage.source_files: if set(selected_targets).issubset(named_targets): raise SparvErrorMessage("No source files available!") +# Sort GPUs visible to modules by available memory +gpus = util.system.gpus() +if gpus: + os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(map(str, gpus)) + # ============================================================================== # Static Snakemake Rules # ============================================================================== From c491f8bc816e544208849653d1a1db9643160c74 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Tue, 17 Jan 2023 16:17:48 +0100 Subject: [PATCH 053/175] Make Stanza select the GPU with the most free memory --- CHANGELOG.md | 1 + sparv/modules/stanza/stanza.py | 10 ++++++++++ sparv/modules/stanza/stanza_swe.py | 10 ++++++++++ 3 files changed, 21 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 82af2fdd..cfc93fd8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ - Added utility function `util.system.gpus()` which returns a list of GPUs, ordered by free memory in descending order. - Sparv will automatically order the GPUs in the environment variable `CUDA_VISIBLE_DEVICES` by the amount of free memory that was available when Sparv started. +- Stanza now always selects the GPU with the most free memory. ### Changed diff --git a/sparv/modules/stanza/stanza.py b/sparv/modules/stanza/stanza.py index 08acb1d2..e641dc96 100644 --- a/sparv/modules/stanza/stanza.py +++ b/sparv/modules/stanza/stanza.py @@ -41,11 +41,21 @@ def annotate(corpus_text: Text = Text(), max_sentence_length: int = Config("stanza.max_sentence_length"), cpu_fallback: bool = Config("stanza.cpu_fallback")): """Do dependency parsing using Stanza.""" + import torch from stanza.pipeline.core import DownloadMethod # cpu_fallback only makes sense if use_gpu is True cpu_fallback = cpu_fallback and use_gpu + # Select the GPU with most free memory available + try: + if use_gpu: + gpus = util.system.gpus() + if gpus: + torch.cuda.set_device(gpus[0]) + except: + pass + # Read corpus_text and text_spans text_data = corpus_text.read() diff --git a/sparv/modules/stanza/stanza_swe.py b/sparv/modules/stanza/stanza_swe.py index eebee249..51e94b9e 100644 --- a/sparv/modules/stanza/stanza_swe.py +++ b/sparv/modules/stanza/stanza_swe.py @@ -37,11 +37,21 @@ def annotate_swe( max_token_length: int = Config("stanza.max_token_length")): """Do dependency parsing using Stanza.""" import stanza + import torch from stanza.pipeline.core import DownloadMethod # cpu_fallback only makes sense if use_gpu is True cpu_fallback = cpu_fallback and use_gpu + # Select the GPU with most free memory available + try: + if use_gpu: + gpus = util.system.gpus() + if gpus: + torch.cuda.set_device(gpus[0]) + except: + pass + sentences_all, orphans = sentence.get_children(token) if orphans: logger.warning(f"Found {len(orphans)} tokens not belonging to any sentence. These will not be annotated with " From 05194bff2b7c6e18088a426eb0189938690d15fe Mon Sep 17 00:00:00 2001 From: Anne Schumacher Date: Tue, 24 Jan 2023 17:01:51 +0100 Subject: [PATCH 054/175] make order in stats files predictable (should fix issue described in PR #174) --- sparv/modules/stats_export/stats_export.py | 2 +- .../stats_freeling-eng-slevel.csv | 2 +- .../stats_export.frequency_list_sbx/stats_freeling-fra-txt.csv | 2 +- .../stats_export.frequency_list_sbx/stats_mini-swe.csv | 2 +- .../stats_export.frequency_list/stats_standard-swe.csv | 2 +- .../stats_export.frequency_list_sbx/stats_stanford-eng.csv | 2 +- .../stats_export.frequency_list_sbx/stats_swe-1800.csv | 2 +- .../stats_export.frequency_list_sbx/stats_swe-fsv.csv | 2 +- .../stats_export.frequency_list_sbx/stats_treetagger-nld.csv | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/sparv/modules/stats_export/stats_export.py b/sparv/modules/stats_export/stats_export.py index e5b46343..10d63b2a 100644 --- a/sparv/modules/stats_export/stats_export.py +++ b/sparv/modules/stats_export/stats_export.py @@ -137,7 +137,7 @@ def write_csv(out, column_names, freq_dict, delimiter, cutoff): with open(out, "w", encoding="utf-8") as csvfile: csv_writer = csv.writer(csvfile, delimiter=delimiter) csv_writer.writerow(column_names) - for annotations, freq in sorted(freq_dict.items(), key=lambda x: -x[1]): + for annotations, freq in sorted(freq_dict.items(), key=lambda x: (-x[1], x[0])): if cutoff and cutoff > freq: break csv_writer.writerow(list(annotations) + [freq]) diff --git a/tests/test_corpora/freeling-eng-slevel/gold_export/stats_export.frequency_list_sbx/stats_freeling-eng-slevel.csv b/tests/test_corpora/freeling-eng-slevel/gold_export/stats_export.frequency_list_sbx/stats_freeling-eng-slevel.csv index cf3c658c..2aebd8b3 100644 --- a/tests/test_corpora/freeling-eng-slevel/gold_export/stats_export.frequency_list_sbx/stats_freeling-eng-slevel.csv +++ b/tests/test_corpora/freeling-eng-slevel/gold_export/stats_export.frequency_list_sbx/stats_freeling-eng-slevel.csv @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9a3e7f92b017e2274af2586b8aa25c86472704353758216abc3d057369625b3c +oid sha256:cda08085dc7769c8e54ebf797859118236a58d288e8d0c5e7ab6924be1fdd176 size 1290 diff --git a/tests/test_corpora/freeling-fra-txt/gold_export/stats_export.frequency_list_sbx/stats_freeling-fra-txt.csv b/tests/test_corpora/freeling-fra-txt/gold_export/stats_export.frequency_list_sbx/stats_freeling-fra-txt.csv index bd137c3f..532840e2 100644 --- a/tests/test_corpora/freeling-fra-txt/gold_export/stats_export.frequency_list_sbx/stats_freeling-fra-txt.csv +++ b/tests/test_corpora/freeling-fra-txt/gold_export/stats_export.frequency_list_sbx/stats_freeling-fra-txt.csv @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8a25bb68b3ef3ca9831611b1d03cf05044720bcc51d669c6fe5fd5d17c4e01e3 +oid sha256:4641b24cdea28008d5b82ec75e90089444c996b54184f930f636ff995119752d size 5312 diff --git a/tests/test_corpora/mini-swe/gold_export/stats_export.frequency_list_sbx/stats_mini-swe.csv b/tests/test_corpora/mini-swe/gold_export/stats_export.frequency_list_sbx/stats_mini-swe.csv index 6fe61151..f8a4aaba 100644 --- a/tests/test_corpora/mini-swe/gold_export/stats_export.frequency_list_sbx/stats_mini-swe.csv +++ b/tests/test_corpora/mini-swe/gold_export/stats_export.frequency_list_sbx/stats_mini-swe.csv @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:eb94e774fd9f8bc648ba72af240a354bc5cca8ca29b05b739f7c6342beb8187c +oid sha256:332464be753f8b7d0e37c983059a937d1cd3c181e4c865eecd50bbcf66b31dfa size 2826 diff --git a/tests/test_corpora/standard-swe/gold_export/stats_export.frequency_list/stats_standard-swe.csv b/tests/test_corpora/standard-swe/gold_export/stats_export.frequency_list/stats_standard-swe.csv index 08593243..3508163d 100644 --- a/tests/test_corpora/standard-swe/gold_export/stats_export.frequency_list/stats_standard-swe.csv +++ b/tests/test_corpora/standard-swe/gold_export/stats_export.frequency_list/stats_standard-swe.csv @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e4ffec4e1f3151749d722e50a8c482b3a48530058d3f87266a5b7a80eb02a7df +oid sha256:8e9f0e0c509a5eabb199e4296256cb04c34a9024d38edec3ede8d31efcf72655 size 7898 diff --git a/tests/test_corpora/stanford-eng/gold_export/stats_export.frequency_list_sbx/stats_stanford-eng.csv b/tests/test_corpora/stanford-eng/gold_export/stats_export.frequency_list_sbx/stats_stanford-eng.csv index 2971b9f7..14efdfcb 100644 --- a/tests/test_corpora/stanford-eng/gold_export/stats_export.frequency_list_sbx/stats_stanford-eng.csv +++ b/tests/test_corpora/stanford-eng/gold_export/stats_export.frequency_list_sbx/stats_stanford-eng.csv @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f5e2fe8f6d57cfd1901f8bea98c80ffdf93e284c42b063b5105152d105b10f06 +oid sha256:8abaed5aa84186d32362c6e401e30bd6c9d362747fb9d8a608079034729c8699 size 4920 diff --git a/tests/test_corpora/swe-1800/gold_export/stats_export.frequency_list_sbx/stats_swe-1800.csv b/tests/test_corpora/swe-1800/gold_export/stats_export.frequency_list_sbx/stats_swe-1800.csv index 85d85b5d..551d6139 100644 --- a/tests/test_corpora/swe-1800/gold_export/stats_export.frequency_list_sbx/stats_swe-1800.csv +++ b/tests/test_corpora/swe-1800/gold_export/stats_export.frequency_list_sbx/stats_swe-1800.csv @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:68d625518ffa859fa5fa33296af1886eddaf369dadb24e68081c85fa4cd4e792 +oid sha256:972a45957c6f75887d5801befc19f26a042c9055becb8d90576467d5331af69d size 2951 diff --git a/tests/test_corpora/swe-fsv/gold_export/stats_export.frequency_list_sbx/stats_swe-fsv.csv b/tests/test_corpora/swe-fsv/gold_export/stats_export.frequency_list_sbx/stats_swe-fsv.csv index 3739da9b..184fce0b 100644 --- a/tests/test_corpora/swe-fsv/gold_export/stats_export.frequency_list_sbx/stats_swe-fsv.csv +++ b/tests/test_corpora/swe-fsv/gold_export/stats_export.frequency_list_sbx/stats_swe-fsv.csv @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8b3437607d8967b50fb5e24028cd61257e36ed80b0f5b45b77568d5a8651bb3c +oid sha256:2ed571fa653ca0ba822a694345f0332656ca932b80285cd91c7a3f0a58fcb698 size 10655 diff --git a/tests/test_corpora/treetagger-nld/gold_export/stats_export.frequency_list_sbx/stats_treetagger-nld.csv b/tests/test_corpora/treetagger-nld/gold_export/stats_export.frequency_list_sbx/stats_treetagger-nld.csv index d7a58da5..bd03dc60 100644 --- a/tests/test_corpora/treetagger-nld/gold_export/stats_export.frequency_list_sbx/stats_treetagger-nld.csv +++ b/tests/test_corpora/treetagger-nld/gold_export/stats_export.frequency_list_sbx/stats_treetagger-nld.csv @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1d0095e10ad9d407308a9edfb70566eb22ae48730b171d39303881ae69f0abaf +oid sha256:c90546b2668ced3b7969f6fcb367e0865b75e5e8ad0fa484382bade9fc773742 size 1855 From 5ff17ed7caf843bf5bd3bd98f06cd38fd85df2d9 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Wed, 25 Jan 2023 13:50:46 +0100 Subject: [PATCH 055/175] Don't use hardcoded reading mode annotations in korp:config No longer needed by the frontend. --- sparv/modules/korp/config.py | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/sparv/modules/korp/config.py b/sparv/modules/korp/config.py index 3ebf2589..21bf5b43 100644 --- a/sparv/modules/korp/config.py +++ b/sparv/modules/korp/config.py @@ -39,13 +39,8 @@ ":stanza.msd_hunpos_backoff_info" ) -# Annotations needed by reading mode (using export names) -READING_MODE_ANNOTATIONS = ( - "text:_id", - "sentence:id", - "_head", - "_tail" -) +# Annotations (using export names) to always include (if they exist), that would normally be excluded +INCLUDED_ANNOTATIONS = () LABELS = { "sentence": { @@ -256,8 +251,10 @@ def config(id: Corpus = Corpus(), # Annotations presets = get_presets(remote_host, config_dir) - token_annotations, struct_annotations, _ = build_annotations(annotation_definitions, annotation_list, export_names, - hidden_annotations, presets, reading_mode, token) + token_annotations, struct_annotations, _ = build_annotations( + annotation_definitions, annotation_list, export_names, hidden_annotations, presets, INCLUDED_ANNOTATIONS, + token + ) config_dict["struct_attributes"] = struct_annotations config_dict["pos_attributes"] = token_annotations @@ -273,7 +270,7 @@ def config(id: Corpus = Corpus(), out_yaml.write(dict_to_yaml(config_dict)) -def build_annotations(annotation_definitions, annotation_list, export_names, hidden_annotations, presets, reading_mode, +def build_annotations(annotation_definitions, annotation_list, export_names, hidden_annotations, presets, include, token, text_annotation=None, cwb_annotations: bool = True): token_annotations = [] struct_annotations = [] @@ -284,9 +281,7 @@ def build_annotations(annotation_definitions, annotation_list, export_names, hid export_name = export_names.get(annotation.name, annotation.name) # Skip certain annotations unless explicitly listed in annotation_definitions if (annotation.name in hidden_annotation_names or annotation.attribute_name is None or export_name.split(":", 1)[ - -1].startswith("_")) and annotation.name not in annotation_definitions and not ( - reading_mode and export_name in READING_MODE_ANNOTATIONS - ): + -1].startswith("_")) and annotation.name not in annotation_definitions and not export_name in include: logger.debug(f"Skipping annotation {annotation.name!r}") continue export_name_cwb = cwb_escape(export_name.replace(":", "_")) From 04e081d90e367eab9b907960803d7d875adadbc0 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Tue, 31 Jan 2023 11:25:05 +0100 Subject: [PATCH 056/175] Update link to CWB in documentation --- docs/user-manual/installation-and-setup.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user-manual/installation-and-setup.md b/docs/user-manual/installation-and-setup.md index a57bad5c..89bf0dbf 100644 --- a/docs/user-manual/installation-and-setup.md +++ b/docs/user-manual/installation-and-setup.md @@ -153,7 +153,7 @@ Download and unpack the zip-file from the [MaltParser webpage](http://www.maltpa | | | |:---|:----------| |**Purpose** |Creating Corpus Workbench binary files. Only needed if you want to be able to search corpora with this tool. -|**Download** |[Corpus Workbench on SourceForge](https://cwb.sourceforge.io/download.php) +|**Download** |[Corpus Workbench on SourceForge](https://cwb.sourceforge.io/install.php) |**License** |[GPL-3.0](https://www.gnu.org/licenses/gpl-3.0.html) |**Version compatible with Sparv** |beta 3.4.21 (most likely works with newer versions) From 55fc5fe4644a5586ac3a0e3b4b66d524c5e31e92 Mon Sep 17 00:00:00 2001 From: Anne Schumacher Date: Wed, 1 Feb 2023 10:39:36 +0100 Subject: [PATCH 057/175] add missing flag in documentation for 'build-models' command --- docs/user-manual/installation-and-setup.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/user-manual/installation-and-setup.md b/docs/user-manual/installation-and-setup.md index 89bf0dbf..5bce9761 100644 --- a/docs/user-manual/installation-and-setup.md +++ b/docs/user-manual/installation-and-setup.md @@ -50,12 +50,12 @@ where to set up its data directory. This will also populate the data directory w If you like, you can pre-build the model files. This step is optional, and the only advantage is that annotating your first corpus will be quicker since all the models are already set up. If you skip this step, models will be downloaded and built automatically on demand when annotating your first corpus. Pre-building models can be done by using the -command `sparv build-models`. If you do this in a directory where there is no +command `sparv build-models --all`. If you do this in a directory where there is no [corpus config](user-manual/corpus-configuration.md) you have to tell Sparv what language the models should be built for (otherwise the language of the corpus config is used). The language is provided as a three-letter code with the `--language` flag (use the `sparv languages` command for a list of available languages and their codes). For example, if you would like to build all the Swedish models you -can run `sparv build-models --language swe`. +can run `sparv build-models --all --language swe`. ## Installing Additional Third-party Software The Sparv Pipeline can be used together with several plugins and third-party software. Installation of the software From 4dafb0c23d685aa66a2d925eae53a64d003b4165 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Tue, 31 Jan 2023 15:14:39 +0100 Subject: [PATCH 058/175] Fix escaping of annotation names in korp:config --- sparv/modules/korp/config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sparv/modules/korp/config.py b/sparv/modules/korp/config.py index 21bf5b43..28a4a867 100644 --- a/sparv/modules/korp/config.py +++ b/sparv/modules/korp/config.py @@ -214,6 +214,7 @@ def config(id: Corpus = Corpus(), config_dict["within"] = [] for v in within: if isinstance(v, str): + v = cwb_escape(v) n = 0 if " " in v: n, _, v = v.partition(" ") @@ -233,6 +234,7 @@ def config(id: Corpus = Corpus(), config_dict["context"] = [] for v in context: if isinstance(v, str): + v = cwb_escape(v) n = 1 if " " in v: n, _, v = v.partition(" ") From 054c6fbaf971a497e926c05d1ab1b4ad527e3838 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Wed, 1 Feb 2023 16:52:01 +0100 Subject: [PATCH 059/175] Fix korp:config not handling nonlocalized descriptions --- sparv/modules/korp/config.py | 37 ++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/sparv/modules/korp/config.py b/sparv/modules/korp/config.py index 28a4a867..505b3295 100644 --- a/sparv/modules/korp/config.py +++ b/sparv/modules/korp/config.py @@ -1,8 +1,9 @@ """Create configuration files for the Korp backend and frontend.""" import itertools import subprocess +from collections import defaultdict from pathlib import Path -from typing import Optional, Union, List +from typing import Dict, List, Optional, Union import yaml @@ -352,23 +353,35 @@ def get_presets(remote_host, config_dir): return presets -def build_description(description, short_description): +def build_description(description, short_description) -> Union[Dict[str, str], str]: """Combine description and short_description if they exist.""" + if isinstance(description, dict): + description_dd = defaultdict(lambda: None, description) + else: + description_dd = defaultdict(lambda: description if description is None else str(description)) + + if isinstance(short_description, dict): + short_description_dd = defaultdict(lambda: None, short_description) + else: + short_description_dd = defaultdict( + lambda: short_description if short_description is None else str(short_description) + ) + lang_dict = {} - description = description or {} - short_description = short_description or {} - langs = set(list(description.keys()) + list(short_description.keys())) + langs = set(list(description_dd.keys()) + list(short_description_dd.keys())) or {None} + for lang in langs: descr = None - if short_description.get(lang) and description.get(lang): - descr = f"{short_description.get(lang)}

{description.get(lang)}" - elif description.get(lang): - descr = description.get(lang) - elif short_description.get(lang): - descr = short_description.get(lang) + if short_description_dd[lang] and description_dd[lang]: + descr = f"{short_description_dd[lang]}

{description_dd[lang]}" + elif description_dd[lang]: + descr = str(description_dd[lang]) + elif short_description_dd[lang]: + descr = str(short_description_dd[lang]) if descr: lang_dict[lang] = descr - return lang_dict + + return lang_dict.get(None) or lang_dict @installer("Install Korp corpus configuration file.", uninstaller="korp:uninstall_config") From 590f93163c21c5c9bbc66b3771316db876d593f1 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Wed, 1 Feb 2023 18:11:14 +0100 Subject: [PATCH 060/175] Fix a bug where Sparv would mistakenly think config variables weren't set --- sparv/core/snake_utils.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/sparv/core/snake_utils.py b/sparv/core/snake_utils.py index df1d910f..40a4508d 100644 --- a/sparv/core/snake_utils.py +++ b/sparv/core/snake_utils.py @@ -520,10 +520,12 @@ def rule_helper(rule: RuleStorage, config: dict, storage: SnakeStorage, config_m rule.export_dirs = [str(p / "_")[:-1] for p in output_dirs] if rule.missing_config: - log_handler.messages["missing_configs"][rule.full_name].update( - [c for c in rule.missing_config if not c.startswith("<")]) - log_handler.messages["missing_classes"][rule.full_name].update( - [c[1:-1] for c in rule.missing_config if c.startswith("<")]) + missing_config = [c for c in rule.missing_config if not c.startswith("<")] + if missing_config: + log_handler.messages["missing_configs"][rule.full_name].update(missing_config) + missing_classes = [c[1:-1] for c in rule.missing_config if c.startswith("<")] + if missing_classes: + log_handler.messages["missing_classes"][rule.full_name].update(missing_classes) if rule.missing_binaries: log_handler.messages["missing_binaries"][rule.full_name].update(rule.missing_binaries) From 6a1791bcc5a187c58c1a9213a8f198a4e585d3f0 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Thu, 2 Feb 2023 14:21:38 +0100 Subject: [PATCH 061/175] Fix quoting of paths in util.system.rsync --- CHANGELOG.md | 1 + sparv/api/util/system.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cfc93fd8..9715c631 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ - Fixed bug where Sparv would hang if an error occurred in a preloaded annotator. - Fixed occasional crash in `cwb:encode` when old CWB export hadn't been removed first. - Fixed bug when using relative socket path while also using `--dir`. +- Fix quoting of paths in `util.system.rsync`. ## [5.1.0] - 2022-11-03 diff --git a/sparv/api/util/system.py b/sparv/api/util/system.py index 7366c255..24a74f23 100644 --- a/sparv/api/util/system.py +++ b/sparv/api/util/system.py @@ -166,10 +166,10 @@ def rsync(local: Union[str, Path], host: Optional[str], remote: Union[str, Path] args = [local] if host: - subprocess.check_call(["ssh", host, f"mkdir -p '{remote_dir}'"]) + subprocess.check_call(["ssh", host, f"mkdir -p {shlex.quote(remote_dir)}"]) subprocess.check_call(["rsync"] + args + [f"{host}:{remote}"]) else: - subprocess.check_call(["mkdir", "-p", f"'{remote_dir}'"]) + subprocess.check_call(["mkdir", "-p", remote_dir]) subprocess.check_call(["rsync"] + args + [remote]) From 24dde86d72b3eb42c9b50386d19ae10b5709790f Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Thu, 2 Feb 2023 14:23:57 +0100 Subject: [PATCH 062/175] Fix quoting of path in korp:config --- sparv/modules/korp/config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sparv/modules/korp/config.py b/sparv/modules/korp/config.py index 505b3295..4cb65c62 100644 --- a/sparv/modules/korp/config.py +++ b/sparv/modules/korp/config.py @@ -1,5 +1,6 @@ """Create configuration files for the Korp backend and frontend.""" import itertools +import shlex import subprocess from collections import defaultdict from pathlib import Path @@ -332,7 +333,8 @@ def get_presets(remote_host, config_dir): """Get list of presets from file system.""" presets = {} if remote_host and remote_host != "localhost": - cmd = ["ssh", remote_host, f"find '{config_dir}/attributes/'"] + remote_path = shlex.quote(f"{config_dir}/attributes/") + cmd = ["ssh", remote_host, f"find {remote_path}"] else: cmd = ["find", f"{config_dir}/attributes/"] logger.debug(f"Getting Korp annotation presets from {remote_host}:{config_dir}") From ac39713b73b3d5bcd76880d4a8480d8ea7432a21 Mon Sep 17 00:00:00 2001 From: Anne Schumacher Date: Fri, 3 Feb 2023 11:16:24 +0100 Subject: [PATCH 063/175] correct path of config_default.yaml in documentation --- docs/user-manual/corpus-configuration.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/user-manual/corpus-configuration.md b/docs/user-manual/corpus-configuration.md index 65e912da..6b3b52e9 100644 --- a/docs/user-manual/corpus-configuration.md +++ b/docs/user-manual/corpus-configuration.md @@ -50,11 +50,11 @@ used by multiple Sparv modules, while others are specific to one particular anno defines the name of the binary the hunpos module uses to run part-of-speech tagging). These module specific config options usually have default values which are defined by the module itself. -When running Sparv your corpus config will be read and combined with Sparv's default config file (`config_default.yaml` -in the [Sparv data directory](user-manual/installation-and-setup.md#setting-up-sparv)) and the default values defined by -different Sparv modules. You can view the resulting configuration by running `sparv config`. Using the `config` command -you can also inspect specific config variables, e.g. `sparv config metadata` or `sparv config metadata.language`. All -default values can be overridden in your own corpus config. +When running Sparv your corpus config will be read and combined with Sparv's default config file +(`config/config_default.yaml` in the [Sparv data directory](user-manual/installation-and-setup.md#setting-up-sparv)) and +the default values defined by different Sparv modules. You can view the resulting configuration by running `sparv +config`. Using the `config` command you can also inspect specific config variables, e.g. `sparv config metadata` or +`sparv config metadata.language`. All default values can be overridden in your own corpus config. There are a few config options that must be set (either through the default config or the corpus config): - `metadata.language` (default: `swe`) From bbc575fc6c1b460f22a6c1daba7573973062882c Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Mon, 6 Feb 2023 17:00:39 +0100 Subject: [PATCH 064/175] Prevent infinite classes loop --- CHANGELOG.md | 3 ++- sparv/core/registry.py | 41 ++++++++++++++++++++++++++--------------- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9715c631..dcf5cea3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,7 +42,8 @@ - Fixed bug where Sparv would hang if an error occurred in a preloaded annotator. - Fixed occasional crash in `cwb:encode` when old CWB export hadn't been removed first. - Fixed bug when using relative socket path while also using `--dir`. -- Fix quoting of paths in `util.system.rsync`. +- Fixed quoting of paths in `util.system.rsync`. +- It's no longer possible to create an infinite loop of classes referring to each other. ## [5.1.0] - 2022-11-03 diff --git a/sparv/core/registry.py b/sparv/core/registry.py index be59b7d6..2039921c 100644 --- a/sparv/core/registry.py +++ b/sparv/core/registry.py @@ -522,26 +522,37 @@ def expand_variables(string, rule_name: Optional[str] = None, is_annotation: boo # Split if list of alternatives strings = [s for s in string.split(", ") for string in strings] + def expand_classes(s: str, parents: set[str]) -> Tuple[Optional[str], Optional[str]]: + classes = find_classes(s, True) + if not classes: + return s, None + for cls in classes: + # Check that cls isn't among its parents + if cls.group(1) in parents: + raise SparvErrorMessage( + f"The class {cls.group()} refers to itself, either directly or indirectly, leading to an infinite " + "loop. Check your class definitions in your config file under the 'classes' section. Also check " + "'import.text_annotation', which automatically sets the class." + ) + real_ann = _expand_class(cls.group(1)) + if real_ann: + final_ann, rest_ = expand_classes(real_ann, parents.union([cls.group(1)])) + if rest_: + return None, rest_ + s = s.replace(cls.group(), final_ann, 1) + else: + return None, cls.group() + return s, None + for string in strings: # Convert class names to real annotations - while True: - clss = find_classes(string, True) - if not clss: - break - for cls in clss: - real_ann = _expand_class(cls.group(1)) - if real_ann: - string = string.replace(cls.group(), real_ann) - else: - rest.append(cls.group()) - break - else: - continue - break + string, unknown = expand_classes(string, set()) + if unknown: + rest.append(unknown) if is_annotation and len(strings) > 1: # If multiple alternative annotations, return the first one that is explicitly used, or the last - if string in explicit_annotations or clss and set(clss).intersection(explicit_annotations): + if string in explicit_annotations: break return string, rest From 06df11f499eab4e0b0872771330e6d53824f1dd4 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 10 Feb 2023 16:52:17 +0100 Subject: [PATCH 065/175] Add ability to stop preloader with interrupt signal --- CHANGELOG.md | 1 + sparv/__main__.py | 1 + sparv/core/Snakefile | 27 ++++++++++++++++----------- sparv/core/preload.py | 10 ++++++---- 4 files changed, 24 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dcf5cea3..7ff04833 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ - Sparv will automatically order the GPUs in the environment variable `CUDA_VISIBLE_DEVICES` by the amount of free memory that was available when Sparv started. - Stanza now always selects the GPU with the most free memory. +- The preloader can now be gracefully stopped by sending an interrupt signal to the Sparv process. ### Changed diff --git a/sparv/__main__.py b/sparv/__main__.py index 07be30c5..895c7eb9 100644 --- a/sparv/__main__.py +++ b/sparv/__main__.py @@ -304,6 +304,7 @@ def main(): config["preloader"] = True config["processes"] = args.processes config["preload_command"] = args.preload_command + config["targets"] = ["preload"] if args.list: snakemake_args["targets"] = ["preload_list"] diff --git a/sparv/core/Snakefile b/sparv/core/Snakefile index b57cf6d3..bfb0cd3a 100644 --- a/sparv/core/Snakefile +++ b/sparv/core/Snakefile @@ -1,5 +1,7 @@ """Snakefile used by Snakemake.""" +import multiprocessing import os +import signal from pathlib import Path from rich import box @@ -495,17 +497,20 @@ rule build_models: input: snake_storage.model_outputs - -rule preload: - run: - from sparv.core import preload - if config["preload_command"] == "start": - preload.serve(config["socket"], config["processes"], snake_storage) - elif config["preload_command"] == "stop": - if not Path(config["socket"]).is_socket(): - raise SparvErrorMessage(f"Socket file '{config['socket']}' doesn't exist or isn't a socket.") - elif not preload.stop(config.get("socket")): - raise SparvErrorMessage(f"Could not connect to socket '{config.get('socket')}'.") +if "preload" in selected_targets: + stop_signal = multiprocessing.Event() + signal.signal(signal.SIGINT, lambda _sig_no, _frame: stop_signal.set()) + + rule preload: + run: + from sparv.core import preload + if config["preload_command"] == "start": + preload.serve(config["socket"], config["processes"], snake_storage, stop_signal) + elif config["preload_command"] == "stop": + if not Path(config["socket"]).is_socket(): + raise SparvErrorMessage(f"Socket file '{config['socket']}' doesn't exist or isn't a socket.") + elif not preload.stop(config.get("socket")): + raise SparvErrorMessage(f"Could not connect to socket '{config.get('socket')}'.") rule preload_list: diff --git a/sparv/core/preload.py b/sparv/core/preload.py index 0c20bc78..57216adb 100644 --- a/sparv/core/preload.py +++ b/sparv/core/preload.py @@ -1,6 +1,7 @@ """Sparv preloader.""" import logging import multiprocessing +import os import pickle import socket import struct @@ -13,8 +14,8 @@ from sparv.core import config, io, log_handler from sparv.core.console import console -from sparv.core.snake_utils import SnakeStorage from sparv.core.misc import SparvErrorMessage +from sparv.core.snake_utils import SnakeStorage INFO = "INFO" STATUS = "STATUS" @@ -220,7 +221,7 @@ def worker(worker_no: int, server_socket, annotators: Dict[str, Preloader], stop client_sock.close() -def serve(socket_path: str, processes: int, storage: SnakeStorage): +def serve(socket_path: str, processes: int, storage: SnakeStorage, stop_signal: multiprocessing.Event): """Start the Sparv preloader socket server.""" socket_file = Path(socket_path) if socket_file.exists(): @@ -285,11 +286,12 @@ def serve(socket_path: str, processes: int, storage: SnakeStorage): log.info(f"The Sparv preloader is ready and waiting for connections using the socket at {socket_file.absolute()}. " "Run Sparv with the command 'sparv run --socket /path/to/socket' to use the preloader. " - "Press Ctrl-C to exit, or run 'sparv preload stop --socket /path/to/socket'.") + "Press Ctrl-C to exit, or run 'sparv preload stop --socket /path/to/socket'. You can also stop the " + f"preloader by sending an interrupt signal to the process with id {os.getpid()}.") # Periodically check whether stop_event is set or not and stop all processes when set while True: - if stop_event.is_set(): + if stop_event.is_set() or stop_signal.is_set(): log.info("Stopping all workers...") for p in workers: if p.is_alive(): From afa3f944a50356e7a19bc1b7a50df9bbb2c9f553 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 10 Feb 2023 17:00:15 +0100 Subject: [PATCH 066/175] Don't limit width of elapsed column in --stats table --- CHANGELOG.md | 1 + sparv/core/log_handler.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ff04833..ae51945b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,7 @@ - Fixed bug when using relative socket path while also using `--dir`. - Fixed quoting of paths in `util.system.rsync`. - It's no longer possible to create an infinite loop of classes referring to each other. +- Elapsed time exceeding 24 hours no longer gets cut off in the `--stats` output. ## [5.1.0] - 2022-11-03 diff --git a/sparv/core/log_handler.py b/sparv/core/log_handler.py index 7bc32e46..103ff735 100644 --- a/sparv/core/log_handler.py +++ b/sparv/core/log_handler.py @@ -696,7 +696,7 @@ def stop(self): spacer = "" table = Table(show_header=False, box=box.SIMPLE) table.add_column("Task", no_wrap=True, min_width=self.jobs_max_len + 2, ratio=1) - table.add_column("Time taken", no_wrap=True, width=10, justify="right", style="progress.remaining") + table.add_column("Time taken", no_wrap=True, justify="right", style="progress.remaining") table.add_column("Percentage", no_wrap=True, justify="right") table.add_row("[b]Task[/]", "[default b]Time taken[/]", "[b]Percentage[/b]") total_time = sum(self.stats_data.values()) From f04f95f925c6cd259313d62d63db7c0e8f9fcaa1 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Mon, 13 Feb 2023 17:50:34 +0100 Subject: [PATCH 067/175] Change how SourceAnnotations works; introduce separate HeaderAnnotations class --- CHANGELOG.md | 3 + docs/developers-guide/sparv-classes.md | 25 ++++- sparv/api/__init__.py | 2 + sparv/api/classes.py | 96 ++++++++++++++++--- sparv/api/util/export.py | 99 ++++++-------------- sparv/api/util/misc.py | 83 +--------------- sparv/core/misc.py | 84 +++++++++++++++++ sparv/core/snake_utils.py | 17 +++- sparv/modules/cwb/cwb.py | 2 +- sparv/modules/korp/config.py | 2 +- sparv/modules/stats_export/stats_export.py | 2 +- sparv/modules/xml_export/preserved_format.py | 21 ++++- sparv/modules/xml_export/pretty.py | 5 +- 13 files changed, 263 insertions(+), 178 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae51945b..89fa8c00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ memory that was available when Sparv started. - Stanza now always selects the GPU with the most free memory. - The preloader can now be gracefully stopped by sending an interrupt signal to the Sparv process. +- Added `HeaderAnnotations` and `HeaderAnnotationsAllSourceFiles` classes. ### Changed @@ -36,6 +37,8 @@ - `korp:relations`, `korp:relations_sql` and `korp:install_relations` has been renamed to `korp:wordpicture`, `korp:wordpicture_sql` and `korp:install_wordpicture` respectively. - Target path is no longer optional for the utility functions `install_path` and `rsync`. +- The classes `SourceAnnotations` and `SourceAnnotationsAllSourceFiles` are now pre-parsed iterables instead of lists + that need parsing and expanding. ### Fixed diff --git a/docs/developers-guide/sparv-classes.md b/docs/developers-guide/sparv-classes.md index f4cf16f2..7422cd67 100644 --- a/docs/developers-guide/sparv-classes.md +++ b/docs/developers-guide/sparv-classes.md @@ -246,6 +246,25 @@ function. - `all_files`: Set to `True` to get the export for all source files. Default: `False` +## HeaderAnnotations +Iterable containing header annotations from the source to be included in the export. This list is defined in the corpus +configuration. + +**Arguments:** + +- `config_name`: The config variable pointing out what header annotations to include. + + +## HeaderAnnotationsAllSourceFiles +Iterable containing header annotations from the source to be included in the export. This list is defined in the corpus +configuration. This differs from `HeaderAnnotations` in that the header annotations file (created by using +`Headers`) of _every_ source file will be added as dependencies. + +**Arguments:** + +- `config_name`: The config variable pointing out what source annotations to include. + + ## Headers List of header annotation names for a given source file. @@ -446,7 +465,7 @@ An instance of this class holds a path to the directory containing input files. ## SourceAnnotations -List of source annotations to be included in the export. This list is defined in the corpus configuration. +Iterable containing source annotations to be included in the export. This list is defined in the corpus configuration. **Arguments:** @@ -454,8 +473,8 @@ List of source annotations to be included in the export. This list is defined in ## SourceAnnotationsAllSourceFiles -List of source annotations to be included in the export. This list is defined in the corpus configuration. This -differs from `SourceAnnotations` in that the source annotations structure file (created by using `SourceStructure`) +Iterable containing source annotations to be included in the export. This list is defined in the corpus configuration. +This differs from `SourceAnnotations` in that the source annotations structure file (created by using `SourceStructure`) of _every_ source file will be added as dependencies. **Arguments:** diff --git a/sparv/api/__init__.py b/sparv/api/__init__.py index 646f2183..ea38596b 100644 --- a/sparv/api/__init__.py +++ b/sparv/api/__init__.py @@ -21,6 +21,8 @@ ExportAnnotationNames, ExportAnnotationsAllSourceFiles, ExportInput, + HeaderAnnotations, + HeaderAnnotationsAllSourceFiles, Headers, Language, Marker, diff --git a/sparv/api/classes.py b/sparv/api/classes.py index 35f62a26..5b81c86a 100644 --- a/sparv/api/classes.py +++ b/sparv/api/classes.py @@ -7,11 +7,11 @@ import urllib.request import zipfile from abc import ABC, abstractmethod -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, Iterable, List, Optional, Tuple, Union import sparv.core from sparv.core import io -from sparv.core.misc import get_logger +from sparv.core.misc import get_logger, parse_annotation_list from sparv.core.paths import models_dir logger = get_logger(__name__) @@ -558,9 +558,9 @@ class SourceStructure(BaseAnnotation): def __init__(self, source_file): super().__init__(io.STRUCTURE_FILE, source_file) - def read(self): + def read(self) -> List[str]: """Read structure file.""" - return io.read_data(self.source_file, self) + return io.read_data(self.source_file, self).split("\n") def write(self, structure): """Sort the source file's structural elements and write structure file.""" @@ -821,20 +821,92 @@ def __init__(self, config_name: str, items=()): self.config_name = config_name -class SourceAnnotations(List[Tuple[Annotation, Optional[str]]]): - """List of source annotations to include in export.""" +class SourceAnnotations(Iterable[Tuple[Annotation, Optional[str]]]): + """Iterable with source annotations to include in export.""" - def __init__(self, config_name: str, items=()): - list.__init__(self, items) + def __init__(self, config_name: str, items: Optional[Iterable[str]] = None, source_file: Optional[str] = None, + _headers: bool = False): self.config_name = config_name + self.raw_list = items + self.annotations = [] + self.source_file = source_file + self.initialized = False + self.headers = _headers + + # If raw_list is an empty list, don't add any source annotations automatically + if not self.raw_list and self.raw_list is not None: + self.initialized = True + + def __iter__(self): + if not self.initialized: + assert self.source_file, "SourceAnnotation is missing source_file" + # Parse annotation list and read available source annotations + if self.headers: + h = Headers(self.source_file) + available_source_annotations = set(h.read()) if h.exists() else set() + else: + available_source_annotations = set(SourceStructure(self.source_file).read()) + parsed_items = parse_annotation_list(self.raw_list, available_source_annotations) + # Only include annotations that are available in source + self.annotations = [ + (Annotation(a[0], self.source_file), a[1]) for a in parsed_items if a[0] in available_source_annotations + ] + self.initialized = True + for annotation in self.annotations: + yield annotation -class SourceAnnotationsAllSourceFiles(List[Tuple[AnnotationAllSourceFiles, Optional[str]]]): - """List of source annotations to include in export.""" +class HeaderAnnotations(SourceAnnotations): + """Iterable with header annotations to include in export.""" - def __init__(self, config_name: str, items=()): - list.__init__(self, items) + def __init__(self, config_name: str, items: Optional[Iterable[str]] = None, source_file: Optional[str] = None): + super().__init__(config_name, items, source_file, _headers=True) + + +class SourceAnnotationsAllSourceFiles(Iterable[Tuple[AnnotationAllSourceFiles, Optional[str]]]): + """Iterable with source annotations to include in export.""" + + def __init__(self, config_name: str, items: Optional[Iterable[str]] = None, source_files: Iterable[str]=(), + headers: bool = False): self.config_name = config_name + self.raw_list = items + self.annotations = [] + self.source_files = source_files + self.initialized = False + self.headers = headers + + # If raw_list is an empty list, don't add any source annotations automatically + if not self.raw_list and self.raw_list is not None: + self.initialized = True + + def __iter__(self): + if not self.initialized: + assert self.source_files, "SourceAnnotationsAllSourceFiles is missing source_files" + # Parse annotation list and, if needed, read available source annotations + available_source_annotations = set() + for f in self.source_files: + if self.headers: + h = Headers(f) + if h.exists(): + available_source_annotations.update(h.read()) + else: + available_source_annotations.update(SourceStructure(f).read()) + + parsed_items = parse_annotation_list(self.raw_list, available_source_annotations) + # Only include annotations that are available in source + self.annotations = [ + (AnnotationAllSourceFiles(a[0]), a[1]) for a in parsed_items if a[0] in available_source_annotations + ] + self.initialized = True + for annotation in self.annotations: + yield annotation + + +class HeaderAnnotationsAllSourceFiles(SourceAnnotationsAllSourceFiles): + """Iterable with header annotations to include in export.""" + + def __init__(self, config_name: str, items: Optional[Iterable[str]] = None, source_files: Iterable[str]=()): + super().__init__(config_name, items, source_files, headers=True) class Language(str): diff --git a/sparv/api/util/export.py b/sparv/api/util/export.py index 42fdf817..d6eb1bd0 100644 --- a/sparv/api/util/export.py +++ b/sparv/api/util/export.py @@ -5,10 +5,23 @@ from collections import OrderedDict, defaultdict from copy import deepcopy from itertools import combinations -from typing import Any, List, Optional, Tuple, Union - -from sparv.api import (Annotation, AnnotationAllSourceFiles, ExportAnnotations, ExportAnnotationsAllSourceFiles, - Headers, Namespaces, SourceStructure, SparvErrorMessage, get_logger, util) +from typing import Any, Dict, List, Optional, Tuple, Union + +from sparv.api import ( + Annotation, + AnnotationAllSourceFiles, + ExportAnnotations, + ExportAnnotationsAllSourceFiles, + HeaderAnnotations, + Headers, + Namespaces, + SourceAnnotations, + SourceAnnotationsAllSourceFiles, + SourceStructure, + SparvErrorMessage, + get_logger, + util +) from sparv.core import io from .constants import SPARV_DEFAULT_NAMESPACE, XML_NAMESPACE_SEP @@ -278,46 +291,9 @@ def calculate_element_hierarchy(source_file, spans_list): return hierarchy -def get_available_source_annotations(source_file: Optional[str] = None, - source_files: Optional[List[str]] = None) -> List[str]: - """Get a list of available annotations generated from the source, either for a single source file or multiple.""" - assert source_file or source_files, "Either 'source_file' or 'source_files' must be provided" - available_source_annotations = set() - if source_files: - for d in source_files: - available_source_annotations.update(SourceStructure(d).read().split()) - else: - available_source_annotations.update(SourceStructure(source_file).read().split()) - - return sorted(available_source_annotations) - - -def get_source_annotations(source_annotation_names: Optional[List[str]], source_file: Optional[str] = None, - source_files: Optional[List[str]] = None): - """Given a list of source annotation names (and possible export names), return a list of annotation objects. - - If no names are provided all available source annotations will be returnd. - """ - # If source_annotation_names is en empty list, do not add any source annotations - if not source_annotation_names and source_annotation_names is not None: - return [] - - # Get list of available source annotation names - available_source_annotations = get_available_source_annotations(source_file, source_files) - - # Parse source_annotation_names - annotation_names = util.misc.parse_annotation_list(source_annotation_names, available_source_annotations) - - # Make sure source_annotations doesn't include annotations not in source - source_annotations = [(Annotation(a[0], source_file) if source_file else AnnotationAllSourceFiles(a[0]), a[1]) for a in - annotation_names if a[0] in available_source_annotations] - - return source_annotations - - def get_annotation_names(annotations: Union[ExportAnnotations, ExportAnnotationsAllSourceFiles], - source_annotations=None, - source_file: Optional[str] = None, source_files: Optional[List[str]] = None, + source_annotations: Union[SourceAnnotations, SourceAnnotationsAllSourceFiles] = None, + source_file: Optional[str] = None, token_name: Optional[str] = None, remove_namespaces=False, keep_struct_names=False, sparv_namespace: Optional[str] = None, @@ -330,7 +306,6 @@ def get_annotation_names(annotations: Union[ExportAnnotations, ExportAnnotations source_annotations: List of elements:attributes from the source file to include. If not specified, everything will be included. source_file: Name of the source file. - source_files: List of names of source files (alternative to `source_file`). token_name: Name of the token annotation. remove_namespaces: Remove all namespaces in export_names unless names are ambiguous. keep_struct_names: For structural attributes (anything other than token), include the annotation base name @@ -342,11 +317,8 @@ def get_annotation_names(annotations: Union[ExportAnnotations, ExportAnnotations A list of annotations, a list of token attribute names, a dictionary with translation from annotation names to export names. """ - # Get source annotations - source_annotations = get_source_annotations(source_annotations, source_file, source_files) - # Combine all annotations - all_annotations = _remove_duplicates(annotations + source_annotations) + all_annotations = _remove_duplicates(annotations + list(source_annotations)) if token_name: # Get the names of all token attributes @@ -359,36 +331,19 @@ def get_annotation_names(annotations: Union[ExportAnnotations, ExportAnnotations xml_namespaces = Namespaces(source_file).read() export_names = _create_export_names(all_annotations, token_name, remove_namespaces, keep_struct_names, - source_annotations, sparv_namespace, source_namespace, xml_namespaces, + list(source_annotations), sparv_namespace, source_namespace, xml_namespaces, xml_mode=xml_mode) return [i[0] for i in all_annotations], token_attributes, export_names -def get_header_names(header_annotation_names: Optional[List[str]], - source_file: Optional[str] = None, - source_files: Optional[List[str]] = None): +def get_header_names( + header_annotations: Optional[HeaderAnnotations], + xml_namespaces: Dict[str, str] +): """Get a list of header annotations and a dictionary for renamed annotations.""" - # Get source_header_names from headers file if it exists - source_header_names = [] - if source_files: - for f in source_files: - h = Headers(f) - if h.exists(): - source_header_names.extend(h.read()) - source_header_names = list(set(source_header_names)) - elif Headers(source_file).exists(): - source_header_names = Headers(source_file).read() - - # Parse header_annotation_names and convert to annotations - annotation_names = util.misc.parse_annotation_list(header_annotation_names, source_header_names) - header_annotations = [(Annotation(a[0], source_file) if source_file else AnnotationAllSourceFiles(a[0]), a[1]) for a in - annotation_names] - - # Get XML namespaces - xml_namespaces = Namespaces(source_file).read() - export_names = _create_export_names(header_annotations, None, False, keep_struct_names=False, + export_names = _create_export_names(list(header_annotations), None, False, keep_struct_names=False, xml_namespaces=xml_namespaces, xml_mode=True) return [a[0] for a in header_annotations], export_names @@ -407,7 +362,7 @@ def _create_export_names(annotations: List[Tuple[Union[Annotation, AnnotationAll token_name: Optional[str], remove_namespaces: bool, keep_struct_names: bool, - source_annotations: list = [], + source_annotations: List[Tuple[Union[Annotation, AnnotationAllSourceFiles], Any]] = [], sparv_namespace: Optional[str] = None, source_namespace: Optional[str] = None, xml_namespaces: Optional[dict] = None, diff --git a/sparv/api/util/misc.py b/sparv/api/util/misc.py index ba175e50..740dfda2 100644 --- a/sparv/api/util/misc.py +++ b/sparv/api/util/misc.py @@ -4,94 +4,15 @@ import re import unicodedata from collections import OrderedDict, defaultdict -from typing import List, Optional, Tuple, Union +from typing import Iterable, List, Optional, Tuple, Union from sparv.api import get_logger from sparv.api.classes import Annotation, Model +from sparv.core.misc import parse_annotation_list # noqa logger = get_logger(__name__) -def parse_annotation_list(annotation_names: Optional[List[str]], all_annotations: Optional[List[str]] = None, - add_plain_annotations: bool = True) -> List[Tuple[str, Optional[str]]]: - """Take a list of annotation names and possible export names, and return a list of tuples. - - Each list item will be split into a tuple by the string ' as '. - Each tuple will contain 2 elements. If there is no ' as ' in the string, the second element will be None. - - If there is an element called '...' everything from all_annotations will be included in the result, except for - the elements that are prefixed with 'not '. - - If an annotation occurs more than once in the list, only the last occurrence will be kept. Similarly, if an - annotation is first included and then excluded (using 'not') it will be excluded from the result. - - If a plain annotation (without attributes) is excluded, all its attributes will be excluded as well. - - Plain annotations (without attributes) will be added if needed, unless add_plain_annotations is set to False. - Make sure to disable add_plain_annotations if the annotation names may include classes or config variables. - """ - if all_annotations is None: - all_annotations = [] - if not annotation_names: - return [(a, None) for a in all_annotations] - - plain_annotations = set() - possible_plain_annotations = set() - omit_annotations = set() - include_rest = False - plain_to_atts = defaultdict(list) - - result: OrderedDict = OrderedDict() - for a in annotation_names: - # Check if this annotation should be omitted - if a.startswith("not ") and " as " not in a: - omit_annotations.add(a[4:]) - elif a == "...": - include_rest = True - else: - name, _, export_name = a.partition(" as ") - if not re.match(r"^<[^>]+>$", name): # Prevent splitting class names - plain_name, attr = Annotation(name).split() - else: - plain_name, attr = None, None - result.pop(name, None) - result[name] = export_name or None - if attr: - possible_plain_annotations.add(plain_name) - plain_to_atts[plain_name].append(name) - else: - plain_annotations.add(name) - - # If only exclusions have been listed, include rest of annotations - if omit_annotations and not result: - include_rest = True - - # Add all_annotations to result if required - if include_rest and all_annotations: - for a in [a for a in all_annotations if not a in omit_annotations]: - if a not in result: - result[a] = None - plain_name, _ = Annotation(a).split() - plain_to_atts[plain_name].append(a) - plain_annotations.add(plain_name) - - # Add annotations names without attributes to result if required - if add_plain_annotations: - for a in sorted(possible_plain_annotations.difference(plain_annotations)): - if a not in result: - result[a] = None - - # Remove any exclusions from final list - if omit_annotations: - for annotation in omit_annotations: - result.pop(annotation, None) - # If we're excluding a plain annotation, also remove all attributes connected to it - for a in plain_to_atts[annotation]: - result.pop(a, None) - - return list(result.items()) - - # TODO: Split into two functions: one for Sparv-internal lists of values, and one used by the CWB module to create the # CWB-specific set format. def cwbset(values, delimiter="|", affix="|", sort=False, maxlength=4095, encoding="UTF-8"): diff --git a/sparv/core/misc.py b/sparv/core/misc.py index 57f5e3d8..be510940 100644 --- a/sparv/core/misc.py +++ b/sparv/core/misc.py @@ -1,6 +1,9 @@ """Miscellaneous classes and methods.""" import logging +import re +from collections import OrderedDict, defaultdict +from typing import Iterable, List, Optional, Tuple class SparvErrorMessage(Exception): @@ -29,3 +32,84 @@ def get_logger(name): name = "sparv.modules." + name return logging.getLogger(name) + +def parse_annotation_list(annotation_names: Optional[Iterable[str]], all_annotations: Optional[Iterable[str]] = None, + add_plain_annotations: bool = True) -> List[Tuple[str, Optional[str]]]: + """Take a list of annotation names and possible export names, and return a list of tuples. + + Each list item will be split into a tuple by the string ' as '. + Each tuple will contain 2 elements. If there is no ' as ' in the string, the second element will be None. + + If there is an element called '...' everything from all_annotations will be included in the result, except for + the elements that are prefixed with 'not '. + + If an annotation occurs more than once in the list, only the last occurrence will be kept. Similarly, if an + annotation is first included and then excluded (using 'not') it will be excluded from the result. + + If a plain annotation (without attributes) is excluded, all its attributes will be excluded as well. + + Plain annotations (without attributes) will be added if needed, unless add_plain_annotations is set to False. + Make sure to disable add_plain_annotations if the annotation names may include classes or config variables. + """ + from sparv.api import Annotation + + if all_annotations is None: + all_annotations = [] + if not annotation_names: + return [(a, None) for a in all_annotations] + + plain_annotations = set() + possible_plain_annotations = set() + omit_annotations = set() + include_rest = False + plain_to_atts = defaultdict(list) + + result: OrderedDict = OrderedDict() + for a in annotation_names: + # Check if this annotation should be omitted + if a.startswith("not ") and " as " not in a: + omit_annotations.add(a[4:]) + elif a == "...": + include_rest = True + else: + name, _, export_name = a.partition(" as ") + if not re.match(r"^<[^>]+>$", name): # Prevent splitting class names + plain_name, attr = Annotation(name).split() + else: + plain_name, attr = None, None + result.pop(name, None) + result[name] = export_name or None + if attr: + possible_plain_annotations.add(plain_name) + plain_to_atts[plain_name].append(name) + else: + plain_annotations.add(name) + + # If only exclusions have been listed, include rest of annotations + if omit_annotations and not result: + include_rest = True + + # Add all_annotations to result if required + if include_rest and all_annotations: + for a in [a for a in all_annotations if not a in omit_annotations]: + if a not in result: + result[a] = None + plain_name, _ = Annotation(a).split() + plain_to_atts[plain_name].append(a) + plain_annotations.add(plain_name) + + # Add annotations names without attributes to result if required + if add_plain_annotations: + for a in sorted(possible_plain_annotations.difference(plain_annotations)): + if a not in result: + result[a] = None + + # Remove any exclusions from final list + if omit_annotations: + for annotation in omit_annotations: + result.pop(annotation, None) + # If we're excluding a plain annotation, also remove all attributes connected to it + for a in plain_to_atts[annotation]: + result.pop(a, None) + + return list(result.items()) diff --git a/sparv/core/snake_utils.py b/sparv/core/snake_utils.py index 40a4508d..b4adf0d6 100644 --- a/sparv/core/snake_utils.py +++ b/sparv/core/snake_utils.py @@ -18,6 +18,7 @@ from sparv.api.classes import (AllSourceFilenames, Annotation, AnnotationAllSourceFiles, AnnotationData, Base, BaseAnnotation, BaseOutput, Binary, BinaryDir, Config, Corpus, SourceFilename, Export, ExportAnnotations, ExportAnnotationNames, ExportAnnotationsAllSourceFiles, ExportInput, + HeaderAnnotations, HeaderAnnotationsAllSourceFiles, Language, Model, ModelOutput, Output, OutputData, Source, SourceAnnotations, SourceAnnotationsAllSourceFiles, Text) @@ -408,13 +409,25 @@ def rule_helper(rule: RuleStorage, config: dict, storage: SnakeStorage, config_m rule.parameters[param_name].append((annotation, export_name)) # SourceAnnotations elif param_type in (SourceAnnotations, SourceAnnotationsAllSourceFiles): - rule.parameters[param_name] = sparv_config.get(param.default.config_name, None) + rule.parameters[param_name] = param_type( + param.default.config_name, + sparv_config.get(param.default.config_name) + ) if param_type == SourceAnnotationsAllSourceFiles: + rule.parameters[param_name].source_files = storage.source_files rule.inputs.extend( expand(escape_wildcards(paths.work_dir / get_annotation_path(io.STRUCTURE_FILE, data=True)), file=storage.source_files)) else: rule.inputs.append(paths.work_dir / get_annotation_path(io.STRUCTURE_FILE, data=True)) + # HeaderAnnotations + elif param_type in (HeaderAnnotations, HeaderAnnotationsAllSourceFiles): + rule.parameters[param_name] = param_type( + param.default.config_name, + sparv_config.get(param.default.config_name) + ) + if param_type == HeaderAnnotationsAllSourceFiles: + rule.parameters[param_name].source_files = storage.source_files # Corpus elif param.annotation == Corpus: rule.parameters[param_name] = Corpus(sparv_config.get("metadata.id")) @@ -624,6 +637,8 @@ def get_params(wildcards): if isinstance(param, (ExportAnnotations, ExportAnnotationNames)): for p in param: p[0].source_file = file + elif isinstance(param, (SourceAnnotations, HeaderAnnotations)): + param.source_file = file else: if not isinstance(param, (list, tuple)): param = [param] diff --git a/sparv/modules/cwb/cwb.py b/sparv/modules/cwb/cwb.py index 257f15cf..205b1b07 100644 --- a/sparv/modules/cwb/cwb.py +++ b/sparv/modules/cwb/cwb.py @@ -184,7 +184,7 @@ def cwb_encode(corpus, annotations, source_annotations, source_files, words, vrt # Get annotation names annotation_list, token_attributes, export_names = util.export.get_annotation_names( - annotations, source_annotations, source_files=source_files, token_name=token_name, + annotations, source_annotations, token_name=token_name, remove_namespaces=remove_namespaces, sparv_namespace=sparv_namespace, source_namespace=source_namespace, keep_struct_names=True) diff --git a/sparv/modules/korp/config.py b/sparv/modules/korp/config.py index 4cb65c62..88e6a914 100644 --- a/sparv/modules/korp/config.py +++ b/sparv/modules/korp/config.py @@ -173,7 +173,7 @@ def config(id: Corpus = Corpus(), # Get annotation names annotation_list, token_attributes, export_names = util.export.get_annotation_names( - annotations, source_annotations, source_files=source_files, token_name=token.name, + annotations, source_annotations, token_name=token.name, remove_namespaces=remove_namespaces, sparv_namespace=sparv_namespace, source_namespace=source_namespace, keep_struct_names=True) diff --git a/sparv/modules/stats_export/stats_export.py b/sparv/modules/stats_export/stats_export.py index 10d63b2a..4e3b8f39 100644 --- a/sparv/modules/stats_export/stats_export.py +++ b/sparv/modules/stats_export/stats_export.py @@ -61,7 +61,7 @@ def freq_list(source_files: AllSourceFilenames = AllSourceFilenames(), # Get annotations list and export names annotation_list, token_attributes, export_names = util.export.get_annotation_names( - annotations, source_annotations or [], source_files=source_files, token_name=token.name, + annotations, source_annotations or [], token_name=token.name, remove_namespaces=remove_namespaces, sparv_namespace=sparv_namespace, source_namespace=source_namespace) # Get all token and struct annotations (except the span annotations) diff --git a/sparv/modules/xml_export/preserved_format.py b/sparv/modules/xml_export/preserved_format.py index db04c3e7..7dc105cc 100644 --- a/sparv/modules/xml_export/preserved_format.py +++ b/sparv/modules/xml_export/preserved_format.py @@ -3,8 +3,21 @@ import os import xml.etree.ElementTree as etree -from sparv.api import (AnnotationData, Config, Export, ExportAnnotations, Namespaces, SourceAnnotations, SourceFilename, - SparvErrorMessage, Text, exporter, get_logger, util) +from sparv.api import ( + AnnotationData, + Config, + Export, + ExportAnnotations, + HeaderAnnotations, + Namespaces, + SourceAnnotations, + SourceFilename, + SparvErrorMessage, + Text, + exporter, + get_logger, + util +) from . import xml_utils logger = get_logger(__name__) @@ -20,7 +33,7 @@ def preserved_format(source_file: SourceFilename = SourceFilename(), out: Export = Export("xml_export.preserved_format/[xml_export.filename_formatted]"), annotations: ExportAnnotations = ExportAnnotations("xml_export.annotations"), source_annotations: SourceAnnotations = SourceAnnotations("xml_export.source_annotations"), - header_annotations: SourceAnnotations = SourceAnnotations("xml_export.header_annotations"), + header_annotations: HeaderAnnotations = HeaderAnnotations("xml_export.header_annotations"), remove_namespaces: bool = Config("export.remove_module_namespaces", False), sparv_namespace: str = Config("export.sparv_namespace"), source_namespace: str = Config("export.source_namespace"), @@ -57,7 +70,7 @@ def preserved_format(source_file: SourceFilename = SourceFilename(), sparv_namespace=sparv_namespace, source_namespace=source_namespace, xml_mode=True) - h_annotations, h_export_names = util.export.get_header_names(header_annotations, source_file=source_file) + h_annotations, h_export_names = util.export.get_header_names(header_annotations, xml_namespaces) export_names.update(h_export_names) span_positions, annotation_dict = util.export.gather_annotations(annotation_list, export_names, h_annotations, source_file=source_file, flatten=False, split_overlaps=True) diff --git a/sparv/modules/xml_export/pretty.py b/sparv/modules/xml_export/pretty.py index 638bf573..a64fc696 100644 --- a/sparv/modules/xml_export/pretty.py +++ b/sparv/modules/xml_export/pretty.py @@ -12,6 +12,7 @@ Export, ExportAnnotations, ExportInput, + HeaderAnnotations, MarkerOptional, Namespaces, OutputMarker, @@ -47,7 +48,7 @@ def pretty(source_file: SourceFilename = SourceFilename(), word: Annotation = Annotation("[export.word]"), annotations: ExportAnnotations = ExportAnnotations("xml_export.annotations"), source_annotations: SourceAnnotations = SourceAnnotations("xml_export.source_annotations"), - header_annotations: SourceAnnotations = SourceAnnotations("xml_export.header_annotations"), + header_annotations: HeaderAnnotations = HeaderAnnotations("xml_export.header_annotations"), remove_namespaces: bool = Config("export.remove_module_namespaces", False), sparv_namespace: str = Config("export.sparv_namespace"), source_namespace: str = Config("export.source_namespace"), @@ -91,7 +92,7 @@ def pretty(source_file: SourceFilename = SourceFilename(), if token not in annotation_list: logger.warning("The 'xml_export:pretty' export requires the annotation for the output to include the " "source text. Make sure to add to the list of export annotations.") - h_annotations, h_export_names = util.export.get_header_names(header_annotations, source_file=source_file) + h_annotations, h_export_names = util.export.get_header_names(header_annotations, xml_namespaces) export_names.update(h_export_names) span_positions, annotation_dict = util.export.gather_annotations(annotation_list, export_names, h_annotations, source_file=source_file, split_overlaps=True) From 1f0848b93c5835ee5d5f7ce6805ec9b3d2632959 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Tue, 14 Feb 2023 16:43:06 +0100 Subject: [PATCH 068/175] Simplify some code --- sparv/modules/korp/timespan.py | 2 +- sparv/modules/stats_export/stats_export.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/sparv/modules/korp/timespan.py b/sparv/modules/korp/timespan.py index b8a18e52..a2eb3992 100644 --- a/sparv/modules/korp/timespan.py +++ b/sparv/modules/korp/timespan.py @@ -92,7 +92,7 @@ def timespan_sql_with_dateinfo(corpus: Corpus = Corpus(), datetimespans = defaultdict(int) for file in source_files: - text_tokens, orphans = Annotation(datefrom.name, source_file=file).get_children(token) + text_tokens, orphans = datefrom.get_children(file, token) if orphans: datespans[("0" * 8, "0" * 8)] += len(orphans) datetimespans[("0" * 14, "0" * 14)] += len(orphans) diff --git a/sparv/modules/stats_export/stats_export.py b/sparv/modules/stats_export/stats_export.py index 4e3b8f39..48c09925 100644 --- a/sparv/modules/stats_export/stats_export.py +++ b/sparv/modules/stats_export/stats_export.py @@ -74,10 +74,9 @@ def freq_list(source_files: AllSourceFilenames = AllSourceFilenames(), # Get values for struct annotations (per token) struct_values = [] for struct_annotation in struct_annotations: - struct_annot = Annotation(struct_annotation.name, source_file=source_file) - token_parents = Annotation(token.name, source_file=source_file).get_parents(struct_annot) + token_parents = token.get_parents(source_file, struct_annotation) try: - struct_annot_list = list(struct_annot.read()) + struct_annot_list = list(struct_annotation.read(source_file)) struct_values.append([struct_annot_list[p] if p is not None else "" for p in token_parents]) # Handle cases where some source files are missing structural source annotations except FileNotFoundError: From 7ff93e9bd97d2c8500f80f80389e08ef88848899 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Thu, 16 Feb 2023 14:04:39 +0100 Subject: [PATCH 069/175] Take advantage of the new SourceAnnotations --- sparv/modules/korp/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sparv/modules/korp/config.py b/sparv/modules/korp/config.py index 88e6a914..175443d6 100644 --- a/sparv/modules/korp/config.py +++ b/sparv/modules/korp/config.py @@ -25,7 +25,6 @@ uninstaller, util ) -from sparv.core.io import split_annotation from sparv.modules.cwb.cwb import cwb_escape logger = get_logger(__name__) @@ -182,7 +181,8 @@ def config(id: Corpus = Corpus(), # Figure out based on available annotations and scrambling within = [] - anns = set([split_annotation(a[0])[0] for a in itertools.chain(annotations, source_annotations or [])]) + anns = set([a[0].split()[0] for a in itertools.chain(annotations, source_annotations or [])]) + if sentence and sentence.name in anns: within.append(export_names[sentence.name]) From 3e53bf5ed3ffad1a3fa7009164b9441f06c9c2dd Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Tue, 17 Jan 2023 12:29:02 +0100 Subject: [PATCH 070/175] ci: add workflow --- .github/workflows/ci.yml | 63 ++++++++++++++++++++++++++++++ Makefile | 80 +++++++++++++++++++++++++++++++++++++++ pytest.ini | 1 + setup.py | 4 +- tests/test_annotations.py | 4 ++ 5 files changed, 150 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 Makefile diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..59f6164f --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,63 @@ +name: CI + +on: [push, pull_request] + + + +jobs: + checks: + name: ${{ matrix.task.name }} py-${{ matrix.python-version }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + + strategy: + max-parallel: 4 + fail-fast: false + matrix: + # TODO: enable if decided to test 3.7 + # python-version: ["3.7", "3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10"] + os: [ubuntu-latest] + task: + - name: Run tests + run: make noexternal-tests + + steps: + - name: Checkout code + uses: nschloe/action-cached-lfs-checkout@v1 + # Use these to explicitly include/exclude files: + # with: + # include: "*" + # exclude: "" + + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + # ---------------------------------------------- + # load cached venv if cache exists + # ---------------------------------------------- + - name: Load cached venv + id: cached-poetry-dependencies + uses: actions/cache@v3.2.3 + with: + path: venv + key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/pyproject.toml') }}-${{ hashFiles('**/setup.py') }}-${{ hashFiles('.github/workflows/ci.yml') }} + + #---------------------------------------------- + # install dependencies if cache does not exist + #---------------------------------------------- + - name: Install dependencies + if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + run: | + make setup-venv + make dev + + - name: Setup Sparv + run: | + venv/bin/sparv setup -d $PWD + tree . + + - name: ${{ matrix.task.name }} + run: ${{ matrix.task.run }} diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..1b194cc2 --- /dev/null +++ b/Makefile @@ -0,0 +1,80 @@ + +.default: help + +.PHONY: help +help: + @echo "usage:" + @echo "" + @echo "setup-venv" + @echo " create virtual environment" + @echo "dev | install-dev" + @echo " setup development environment" + @echo "" + @echo "test | run-all-tests" + @echo " run all tests" + @echo "" + @echo "run-doc-tests" + @echo " run all tests" + @echo "" + @echo "run-all-tests-w-coverage" + @echo " run all tests with coverage collection" + @echo "" + @echo "lint" + @echo " lint the code" + @echo "" + @echo "type-check" + @echo " check types" + @echo "" + @echo "fmt" + @echo " run formatter(s)" + @echo "" + @echo "check-fmt" + @echo " check formatting" + @echo "" + +PLATFORM := ${shell uname -o} +PROJECT := sparv + +ifeq (${VIRTUAL_ENV},) + VENV_NAME = venv + INVENV = export VIRTUAL_ENV="${VENV_NAME}"; export PATH="${VENV_NAME}/bin:${PATH}"; unset PYTHON_HOME; +else + VENV_NAME = ${VIRTUAL_ENV} + INVENV = +endif + +${info Platform: ${PLATFORM}} + +setup-venv: + python3 -m venv ${VENV_NAME} + ${VENV_NAME}/bin/pip install wheel + +dev: install-dev +install-dev: + ${VENV_NAME}/bin/pip install -e .[dev] + +setup-sparv: + ${INVENV} sparv setup + +setup-sparv-from-env: + ${INVENV} sparv setup --dir ${SPARV_DATADIR} + +.PHONY: test +test: run-all-tests +.PHONY: run-all-tests +run-all-tests: + ${INVENV} pytest + +.PHONY: noexternal-tests +noexternal-tests: + ${INVENV} pytest -m noexternal + +.PHONY: run-all-tests-w-coverage +run-all-tests-w-coverage: + ${INVENV} pytest -vv --cov=${PROJECT} --cov-report=xml tests + +.PHONY: run-all-tests-w-coverage +run-noexternal-tests-w-coverage: + ${INVENV} pytest -vv --cov=${PROJECT} --cov-report=xml -m noexternal + + diff --git a/pytest.ini b/pytest.ini index ab6652dc..75227a57 100644 --- a/pytest.ini +++ b/pytest.ini @@ -9,3 +9,4 @@ markers = treetagger: tests for Treetagger corpora freeling: tests for FreeLing corpora stanford: tests for Stanford Parser corpora + noexternal: tests that can be runned directly diff --git a/setup.py b/setup.py index af473c9a..7c1dfb73 100644 --- a/setup.py +++ b/setup.py @@ -55,8 +55,8 @@ def get_readme(readme_path): extras_require={ "dev": [ "pandocfilters==1.5.0", - "pytest==6.2.5", - "pytest-sugar==0.9.4" + "pytest", + "pytest-sugar>=0.9.6" ] }, entry_points={ diff --git a/tests/test_annotations.py b/tests/test_annotations.py index 255b03c7..5df6f7a8 100644 --- a/tests/test_annotations.py +++ b/tests/test_annotations.py @@ -9,6 +9,7 @@ @pytest.mark.swe +# @pytest.mark.noexternal TODO: this test fails in CI, enable when fixed def test_mini_swe(tmp_path): """Run corpus mini-swe and compare the annotations and exports to gold standard.""" gold_corpus_dir = pathlib.Path("tests/test_corpora/mini-swe") @@ -18,6 +19,7 @@ def test_mini_swe(tmp_path): @pytest.mark.swe +@pytest.mark.noexternal def test_special_swe(tmp_path): """Run corpus special-swe and compare the annotations and exports to gold standard.""" gold_corpus_dir = pathlib.Path("tests/test_corpora/special-swe") @@ -27,6 +29,7 @@ def test_special_swe(tmp_path): @pytest.mark.swe +@pytest.mark.noexternal def test_txt_swe(tmp_path): """Run corpus txt-swe and compare the annotations and exports to gold standard.""" gold_corpus_dir = pathlib.Path("tests/test_corpora/txt-swe") @@ -58,6 +61,7 @@ def test_swe_1800(tmp_path): @pytest.mark.swe @pytest.mark.swehist +@pytest.mark.noexternal def test_swe_fsv(tmp_path): """Run corpus swe-fsv and compare the annotations and exports to gold standard.""" gold_corpus_dir = pathlib.Path("tests/test_corpora/swe-fsv") From 57054379726e75de032e8747d94657bb9246b827 Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Thu, 19 Jan 2023 12:37:03 +0100 Subject: [PATCH 071/175] refactor: move lang code to LanguageRegistry --- setup.py | 2 +- sparv/core/language_registry.py | 21 +++++++++++++++++++++ sparv/core/registry.py | 13 +++---------- tests/unit/test_language_registry.py | 22 ++++++++++++++++++++++ 4 files changed, 47 insertions(+), 11 deletions(-) create mode 100644 sparv/core/language_registry.py create mode 100644 tests/unit/test_language_registry.py diff --git a/setup.py b/setup.py index 7c1dfb73..f954a820 100644 --- a/setup.py +++ b/setup.py @@ -39,10 +39,10 @@ def get_readme(readme_path): python_requires=">=3.6.2", install_requires=[ "appdirs==1.4.4", - "iso-639==0.4.5", "docx2python==1.27.1", "nltk==3.6.7", "protobuf~=3.19.0", # Used by Stanza; see https://github.com/spraakbanken/sparv-pipeline/issues/161 + "pycountry==22.3.5", "python-dateutil==2.8.2", "PyYAML==6.0", "questionary==1.10.0", diff --git a/sparv/core/language_registry.py b/sparv/core/language_registry.py new file mode 100644 index 00000000..3eab9226 --- /dev/null +++ b/sparv/core/language_registry.py @@ -0,0 +1,21 @@ +from typing import Optional +import pycountry + + +class LanguageRegistry(dict): + """Registry for installed languages.""" + + def add_language(self, lang: str) -> str: + if lang not in self: + langcode, _, suffix = lang.partition("-") + iso_lang = self.get_language_name_by_part3(langcode) + if iso_lang: + self[lang] = f"{iso_lang} ({suffix})" if suffix else iso_lang + else: + self[lang] = lang + return self[lang] + + @staticmethod + def get_language_name_by_part3(part3: str) -> Optional[str]: + lang = pycountry.languages.get(alpha_3=part3) + return lang.name if lang else None diff --git a/sparv/core/registry.py b/sparv/core/registry.py index 2039921c..7ef25897 100644 --- a/sparv/core/registry.py +++ b/sparv/core/registry.py @@ -7,12 +7,12 @@ from enum import Enum from typing import Callable, Dict, List, Optional, Tuple, Type, TypeVar -import iso639 import typing_inspect from sparv.core import config as sparv_config from sparv.core import paths from sparv.core.console import console +from sparv.core.language_registry import LanguageRegistry from sparv.core.misc import SparvErrorMessage from sparv.api.classes import (BaseOutput, Config, Export, ExportAnnotations, ExportAnnotationsAllSourceFiles, SourceAnnotations, SourceStructureParser, ModelOutput, OutputMarker, Wildcard) @@ -64,7 +64,7 @@ def __init__(self, name): wizards = [] # All supported languages -languages = {} +languages = LanguageRegistry() # All config keys containing lists of automatic annotations (i.e. ExportAnnotations) annotation_sources = {"export.annotations"} @@ -297,14 +297,7 @@ def _add_to_registry(annotator): if annotator["language"]: # Add to set of supported languages... for lang in annotator["language"]: - if lang not in languages: - langcode, _, suffix = lang.partition("-") - if suffix: - suffix = f" ({suffix})" - if langcode in iso639.languages.part3: - languages[lang] = iso639.languages.get(part3=langcode).name + suffix - else: - languages[lang] = lang + languages.add_language(lang) # ... but skip annotators for other languages than the one specified in the config if sparv_config.get("metadata.language") and not check_language( sparv_config.get("metadata.language"), annotator["language"], sparv_config.get("metadata.variety")): diff --git a/tests/unit/test_language_registry.py b/tests/unit/test_language_registry.py new file mode 100644 index 00000000..010366d7 --- /dev/null +++ b/tests/unit/test_language_registry.py @@ -0,0 +1,22 @@ +import pytest + +from sparv.core.language_registry import LanguageRegistry + + +@pytest.fixture() +def language_registry() -> LanguageRegistry: + return LanguageRegistry() + + +class TestLanguageRegistry: + def check(self, tested: str, expected: str): + assert tested == expected + + @pytest.mark.parametrize("lang, expected", [("swe", "Swedish"), ("xxx", "xxx")]) + def test_add_language_succeeds(self, language_registry: LanguageRegistry, lang: str, expected: str): + res = language_registry.add_language(lang) + self.check(res, expected) + self.check(language_registry[lang], expected) + + + From 25582ea06223ca5b67d3e31fdf186c3a917e5ac2 Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Thu, 19 Jan 2023 13:03:20 +0100 Subject: [PATCH 072/175] build: add pytest marker 'unit' --- Makefile | 14 ++++++++++++-- pytest.ini | 1 + 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 1b194cc2..718d853b 100644 --- a/Makefile +++ b/Makefile @@ -13,12 +13,18 @@ help: @echo "test | run-all-tests" @echo " run all tests" @echo "" - @echo "run-doc-tests" - @echo " run all tests" + @echo "noexternal-tests" + @echo " run all tests marked with 'noexternal'" + @echo "" + @echo "unit-tests" + @echo " run all tests marked with 'unit'" @echo "" @echo "run-all-tests-w-coverage" @echo " run all tests with coverage collection" @echo "" + @echo "run-noexternal-tests-w-coverage" + @echo " run all tests marked with 'noexternal' with coverage collection" + @echo "" @echo "lint" @echo " lint the code" @echo "" @@ -69,6 +75,10 @@ run-all-tests: noexternal-tests: ${INVENV} pytest -m noexternal +.PHONY: unit-tests +unit-tests: + ${INVENV} pytest -m unit + .PHONY: run-all-tests-w-coverage run-all-tests-w-coverage: ${INVENV} pytest -vv --cov=${PROJECT} --cov-report=xml tests diff --git a/pytest.ini b/pytest.ini index 75227a57..4dadce8f 100644 --- a/pytest.ini +++ b/pytest.ini @@ -10,3 +10,4 @@ markers = freeling: tests for FreeLing corpora stanford: tests for Stanford Parser corpora noexternal: tests that can be runned directly + unit: unit tests From 0fb4915b6fb95212b88317aac90e6fd89a9404ad Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Thu, 19 Jan 2023 13:03:46 +0100 Subject: [PATCH 073/175] refactor: use LangaugeRegistry --- sparv/core/language_registry.py | 5 +++++ sparv/modules/stanza/models.py | 7 +++---- sparv/modules/stanza/stanza.py | 5 ++--- tests/unit/test_language_registry.py | 10 +++++++++- 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/sparv/core/language_registry.py b/sparv/core/language_registry.py index 3eab9226..c3887b8d 100644 --- a/sparv/core/language_registry.py +++ b/sparv/core/language_registry.py @@ -19,3 +19,8 @@ def add_language(self, lang: str) -> str: def get_language_name_by_part3(part3: str) -> Optional[str]: lang = pycountry.languages.get(alpha_3=part3) return lang.name if lang else None + + @staticmethod + def get_language_part1_by_part3(part3: str) -> Optional[str]: + lang = pycountry.languages.get(alpha_3=part3) + return lang.alpha_2 if lang else None diff --git a/sparv/modules/stanza/models.py b/sparv/modules/stanza/models.py index fdc4285a..ec6dcd7f 100644 --- a/sparv/modules/stanza/models.py +++ b/sparv/modules/stanza/models.py @@ -3,9 +3,8 @@ import json import logging -import iso639 - from sparv.api import Language, Model, ModelOutput, modelbuilder, get_logger +from sparv.core.language_registry import LanguageRegistry logger = get_logger(__name__) @@ -70,8 +69,8 @@ def get_model(lang: Language = Language(), resources_file: ModelOutput = ModelOutput("stanza/[metadata.language]/resources.json")): """Download Stanza language models.""" import stanza - lang_name = iso639.languages.get(part3=lang).name if lang in iso639.languages.part3 else lang - stanza_lang = iso639.languages.get(part3=lang).part1 + lang_name = LanguageRegistry.get_language_name_by_part3(lang) or lang + stanza_lang = LanguageRegistry.get_language_part1_by_part3(lang) logger.info(f"Downloading Stanza language model for {lang_name}") stanza.download(lang=stanza_lang, model_dir=str(resources_file.path.parent), verbose=False, logging_level=logging.WARNING) diff --git a/sparv/modules/stanza/stanza.py b/sparv/modules/stanza/stanza.py index e641dc96..110662a8 100644 --- a/sparv/modules/stanza/stanza.py +++ b/sparv/modules/stanza/stanza.py @@ -2,9 +2,8 @@ from typing import Optional -import iso639 - from sparv.api import Annotation, Config, Language, Model, Output, Text, annotator, get_logger, util +from sparv.core.language_registry import LanguageRegistry from . import stanza_utils logger = get_logger(__name__) @@ -61,7 +60,7 @@ def annotate(corpus_text: Text = Text(), # Define some values needed for Stanza Pipeline nlp_args = { - "lang": iso639.languages.get(part3=lang).part1, + "lang": LanguageRegistry.get_language_part1_by_part3(lang), "processors": "tokenize,mwt,pos,lemma,depparse,ner", # Comma-separated list of processors to use "dir": str(resources_file.path.parent), "depparse_max_sentence_size": 200, # Create new batch when encountering sentences larger than this diff --git a/tests/unit/test_language_registry.py b/tests/unit/test_language_registry.py index 010366d7..568ade6b 100644 --- a/tests/unit/test_language_registry.py +++ b/tests/unit/test_language_registry.py @@ -1,3 +1,4 @@ +from typing import Optional import pytest from sparv.core.language_registry import LanguageRegistry @@ -9,14 +10,21 @@ def language_registry() -> LanguageRegistry: class TestLanguageRegistry: - def check(self, tested: str, expected: str): + def check(self, tested: Optional[str], expected: str): assert tested == expected @pytest.mark.parametrize("lang, expected", [("swe", "Swedish"), ("xxx", "xxx")]) + @pytest.mark.unit + @pytest.mark.noexternal def test_add_language_succeeds(self, language_registry: LanguageRegistry, lang: str, expected: str): res = language_registry.add_language(lang) self.check(res, expected) self.check(language_registry[lang], expected) + @pytest.mark.parametrize("lang, expected", [("swe", "sv"), ("xxx", None)]) + @pytest.mark.unit + @pytest.mark.noexternal + def test_get_part1_by_part3(self, lang: str, expected: str): + self.check(LanguageRegistry.get_language_part1_by_part3(lang), expected) From b80cad667b66874c22f28c3d8be3e82f84cb8d35 Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Thu, 19 Jan 2023 13:58:48 +0100 Subject: [PATCH 074/175] build: bump typing-inspect --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f954a820..6798194d 100644 --- a/setup.py +++ b/setup.py @@ -50,7 +50,7 @@ def get_readme(readme_path): "snakemake==6.3.0", "stanza==1.4.0", "torch>=1.9.1", # Used by Stanza; see https://github.com/spraakbanken/sparv-pipeline/issues/82 - "typing-inspect==0.7.1" + "typing-inspect==0.8.0" ], extras_require={ "dev": [ From 77a9b588b8002627084cfa1412920e805a797e6b Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Mon, 23 Jan 2023 09:46:07 +0100 Subject: [PATCH 075/175] docs: update changelog --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 89fa8c00..893a03da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,8 @@ ### Changed +- Added support for Python 3.10. +- Dropped support for Python 3.6. - `AnnotationAllSourceFiles` now have the same methods as `Annotation`. - The util function `install_mysql` can now install locally as well as to a remote server. - Pre-built SALDO models are now downloaded instead of being built on demand. @@ -139,7 +141,7 @@ - `korp.remote_cwb_registry` is now called `cwb.remote_registry_dir` - `korp.remote_host` has been split into `korp.remote_host` (host for SQL files) and `cwb.remote_host` (host for CWB files) - - install target `korp:install_corpus` has been renamed and split into `cwb:install_corpus` and + - install target `korp:install_corpus` has been renamed and split into `cwb:install_corpus` and `cwb:install_corpus_scrambled` - Renamed the following stats exports: `stats_export:freq_list` is now called `stats_export:sbx_freq_list` From f936c1d780a4e8e548384e3286e0ddccaadb8492 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 17 Feb 2023 16:28:28 +0100 Subject: [PATCH 076/175] Move ISO 639 functions to util package --- sparv/api/util/misc.py | 13 +++++++++++++ sparv/core/language_registry.py | 26 -------------------------- sparv/core/registry.py | 16 +++++++++++++++- sparv/modules/stanza/models.py | 7 +++---- sparv/modules/stanza/stanza.py | 3 +-- tests/unit/test_language_registry.py | 11 ++--------- 6 files changed, 34 insertions(+), 42 deletions(-) delete mode 100644 sparv/core/language_registry.py diff --git a/sparv/api/util/misc.py b/sparv/api/util/misc.py index 740dfda2..b78d9074 100644 --- a/sparv/api/util/misc.py +++ b/sparv/api/util/misc.py @@ -1,6 +1,7 @@ """Misc util functions.""" import pathlib +import pycountry import re import unicodedata from collections import OrderedDict, defaultdict @@ -134,3 +135,15 @@ def indent_xml(elem, level=0, indentation=" ") -> None: else: if level and (not elem.tail or not elem.tail.strip()): elem.tail = i + + +def get_language_name_by_part3(part3: str) -> Optional[str]: + """Return language name in English given an ISO 639-3 code.""" + lang = pycountry.languages.get(alpha_3=part3) + return lang.name if lang else None + + +def get_language_part1_by_part3(part3: str) -> Optional[str]: + """Return ISO 639-1 code given an ISO 639-3 code.""" + lang = pycountry.languages.get(alpha_3=part3) + return lang.alpha_2 if lang else None diff --git a/sparv/core/language_registry.py b/sparv/core/language_registry.py deleted file mode 100644 index c3887b8d..00000000 --- a/sparv/core/language_registry.py +++ /dev/null @@ -1,26 +0,0 @@ -from typing import Optional -import pycountry - - -class LanguageRegistry(dict): - """Registry for installed languages.""" - - def add_language(self, lang: str) -> str: - if lang not in self: - langcode, _, suffix = lang.partition("-") - iso_lang = self.get_language_name_by_part3(langcode) - if iso_lang: - self[lang] = f"{iso_lang} ({suffix})" if suffix else iso_lang - else: - self[lang] = lang - return self[lang] - - @staticmethod - def get_language_name_by_part3(part3: str) -> Optional[str]: - lang = pycountry.languages.get(alpha_3=part3) - return lang.name if lang else None - - @staticmethod - def get_language_part1_by_part3(part3: str) -> Optional[str]: - lang = pycountry.languages.get(alpha_3=part3) - return lang.alpha_2 if lang else None diff --git a/sparv/core/registry.py b/sparv/core/registry.py index 7ef25897..01ed555a 100644 --- a/sparv/core/registry.py +++ b/sparv/core/registry.py @@ -12,7 +12,6 @@ from sparv.core import config as sparv_config from sparv.core import paths from sparv.core.console import console -from sparv.core.language_registry import LanguageRegistry from sparv.core.misc import SparvErrorMessage from sparv.api.classes import (BaseOutput, Config, Export, ExportAnnotations, ExportAnnotationsAllSourceFiles, SourceAnnotations, SourceStructureParser, ModelOutput, OutputMarker, Wildcard) @@ -42,6 +41,21 @@ def __init__(self, name): self.description = None +class LanguageRegistry(dict): + """Registry for supported languages.""" + + def add_language(self, lang: str) -> str: + from sparv.api import util + if lang not in self: + langcode, _, suffix = lang.partition("-") + iso_lang = util.misc.get_language_name_by_part3(langcode) + if iso_lang: + self[lang] = f"{iso_lang} ({suffix})" if suffix else iso_lang + else: + self[lang] = lang + return self[lang] + + # All loaded Sparv modules with their functions (possibly limited by the selected language) modules: Dict[str, Module] = {} diff --git a/sparv/modules/stanza/models.py b/sparv/modules/stanza/models.py index ec6dcd7f..2270f27f 100644 --- a/sparv/modules/stanza/models.py +++ b/sparv/modules/stanza/models.py @@ -3,8 +3,7 @@ import json import logging -from sparv.api import Language, Model, ModelOutput, modelbuilder, get_logger -from sparv.core.language_registry import LanguageRegistry +from sparv.api import Language, Model, ModelOutput, modelbuilder, get_logger, util logger = get_logger(__name__) @@ -69,8 +68,8 @@ def get_model(lang: Language = Language(), resources_file: ModelOutput = ModelOutput("stanza/[metadata.language]/resources.json")): """Download Stanza language models.""" import stanza - lang_name = LanguageRegistry.get_language_name_by_part3(lang) or lang - stanza_lang = LanguageRegistry.get_language_part1_by_part3(lang) + lang_name = util.misc.get_language_name_by_part3(lang) or lang + stanza_lang = util.misc.get_language_part1_by_part3(lang) logger.info(f"Downloading Stanza language model for {lang_name}") stanza.download(lang=stanza_lang, model_dir=str(resources_file.path.parent), verbose=False, logging_level=logging.WARNING) diff --git a/sparv/modules/stanza/stanza.py b/sparv/modules/stanza/stanza.py index 110662a8..bcbf43d8 100644 --- a/sparv/modules/stanza/stanza.py +++ b/sparv/modules/stanza/stanza.py @@ -3,7 +3,6 @@ from typing import Optional from sparv.api import Annotation, Config, Language, Model, Output, Text, annotator, get_logger, util -from sparv.core.language_registry import LanguageRegistry from . import stanza_utils logger = get_logger(__name__) @@ -60,7 +59,7 @@ def annotate(corpus_text: Text = Text(), # Define some values needed for Stanza Pipeline nlp_args = { - "lang": LanguageRegistry.get_language_part1_by_part3(lang), + "lang": util.misc.get_language_part1_by_part3(lang), "processors": "tokenize,mwt,pos,lemma,depparse,ner", # Comma-separated list of processors to use "dir": str(resources_file.path.parent), "depparse_max_sentence_size": 200, # Create new batch when encountering sentences larger than this diff --git a/tests/unit/test_language_registry.py b/tests/unit/test_language_registry.py index 568ade6b..18375565 100644 --- a/tests/unit/test_language_registry.py +++ b/tests/unit/test_language_registry.py @@ -1,7 +1,8 @@ from typing import Optional + import pytest -from sparv.core.language_registry import LanguageRegistry +from sparv.core.registry import LanguageRegistry @pytest.fixture() @@ -20,11 +21,3 @@ def test_add_language_succeeds(self, language_registry: LanguageRegistry, lang: res = language_registry.add_language(lang) self.check(res, expected) self.check(language_registry[lang], expected) - - @pytest.mark.parametrize("lang, expected", [("swe", "sv"), ("xxx", None)]) - @pytest.mark.unit - @pytest.mark.noexternal - def test_get_part1_by_part3(self, lang: str, expected: str): - self.check(LanguageRegistry.get_language_part1_by_part3(lang), expected) - - From f93a041c9f51fc1ed0eb4ceeb0c460d358d1b47d Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Mon, 20 Feb 2023 15:00:45 +0100 Subject: [PATCH 077/175] Fix Python 3.8 compatibility --- sparv/core/registry.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sparv/core/registry.py b/sparv/core/registry.py index 01ed555a..62219aa5 100644 --- a/sparv/core/registry.py +++ b/sparv/core/registry.py @@ -5,7 +5,7 @@ import re from collections import defaultdict from enum import Enum -from typing import Callable, Dict, List, Optional, Tuple, Type, TypeVar +from typing import Callable, Dict, List, Optional, Set, Tuple, Type, TypeVar import typing_inspect @@ -529,7 +529,7 @@ def expand_variables(string, rule_name: Optional[str] = None, is_annotation: boo # Split if list of alternatives strings = [s for s in string.split(", ") for string in strings] - def expand_classes(s: str, parents: set[str]) -> Tuple[Optional[str], Optional[str]]: + def expand_classes(s: str, parents: Set[str]) -> Tuple[Optional[str], Optional[str]]: classes = find_classes(s, True) if not classes: return s, None From 03ba07f22b036d033ffb8918b0531e28c9a664e0 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Mon, 20 Feb 2023 17:41:12 +0100 Subject: [PATCH 078/175] Small code improvements --- sparv/modules/dateformat/dateformat.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/sparv/modules/dateformat/dateformat.py b/sparv/modules/dateformat/dateformat.py index a3ada9a5..6f78b225 100644 --- a/sparv/modules/dateformat/dateformat.py +++ b/sparv/modules/dateformat/dateformat.py @@ -66,7 +66,7 @@ def dateformat_pretty(in_date: Annotation = Annotation("[dateformat.datetime_fro splitter: Optional[str] = Config("dateformat.splitter"), regex: Optional[str] = Config("dateformat.regex")): """Convert existing dates to format YYYY-MM-DD.""" - _formatter(in_date, Annotation(), out, Annotation(), informat, "%Y-%m-%d", splitter, regex) + _formatter(in_date, None, out, None, informat, "%Y-%m-%d", splitter, regex) @annotator("Convert existing times to specified output format", config=[ @@ -143,7 +143,7 @@ def resolution(out_resolution: OutputCommonData = OutputCommonData("dateformat.r out_resolution.write(resolutions) -def _formatter(in_from: Annotation, in_to: Optional[Annotation], out_from: Output, out_to: Output, +def _formatter(in_from: Annotation, in_to: Optional[Annotation], out_from: Output, out_to: Optional[Output], informat: str, outformat: str, splitter: str, regex: str): """Take existing dates/times and input formats and convert to specified output format.""" def get_smallest_unit(informat): @@ -192,8 +192,8 @@ def get_date_length(informat): return length # Check that the input annotation matches the output - if (in_from.annotation_name != out_from.annotation_name) or ( - in_to.annotation_name != out_to.annotation_name): + if (in_from and in_from.annotation_name != out_from.annotation_name) or ( + in_to and in_to.annotation_name != out_to.annotation_name): raise SparvErrorMessage("The 'dateformat' attributes must be attached to the same annotation as the input" f" (in this case the '{in_from.annotation_name}' annotation)") @@ -202,8 +202,6 @@ def get_date_length(informat): informat = informat.split("|") outformat = outformat.split("|") - if splitter: - splitter = splitter assert len(outformat) == 1 or (len(outformat) == len(informat)), "The number of out-formats must be equal to one " \ "or the number of in-formats." From 97284a451b9855fdca5399fb03cf88268b371e2a Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Wed, 22 Feb 2023 13:02:45 +0100 Subject: [PATCH 079/175] Clean up CI workflow; don't use make --- .github/workflows/ci.yml | 39 +++++++---------- Makefile | 90 ---------------------------------------- pytest.ini | 2 +- 3 files changed, 16 insertions(+), 115 deletions(-) delete mode 100644 Makefile diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 59f6164f..37a7285e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,9 +1,6 @@ name: CI - on: [push, pull_request] - - jobs: checks: name: ${{ matrix.task.name }} py-${{ matrix.python-version }} on ${{ matrix.os }} @@ -13,51 +10,45 @@ jobs: max-parallel: 4 fail-fast: false matrix: - # TODO: enable if decided to test 3.7 - # python-version: ["3.7", "3.8", "3.9", "3.10"] python-version: ["3.8", "3.9", "3.10"] os: [ubuntu-latest] task: - name: Run tests - run: make noexternal-tests + run: | + source venv/bin/activate + pytest -m noexternal steps: - name: Checkout code uses: nschloe/action-cached-lfs-checkout@v1 - # Use these to explicitly include/exclude files: - # with: - # include: "*" - # exclude: "" - - name: Set up Python ${{ matrix.python-version }} + id: setup-python uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - # ---------------------------------------------- - # load cached venv if cache exists - # ---------------------------------------------- + # Load cached venv if cache exists - name: Load cached venv - id: cached-poetry-dependencies + id: cached-dependencies uses: actions/cache@v3.2.3 with: path: venv key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/pyproject.toml') }}-${{ hashFiles('**/setup.py') }}-${{ hashFiles('.github/workflows/ci.yml') }} - #---------------------------------------------- - # install dependencies if cache does not exist - #---------------------------------------------- - - name: Install dependencies - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + # Create virtual environment and install dependencies if cache does not exist + - name: Create venv and install dependencies + if: steps.cached-dependencies.outputs.cache-hit != 'true' run: | - make setup-venv - make dev + python3 -m venv venv + source venv/bin/activate + pip install wheel + pip install -e .[dev] - name: Setup Sparv run: | - venv/bin/sparv setup -d $PWD - tree . + source venv/bin/activate + sparv setup -d $PWD - name: ${{ matrix.task.name }} run: ${{ matrix.task.run }} diff --git a/Makefile b/Makefile deleted file mode 100644 index 718d853b..00000000 --- a/Makefile +++ /dev/null @@ -1,90 +0,0 @@ - -.default: help - -.PHONY: help -help: - @echo "usage:" - @echo "" - @echo "setup-venv" - @echo " create virtual environment" - @echo "dev | install-dev" - @echo " setup development environment" - @echo "" - @echo "test | run-all-tests" - @echo " run all tests" - @echo "" - @echo "noexternal-tests" - @echo " run all tests marked with 'noexternal'" - @echo "" - @echo "unit-tests" - @echo " run all tests marked with 'unit'" - @echo "" - @echo "run-all-tests-w-coverage" - @echo " run all tests with coverage collection" - @echo "" - @echo "run-noexternal-tests-w-coverage" - @echo " run all tests marked with 'noexternal' with coverage collection" - @echo "" - @echo "lint" - @echo " lint the code" - @echo "" - @echo "type-check" - @echo " check types" - @echo "" - @echo "fmt" - @echo " run formatter(s)" - @echo "" - @echo "check-fmt" - @echo " check formatting" - @echo "" - -PLATFORM := ${shell uname -o} -PROJECT := sparv - -ifeq (${VIRTUAL_ENV},) - VENV_NAME = venv - INVENV = export VIRTUAL_ENV="${VENV_NAME}"; export PATH="${VENV_NAME}/bin:${PATH}"; unset PYTHON_HOME; -else - VENV_NAME = ${VIRTUAL_ENV} - INVENV = -endif - -${info Platform: ${PLATFORM}} - -setup-venv: - python3 -m venv ${VENV_NAME} - ${VENV_NAME}/bin/pip install wheel - -dev: install-dev -install-dev: - ${VENV_NAME}/bin/pip install -e .[dev] - -setup-sparv: - ${INVENV} sparv setup - -setup-sparv-from-env: - ${INVENV} sparv setup --dir ${SPARV_DATADIR} - -.PHONY: test -test: run-all-tests -.PHONY: run-all-tests -run-all-tests: - ${INVENV} pytest - -.PHONY: noexternal-tests -noexternal-tests: - ${INVENV} pytest -m noexternal - -.PHONY: unit-tests -unit-tests: - ${INVENV} pytest -m unit - -.PHONY: run-all-tests-w-coverage -run-all-tests-w-coverage: - ${INVENV} pytest -vv --cov=${PROJECT} --cov-report=xml tests - -.PHONY: run-all-tests-w-coverage -run-noexternal-tests-w-coverage: - ${INVENV} pytest -vv --cov=${PROJECT} --cov-report=xml -m noexternal - - diff --git a/pytest.ini b/pytest.ini index 4dadce8f..6c646927 100644 --- a/pytest.ini +++ b/pytest.ini @@ -9,5 +9,5 @@ markers = treetagger: tests for Treetagger corpora freeling: tests for FreeLing corpora stanford: tests for Stanford Parser corpora - noexternal: tests that can be runned directly + noexternal: tests that don't rely on external tools unit: unit tests From c773691d1b187fa27b96c14dc330a4c82dfeca2b Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Wed, 22 Feb 2023 14:41:53 +0100 Subject: [PATCH 080/175] Require Python 3.8 or higher --- CHANGELOG.md | 2 +- README.md | 2 +- docs/user-manual/installation-and-setup.md | 2 +- docs/user-manual/quick-start.md | 2 +- setup.py | 2 +- sparv/__main__.py | 5 +++-- 6 files changed, 8 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 893a03da..61b44dd4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,7 @@ ### Changed - Added support for Python 3.10. -- Dropped support for Python 3.6. +- Dropped support for Python 3.6 and 3.7. - `AnnotationAllSourceFiles` now have the same methods as `Annotation`. - The util function `install_mysql` can now install locally as well as to a remote server. - Pre-built SALDO models are now downloaded instead of being built on demand. diff --git a/README.md b/README.md index 4c7eedb7..bf5affdd 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ If you have any questions, problems or suggestions please contact [!NOTE] > Most of Sparv's features should work in a Windows environment as well, but since we don't do any testing on Windows diff --git a/docs/user-manual/quick-start.md b/docs/user-manual/quick-start.md index f6ce151e..62381849 100644 --- a/docs/user-manual/quick-start.md +++ b/docs/user-manual/quick-start.md @@ -13,7 +13,7 @@ user guide, please refer to the full documentation. ## Installation -Begin by making sure that you have [Python 3.6.2](http://python.org/) or newer installed by running the following +Begin by making sure that you have [Python 3.8](https://python.org/) or newer (up to 3.10) installed by running the following in your terminal: ``` python3 --version diff --git a/setup.py b/setup.py index 6798194d..d3d037b3 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ def get_readme(readme_path): license="MIT", packages=setuptools.find_namespace_packages(include=["sparv", "sparv.*"]), zip_safe=False, - python_requires=">=3.6.2", + python_requires=">=3.8,<3.11", install_requires=[ "appdirs==1.4.4", "docx2python==1.27.1", diff --git a/sparv/__main__.py b/sparv/__main__.py index 895c7eb9..a2e86ce2 100644 --- a/sparv/__main__.py +++ b/sparv/__main__.py @@ -7,8 +7,9 @@ from sparv import __version__ # Check Python version -if sys.version_info < (3, 6, 2): - raise Exception("Python 3.6.2 or higher is required.") +if sys.version_info < (3, 8, 0) or sys.version_info >= (3, 11, 0): + print("Python 3.8–3.10 is required.") + sys.exit(1) class CustomArgumentParser(argparse.ArgumentParser): From 633d8ebe5033ebf3a087ab92e2f91e362a7b8b07 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Wed, 22 Feb 2023 14:46:37 +0100 Subject: [PATCH 081/175] Replace old Python 3.7 compatibility code --- sparv/core/io.py | 20 +++++++------------- sparv/modules/korp/config.py | 7 +------ 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/sparv/core/io.py b/sparv/core/io.py index 7d753c64..0dc36146 100644 --- a/sparv/core/io.py +++ b/sparv/core/io.py @@ -43,10 +43,7 @@ def annotation_exists(annotation: BaseAnnotation, source_file: Optional[str] = N def remove_annotation(annotation: BaseAnnotation, source_file: Optional[str] = None): """Remove an annotation file.""" annotation_path = get_annotation_path(source_file or annotation.source_file, annotation, data=annotation.data) - try: - annotation_path.unlink() # TODO: Use missing_ok=True once we require Python 3.8 - except FileNotFoundError: - pass + annotation_path.unlink(missing_ok=True) def write_annotation(source_file: str, annotation: BaseOutput, values, append: bool = False, @@ -135,9 +132,8 @@ def _generator(reader_): with open_annotation_file(ann_file, mode="rb") as f: reader = f.raw.read if hasattr(f, "raw") and hasattr(f.raw, "read") else f.read count += sum(buf.count(b"\n") for buf in _generator(reader)) - except (OSError, lzma.LZMAError, UnicodeDecodeError) as e: - # TODO: Use gzip.BadGzipFile instead of checking for "Not a gzipped file" once we require Python 3.8 - if isinstance(e, OSError) and not ("Not a gzipped file" in str(e) or str(e) == "Invalid data stream"): + except (gzip.BadGzipFile, OSError, lzma.LZMAError, UnicodeDecodeError) as e: + if isinstance(e, OSError) and str(e) != "Invalid data stream": raise e raise_format_error(ann_file) @@ -215,9 +211,8 @@ def _read_single_annotation(source_file: str, annotation: str, with_annotation_n value = re.sub(r"((? Date: Thu, 23 Feb 2023 14:15:44 +0100 Subject: [PATCH 082/175] Improve some comments and documentation --- docs/developers-guide/config-parameters.md | 4 ++-- docs/developers-guide/general-concepts.md | 4 ++-- docs/developers-guide/sparv-decorators.md | 4 ++-- docs/developers-guide/writing-sparv-plugins.md | 8 ++++---- docs/docsify/index.html | 2 +- docs/user-manual/corpus-configuration.md | 2 +- docs/user-manual/running-sparv.md | 2 +- sparv/api/util/system.py | 2 +- sparv/modules/hist/__init__.py | 2 +- sparv/modules/hist/models.py | 2 +- sparv/modules/hunpos/morphtable.py | 10 +++++----- sparv/modules/lexical_classes/token.py | 2 +- sparv/modules/misc/misc.py | 4 ++-- sparv/modules/saldo/compound.py | 2 +- sparv/modules/saldo/saldo.py | 8 ++++---- sparv/modules/segment/segment.py | 2 +- sparv/modules/sensaldo/sensaldo.py | 2 +- sparv/modules/xml_import/xml_import.py | 2 +- 18 files changed, 32 insertions(+), 32 deletions(-) diff --git a/docs/developers-guide/config-parameters.md b/docs/developers-guide/config-parameters.md index 5045bf55..39e8bcff 100644 --- a/docs/developers-guide/config-parameters.md +++ b/docs/developers-guide/config-parameters.md @@ -47,8 +47,8 @@ When Sparv processes the corpus configuration it will look for config values in priority order: 1. the corpus configuration file 2. a parent corpus configuration file -2. the default configuration file in the [Sparv data directory](user-manual/installation-and-setup.md#setting-up-sparv) -3. config default values defined in the Sparv decorators (as shown above) +3. the default configuration file in the [Sparv data directory](user-manual/installation-and-setup.md#setting-up-sparv) +4. config default values defined in the Sparv decorators (as shown above) This means that if a config parameter is given a default value in a Sparv decorator it can be overridden by the default configuration file which in turn can be overridden by the user's corpus config file. diff --git a/docs/developers-guide/general-concepts.md b/docs/developers-guide/general-concepts.md index 9fc98a70..ac565ced 100644 --- a/docs/developers-guide/general-concepts.md +++ b/docs/developers-guide/general-concepts.md @@ -2,13 +2,13 @@ This section will give a brief overview of how Sparv modules work and introduce some general concepts. More details are provided in the following chapters. -The Sparv Pipeline is comprised of some core functionality and many different modules containing Sparv functions that +The Sparv Pipeline is made up of a core and different modules. The modules contain Sparv functions that serve different purposes like reading and parsing source files, building or downloading models, producing annotations and producing output files that contain the source text and annotations. All of these modules (i.e. the code inside the `sparv/modules` directory) are replacable. A Sparv function is decorated with a special [decorator](developers-guide/sparv-decorators) that tells Sparv what purpose it serves. A function's parameters hold information about what input is needed in order to run the function and what output is produced by it. The Sparv core -automatically finds all decorated functions, scans their parameters and builds a registry for what modules are available +automatically finds all decorated functions, scans their parameters and builds a registry of what modules are available and how they depend on each other. diff --git a/docs/developers-guide/sparv-decorators.md b/docs/developers-guide/sparv-decorators.md index 2d1b0227..8316e4c9 100644 --- a/docs/developers-guide/sparv-decorators.md +++ b/docs/developers-guide/sparv-decorators.md @@ -83,7 +83,7 @@ def parse(source_file: SourceFilename = SourceFilename(), ## @exporter A function decorated with `@exporter` is used to produce "final" output (also called export), typically combining information from multiple annotations into one file. Output produced by an exporter is usually not used as input in any -another module. An export can consist of one file per input corpus file or it can combine information from all input +another module. An export can consist of one file per input corpus file, or it can combine information from all input files into one output file. **Arguments:** @@ -205,7 +205,7 @@ A function decorated with `@wizard` is used to generate questions for the corpus - `config_keys`: a list of config keys to be set or changed by this function. - `source_structure`: Set to `True` if the function needs access to a `SourceStructureParser` instance (the one holding - information on the structure of the source files. Default: `False` + information on the structure of the source files). Default: `False` **Example:** ```python diff --git a/docs/developers-guide/writing-sparv-plugins.md b/docs/developers-guide/writing-sparv-plugins.md index b642afdf..415e68df 100644 --- a/docs/developers-guide/writing-sparv-plugins.md +++ b/docs/developers-guide/writing-sparv-plugins.md @@ -1,6 +1,6 @@ # Writing Sparv Plugins -The Sparv Pipeline is comprised of different modules like importers, annotators and exporters. Although many modules are -shipped with the main Sparv package none of these modules are hard-coded into the Sparv Pipeline and therefore it can +The Sparv Pipeline is made up of different modules like importers, annotators and exporters. Although many modules are +shipped with the main Sparv package, none of these modules are hard-coded into the Sparv Pipeline, and therefore it can easily be extended with plugins. A plugin is a Sparv module that is not part of the main Sparv package. Writing a plugin is the recommended way of adding a new module to Sparv. @@ -30,7 +30,7 @@ In the above example the `sbx_uppercase` directory is a Sparv module containing `uppercase.py` and the mandatory [init file](#init-file) `__init__.py`. The [setup file](#setup-file) `setup.py` in the root directory is needed in order to install the plugin. -The readme and license files are not strictly necessary for the plugin to work but we strongly recommend that you +The readme and license files are not strictly necessary for the plugin to work, but we strongly recommend that you include these if you want to publish your plugin. @@ -235,7 +235,7 @@ you should be able to inject your plugin into the Sparv Pipeline code using pipx pipx inject sparv-pipeline [pointer-to-sparv-plugin] ``` -So if you are trying to install the `sparv-sbx-uppercase` plugin and it exists on PyPI you can install it like this: +So if you are trying to install the `sparv-sbx-uppercase` plugin and it exists on PyPI, you can install it like this: ``` pipx inject sparv-pipeline sparv-sbx-uppercase ``` diff --git a/docs/docsify/index.html b/docs/docsify/index.html index d87c2ad3..e279152d 100644 --- a/docs/docsify/index.html +++ b/docs/docsify/index.html @@ -77,7 +77,7 @@ placeholder: "Type to search", noData: "No Results!", // depth: 6, // Headline depth, 1 - 6 - hideOtherSidebarContent: true, // whether or not to hide other sidebar content on search + hideOtherSidebarContent: true, // Whether to hide other sidebar content on search }, plugins: [ function(hook) { diff --git a/docs/user-manual/corpus-configuration.md b/docs/user-manual/corpus-configuration.md index 6b3b52e9..88dfc050 100644 --- a/docs/user-manual/corpus-configuration.md +++ b/docs/user-manual/corpus-configuration.md @@ -70,7 +70,7 @@ The `metadata` section of your corpus config contains metadata about your corpus - `metadata.id` defines the machine name of the corpus. It is required by some exporter modules. This string may contain ascii letters, digits and dashes. -- `metadata.name` is an optional human readable name of the corpus. This option is split into two fields, `eng` and +- `metadata.name` is an optional human-readable name of the corpus. This option is split into two fields, `eng` and `swe` for defining a name in English and in Swedish. - `metadata.language` defines the language of the source files in the corpus. This should be an ISO 639-3 code. If not diff --git a/docs/user-manual/running-sparv.md b/docs/user-manual/running-sparv.md index 68662261..4064f9d9 100644 --- a/docs/user-manual/running-sparv.md +++ b/docs/user-manual/running-sparv.md @@ -61,7 +61,7 @@ Therefore, you do not need to annotate the corpus before installing. You can lis with `sparv install -l`. **`sparv clean`:** While annotating, Sparv will create a directory called `sparv-workdir` inside your corpus directory. -You normally don't need to touch the files stored here. Leaving this directory as it is will usually lead to faster +You normally don't need to touch the files stored here. Leaving this directory as it is usually leads to faster processing of your corpus if you for example want to add a new output format. However, if you would like to delete this directory (e.g. because you want to save disk space or because you want to rerun all annotations from scratch) you can do so by running `sparv clean`. The export directory and log files can also be removed with the `clean` command diff --git a/sparv/api/util/system.py b/sparv/api/util/system.py index 24a74f23..7ad75c93 100644 --- a/sparv/api/util/system.py +++ b/sparv/api/util/system.py @@ -144,7 +144,7 @@ def find_binary(name: Union[str, list], search_paths=(), executable: bool = True if raise_error: err_msg = f"Couldn't find binary: {name[0]}\nSearched in: {', '.join(search_paths)}\n" if len(name) > 1: - err_msg += f"For binary names: {', '.join(name)}" + err_msg += f"for binary names: {', '.join(name)}" raise SparvErrorMessage(err_msg) else: return None diff --git a/sparv/modules/hist/__init__.py b/sparv/modules/hist/__init__.py index 01de8882..379ddb8a 100644 --- a/sparv/modules/hist/__init__.py +++ b/sparv/modules/hist/__init__.py @@ -10,7 +10,7 @@ Config("hist.fsv_model", default="hist/fsvm.pickle", description="Path to model for fornsvenska morphology"), Config("hist.fsv_spelling", default="hist/fsv-spelling-variants.txt", description="Path to model for fornsvenska spelling variants"), - # Set max_mwe_gaps to 0 since many (most?) multi-word in the old lexicons are unseparable (half öre etc) + # Set max_mwe_gaps to 0 since many (most?) multi-word in the old lexicons are unseparable (half öre etc.) Config("hist.max_mwe_gaps", default=0, description="Max amount of gaps allowed within a multiword expression"), Config("hist.delimiter", default=util.constants.DELIM, description="Character to put between ambiguous results"), Config("hist.affix", default=util.constants.AFFIX, description="Character to put before and after sets of results"), diff --git a/sparv/modules/hist/models.py b/sparv/modules/hist/models.py index 01985a24..3a99d891 100644 --- a/sparv/modules/hist/models.py +++ b/sparv/modules/hist/models.py @@ -140,7 +140,7 @@ def read_lmf(xml, annotation_elements=("writtenForm", "lemgram"), verbose=True, particle = False mwe_gap = False - # Is it the last word in the multi word expression? + # Is it the last word in the multi-word expression? if i == len(wordparts) - 1: lexicon.setdefault(multiwords[0], {}).setdefault(annotations, (set(), set(), mwe_gap, particle))[1].add(tuple(multiwords[1:])) multiwords = [] diff --git a/sparv/modules/hunpos/morphtable.py b/sparv/modules/hunpos/morphtable.py index 9269234c..5473b8f7 100644 --- a/sparv/modules/hunpos/morphtable.py +++ b/sparv/modules/hunpos/morphtable.py @@ -29,7 +29,7 @@ def saldo_morphtable(out: ModelOutput = ModelOutput("hunpos/saldo_suc-tags.morph """Create a morphtable file for use with Hunpos. A morphtable contains wordforms from SALDO's morphology (with accompanying tags) which are missing in SUC3. - Since the morphtable is case sensitive, both the original form and a capitalized form + Since the morphtable is case-sensitive, both the original form and a capitalized form is saved. Args: @@ -38,16 +38,16 @@ def saldo_morphtable(out: ModelOutput = ModelOutput("hunpos/saldo_suc-tags.morph suc: Tab-separated file with wordforms from SUC, containing: frequency, wordform, tag. morphtable_base: Existing morphtable file, whose contents will be included in the new one. morphtable_patterns: Optional file with regular expressions. - add_capitalized: Whether or not capitalized word forms should be added. - add_lowercase: Whether or not lower case word forms should be added. + add_capitalized: Whether capitalized word forms should be added. + add_lowercase: Whether lower case word forms should be added. """ lex = saldo.SaldoLexicon(saldo_model.path) tags = defaultdict(set) - # Get all wordforms from SALDO + # Get all word forms from SALDO for word in list(lex.lexicon.keys()): words = lex.lookup(word) - # Filter out multi word expressions + # Filter out multi-word expressions words = [x for x in words if len(x[2]) == 0] if words: # Only use MSD not containing "-" diff --git a/sparv/modules/lexical_classes/token.py b/sparv/modules/lexical_classes/token.py index e0f36fd7..aee360dd 100644 --- a/sparv/modules/lexical_classes/token.py +++ b/sparv/modules/lexical_classes/token.py @@ -106,7 +106,7 @@ def annotate_words(out: Output, model: Model, saldoids: Annotation, pos: Annotat """ if not lexicon: lexicon = util.misc.PickledLexicon(model.path) - # Otherwise use pre-loaded lexicon (from catapult) + # Otherwise use preloaded lexicon (from catapult) sense = saldoids.read() token_pos = list(pos.read()) diff --git a/sparv/modules/misc/misc.py b/sparv/modules/misc/misc.py index 8276e0ad..e125872c 100644 --- a/sparv/modules/misc/misc.py +++ b/sparv/modules/misc/misc.py @@ -331,7 +331,7 @@ def backoff_with_info( def override(chunk: Annotation, repl: Annotation, out: Output): - """Replace values in 'chunk' with non empty values from 'repl'.""" + """Replace values in 'chunk' with non-empty values from 'repl'.""" def empty(val): if not val: return True @@ -385,7 +385,7 @@ def source(out: Output = Output(":misc.source"), @annotator("Get the first annotation from a cwb set") def first_from_set(out: Output, chunk: Annotation): - """"Get the first annotation from a set.""" + """Get the first annotation from a set.""" out_annotation = [] for val in chunk.read(): out_annotation.append(util.misc.set_to_list(val)[0] if util.misc.set_to_list(val) else "") diff --git a/sparv/modules/saldo/compound.py b/sparv/modules/saldo/compound.py index db318e87..8892d7e4 100644 --- a/sparv/modules/saldo/compound.py +++ b/sparv/modules/saldo/compound.py @@ -437,7 +437,7 @@ def three_consonant_rule(compound): def rank_compounds(compounds, nst_model, stats_lexicon): """Return a list of compounds, ordered according to their ranks. - Ranking is being done according to the amount of affixes (the fewer the higher) + Ranking is being done according to the amount of affixes (the fewer, the higher) and the compound probability which is calculated as follows: p((w1, tag1)..(wn, tag1)) = p(w1, tag1) ... * p(wn, tagn) * p(tag1, ...tagn) diff --git a/sparv/modules/saldo/saldo.py b/sparv/modules/saldo/saldo.py index 97dac8a4..1ff3b39f 100644 --- a/sparv/modules/saldo/saldo.py +++ b/sparv/modules/saldo/saldo.py @@ -78,7 +78,7 @@ def annotate(token: Annotation = Annotation(""), affix: Character to put before and after sets of results. precision: Optional format string for appending precision to each value (e.g. ':%.3f'). precision_filter: Precision filter with values 'max' (only use the annotations that are most probable), - 'first' (only use the most probable annotation(s)), 'none' (use all annotations)". + 'first' (only use the most probable annotation(s)), 'none' (use all annotations). min_precision: Only use annotations with a probability score higher than this. skip_multiword: Whether to disable annotation of multiword expressions. max_gaps (int): Max amount of gaps allowed within a multiword expression. @@ -110,7 +110,7 @@ def main(token, word, sentence, reference, out_sense, out_lemgram, out_baseform, assert models_preloaded.get(name, None) is not None, "Lexicon %s not found!" % name lexicon_list.append((name, models_preloaded[name])) - # Combine annotation names in SALDO lexicon with out annotations + # Combine annotation names in SALDO lexicon without annotations annotations = [] if out_baseform: annotations.append((out_baseform, "gf")) @@ -179,7 +179,7 @@ def main(token, word, sentence, reference, out_sense, out_lemgram, out_baseform, # Check that we don't have any unwanted overlaps _remove_unwanted_overlaps(complete_multis) - # Then save the rest of the multi word expressions in sentence_tokens + # Then save the rest of the multi-word expressions in sentence_tokens _save_multiwords(complete_multis, sentence_tokens) for tok in list(sentence_tokens.values()): @@ -381,7 +381,7 @@ def get_precision(msd, msdtags): """ Calculate the precision of a Saldo annotation. - If the the word's msdtag is among the annotation's possible msdtags, + If the word's msdtag is among the annotation's possible msdtags, we return a high value (0.75), a partial match returns 0.66, missing MSD returns 0.5, and otherwise a low value (0.25). """ diff --git a/sparv/modules/segment/segment.py b/sparv/modules/segment/segment.py index ce07c972..38ce39ae 100644 --- a/sparv/modules/segment/segment.py +++ b/sparv/modules/segment/segment.py @@ -183,7 +183,7 @@ def build_tokenlist(saldo_model: Model = Model("saldo/saldo.pickle"), wordforms = set() # Skip strings already handled by the tokenizer. - # Also skip words ending in comma (used by some multi word expressions in SALDO). + # Also skip words ending in comma (used by some multi-word expressions in SALDO). with open(saldo_model.path, "rb") as F: lexicon = pickle.load(F) for w in lexicon: diff --git a/sparv/modules/sensaldo/sensaldo.py b/sparv/modules/sensaldo/sensaldo.py index 3ef80ff8..1ad7bb30 100644 --- a/sparv/modules/sensaldo/sensaldo.py +++ b/sparv/modules/sensaldo/sensaldo.py @@ -31,7 +31,7 @@ def annotate(sense: Annotation = Annotation(""), """ if not lexicon: lexicon = util.misc.PickledLexicon(model.path) - # Otherwise use pre-loaded lexicon (from catapult) + # Otherwise use preloaded lexicon (from catapult) result_scores = [] result_labels = [] diff --git a/sparv/modules/xml_import/xml_import.py b/sparv/modules/xml_import/xml_import.py index 7d4c0b5f..c9fc370a 100644 --- a/sparv/modules/xml_import/xml_import.py +++ b/sparv/modules/xml_import/xml_import.py @@ -275,7 +275,7 @@ def iter_ns_declarations(): yield prefix, uri def get_sparv_name(xml_name: str): - """Get the sparv notation of a tag or attr name with regards to XML namespaces.""" + """Get the sparv notation of a tag or attr name with regard to XML namespaces.""" ns_uri, tag = get_namespace(xml_name) if self.remove_namespaces: return tag From e10d1475f2ad4dd3a82dc427c35195b94cee4811 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Thu, 23 Feb 2023 16:12:00 +0100 Subject: [PATCH 083/175] Clean up imports --- sparv/api/__init__.py | 18 +++++++++++++----- sparv/api/util/export.py | 3 --- sparv/api/util/misc.py | 9 ++++----- sparv/api/util/mysql_wrapper.py | 1 - sparv/modules/hunpos/hunpos.py | 7 ++++--- sparv/modules/korp/timespan.py | 1 - sparv/modules/stanza/stanza_utils.py | 2 +- sparv/modules/stats_export/stats_export.py | 1 - 8 files changed, 22 insertions(+), 20 deletions(-) diff --git a/sparv/api/__init__.py b/sparv/api/__init__.py index ea38596b..0b5674e1 100644 --- a/sparv/api/__init__.py +++ b/sparv/api/__init__.py @@ -1,10 +1,18 @@ """Classes and methods for use by plugin modules.""" -import sparv.core.io # Needed to avoid a circular import problem when importing the classes below -from sparv.core.misc import SparvErrorMessage, get_logger -from sparv.core.registry import annotator, exporter, importer, installer, modelbuilder, uninstaller, wizard +import sparv.core.io # noqa: F401; Needed to avoid a circular import problem when importing the classes below +from sparv.core.misc import SparvErrorMessage, get_logger # noqa: F401 +from sparv.core.registry import ( # noqa: F401 + annotator, + exporter, + importer, + installer, + modelbuilder, + uninstaller, + wizard, +) -from .classes import ( +from .classes import ( # noqa: F401 AllSourceFilenames, Annotation, AnnotationAllSourceFiles, @@ -43,5 +51,5 @@ SourceStructure, SourceStructureParser, Text, - Wildcard + Wildcard, ) diff --git a/sparv/api/util/export.py b/sparv/api/util/export.py index d6eb1bd0..3efc71cf 100644 --- a/sparv/api/util/export.py +++ b/sparv/api/util/export.py @@ -13,17 +13,14 @@ ExportAnnotations, ExportAnnotationsAllSourceFiles, HeaderAnnotations, - Headers, Namespaces, SourceAnnotations, SourceAnnotationsAllSourceFiles, - SourceStructure, SparvErrorMessage, get_logger, util ) from sparv.core import io - from .constants import SPARV_DEFAULT_NAMESPACE, XML_NAMESPACE_SEP logger = get_logger(__name__) diff --git a/sparv/api/util/misc.py b/sparv/api/util/misc.py index b78d9074..aa8f2383 100644 --- a/sparv/api/util/misc.py +++ b/sparv/api/util/misc.py @@ -1,14 +1,13 @@ """Misc util functions.""" import pathlib -import pycountry -import re import unicodedata -from collections import OrderedDict, defaultdict -from typing import Iterable, List, Optional, Tuple, Union +from typing import Optional, Union + +import pycountry from sparv.api import get_logger -from sparv.api.classes import Annotation, Model +from sparv.api.classes import Model from sparv.core.misc import parse_annotation_list # noqa logger = get_logger(__name__) diff --git a/sparv/api/util/mysql_wrapper.py b/sparv/api/util/mysql_wrapper.py index d7ed1a53..d752adaa 100644 --- a/sparv/api/util/mysql_wrapper.py +++ b/sparv/api/util/mysql_wrapper.py @@ -1,7 +1,6 @@ """Util function for creating mysql files.""" import os -import shlex from sparv.api import get_logger from . import system diff --git a/sparv/modules/hunpos/hunpos.py b/sparv/modules/hunpos/hunpos.py index d26c39ab..3d807abe 100644 --- a/sparv/modules/hunpos/hunpos.py +++ b/sparv/modules/hunpos/hunpos.py @@ -3,8 +3,10 @@ import re from typing import Optional -from sparv.api import (Annotation, Binary, Config, Model, ModelOutput, Output, SparvErrorMessage, annotator, get_logger, - modelbuilder, util) +from sparv.api import ( + Annotation, Binary, Config, Model, ModelOutput, Output, SparvErrorMessage, annotator, get_logger, + modelbuilder, util +) from sparv.api.util.tagsets import tagmappings logger = get_logger(__name__) @@ -101,7 +103,6 @@ def postag(out: Output = Output(":hunpos.pos", cls="token:pos", descripti def hunpos_model(model: ModelOutput = ModelOutput("hunpos/suc3_suc-tags_default-setting_utf8.model"), binary: Binary = Binary("[hunpos.binary]")): """Download the Hunpos model.""" - from sys import platform def test_hunpos(model): stdin = TOK_SEP.join(["jag", "och", "du"]) + SENT_SEP diff --git a/sparv/modules/korp/timespan.py b/sparv/modules/korp/timespan.py index a2eb3992..c224232a 100644 --- a/sparv/modules/korp/timespan.py +++ b/sparv/modules/korp/timespan.py @@ -5,7 +5,6 @@ from sparv.api import ( AllSourceFilenames, - Annotation, AnnotationAllSourceFiles, Config, Corpus, diff --git a/sparv/modules/stanza/stanza_utils.py b/sparv/modules/stanza/stanza_utils.py index f5419468..a25558e7 100644 --- a/sparv/modules/stanza/stanza_utils.py +++ b/sparv/modules/stanza/stanza_utils.py @@ -1,6 +1,6 @@ """Util functions used in stanza.""" -from sparv.api import Annotation, Output, SparvErrorMessage, annotator, util +from sparv.api import Annotation, Output, SparvErrorMessage, annotator @annotator("Annotate tokens with IDs relative to their sentences") diff --git a/sparv/modules/stats_export/stats_export.py b/sparv/modules/stats_export/stats_export.py index 48c09925..f644e6ce 100644 --- a/sparv/modules/stats_export/stats_export.py +++ b/sparv/modules/stats_export/stats_export.py @@ -7,7 +7,6 @@ from sparv.api import ( AllSourceFilenames, - Annotation, AnnotationAllSourceFiles, Config, Corpus, From d222cb77dd69a5b154dc19175d6ba75b757f30ff Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 24 Feb 2023 14:46:00 +0100 Subject: [PATCH 084/175] Make expand_variables always return a string --- sparv/core/registry.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sparv/core/registry.py b/sparv/core/registry.py index 62219aa5..6b27189a 100644 --- a/sparv/core/registry.py +++ b/sparv/core/registry.py @@ -545,10 +545,10 @@ def expand_classes(s: str, parents: Set[str]) -> Tuple[Optional[str], Optional[s if real_ann: final_ann, rest_ = expand_classes(real_ann, parents.union([cls.group(1)])) if rest_: - return None, rest_ + return "", rest_ s = s.replace(cls.group(), final_ann, 1) else: - return None, cls.group() + return "", cls.group() return s, None for string in strings: From 0150666a96f6a0a59c70e0fa0a514bebe5647548 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 24 Feb 2023 15:56:18 +0100 Subject: [PATCH 085/175] Add Black configuration --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 1cad53ef..e43b7fb7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,7 @@ [build-system] requires = ["setuptools >= 40.1.0", "wheel"] build-backend = "setuptools.build_meta" + +[tool.black] +line-length = 120 +target-version = ["py38", "py39", "py310"] From 9cd6a2e34921f1452833480af683fddfaaeac9bf Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 24 Feb 2023 16:01:50 +0100 Subject: [PATCH 086/175] Remove redundant wheel installation from workflow wheel is already required in pyproject.toml --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 37a7285e..3109b455 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,7 +42,6 @@ jobs: run: | python3 -m venv venv source venv/bin/activate - pip install wheel pip install -e .[dev] - name: Setup Sparv From 7f6eb90cf8b9461691e02c5eafb07c101f714dab Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 24 Feb 2023 18:07:52 +0100 Subject: [PATCH 087/175] Update misc URLs --- docs/README.md | 4 ++-- docs/user-manual/available-analyses.md | 10 ++++---- docs/user-manual/installation-and-setup.md | 16 ++++++------- sparv/modules/dateformat/dateformat.py | 4 ++-- sparv/modules/korp/wordpicture.py | 2 +- sparv/modules/malt/malt.py | 2 +- sparv/modules/treetagger/treetagger.py | 28 +++++++++++----------- 7 files changed, 33 insertions(+), 33 deletions(-) diff --git a/docs/README.md b/docs/README.md index b88cb291..93053d6a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -53,8 +53,8 @@ cd md2pdf ./make_pdf.sh ``` + diff --git a/docs/user-manual/available-analyses.md b/docs/user-manual/available-analyses.md index 320f1045..c16107ae 100644 --- a/docs/user-manual/available-analyses.md +++ b/docs/user-manual/available-analyses.md @@ -64,7 +64,7 @@ preset](user-manual/corpus-configuration.md#annotation-presets) called `SWE_DEFA | | | |:---|:-----------| |**Description** | Sentence segments are analysed to enrich tokens varje token with part-of-speech tags and morphosyntactic information. No longer used by default because Stanza's POS-tagging yields better results. -|**Tool** | [Hunpos](http://code.google.com/p/hunpos/) +|**Tool** | [Hunpos](https://code.google.com/archive/p/hunpos/) |**Model** | [suc3_suc-tags_default-setting_utf8.model](https://github.com/spraakbanken/sparv-models/blob/master/hunpos/suc3_suc-tags_default-setting_utf8.model?raw=true) trained on [SUC 3.0](https://spraakbanken.gu.se/resurser/suc3) |**Tagset** | [SUC MSD tags](https://spraakbanken.gu.se/korp/markup/msdtags.html) |**Annotations** | - `:hunpos.msd` (morphosyntactic tag)
- `:hunpos.pos` (part-of-speech tag) @@ -84,8 +84,8 @@ preset](user-manual/corpus-configuration.md#annotation-presets) called `SWE_DEFA | | | |:---|:-----------| |**Description** | Sentence segments are analysed to enrich tokens with dependency information. No longer used by default because Stanza's dependency parsing yields better results. -|**Tool** | [MaltParser](http://www.maltparser.org/download.html) -|**Model** | [swemalt](http://www.maltparser.org/mco/swedish_parser/swemalt.html) trained on [Svensk trädbank](https://spraakbanken.gu.se/resurser/sv-treebank) +|**Tool** | [MaltParser](https://www.maltparser.org/download.html) +|**Model** | [swemalt](https://www.maltparser.org/mco/swedish_parser/swemalt.html) trained on [Svensk trädbank](https://spraakbanken.gu.se/resurser/sv-treebank) |**Tagset** | [Mamba-Dep](https://svn.spraakdata.gu.se/sb-arkiv/pub/mamba.html) |**Annotations** | - `:malt.ref` (the token position within the sentence)
- `:malt.dephead_ref` (dependency head, the ref of the word which the current word modifies or is dependent of)
- `:malt.deprel` (dependency relation, the relation of the current word to its dependency head) |**Annotators** | - `malt:annotate`
- `malt:make_ref` @@ -146,7 +146,7 @@ preset](user-manual/corpus-configuration.md#annotation-presets) called `SWE_DEFA | | | |:---|:-----------| |**Description** | Sentence segments are analysed and enriched with named entities. -|**Tool** | [hfst-SweNER](http://urn.fi/urn%3Anbn%3Afi%3Alb-2021101202) +|**Tool** | [hfst-SweNER](https://urn.fi/urn%3Anbn%3Afi%3Alb-2021101202) |**Model** | included in the tool |**referenser** | - [HFST-SweNER – A New NER Resource for Swedish](http://www.lrec-conf.org/proceedings/lrec2014/pdf/391_Paper.pdf)
- [Reducing the effect of name explosion](http://demo.spraakdata.gu.se/svedk/pbl/kokkinakisBNER.pdf) |**Tagset** | [HFST-SweNER tags](https://svn.spraakdata.gu.se/sb-arkiv/pub/swener-tags.html) @@ -187,7 +187,7 @@ for Swedish from the 1800's: | | | |:---|:-----------| |**Description** | Sentence segments are analysed to enrich tokens with part-of-speech tags and morphosyntactic information. -|**Tool** | [Hunpos](http://code.google.com/p/hunpos/) +|**Tool** | [Hunpos](https://code.google.com/archive/p/hunpos/) |**Model** | - [suc3_suc-tags_default-setting_utf8.model](https://github.com/spraakbanken/sparv-models/blob/master/hunpos/suc3_suc-tags_default-setting_utf8.model?raw=true) trained on [SUC 3.0](https://spraakbanken.gu.se/resurser/suc3)
- a word list along with the words' morphosyntactig information generated from the [Dalin morphology](https://spraakbanken.gu.se/resurser/dalinm) and the [Swedberg morphology](https://spraakbanken.gu.se/resurser/swedbergm) |**Tagset** | [SUC MSD tags](https://spraakbanken.gu.se/korp/markup/msdtags.html) |**Annotations** | - `:hunpos.msd` (morphosyntactic tag)
- `:hunpos.pos` (part-of-speech tag diff --git a/docs/user-manual/installation-and-setup.md b/docs/user-manual/installation-and-setup.md index 8a08eb45..304c2db2 100644 --- a/docs/user-manual/installation-and-setup.md +++ b/docs/user-manual/installation-and-setup.md @@ -72,7 +72,7 @@ you prior to annotating data. |**Purpose** |Swedish word-sense disambiguation. Recommended for standard Swedish annotations. |**Download** |[Sparv wsd](https://github.com/spraakbanken/sparv-wsd/raw/master/bin/saldowsd.jar) |**License** |[MIT](https://opensource.org/licenses/MIT) -|**Dependencies** |[Java](http://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html) +|**Dependencies** |[Java](https://www.java.com/en/download/) [Sparv wsd](https://github.com/spraakbanken/sparv-wsd) is developed at Språkbanken and runs under the same license as the Sparv Pipeline. In order to use it within the Sparv Pipeline it is enough to download the saldowsd.jar from GitHub @@ -82,7 +82,7 @@ the Sparv Pipeline. In order to use it within the Sparv Pipeline it is enough to | | | |:---|:----------| |**Purpose** |Swedish named-entity recognition. Recommended for standard Swedish annotations. -|**Download** |[hfst-SweNER](http://urn.fi/urn%3Anbn%3Afi%3Alb-2021101202) +|**Download** |[hfst-SweNER](https://urn.fi/urn%3Anbn%3Afi%3Alb-2021101202) |**Version compatible with Sparv** |0.9.3 > [!NOTE] @@ -141,12 +141,12 @@ with Hunpos to your path or copy it into your [Sparv data directory](#setting-up | | | |:---|:----------| |**Purpose** |Alternative Swedish dependency parser (if you don't want to use Stanza) -|**Download** |[MaltParser webpage](http://www.maltparser.org/download.html) -|**License** |[MaltParser license](http://www.maltparser.org/license.html) (open source) +|**Download** |[MaltParser webpage](https://www.maltparser.org/download.html) +|**License** |[MaltParser license](https://www.maltparser.org/license.html) (open source) |**Version compatible with Sparv** |1.7.2 -|**Dependencies** |[Java](http://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html) +|**Dependencies** |[Java](https://www.java.com/en/download/) -Download and unpack the zip-file from the [MaltParser webpage](http://www.maltparser.org/download.html) and place the +Download and unpack the zip-file from the [MaltParser webpage](https://www.maltparser.org/download.html) and place the `maltparser-1.7.2` directory inside the `bin` directory of the [Sparv data directory](#setting-up-sparv). ### Corpus Workbench @@ -196,7 +196,7 @@ Swedish |swe |Sparv | | | |:---|:----------| |**Purpose** |POS-tagging and lemmatisation for [some languages](#software-for-analysing-other-languages-than-swedish) -|**Download** |[TreeTagger webpage](http://www.cis.uni-muenchen.de/~schmid/tools/TreeTagger/) +|**Download** |[TreeTagger webpage](https://www.cis.uni-muenchen.de/~schmid/tools/TreeTagger/) |**License** |[TreeTagger license](https://www.cis.uni-muenchen.de/~schmid/tools/TreeTagger/Tagger-Licence) (freely available for research, education and evaluation) |**Version compatible with Sparv** |3.2.3 (may work with newer versions) @@ -210,7 +210,7 @@ After downloading the software you need to have the `tree-tagger` binary in your |**Download** |[Stanford CoreNLP webpage](https://stanfordnlp.github.io/CoreNLP/history.html) |**License** |[GPL-2.0](https://www.gnu.org/licenses/old-licenses/gpl-2.0.html) |**Version compatible with Sparv** |4.0.0 (may work with newer versions) -|**Dependencies** |[Java](http://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html) +|**Dependencies** |[Java](https://www.java.com/en/download/) Please download, unzip and place contents inside the [Sparv data directory](#setting-up-sparv) under `bin/stanford_parser`. diff --git a/sparv/modules/dateformat/dateformat.py b/sparv/modules/dateformat/dateformat.py index 6f78b225..5b1d9848 100644 --- a/sparv/modules/dateformat/dateformat.py +++ b/sparv/modules/dateformat/dateformat.py @@ -39,7 +39,7 @@ def dateformat(in_from: Annotation = Annotation("[dateformat.datetime_from]"), regex: Optional[str] = Config("dateformat.regex")): """Convert existing dates/times to specified date output format. - http://docs.python.org/library/datetime.html#strftime-and-strptime-behavior + https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior Args: in_from: Annotation containing from-dates (and times). @@ -86,7 +86,7 @@ def timeformat(in_from: Annotation = Annotation("[dateformat.datetime_from]"), regex: Optional[str] = Config("dateformat.regex")): """Convert existing dates/times to specified time output format. - http://docs.python.org/library/datetime.html#strftime-and-strptime-behavior + https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior Args: in_from: Annotation containing from-dates (and times). diff --git a/sparv/modules/korp/wordpicture.py b/sparv/modules/korp/wordpicture.py index 6fc77e18..5321aebf 100644 --- a/sparv/modules/korp/wordpicture.py +++ b/sparv/modules/korp/wordpicture.py @@ -107,7 +107,7 @@ def wordpicture( annotations = list(word.read_attributes((word, pos, lemgram, dephead, deprel, ref, baseform))) - # http://stp.ling.uu.se/~nivre/swedish_treebank/dep.html + # https://cl.lingfil.uu.se/~nivre/swedish_treebank/dep.html # Tuples with relations (head, rel, dep) to be found (with indexes) and an optional tuple specifying which info # should be stored and how rels = [ diff --git a/sparv/modules/malt/malt.py b/sparv/modules/malt/malt.py index 87d8d8d3..4212ce77 100644 --- a/sparv/modules/malt/malt.py +++ b/sparv/modules/malt/malt.py @@ -196,4 +196,4 @@ def build_model(out: ModelOutput = ModelOutput("malt/swemalt-1.7.2.mco"), Won't download model unless maltjar has been installed. """ - out.download("http://maltparser.org/mco/swedish_parser/swemalt-1.7.2.mco") + out.download("https://maltparser.org/mco/swedish_parser/swemalt-1.7.2.mco") diff --git a/sparv/modules/treetagger/treetagger.py b/sparv/modules/treetagger/treetagger.py index 9da1e504..84749f7c 100644 --- a/sparv/modules/treetagger/treetagger.py +++ b/sparv/modules/treetagger/treetagger.py @@ -100,7 +100,7 @@ def get_bul_model(out: ModelOutput = ModelOutput("treetagger/bul.par"), tt_binary: Binary = Binary("[treetagger.binary]")): """Download TreeTagger language model.""" gzip = "treetagger/bulgarian.par.gz" - url = "http://www.cis.uni-muenchen.de/~schmid/tools/TreeTagger/data/bulgarian.par.gz" + url = "https://www.cis.uni-muenchen.de/~schmid/tools/TreeTagger/data/bulgarian.par.gz" _download(url, gzip, out) @@ -109,7 +109,7 @@ def get_est_model(out: ModelOutput = ModelOutput("treetagger/est.par"), tt_binary: Binary = Binary("[treetagger.binary]")): """Download TreeTagger language model.""" gzip = "treetagger/estonian.par.gz" - url = "http://www.cis.uni-muenchen.de/~schmid/tools/TreeTagger/data/estonian.par.gz" + url = "https://www.cis.uni-muenchen.de/~schmid/tools/TreeTagger/data/estonian.par.gz" _download(url, gzip, out) @@ -118,7 +118,7 @@ def get_fin_model(out: ModelOutput = ModelOutput("treetagger/fin.par"), tt_binary: Binary = Binary("[treetagger.binary]")): """Download TreeTagger language model.""" gzip = "treetagger/finnish.par.gz" - url = "http://www.cis.uni-muenchen.de/~schmid/tools/TreeTagger/data/finnish.par.gz" + url = "https://www.cis.uni-muenchen.de/~schmid/tools/TreeTagger/data/finnish.par.gz" _download(url, gzip, out) @@ -127,7 +127,7 @@ def get_lat_model(out: ModelOutput = ModelOutput("treetagger/lat.par"), tt_binary: Binary = Binary("[treetagger.binary]")): """Download TreeTagger language model.""" gzip = "treetagger/latin.par.gz" - url = "http://www.cis.uni-muenchen.de/~schmid/tools/TreeTagger/data/latin.par.gz" + url = "https://www.cis.uni-muenchen.de/~schmid/tools/TreeTagger/data/latin.par.gz" _download(url, gzip, out) @@ -136,7 +136,7 @@ def get_nld_model(out: ModelOutput = ModelOutput("treetagger/nld.par"), tt_binary: Binary = Binary("[treetagger.binary]")): """Download TreeTagger language model.""" gzip = "treetagger/dutch.par.gz" - url = "http://www.cis.uni-muenchen.de/~schmid/tools/TreeTagger/data/dutch.par.gz" + url = "https://www.cis.uni-muenchen.de/~schmid/tools/TreeTagger/data/dutch.par.gz" _download(url, gzip, out) @@ -145,7 +145,7 @@ def get_pol_model(out: ModelOutput = ModelOutput("treetagger/pol.par"), tt_binary: Binary = Binary("[treetagger.binary]")): """Download TreeTagger language model.""" gzip = "treetagger/polish.par.gz" - url = "http://www.cis.uni-muenchen.de/~schmid/tools/TreeTagger/data/polish.par.gz" + url = "https://www.cis.uni-muenchen.de/~schmid/tools/TreeTagger/data/polish.par.gz" _download(url, gzip, out) @@ -154,7 +154,7 @@ def get_ron_model(out: ModelOutput = ModelOutput("treetagger/ron.par"), tt_binary: Binary = Binary("[treetagger.binary]")): """Download TreeTagger language model.""" gzip = "treetagger/romanian.par.gz" - url = "http://www.cis.uni-muenchen.de/~schmid/tools/TreeTagger/data/romanian.par.gz" + url = "https://www.cis.uni-muenchen.de/~schmid/tools/TreeTagger/data/romanian.par.gz" _download(url, gzip, out) @@ -163,7 +163,7 @@ def get_slk_model(out: ModelOutput = ModelOutput("treetagger/slk.par"), tt_binary: Binary = Binary("[treetagger.binary]")): """Download TreeTagger language model.""" gzip = "treetagger/slovak.par.gz" - url = "http://www.cis.uni-muenchen.de/~schmid/tools/TreeTagger/data/slovak.par.gz" + url = "https://www.cis.uni-muenchen.de/~schmid/tools/TreeTagger/data/slovak.par.gz" _download(url, gzip, out) @@ -175,7 +175,7 @@ def get_spa_model(out: ModelOutput = ModelOutput("treetagger/spa.par"), tt_binary: Binary = Binary("[treetagger.binary]")): """Download TreeTagger language model.""" gzip = "treetagger/spanish.par.gz" - url = "http://www.cis.uni-muenchen.de/~schmid/tools/TreeTagger/data/spanish.par.gz" + url = "https://www.cis.uni-muenchen.de/~schmid/tools/TreeTagger/data/spanish.par.gz" _download(url, gzip, out) @@ -184,7 +184,7 @@ def get_deu_model(out: ModelOutput = ModelOutput("treetagger/deu.par"), tt_binary: Binary = Binary("[treetagger.binary]")): """Download TreeTagger language model.""" gzip = "treetagger/german.par.gz" - url = "http://www.cis.uni-muenchen.de/~schmid/tools/TreeTagger/data/german.par.gz" + url = "https://www.cis.uni-muenchen.de/~schmid/tools/TreeTagger/data/german.par.gz" _download(url, gzip, out) @@ -193,7 +193,7 @@ def get_eng_model(out: ModelOutput = ModelOutput("treetagger/eng.par"), tt_binary: Binary = Binary("[treetagger.binary]")): """Download TreeTagger language model.""" gzip = "treetagger/english.par.gz" - url = "http://www.cis.uni-muenchen.de/~schmid/tools/TreeTagger/data/english.par.gz" + url = "https://www.cis.uni-muenchen.de/~schmid/tools/TreeTagger/data/english.par.gz" _download(url, gzip, out) @@ -202,7 +202,7 @@ def get_fra_model(out: ModelOutput = ModelOutput("treetagger/fra.par"), tt_binary: Binary = Binary("[treetagger.binary]")): """Download TreeTagger language model.""" gzip = "treetagger/french.par.gz" - url = "http://www.cis.uni-muenchen.de/~schmid/tools/TreeTagger/data/french.par.gz" + url = "https://www.cis.uni-muenchen.de/~schmid/tools/TreeTagger/data/french.par.gz" _download(url, gzip, out) @@ -211,7 +211,7 @@ def get_ita_model(out: ModelOutput = ModelOutput("treetagger/ita.par"), tt_binary: Binary = Binary("[treetagger.binary]")): """Download TreeTagger language model.""" gzip = "treetagger/italian.par.gz" - url = "http://www.cis.uni-muenchen.de/~schmid/tools/TreeTagger/data/italian.par.gz" + url = "https://www.cis.uni-muenchen.de/~schmid/tools/TreeTagger/data/italian.par.gz" _download(url, gzip, out) @@ -220,7 +220,7 @@ def get_rus_model(out: ModelOutput = ModelOutput("treetagger/rus.par"), tt_binary: Binary = Binary("[treetagger.binary]")): """Download TreeTagger language model.""" gzip = "treetagger/russian.par.gz" - url = "http://www.cis.uni-muenchen.de/~schmid/tools/TreeTagger/data/russian.par.gz" + url = "https://www.cis.uni-muenchen.de/~schmid/tools/TreeTagger/data/russian.par.gz" _download(url, gzip, out) From d1eba3ac0028dbfb20303d3e22afb8a4c3935884 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 24 Feb 2023 18:42:31 +0100 Subject: [PATCH 088/175] Add some type hints --- sparv/api/util/export.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sparv/api/util/export.py b/sparv/api/util/export.py index 3efc71cf..c0a1f04a 100644 --- a/sparv/api/util/export.py +++ b/sparv/api/util/export.py @@ -295,7 +295,8 @@ def get_annotation_names(annotations: Union[ExportAnnotations, ExportAnnotations remove_namespaces=False, keep_struct_names=False, sparv_namespace: Optional[str] = None, source_namespace: Optional[str] = None, - xml_mode: Optional[bool] = False): + xml_mode: Optional[bool] = False) -> Tuple[List[Union[Annotation, AnnotationAllSourceFiles]], + List[str], Dict[str, str]]: """Get a list of annotations, token attributes and a dictionary for renamed annotations. Args: @@ -363,7 +364,7 @@ def _create_export_names(annotations: List[Tuple[Union[Annotation, AnnotationAll sparv_namespace: Optional[str] = None, source_namespace: Optional[str] = None, xml_namespaces: Optional[dict] = None, - xml_mode: Optional[bool] = False): + xml_mode: Optional[bool] = False) -> Dict[str, str]: """Create dictionary for renamed annotations.""" if remove_namespaces: def shorten(annotation): From 5da84198789aea741deff6400263f3cbded258d0 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Thu, 2 Mar 2023 13:30:15 +0100 Subject: [PATCH 089/175] Add korp.keep_undefined_annotations setting --- CHANGELOG.md | 1 + sparv/modules/korp/config.py | 37 +++++++++++++++++++++++++++--------- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 61b44dd4..c6063c3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ - Stanza now always selects the GPU with the most free memory. - The preloader can now be gracefully stopped by sending an interrupt signal to the Sparv process. - Added `HeaderAnnotations` and `HeaderAnnotationsAllSourceFiles` classes. +- Added `korp.keep_undefined_annotations` setting, to include even undefined annotations in the Korp config. ### Changed diff --git a/sparv/modules/korp/config.py b/sparv/modules/korp/config.py index a5e833a3..09ff5cd1 100644 --- a/sparv/modules/korp/config.py +++ b/sparv/modules/korp/config.py @@ -72,7 +72,10 @@ Config("korp.reading_mode", description="Reading mode configuration"), Config("korp.filters", description="List of annotations to use for filtering in Korp"), Config("korp.hidden_annotations", description="List of annotations not to include in corpus config", - default=HIDDEN_ANNOTATIONS) + default=HIDDEN_ANNOTATIONS), + Config("korp.keep_undefined_annotations", + description="Include all annotations in config, even those without an annotation definition/preset.", + default=False) ]) def config(id: Corpus = Corpus(), name: dict = Config("metadata.name"), @@ -107,6 +110,7 @@ def config(id: Corpus = Corpus(), source_namespace: str = Config("export.source_namespace"), remote_host: Optional[str] = Config("korp.remote_host"), config_dir: str = Config("korp.config_dir"), + keep_undefined_annotations: bool = Config("korp.keep_undefined_annotations"), out: Export = Export("korp.config/[metadata.id].yaml")): """Create Korp config file for the corpus, to be served by the Korp backend and used by the frontend. @@ -142,6 +146,8 @@ def config(id: Corpus = Corpus(), source_namespace: The namespace to be added to all annotations present in the source. remote_host: Host where Korp configuration files are installed. config_dir: Path on remote host where Korp configuration files are located. + keep_undefined_annotations: Set to Trye to include all annotations in config, even those without an annotation + definition/preset. out: YAML file to create. """ config_dict = { @@ -257,7 +263,7 @@ def config(id: Corpus = Corpus(), presets = get_presets(remote_host, config_dir) token_annotations, struct_annotations, _ = build_annotations( annotation_definitions, annotation_list, export_names, hidden_annotations, presets, INCLUDED_ANNOTATIONS, - token + token, keep_undefined_annotations=keep_undefined_annotations ) config_dict["struct_attributes"] = struct_annotations @@ -274,8 +280,18 @@ def config(id: Corpus = Corpus(), out_yaml.write(dict_to_yaml(config_dict)) -def build_annotations(annotation_definitions, annotation_list, export_names, hidden_annotations, presets, include, - token, text_annotation=None, cwb_annotations: bool = True): +def build_annotations( + annotation_definitions, + annotation_list, + export_names, + hidden_annotations, + presets, + include, + token, + text_annotation=None, + cwb_annotations: bool = True, + keep_undefined_annotations: bool = False +): token_annotations = [] struct_annotations = [] text_annotations = [] @@ -296,11 +312,14 @@ def build_annotations(annotation_definitions, annotation_list, export_names, hid if isinstance(definition, str): # Referring to a preset # Check that preset exists if definition not in presets: - logger.warning( - f"{annotation.name!r} is missing a definition, and {definition!r} is not available as a " - "preset. Annotation will not be included.") - continue - if not is_token and presets[definition] == "positional": + if keep_undefined_annotations: + definition = {"label": definition.replace("_", " ")} + else: + logger.warning( + f"{annotation.name!r} is missing a definition, and {definition!r} is not available as a " + "preset. Annotation will not be included.") + continue + elif not is_token and presets[definition] == "positional": # Non-token annotation used as a token-annotation in Korp is_token = True elif "preset" in definition: # Extending a preset From dc6ea99afa903e54e2363f368b323c2a27c43a06 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Thu, 2 Mar 2023 16:20:52 +0100 Subject: [PATCH 090/175] Reformat korp:config --- sparv/modules/korp/config.py | 222 ++++++++++++++++++++--------------- 1 file changed, 127 insertions(+), 95 deletions(-) diff --git a/sparv/modules/korp/config.py b/sparv/modules/korp/config.py index 09ff5cd1..85235604 100644 --- a/sparv/modules/korp/config.py +++ b/sparv/modules/korp/config.py @@ -9,7 +9,6 @@ import yaml from sparv.api import ( - AllSourceFilenames, AnnotationName, Config, Corpus, @@ -23,22 +22,22 @@ get_logger, installer, uninstaller, - util + util, ) from sparv.modules.cwb.cwb import cwb_escape logger = get_logger(__name__) # Annotations that should not be shown in Korp -HIDDEN_ANNOTATIONS = ( +HIDDEN_ANNOTATIONS = [ ":dateformat.datefrom", ":dateformat.dateto", ":dateformat.timefrom", ":dateformat.timeto", ":misc.id", ":sensaldo.sentiment_score", - ":stanza.msd_hunpos_backoff_info" -) + ":stanza.msd_hunpos_backoff_info", +] # Annotations (using export names) to always include (if they exist), that would normally be excluded INCLUDED_ANNOTATIONS = () @@ -46,72 +45,88 @@ LABELS = { "sentence": { "swe": ("mening", "meningar"), - "eng": ("sentence", "sentences") + "eng": ("sentence", "sentences"), }, "paragraph": { "swe": ("stycke", "stycken"), - "eng": ("paragraph", "paragraphs") - } + "eng": ("paragraph", "paragraphs"), + }, } -@exporter("Create Korp config file for the corpus.", config=[ - Config("korp.annotations", description="Sparv annotations to include. Leave blank to use cwb.annotations."), - Config("korp.source_annotations", - description="List of annotations and attributes from the source data to include. Leave blank to use " - "cwb.source_annotations."), - Config("korp.annotation_definitions", description="Frontend definitions of annotations in 'annotations' and " - "'source_annotations'. Classes and config keys are currently " - "not supported."), - Config("korp.context", description="Contexts to use in Korp, from smaller to bigger. " - "Leave blank to detect automatically."), - Config("korp.within", description="Search boundaries to use in Korp, from smaller to bigger. " - "Leave blank to detect automatically."), - Config("korp.custom_annotations", description="Custom Korp-annotations."), - Config("korp.morphology", description="Morphologies"), - Config("korp.reading_mode", description="Reading mode configuration"), - Config("korp.filters", description="List of annotations to use for filtering in Korp"), - Config("korp.hidden_annotations", description="List of annotations not to include in corpus config", - default=HIDDEN_ANNOTATIONS), - Config("korp.keep_undefined_annotations", - description="Include all annotations in config, even those without an annotation definition/preset.", - default=False) -]) -def config(id: Corpus = Corpus(), - name: dict = Config("metadata.name"), - description: Optional[dict] = Config("metadata.description"), - short_description: Optional[dict] = Config("metadata.short_description"), - language: str = Config("metadata.language"), - modes: list = Config("korp.modes"), - protected: bool = Config("korp.protected"), - annotations: ExportAnnotationNames = ExportAnnotationNames("korp.annotations"), - source_annotations: SourceAnnotationsAllSourceFiles = SourceAnnotationsAllSourceFiles( - "korp.source_annotations"), - cwb_annotations: ExportAnnotationNames = ExportAnnotationNames("cwb.annotations"), - cwb_source_annotations: SourceAnnotationsAllSourceFiles = SourceAnnotationsAllSourceFiles( - "cwb.source_annotations"), - annotation_definitions: Optional[dict] = Config("korp.annotation_definitions"), - custom_annotations: Optional[list] = Config("korp.custom_annotations"), - morphology: Optional[list] = Config("korp.morphology"), - reading_mode: Optional[dict] = Config("korp.reading_mode"), - hidden_annotations: List[AnnotationName] = Config("korp.hidden_annotations"), - filters: Optional[list] = Config("korp.filters"), - sentence: Optional[AnnotationName] = AnnotationName(""), - paragraph: Optional[AnnotationName] = AnnotationName(""), - installations: Optional[list] = Config("install"), - exports: Optional[list] = Config("export.default"), - scramble_on: Optional[AnnotationName] = AnnotationName("[cwb.scramble_on]"), - context: Optional[list] = Config("korp.context"), - within: Optional[list] = Config("korp.within"), - source_files: AllSourceFilenames = AllSourceFilenames(), - token: AnnotationName = AnnotationName(""), - remove_namespaces: bool = Config("export.remove_module_namespaces", False), - sparv_namespace: str = Config("export.sparv_namespace"), - source_namespace: str = Config("export.source_namespace"), - remote_host: Optional[str] = Config("korp.remote_host"), - config_dir: str = Config("korp.config_dir"), - keep_undefined_annotations: bool = Config("korp.keep_undefined_annotations"), - out: Export = Export("korp.config/[metadata.id].yaml")): +@exporter( + "Create Korp config file for the corpus.", + config=[ + Config("korp.annotations", description="Sparv annotations to include. Leave blank to use cwb.annotations."), + Config( + "korp.source_annotations", + description="List of annotations and attributes from the source data to include. Leave blank to use " + "cwb.source_annotations.", + ), + Config( + "korp.annotation_definitions", + description="Frontend definitions of annotations in 'annotations' and 'source_annotations'. Classes and " + "config keys are currently not supported.", + ), + Config( + "korp.context", + description="Contexts to use in Korp, from smaller to bigger. " "Leave blank to detect automatically.", + ), + Config( + "korp.within", + description="Search boundaries to use in Korp, from smaller to bigger. " + "Leave blank to detect automatically.", + ), + Config("korp.custom_annotations", description="Custom Korp-annotations."), + Config("korp.morphology", description="Morphologies"), + Config("korp.reading_mode", description="Reading mode configuration"), + Config("korp.filters", description="List of annotations to use for filtering in Korp"), + Config( + "korp.hidden_annotations", + description="List of annotations not to include in corpus config", + default=HIDDEN_ANNOTATIONS, + ), + Config( + "korp.keep_undefined_annotations", + description="Include all annotations in config, even those without an annotation definition/preset.", + default=False, + ), + ], +) +def config( + id: Corpus = Corpus(), + name: dict = Config("metadata.name"), + description: Optional[dict] = Config("metadata.description"), + short_description: Optional[dict] = Config("metadata.short_description"), + language: str = Config("metadata.language"), + modes: list = Config("korp.modes"), + protected: bool = Config("korp.protected"), + annotations: ExportAnnotationNames = ExportAnnotationNames("korp.annotations"), + source_annotations: SourceAnnotationsAllSourceFiles = SourceAnnotationsAllSourceFiles("korp.source_annotations"), + cwb_annotations: ExportAnnotationNames = ExportAnnotationNames("cwb.annotations"), + cwb_source_annotations: SourceAnnotationsAllSourceFiles = SourceAnnotationsAllSourceFiles("cwb.source_annotations"), + annotation_definitions: Optional[dict] = Config("korp.annotation_definitions"), + custom_annotations: Optional[list] = Config("korp.custom_annotations"), + morphology: Optional[list] = Config("korp.morphology"), + reading_mode: Optional[dict] = Config("korp.reading_mode"), + hidden_annotations: List[AnnotationName] = Config("korp.hidden_annotations"), + filters: Optional[list] = Config("korp.filters"), + sentence: Optional[AnnotationName] = AnnotationName(""), + paragraph: Optional[AnnotationName] = AnnotationName(""), + installations: Optional[list] = Config("install"), + exports: Optional[list] = Config("export.default"), + scramble_on: Optional[AnnotationName] = AnnotationName("[cwb.scramble_on]"), + context: Optional[list] = Config("korp.context"), + within: Optional[list] = Config("korp.within"), + token: AnnotationName = AnnotationName(""), + remove_namespaces: bool = Config("export.remove_module_namespaces", False), + sparv_namespace: str = Config("export.sparv_namespace"), + source_namespace: str = Config("export.source_namespace"), + remote_host: Optional[str] = Config("korp.remote_host"), + config_dir: str = Config("korp.config_dir"), + keep_undefined_annotations: bool = Config("korp.keep_undefined_annotations"), + out: Export = Export("korp.config/[metadata.id].yaml"), +): """Create Korp config file for the corpus, to be served by the Korp backend and used by the frontend. Args: @@ -139,7 +154,6 @@ def config(id: Corpus = Corpus(), scramble_on: Annotation to scramble on. context: List of annotations to use for context in the Korp frontend. within: List of annotations to use as search boundaries in the Korp frontend. - source_files: List of source files. token: The token annotation. remove_namespaces: Remove all namespaces in export_names unless names are ambiguous. sparv_namespace: The namespace to be added to all Sparv annotations. @@ -153,7 +167,7 @@ def config(id: Corpus = Corpus(), config_dict = { "id": str(id), "lang": language, - "mode": modes + "mode": modes, } optional = { "description": build_description(description, short_description), @@ -161,7 +175,7 @@ def config(id: Corpus = Corpus(), "limited_access": protected, "custom_attributes": custom_annotations, "morphology": morphology, - "reading_mode": reading_mode + "reading_mode": reading_mode, } config_dict.update({k: v for k, v in optional.items() if v}) @@ -178,9 +192,14 @@ def config(id: Corpus = Corpus(), # Get annotation names annotation_list, token_attributes, export_names = util.export.get_annotation_names( - annotations, source_annotations, token_name=token.name, - remove_namespaces=remove_namespaces, sparv_namespace=sparv_namespace, source_namespace=source_namespace, - keep_struct_names=True) + annotations, + source_annotations, + token_name=token.name, + remove_namespaces=remove_namespaces, + sparv_namespace=sparv_namespace, + source_namespace=source_namespace, + keep_struct_names=True, + ) # Context and within if not within and not context: @@ -215,8 +234,9 @@ def config(id: Corpus = Corpus(), elif context and not within: within = [v.split(" ", 1)[1] if isinstance(v, str) else v["value"].split(" ", 1)[1] for v in context] elif not within and not context: - logger.warning("Couldn't figure out 'context' and 'within' automatically. Set at least one of them manually in " - "the config.") + logger.warning( + "Couldn't figure out 'context' and 'within' automatically. Set at least one of them manually in the config." + ) if within: config_dict["within"] = [] @@ -231,10 +251,7 @@ def config(id: Corpus = Corpus(), label = {lang: f"{n} {val[i]}" if n else val[i] for lang, val in LABELS[v].items()} else: label = {"swe": f"{n} {v}" if n else v, "eng": f"{n} {v}" if n else v} - w = { - "value": f"{n} {v}" if n else v, - "label": label - } + w = {"value": f"{n} {v}" if n else v, "label": label} else: w = v config_dict["within"].append(w) @@ -251,10 +268,7 @@ def config(id: Corpus = Corpus(), label = {lang: f"{n} {val[i]}" for lang, val in LABELS[v].items()} else: label = {"swe": f"{n} {v}", "eng": f"{n} {v}"} - c = { - "value": f"{n} {v}", - "label": label - } + c = {"value": f"{n} {v}", "label": label} else: c = v config_dict["context"].append(c) @@ -262,8 +276,14 @@ def config(id: Corpus = Corpus(), # Annotations presets = get_presets(remote_host, config_dir) token_annotations, struct_annotations, _ = build_annotations( - annotation_definitions, annotation_list, export_names, hidden_annotations, presets, INCLUDED_ANNOTATIONS, - token, keep_undefined_annotations=keep_undefined_annotations + annotation_definitions, + annotation_list, + export_names, + hidden_annotations, + presets, + INCLUDED_ANNOTATIONS, + token, + keep_undefined_annotations=keep_undefined_annotations, ) config_dict["struct_attributes"] = struct_annotations @@ -275,8 +295,10 @@ def config(id: Corpus = Corpus(), config_dict["attribute_filters"].append(cwb_escape(export_names[a].replace(":", "_"))) with open(out, "w", encoding="utf-8") as out_yaml: - out_yaml.write("# This file was automatically generated by Sparv. Do not make changes directly to this file as " - "they will get overwritten.\n") + out_yaml.write( + "# This file was automatically generated by Sparv. Do not make changes directly to this file as they will " + "get overwritten.\n" + ) out_yaml.write(dict_to_yaml(config_dict)) @@ -290,7 +312,7 @@ def build_annotations( token, text_annotation=None, cwb_annotations: bool = True, - keep_undefined_annotations: bool = False + keep_undefined_annotations: bool = False, ): token_annotations = [] struct_annotations = [] @@ -300,8 +322,15 @@ def build_annotations( for annotation in annotation_list: export_name = export_names.get(annotation.name, annotation.name) # Skip certain annotations unless explicitly listed in annotation_definitions - if (annotation.name in hidden_annotation_names or annotation.attribute_name is None or export_name.split(":", 1)[ - -1].startswith("_")) and annotation.name not in annotation_definitions and not export_name in include: + if ( + ( + annotation.name in hidden_annotation_names + or annotation.attribute_name is None + or export_name.split(":", 1)[-1].startswith("_") + ) + and annotation.name not in annotation_definitions + and not export_name in include + ): logger.debug(f"Skipping annotation {annotation.name!r}") continue export_name_cwb = cwb_escape(export_name.replace(":", "_")) @@ -317,7 +346,8 @@ def build_annotations( else: logger.warning( f"{annotation.name!r} is missing a definition, and {definition!r} is not available as a " - "preset. Annotation will not be included.") + "preset. Annotation will not be included." + ) continue elif not is_token and presets[definition] == "positional": # Non-token annotation used as a token-annotation in Korp @@ -406,7 +436,7 @@ def install_config( config_dir: str = Config("korp.config_dir"), config_file: ExportInput = ExportInput("korp.config/[metadata.id].yaml"), marker: OutputMarker = OutputMarker("korp.install_config_marker"), - uninstall_marker: MarkerOptional = MarkerOptional("korp.uninstall_config_marker") + uninstall_marker: MarkerOptional = MarkerOptional("korp.uninstall_config_marker"), ): """Install Korp corpus configuration file.""" corpus_dir = Path(config_dir) / "corpora" @@ -422,12 +452,13 @@ def uninstall_config( config_dir: str = Config("korp.config_dir"), corpus_id: Corpus = Corpus(), marker: OutputMarker = OutputMarker("korp.uninstall_config_marker"), - install_marker: MarkerOptional = MarkerOptional("korp.install_config_marker") + install_marker: MarkerOptional = MarkerOptional("korp.install_config_marker"), ): """Uninstall Korp corpus configuration file.""" corpus_file = Path(config_dir) / "corpora" / f"{corpus_id}.yaml" - logger.info(f"Uninstalling Korp corpus configuration file from {remote_host + ':' if remote_host else ''}" - f"{corpus_file}") + logger.info( + f"Uninstalling Korp corpus configuration file from {remote_host + ':' if remote_host else ''}{corpus_file}" + ) util.install.uninstall_path(corpus_file, host=remote_host) install_marker.remove() marker.write() @@ -439,6 +470,7 @@ def dict_to_yaml(data): Args: data: The dict to be converted. """ + class IndentDumper(yaml.Dumper): """Customized YAML dumper that indents lists.""" @@ -451,7 +483,7 @@ def str_representer(dumper, data): if len(data.splitlines()) > 1: # Check for multiline string return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|") return dumper.represent_scalar("tag:yaml.org,2002:str", data) + yaml.add_representer(str, str_representer) - return yaml.dump(data, sort_keys=False, allow_unicode=True, Dumper=IndentDumper, indent=4, - default_flow_style=False) + return yaml.dump(data, sort_keys=False, allow_unicode=True, Dumper=IndentDumper, indent=4, default_flow_style=False) From f08ee07252fbba55f786df52ac607b040e8d1359 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 3 Mar 2023 10:08:44 +0100 Subject: [PATCH 091/175] Add dateformat.pre_regex setting --- CHANGELOG.md | 1 + sparv/modules/dateformat/dateformat.py | 36 +++++++++++++++++++++----- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c6063c3e..c56b7593 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ - The preloader can now be gracefully stopped by sending an interrupt signal to the Sparv process. - Added `HeaderAnnotations` and `HeaderAnnotationsAllSourceFiles` classes. - Added `korp.keep_undefined_annotations` setting, to include even undefined annotations in the Korp config. +- Added `dateformat.pre_regex` setting. ### Changed diff --git a/sparv/modules/dateformat/dateformat.py b/sparv/modules/dateformat/dateformat.py index 5b1d9848..b81fc017 100644 --- a/sparv/modules/dateformat/dateformat.py +++ b/sparv/modules/dateformat/dateformat.py @@ -19,8 +19,12 @@ "by |. They will be tried in order."), Config("dateformat.splitter", description="One or more characters separating two dates in 'datetime_from', " "treating them as from-date and to-date."), - Config("dateformat.regex", description="Regular expression with a catching group whose content will be used in the " - "parsing instead of the whole string."), + Config("dateformat.pre_regex", + description="Regular expression with a catching group whose content will be used in the parsing instead of " + "the whole string. Applied before splitting."), + Config("dateformat.regex", + description="Regular expression with a catching group whose content will be used in the parsing instead of " + "the whole string. Applied on each value after splitting."), Config("dateformat.date_outformat", default="%Y%m%d", description="Desired format of the formatted dates. Several formats can be specified separated " "by |. They will be tied to their respective in-format."), @@ -36,6 +40,7 @@ def dateformat(in_from: Annotation = Annotation("[dateformat.datetime_from]"), informat: str = Config("dateformat.datetime_informat"), outformat: str = Config("dateformat.date_outformat"), splitter: Optional[str] = Config("dateformat.splitter"), + pre_regex: Optional[str] = Config("dateformat.pre_regex"), regex: Optional[str] = Config("dateformat.regex")): """Convert existing dates/times to specified date output format. @@ -52,10 +57,12 @@ def dateformat(in_from: Annotation = Annotation("[dateformat.datetime_from]"), Several formats can be specified separated by |. They will be tied to their respective in-format. splitter: One or more characters separating two dates in 'in_from', treating them as from-date and to-date. + pre_regex: Regular expression with a catching group whose content will be used in the parsing + instead of the whole string. Applied before splitting. regex: Regular expression with a catching group whose content will be used in the parsing - instead of the whole string. + instead of the whole string. Applied on each value after splitting. """ - _formatter(in_from, in_to, out_from, out_to, informat, outformat, splitter, regex) + _formatter(in_from, in_to, out_from, out_to, informat, outformat, splitter, pre_regex, regex) @annotator("Convert existing dates to format YYYY-MM-DD") @@ -64,9 +71,10 @@ def dateformat_pretty(in_date: Annotation = Annotation("[dateformat.datetime_fro description="Date without timestamp 'YYYY-MM-DD'"), informat: str = Config("dateformat.datetime_informat"), splitter: Optional[str] = Config("dateformat.splitter"), + pre_regex: Optional[str] = Config("dateformat.pre_regex"), regex: Optional[str] = Config("dateformat.regex")): """Convert existing dates to format YYYY-MM-DD.""" - _formatter(in_date, None, out, None, informat, "%Y-%m-%d", splitter, regex) + _formatter(in_date, None, out, None, informat, "%Y-%m-%d", splitter, pre_regex, regex) @annotator("Convert existing times to specified output format", config=[ @@ -144,7 +152,7 @@ def resolution(out_resolution: OutputCommonData = OutputCommonData("dateformat.r def _formatter(in_from: Annotation, in_to: Optional[Annotation], out_from: Output, out_to: Optional[Output], - informat: str, outformat: str, splitter: str, regex: str): + informat: str, outformat: str, splitter: str, pre_regex: str, regex: str): """Take existing dates/times and input formats and convert to specified output format.""" def get_smallest_unit(informat): smallest_unit = 0 # No date @@ -215,6 +223,14 @@ def get_date_length(informat): ofrom[index] = None continue + if pre_regex: + matches = re.match(pre_regex, val) + val = [v for v in matches.groups() if v][0] + if not val: + # If the regex doesn't match, treat as no date + ofrom[index] = None + continue + tries = 0 for inf in informat: if splitter and splitter in inf: @@ -279,6 +295,14 @@ def get_date_length(informat): oto[index] = None continue + if pre_regex: + matches = re.match(pre_regex, val) + val = [v for v in matches.groups() if v][0] + if not val: + # If the regex doesn't match, treat as no date + oto[index] = None + continue + tries = 0 for inf in informat: if splitter and splitter in inf: From 856951ea7a444ee0483cfd0bead95a92d53512c3 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 3 Mar 2023 12:05:21 +0100 Subject: [PATCH 092/175] Fix dateformat:timeformat --- sparv/modules/dateformat/dateformat.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sparv/modules/dateformat/dateformat.py b/sparv/modules/dateformat/dateformat.py index b81fc017..b7fb831f 100644 --- a/sparv/modules/dateformat/dateformat.py +++ b/sparv/modules/dateformat/dateformat.py @@ -91,6 +91,7 @@ def timeformat(in_from: Annotation = Annotation("[dateformat.datetime_from]"), informat: str = Config("dateformat.datetime_informat"), outformat: str = Config("dateformat.time_outformat"), splitter: Optional[str] = Config("dateformat.splitter"), + pre_regex: Optional[str] = Config("dateformat.pre_regex"), regex: Optional[str] = Config("dateformat.regex")): """Convert existing dates/times to specified time output format. @@ -107,10 +108,12 @@ def timeformat(in_from: Annotation = Annotation("[dateformat.datetime_from]"), Several formats can be specified separated by |. They will be tied to their respective in-format. splitter: One or more characters separating two dates in 'in_from', treating them as from-date and to-date. + pre_regex: Regular expression with a catching group whose content will be used in the parsing + instead of the whole string. Applied before splitting. regex: Regular expression with a catching group whose content will be used in the parsing - instead of the whole string. + instead of the whole string. Applied on each value after splitting. """ - _formatter(in_from, in_to, out_from, out_to, informat, outformat, splitter, regex) + _formatter(in_from, in_to, out_from, out_to, informat, outformat, splitter, pre_regex, regex) @annotator("Get datetime resolutions from informat") From f38a41bde3b9c2929b3f6e228e6e9040f7bbc4f0 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 3 Mar 2023 17:10:06 +0100 Subject: [PATCH 093/175] Change all list subclasses to immutable sequences --- CHANGELOG.md | 6 +- docs/developers-guide/sparv-classes.md | 12 +- sparv/api/classes.py | 158 +++++++++++++-------- sparv/api/util/export.py | 26 ++-- sparv/core/snake_utils.py | 51 ++++--- sparv/modules/cwb/cwb.py | 2 +- sparv/modules/misc/ids.py | 2 +- sparv/modules/stats_export/stats_export.py | 2 +- 8 files changed, 154 insertions(+), 105 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c56b7593..443cbf64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,8 +41,10 @@ - `korp:relations`, `korp:relations_sql` and `korp:install_relations` has been renamed to `korp:wordpicture`, `korp:wordpicture_sql` and `korp:install_wordpicture` respectively. - Target path is no longer optional for the utility functions `install_path` and `rsync`. -- The classes `SourceAnnotations` and `SourceAnnotationsAllSourceFiles` are now pre-parsed iterables instead of lists - that need parsing and expanding. +- The classes `SourceAnnotations` and `SourceAnnotationsAllSourceFiles` are now pre-parsed, immutable iterables instead + of lists that need parsing and expanding. +- The classes `AllSourceFilenames`, `ExportAnnotations`, `ExportAnnotationsAllSourceFiles` and `ExportAnnotationNames` + are now immutable iterables instead of lists. ### Fixed diff --git a/docs/developers-guide/sparv-classes.md b/docs/developers-guide/sparv-classes.md index 7422cd67..b5c4ec67 100644 --- a/docs/developers-guide/sparv-classes.md +++ b/docs/developers-guide/sparv-classes.md @@ -8,7 +8,7 @@ the available Sparv classes, their arguments, properties, and public methods. ## AllSourceFilenames -An instance of this class holds a list with the names of all source files. It is typically used by exporter +An instance of this class holds an iterable with the names of all source files. It is typically used by exporter functions that combine annotations from all source files. @@ -210,8 +210,8 @@ An instance of this class represents an export file. This class is used to defin ## ExportAnnotations -List of annotations to be included in the export. This list is defined in the corpus configuration. Annotation files -for the current source file will automatically be added as dependencies when using this class. +Iterable with annotations to be included in the export. This list is defined in the corpus configuration. +Annotation files for the current source file will automatically be added as dependencies when using this class. **Arguments:** @@ -219,8 +219,8 @@ for the current source file will automatically be added as dependencies when usi ## ExportAnnotationsAllSourceFiles -List of annotations to be included in the export. This list is defined in the corpus configuration. Annotation files -for _all_ source files will automatically be added as dependencies when using this class. +Iterable with annotations to be included in the export. This list is defined in the corpus configuration. +Annotation files for _all_ source files will automatically be added as dependencies when using this class. **Arguments:** @@ -228,7 +228,7 @@ for _all_ source files will automatically be added as dependencies when using th ## ExportAnnotationNames -List of annotations to be included in the export. This list is defined in the corpus configuration. Unlike +Iterable with annotations to be included in the export. This list is defined in the corpus configuration. Unlike `ExportAnnotations`, the annotations will not be added as dependencies when using this class. **Arguments:** diff --git a/sparv/api/classes.py b/sparv/api/classes.py index 5b81c86a..a7484374 100644 --- a/sparv/api/classes.py +++ b/sparv/api/classes.py @@ -7,7 +7,7 @@ import urllib.request import zipfile from abc import ABC, abstractmethod -from typing import Any, Dict, Iterable, List, Optional, Tuple, Union +from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple, Union import sparv.core from sparv.core import io @@ -614,9 +614,18 @@ class Corpus(str): """Name of the corpus.""" -class AllSourceFilenames(List[str]): +class AllSourceFilenames(Sequence[str]): """List with names of all source files.""" + def __init__(self): + self.items: Sequence[str] = [] + + def __getitem__(self, index: int): + return self.items[index] + + def __len__(self): + return len(self.items) + class Config(str): """Class holding configuration key names.""" @@ -785,18 +794,24 @@ def __init__(self, val: str, all_files: bool = False): self.all_files = all_files -class ExportAnnotations(List[Tuple[Annotation, Optional[str]]]): - """List of annotations to include in export.""" +class ExportAnnotations(Sequence[Tuple[Annotation, Optional[str]]]): + """Iterable with annotations to include in export.""" # If is_input = False the annotations won't be added to the rule's input. is_input = True - def __init__(self, config_name: str, items=(), is_input: Optional[bool] = None): - list.__init__(self, items) + def __init__(self, config_name: str, is_input: Optional[bool] = None): self.config_name = config_name + self.items = [] if is_input is not None: self.is_input = is_input + def __getitem__(self, index: int): + return self.items[index] + + def __len__(self): + return len(self.items) + class ExportAnnotationNames(ExportAnnotations): """List of annotations to include in export. @@ -806,107 +821,128 @@ class ExportAnnotationNames(ExportAnnotations): is_input = False - def __init__(self, config_name: str, items=()): - super().__init__(config_name, items=items) + def __init__(self, config_name: str): + super().__init__(config_name) -class ExportAnnotationsAllSourceFiles(List[Tuple[AnnotationAllSourceFiles, Optional[str]]]): +class ExportAnnotationsAllSourceFiles(Sequence[Tuple[AnnotationAllSourceFiles, Optional[str]]]): """List of annotations to include in export.""" # Always true for ExportAnnotationsAllSourceFiles is_input = True - def __init__(self, config_name: str, items=()): - list.__init__(self, items) + def __init__(self, config_name: str): self.config_name = config_name + self.items: Sequence[Tuple[AnnotationAllSourceFiles, Optional[str]]] = [] + + def __getitem__(self, index: int): + return self.items[index] + + def __len__(self): + return len(self.items) -class SourceAnnotations(Iterable[Tuple[Annotation, Optional[str]]]): +class SourceAnnotations(Sequence[Tuple[Annotation, Optional[str]]]): """Iterable with source annotations to include in export.""" - def __init__(self, config_name: str, items: Optional[Iterable[str]] = None, source_file: Optional[str] = None, - _headers: bool = False): + def __init__(self, config_name: str, source_file: Optional[str] = None, _headers: bool = False): self.config_name = config_name - self.raw_list = items - self.annotations = [] + self.raw_list = None + self.annotations: Sequence[Tuple[Annotation, Optional[str]]] = [] self.source_file = source_file self.initialized = False self.headers = _headers + def _initialize(self): + assert self.source_file, "SourceAnnotation is missing source_file" + # If raw_list is an empty list, don't add any source annotations automatically if not self.raw_list and self.raw_list is not None: self.initialized = True + return - def __iter__(self): + # Parse annotation list and read available source annotations + if self.headers: + h = Headers(self.source_file) + available_source_annotations = set(h.read()) if h.exists() else set() + else: + available_source_annotations = set(SourceStructure(self.source_file).read()) + parsed_items = parse_annotation_list(self.raw_list, available_source_annotations) + # Only include annotations that are available in source + self.annotations = [ + (Annotation(a[0], self.source_file), a[1]) for a in parsed_items if a[0] in available_source_annotations + ] + self.initialized = True + + def __getitem__(self, index: int): if not self.initialized: - assert self.source_file, "SourceAnnotation is missing source_file" - # Parse annotation list and read available source annotations - if self.headers: - h = Headers(self.source_file) - available_source_annotations = set(h.read()) if h.exists() else set() - else: - available_source_annotations = set(SourceStructure(self.source_file).read()) - parsed_items = parse_annotation_list(self.raw_list, available_source_annotations) - # Only include annotations that are available in source - self.annotations = [ - (Annotation(a[0], self.source_file), a[1]) for a in parsed_items if a[0] in available_source_annotations - ] - self.initialized = True - for annotation in self.annotations: - yield annotation + self._initialize() + return self.annotations[index] + + def __len__(self): + if not self.initialized: + self._initialize() + return len(self.annotations) class HeaderAnnotations(SourceAnnotations): """Iterable with header annotations to include in export.""" - def __init__(self, config_name: str, items: Optional[Iterable[str]] = None, source_file: Optional[str] = None): - super().__init__(config_name, items, source_file, _headers=True) + def __init__(self, config_name: str, source_file: Optional[str] = None): + super().__init__(config_name, source_file, _headers=True) -class SourceAnnotationsAllSourceFiles(Iterable[Tuple[AnnotationAllSourceFiles, Optional[str]]]): +class SourceAnnotationsAllSourceFiles(Sequence[Tuple[AnnotationAllSourceFiles, Optional[str]]]): """Iterable with source annotations to include in export.""" - def __init__(self, config_name: str, items: Optional[Iterable[str]] = None, source_files: Iterable[str]=(), - headers: bool = False): + def __init__(self, config_name: str, source_files: Iterable[str]=(), headers: bool = False): self.config_name = config_name - self.raw_list = items - self.annotations = [] + self.raw_list = None + self.annotations: Sequence[Tuple[AnnotationAllSourceFiles, Optional[str]]] = [] self.source_files = source_files self.initialized = False self.headers = headers + def _initialize(self): + assert self.source_files, "SourceAnnotationsAllSourceFiles is missing source_files" + # If raw_list is an empty list, don't add any source annotations automatically if not self.raw_list and self.raw_list is not None: self.initialized = True - def __iter__(self): + # Parse annotation list and, if needed, read available source annotations + available_source_annotations = set() + for f in self.source_files: + if self.headers: + h = Headers(f) + if h.exists(): + available_source_annotations.update(h.read()) + else: + available_source_annotations.update(SourceStructure(f).read()) + + parsed_items = parse_annotation_list(self.raw_list, available_source_annotations) + # Only include annotations that are available in source + self.annotations = [ + (AnnotationAllSourceFiles(a[0]), a[1]) for a in parsed_items if a[0] in available_source_annotations + ] + self.initialized = True + + def __getitem__(self, index: int): if not self.initialized: - assert self.source_files, "SourceAnnotationsAllSourceFiles is missing source_files" - # Parse annotation list and, if needed, read available source annotations - available_source_annotations = set() - for f in self.source_files: - if self.headers: - h = Headers(f) - if h.exists(): - available_source_annotations.update(h.read()) - else: - available_source_annotations.update(SourceStructure(f).read()) - - parsed_items = parse_annotation_list(self.raw_list, available_source_annotations) - # Only include annotations that are available in source - self.annotations = [ - (AnnotationAllSourceFiles(a[0]), a[1]) for a in parsed_items if a[0] in available_source_annotations - ] - self.initialized = True - for annotation in self.annotations: - yield annotation + self._initialize() + return self.annotations[index] + + def __len__(self): + if not self.initialized: + self._initialize() + return len(self.annotations) class HeaderAnnotationsAllSourceFiles(SourceAnnotationsAllSourceFiles): """Iterable with header annotations to include in export.""" - def __init__(self, config_name: str, items: Optional[Iterable[str]] = None, source_files: Iterable[str]=()): - super().__init__(config_name, items, source_files, headers=True) + def __init__(self, config_name: str, source_files: Iterable[str]=()): + super().__init__(config_name, source_files, headers=True) class Language(str): diff --git a/sparv/api/util/export.py b/sparv/api/util/export.py index c0a1f04a..3dd5807e 100644 --- a/sparv/api/util/export.py +++ b/sparv/api/util/export.py @@ -288,15 +288,21 @@ def calculate_element_hierarchy(source_file, spans_list): return hierarchy -def get_annotation_names(annotations: Union[ExportAnnotations, ExportAnnotationsAllSourceFiles], - source_annotations: Union[SourceAnnotations, SourceAnnotationsAllSourceFiles] = None, - source_file: Optional[str] = None, - token_name: Optional[str] = None, - remove_namespaces=False, keep_struct_names=False, - sparv_namespace: Optional[str] = None, - source_namespace: Optional[str] = None, - xml_mode: Optional[bool] = False) -> Tuple[List[Union[Annotation, AnnotationAllSourceFiles]], - List[str], Dict[str, str]]: +def get_annotation_names( + annotations: Union[ + ExportAnnotations, + ExportAnnotationsAllSourceFiles, + List[Tuple[Union[Annotation, AnnotationAllSourceFiles], Optional[str]]], + ], + source_annotations: Union[SourceAnnotations, SourceAnnotationsAllSourceFiles] = None, + source_file: Optional[str] = None, + token_name: Optional[str] = None, + remove_namespaces=False, + keep_struct_names=False, + sparv_namespace: Optional[str] = None, + source_namespace: Optional[str] = None, + xml_mode: Optional[bool] = False, +) -> Tuple[List[Union[Annotation, AnnotationAllSourceFiles]], List[str], Dict[str, str]]: """Get a list of annotations, token attributes and a dictionary for renamed annotations. Args: @@ -316,7 +322,7 @@ def get_annotation_names(annotations: Union[ExportAnnotations, ExportAnnotations export names. """ # Combine all annotations - all_annotations = _remove_duplicates(annotations + list(source_annotations)) + all_annotations = _remove_duplicates(list(annotations) + list(source_annotations)) if token_name: # Get the names of all token attributes diff --git a/sparv/core/snake_utils.py b/sparv/core/snake_utils.py index b4adf0d6..cc9ec8b4 100644 --- a/sparv/core/snake_utils.py +++ b/sparv/core/snake_utils.py @@ -6,7 +6,7 @@ from collections import OrderedDict, defaultdict from itertools import combinations from pathlib import Path -from typing import Any, List, Optional, Set, Tuple +from typing import Any, List, Optional, Set, Tuple, Union import snakemake from snakemake.io import expand @@ -370,9 +370,8 @@ def rule_helper(rule: RuleStorage, config: dict, storage: SnakeStorage, config_m elif param_type in (ExportAnnotations, ExportAnnotationNames, ExportAnnotationsAllSourceFiles): if not isinstance(param_value, param_type): param_value = param_type(param_value) - rule.parameters[param_name] = param_value - source = param.default.config_name + source = param_value.config_name annotations = sparv_config.get(source, []) if not annotations: rule.missing_config.add(source) @@ -398,21 +397,26 @@ def rule_helper(rule: RuleStorage, config: dict, storage: SnakeStorage, config_m if a not in plain_annotations: export_annotations.append((annotation_type(a), None)) + items = [] + for annotation, export_name in export_annotations: - if param.default.is_input: + if param_value.is_input: if param_type == ExportAnnotationsAllSourceFiles: rule.inputs.extend( expand(escape_wildcards(paths.work_dir / get_annotation_path(annotation.name)), file=storage.source_files)) else: rule.inputs.append(paths.work_dir / get_annotation_path(annotation.name)) - rule.parameters[param_name].append((annotation, export_name)) + items.append((annotation, export_name)) + param_value.items = items + rule.parameters[param_name] = param_value # SourceAnnotations elif param_type in (SourceAnnotations, SourceAnnotationsAllSourceFiles): - rule.parameters[param_name] = param_type( - param.default.config_name, - sparv_config.get(param.default.config_name) - ) + if not isinstance(param_value, param_type): + param_value = param_type(param_value) + param_value: Union[SourceAnnotations, SourceAnnotationsAllSourceFiles] + param_value.raw_list = sparv_config.get(param_value.config_name) + rule.parameters[param_name] = param_value if param_type == SourceAnnotationsAllSourceFiles: rule.parameters[param_name].source_files = storage.source_files rule.inputs.extend( @@ -422,24 +426,25 @@ def rule_helper(rule: RuleStorage, config: dict, storage: SnakeStorage, config_m rule.inputs.append(paths.work_dir / get_annotation_path(io.STRUCTURE_FILE, data=True)) # HeaderAnnotations elif param_type in (HeaderAnnotations, HeaderAnnotationsAllSourceFiles): - rule.parameters[param_name] = param_type( - param.default.config_name, - sparv_config.get(param.default.config_name) - ) + if not isinstance(param_value, param_type): + param_value = param_type(param_value) + param_value.raw_list = sparv_config.get(param_value.config_name) + rule.parameters[param_name] = param_value if param_type == HeaderAnnotationsAllSourceFiles: rule.parameters[param_name].source_files = storage.source_files # Corpus - elif param.annotation == Corpus: + elif param_type == Corpus: rule.parameters[param_name] = Corpus(sparv_config.get("metadata.id")) # Language - elif param.annotation == Language: + elif param_type == Language: rule.parameters[param_name] = Language(sparv_config.get("metadata.language")) # SourceFilename - elif param.annotation == SourceFilename: + elif param_type == SourceFilename: rule.file_parameters.append(param_name) # AllSourceFilenames (all source filenames) elif param_type == AllSourceFilenames: - rule.parameters[param_name] = AllSourceFilenames(storage.source_files) + param_value.items = storage.source_files + rule.parameters[param_name] = param_value # Text elif param_type == Text: text_path = Path("{file}") / io.TEXT_FILE @@ -467,22 +472,22 @@ def rule_helper(rule: RuleStorage, config: dict, storage: SnakeStorage, config_m else: rule.parameters[param_name] = model_param[0] # Binary - elif param.annotation in (Binary, BinaryDir): + elif param_type in (Binary, BinaryDir): rule.configs.update(registry.find_config_variables(param.default)) rule.classes.update(registry.find_classes(param.default)) param_value, missing_configs = registry.expand_variables(param.default, rule.full_name) rule.missing_config.update(missing_configs) - binary = util.system.find_binary(param_value, executable=False, allow_dir=param.annotation == BinaryDir) + binary = util.system.find_binary(param_value, executable=False, allow_dir=param_type == BinaryDir) if not binary: rule.missing_binaries.add(param_value) binary = Path(binary if binary else param_value) rule.inputs.append(binary) - rule.parameters[param_name] = param.annotation(binary) + rule.parameters[param_name] = param_type(binary) # Source - elif param.annotation == Source: + elif param_type == Source: rule.parameters[param_name] = Source(get_source_path()) # Export - elif param.annotation == Export: + elif param_type == Export: rule.configs.update(registry.find_config_variables(param.default)) rule.classes.update(registry.find_classes(param.default)) param_value, missing_configs = registry.expand_variables(param.default, rule.full_name) @@ -496,7 +501,7 @@ def rule_helper(rule: RuleStorage, config: dict, storage: SnakeStorage, config_m if "{" in param_value: rule.wildcard_annotations.append(param_name) # ExportInput - elif param.annotation == ExportInput: + elif param_type == ExportInput: rule.configs.update(registry.find_config_variables(param.default)) rule.classes.update(registry.find_classes(param.default)) param_value, missing_configs = registry.expand_variables(param.default, rule.full_name) diff --git a/sparv/modules/cwb/cwb.py b/sparv/modules/cwb/cwb.py index 205b1b07..af45233b 100644 --- a/sparv/modules/cwb/cwb.py +++ b/sparv/modules/cwb/cwb.py @@ -180,7 +180,7 @@ def cwb_encode(corpus, annotations, source_annotations, source_files, words, vrt vrtfiles.sort() # Word annotation should always be included in CWB export - annotations.insert(0, (words, None)) + annotations = [(words, None)] + list(annotations) # Get annotation names annotation_list, token_attributes, export_names = util.export.get_annotation_names( diff --git a/sparv/modules/misc/ids.py b/sparv/modules/misc/ids.py index ff83992d..fbc240ee 100644 --- a/sparv/modules/misc/ids.py +++ b/sparv/modules/misc/ids.py @@ -29,7 +29,7 @@ def file_id(out: OutputDataAllSourceFiles = OutputDataAllSourceFiles("misc.filei with open(source_files_list, encoding="utf-8") as f: source_files = f.read().strip().splitlines() - source_files.sort() + source_files = sorted(source_files) logger.progress(total=len(source_files)) numfiles = len(source_files) * 2 diff --git a/sparv/modules/stats_export/stats_export.py b/sparv/modules/stats_export/stats_export.py index f644e6ce..cf6e9eb1 100644 --- a/sparv/modules/stats_export/stats_export.py +++ b/sparv/modules/stats_export/stats_export.py @@ -56,7 +56,7 @@ def freq_list(source_files: AllSourceFilenames = AllSourceFilenames(), cutoff: The minimum frequency a word must have in order to be included in the result. """ # Add "word" to annotations - annotations = [(word, None)] + annotations + annotations = [(word, None)] + list(annotations) # Get annotations list and export names annotation_list, token_attributes, export_names = util.export.get_annotation_names( From 549a1ed8557c72c125549be6277c5de0745fc29b Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 3 Mar 2023 17:40:10 +0100 Subject: [PATCH 094/175] Fix bug where expand_variables would return empty string --- sparv/core/registry.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sparv/core/registry.py b/sparv/core/registry.py index 6b27189a..f7fdefd9 100644 --- a/sparv/core/registry.py +++ b/sparv/core/registry.py @@ -544,11 +544,11 @@ def expand_classes(s: str, parents: Set[str]) -> Tuple[Optional[str], Optional[s real_ann = _expand_class(cls.group(1)) if real_ann: final_ann, rest_ = expand_classes(real_ann, parents.union([cls.group(1)])) - if rest_: - return "", rest_ s = s.replace(cls.group(), final_ann, 1) + if rest_: + return s, rest_ else: - return "", cls.group() + return s, cls.group() return s, None for string in strings: From 449a2b9918525f892267698dc8dfd20370000fd6 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Wed, 8 Mar 2023 17:33:05 +0100 Subject: [PATCH 095/175] Make SourceAnnotations order predictable --- sparv/api/classes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sparv/api/classes.py b/sparv/api/classes.py index a7484374..106ce6bb 100644 --- a/sparv/api/classes.py +++ b/sparv/api/classes.py @@ -864,9 +864,9 @@ def _initialize(self): # Parse annotation list and read available source annotations if self.headers: h = Headers(self.source_file) - available_source_annotations = set(h.read()) if h.exists() else set() + available_source_annotations = h.read() if h.exists() else [] else: - available_source_annotations = set(SourceStructure(self.source_file).read()) + available_source_annotations = SourceStructure(self.source_file).read() parsed_items = parse_annotation_list(self.raw_list, available_source_annotations) # Only include annotations that are available in source self.annotations = [ From fb8344c9c46cc8dd55b88507a8cc81c2c7e06656 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Thu, 9 Mar 2023 16:07:54 +0100 Subject: [PATCH 096/175] Make sure all CWB/VRT exports use the same token attributes in the same order --- sparv/modules/cwb/cwb.py | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/sparv/modules/cwb/cwb.py b/sparv/modules/cwb/cwb.py index af45233b..b3553275 100644 --- a/sparv/modules/cwb/cwb.py +++ b/sparv/modules/cwb/cwb.py @@ -28,6 +28,8 @@ def vrt(source_file: SourceFilename = SourceFilename(), word: Annotation = Annotation("[export.word]"), annotations: ExportAnnotations = ExportAnnotations("cwb.annotations"), source_annotations: SourceAnnotations = SourceAnnotations("cwb.source_annotations"), + all_source_annotations: SourceAnnotationsAllSourceFiles = SourceAnnotationsAllSourceFiles( + "cwb.source_annotations"), remove_namespaces: bool = Config("export.remove_module_namespaces", False), sparv_namespace: str = Config("export.sparv_namespace"), source_namespace: str = Config("export.source_namespace")): @@ -43,8 +45,17 @@ def vrt(source_file: SourceFilename = SourceFilename(), # Read words word_annotation = list(word.read()) + # Make a list of all token attributes for all source files. We need this because all VRT files need to have the + # exact same token attributes in the same order. + all_token_attributes = [ + a[0].attribute_name + for a in set(annotations).union(all_source_annotations) + if a[0].annotation_name == token.annotation_name and a[0].has_attribute() + ] + all_token_attributes.sort() + # Get annotation spans, annotations list etc. - annotation_list, token_attributes, export_names = util.export.get_annotation_names( + annotation_list, _token_attributes, export_names = util.export.get_annotation_names( annotations, source_annotations, source_file=source_file, token_name=token.name, remove_namespaces=remove_namespaces, sparv_namespace=sparv_namespace, source_namespace=source_namespace) if token not in annotation_list: @@ -52,7 +63,7 @@ def vrt(source_file: SourceFilename = SourceFilename(), "the source text. Make sure to add to the list of export annotations.") span_positions, annotation_dict = util.export.gather_annotations(annotation_list, export_names, source_file=source_file) - vrt_data = create_vrt(span_positions, token.name, word_annotation, token_attributes, annotation_dict, + vrt_data = create_vrt(span_positions, token.name, word_annotation, all_token_attributes, annotation_dict, export_names) # Write result to file @@ -72,13 +83,15 @@ def vrt_scrambled(source_file: SourceFilename = SourceFilename(), word: Annotation = Annotation("[export.word]"), annotations: ExportAnnotations = ExportAnnotations("cwb.annotations"), source_annotations: SourceAnnotations = SourceAnnotations("cwb.source_annotations"), + all_source_annotations: SourceAnnotationsAllSourceFiles = SourceAnnotationsAllSourceFiles( + "cwb.source_annotations"), remove_namespaces: bool = Config("export.remove_module_namespaces", False), sparv_namespace: str = Config("export.sparv_namespace"), source_namespace: str = Config("export.source_namespace")): """Export annotations to vrt in scrambled order.""" logger.progress(total=6) # Get annotation spans, annotations list etc. - annotation_list, token_attributes, export_names = util.export.get_annotation_names( + annotation_list, _token_attributes, export_names = util.export.get_annotation_names( annotations, source_annotations, source_file=source_file, token_name=token.name, remove_namespaces=remove_namespaces, sparv_namespace=sparv_namespace, source_namespace=source_namespace) logger.progress() @@ -98,11 +111,20 @@ def vrt_scrambled(source_file: SourceFilename = SourceFilename(), logger.progress() + # Make a list of all token attributes for all source files. We need this because all VRT files need to have the + # exact same token attributes in the same order. + all_token_attributes = [ + a[0].attribute_name + for a in set(annotations).union(all_source_annotations) + if a[0].annotation_name == token.annotation_name and a[0].has_attribute() + ] + all_token_attributes.sort() + # Reorder chunks and open/close tags in correct order new_span_positions = util.export.scramble_spans(span_positions, chunk.name, chunk_order_data) logger.progress() # Make vrt format - vrt_data = create_vrt(new_span_positions, token.name, word_annotation, token_attributes, annotation_dict, + vrt_data = create_vrt(new_span_positions, token.name, word_annotation, all_token_attributes, annotation_dict, export_names) logger.progress() # Create export dir @@ -188,6 +210,9 @@ def cwb_encode(corpus, annotations, source_annotations, source_files, words, vrt remove_namespaces=remove_namespaces, sparv_namespace=sparv_namespace, source_namespace=source_namespace, keep_struct_names=True) + # Sort token attributes (but keep word first) to be in the same order as the VRT columns + token_attributes = token_attributes[:1] + sorted(token_attributes[1:]) + # Get VRT columns token_attributes = [(token_name + ":" + i) for i in token_attributes] # First column must be called "word" From 33845a4eca1b7d5f7204611915e14b2ef96b1dbb Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Thu, 9 Mar 2023 17:21:14 +0100 Subject: [PATCH 097/175] Small improvements to xml_import --- sparv/modules/xml_import/xml_import.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/sparv/modules/xml_import/xml_import.py b/sparv/modules/xml_import/xml_import.py index c9fc370a..f8c3ffea 100644 --- a/sparv/modules/xml_import/xml_import.py +++ b/sparv/modules/xml_import/xml_import.py @@ -1,4 +1,4 @@ -"""Parse XML source file.""" +"""Parse XML source files.""" import copy import re @@ -85,6 +85,7 @@ def parse(filename: SourceFilename = SourceFilename(), header_elements: Elements containing header metadata. Contents will not be included in corpus text. header_data: List of header elements and attributes from which to extract metadata. prefix: Optional prefix to add to annotations. + remove_namespaces: Set to True to remove any namespaces. encoding: Encoding of source file. Defaults to UTF-8. keep_control_chars: Set to True to keep control characters in the text. normalize: Normalize input using any of the following forms: 'NFC', 'NFKC', 'NFD', and 'NFKD'. @@ -185,14 +186,15 @@ def handle_element(element): start, start_subpos, end, end_subpos, name_orig, attrs = element # Handle possible skipping of element and attributes - if (name_orig, "") in self.skipped_elems: - return - if (name_orig, "*") in self.skipped_elems: - attrs = {} - for attr in attrs.copy(): - attr_name = get_sparv_name(attr) - if (name_orig, attr_name) in self.skipped_elems: - attrs.pop(attr) + if self.skipped_elems: + if (name_orig, "") in self.skipped_elems: + return + if (name_orig, "*") in self.skipped_elems: + attrs = {} + for attr in attrs.copy(): + attr_name = get_sparv_name(attr) + if (name_orig, attr_name) in self.skipped_elems: + attrs.pop(attr) if name_orig in self.targets: # Rename element and/or attributes @@ -434,7 +436,7 @@ def save(self): def get_namespace(xml_name: str): """Search for a namespace in tag and return a tuple (URI, tagname).""" - m = re.match(r"\{(.*)\}(.+)", xml_name) + m = re.match(r"\{(.*)}(.+)", xml_name) return (m.group(1), m.group(2)) if m else ("", xml_name) From f1cd0c7c8cd6c5d4c3bc2d7e854e01f166e423cc Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 10 Mar 2023 11:45:14 +0100 Subject: [PATCH 098/175] Update tests with reordered VRT columns --- .../freeling-eng-slevel/gold_export/cwb.vrt/document1.vrt | 2 +- .../gold_export/cwb.vrt_scrambled/document1.vrt | 2 +- .../freeling-fra-txt/gold_export/cwb.vrt/wikipedia.vrt | 2 +- .../gold_export/cwb.vrt_scrambled/wikipedia.vrt | 2 +- tests/test_corpora/mini-swe/gold_export/cwb.vrt/dokument1.vrt | 2 +- tests/test_corpora/mini-swe/gold_export/cwb.vrt/dokument2.vrt | 2 +- .../mini-swe/gold_export/cwb.vrt_scrambled/dokument1.vrt | 2 +- .../mini-swe/gold_export/cwb.vrt_scrambled/dokument2.vrt | 2 +- .../standard-swe/gold_export/cwb.vrt/r\303\244var.vrt" | 2 +- .../gold_export/cwb.vrt_scrambled/r\303\244var.vrt" | 2 +- .../test_corpora/stanford-eng/gold_export/cwb.vrt/wikipedia.vrt | 2 +- .../stanford-eng/gold_export/cwb.vrt_scrambled/wikipedia.vrt | 2 +- tests/test_corpora/swe-1800/gold_export/cwb.vrt/boaorm.vrt | 2 +- tests/test_corpora/swe-fsv/gold_export/cwb.vrt/fsv.vrt | 2 +- .../treetagger-nld/gold_export/cwb.vrt/document1.vrt | 2 +- .../treetagger-nld/gold_export/cwb.vrt_scrambled/document1.vrt | 2 +- 16 files changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/test_corpora/freeling-eng-slevel/gold_export/cwb.vrt/document1.vrt b/tests/test_corpora/freeling-eng-slevel/gold_export/cwb.vrt/document1.vrt index 8d672e76..31a39f43 100644 --- a/tests/test_corpora/freeling-eng-slevel/gold_export/cwb.vrt/document1.vrt +++ b/tests/test_corpora/freeling-eng-slevel/gold_export/cwb.vrt/document1.vrt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a44d4832d7d90be76169806f746ad3d09c1e9804b451eaf16949fa5b22b56c99 +oid sha256:0a09f56a05180cf44398e7ecf7054963a26086a9312edf4562e85c337cb3f2a1 size 2311 diff --git a/tests/test_corpora/freeling-eng-slevel/gold_export/cwb.vrt_scrambled/document1.vrt b/tests/test_corpora/freeling-eng-slevel/gold_export/cwb.vrt_scrambled/document1.vrt index 444b7e54..1d0c52d2 100644 --- a/tests/test_corpora/freeling-eng-slevel/gold_export/cwb.vrt_scrambled/document1.vrt +++ b/tests/test_corpora/freeling-eng-slevel/gold_export/cwb.vrt_scrambled/document1.vrt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:68765d77780206c7ff4d2e616b10ae63f2986a67df29cbcc997e8f36b9862821 +oid sha256:1f7eb32a5899af883100054a032c67b773e81aaea8a98aea88c5721702e5deea size 2559 diff --git a/tests/test_corpora/freeling-fra-txt/gold_export/cwb.vrt/wikipedia.vrt b/tests/test_corpora/freeling-fra-txt/gold_export/cwb.vrt/wikipedia.vrt index ea54fe27..6eaa5156 100644 --- a/tests/test_corpora/freeling-fra-txt/gold_export/cwb.vrt/wikipedia.vrt +++ b/tests/test_corpora/freeling-fra-txt/gold_export/cwb.vrt/wikipedia.vrt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:074609bfef8a4936cf5a7ed3500641dbee904db450b253d567d9cd45b9429a3b +oid sha256:ea87d8e0526f3f06e2dd7899468579b1228a3ad19774f7e50ccdba815b29a03d size 10178 diff --git a/tests/test_corpora/freeling-fra-txt/gold_export/cwb.vrt_scrambled/wikipedia.vrt b/tests/test_corpora/freeling-fra-txt/gold_export/cwb.vrt_scrambled/wikipedia.vrt index 019db489..36ca2ba6 100644 --- a/tests/test_corpora/freeling-fra-txt/gold_export/cwb.vrt_scrambled/wikipedia.vrt +++ b/tests/test_corpora/freeling-fra-txt/gold_export/cwb.vrt_scrambled/wikipedia.vrt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b0b2db79331117fd9442e9902c25a7cd07c237b1ca4540174e4e83c2ee511793 +oid sha256:c287916619ec57797ea0151ccf59601ece6a4fb718c2656d083c3957e88fa3d3 size 10484 diff --git a/tests/test_corpora/mini-swe/gold_export/cwb.vrt/dokument1.vrt b/tests/test_corpora/mini-swe/gold_export/cwb.vrt/dokument1.vrt index 4233f5fa..e9136207 100644 --- a/tests/test_corpora/mini-swe/gold_export/cwb.vrt/dokument1.vrt +++ b/tests/test_corpora/mini-swe/gold_export/cwb.vrt/dokument1.vrt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d3b94732f013f33cae07a76c10e3258b942d44584b5d90362314aaf76c99cbb3 +oid sha256:a856151cc837af78ddb16c7b4bf78f48a7a51f6880f6050babed3b6d08a016c9 size 4465 diff --git a/tests/test_corpora/mini-swe/gold_export/cwb.vrt/dokument2.vrt b/tests/test_corpora/mini-swe/gold_export/cwb.vrt/dokument2.vrt index f681daa5..d66ccf29 100644 --- a/tests/test_corpora/mini-swe/gold_export/cwb.vrt/dokument2.vrt +++ b/tests/test_corpora/mini-swe/gold_export/cwb.vrt/dokument2.vrt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9e8c965707e3491203894dcf306e09ddfd2985dc83f48aa9829be085162ece74 +oid sha256:fb545e61ebc59b3d98fad848713c4ff232a3dc005500c5f46b96969f120579ef size 543 diff --git a/tests/test_corpora/mini-swe/gold_export/cwb.vrt_scrambled/dokument1.vrt b/tests/test_corpora/mini-swe/gold_export/cwb.vrt_scrambled/dokument1.vrt index 885277d9..4d3c0a13 100644 --- a/tests/test_corpora/mini-swe/gold_export/cwb.vrt_scrambled/dokument1.vrt +++ b/tests/test_corpora/mini-swe/gold_export/cwb.vrt_scrambled/dokument1.vrt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:499476bc8dc8c905becdd4a972d4ef17ff01c8dcf68bf7516ddc1a05c69c4b59 +oid sha256:8eac9ae29d9a44673af48bf7824a9c4d310b6731df1916bd2cb2963a52207b10 size 4795 diff --git a/tests/test_corpora/mini-swe/gold_export/cwb.vrt_scrambled/dokument2.vrt b/tests/test_corpora/mini-swe/gold_export/cwb.vrt_scrambled/dokument2.vrt index a878a702..c7b88ccf 100644 --- a/tests/test_corpora/mini-swe/gold_export/cwb.vrt_scrambled/dokument2.vrt +++ b/tests/test_corpora/mini-swe/gold_export/cwb.vrt_scrambled/dokument2.vrt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2c6571115a6b0015aa1aa78fe47a7f30562cf3fe3557dde6b7070f03a52e4805 +oid sha256:029ba3dd0119058d180dc53edaa5ec0e41af03ed67d7213082f2ae6e7dc75473 size 543 diff --git "a/tests/test_corpora/standard-swe/gold_export/cwb.vrt/r\303\244var.vrt" "b/tests/test_corpora/standard-swe/gold_export/cwb.vrt/r\303\244var.vrt" index 6a4593e5..1354b187 100644 --- "a/tests/test_corpora/standard-swe/gold_export/cwb.vrt/r\303\244var.vrt" +++ "b/tests/test_corpora/standard-swe/gold_export/cwb.vrt/r\303\244var.vrt" @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3bb9fd6ee6616de333efcb039e46318965e2cdaed57c2343d67d9b75c6c65c44 +oid sha256:c9e9cebaa899b92d805c2f587d43e8dba915d6a2c98c888d1ee41f95ecdf389b size 77904 diff --git "a/tests/test_corpora/standard-swe/gold_export/cwb.vrt_scrambled/r\303\244var.vrt" "b/tests/test_corpora/standard-swe/gold_export/cwb.vrt_scrambled/r\303\244var.vrt" index e565921f..03388ecc 100644 --- "a/tests/test_corpora/standard-swe/gold_export/cwb.vrt_scrambled/r\303\244var.vrt" +++ "b/tests/test_corpora/standard-swe/gold_export/cwb.vrt_scrambled/r\303\244var.vrt" @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e809f1a9beb436ec07bf7ff78fb83370eeb85294aeb4ef2abcff851013e8eae0 +oid sha256:c000d61306dc05fec307afc8d88119a99c6cdcf3b972a9a4da1117accadf4d39 size 86864 diff --git a/tests/test_corpora/stanford-eng/gold_export/cwb.vrt/wikipedia.vrt b/tests/test_corpora/stanford-eng/gold_export/cwb.vrt/wikipedia.vrt index 356c7f5c..148761ce 100644 --- a/tests/test_corpora/stanford-eng/gold_export/cwb.vrt/wikipedia.vrt +++ b/tests/test_corpora/stanford-eng/gold_export/cwb.vrt/wikipedia.vrt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c48b5caba604481c437d2be2df202a26626eeb5692c938819da62291241f426a +oid sha256:348c7b1967b467d067ccd81dc1254b490ff1af5f5396059ee9d41a3cd8897e88 size 17378 diff --git a/tests/test_corpora/stanford-eng/gold_export/cwb.vrt_scrambled/wikipedia.vrt b/tests/test_corpora/stanford-eng/gold_export/cwb.vrt_scrambled/wikipedia.vrt index d62182d7..0dbca000 100644 --- a/tests/test_corpora/stanford-eng/gold_export/cwb.vrt_scrambled/wikipedia.vrt +++ b/tests/test_corpora/stanford-eng/gold_export/cwb.vrt_scrambled/wikipedia.vrt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a3467515df64df0c6cf76165faff25636a5232ff10413d360850f3ec7239b3e8 +oid sha256:f4c46c9a1763b9e091c87e190a30df7471328236dfe13d4fad5561ae973a4cdd size 17378 diff --git a/tests/test_corpora/swe-1800/gold_export/cwb.vrt/boaorm.vrt b/tests/test_corpora/swe-1800/gold_export/cwb.vrt/boaorm.vrt index 9c9c03b4..48602b7e 100644 --- a/tests/test_corpora/swe-1800/gold_export/cwb.vrt/boaorm.vrt +++ b/tests/test_corpora/swe-1800/gold_export/cwb.vrt/boaorm.vrt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4162d59fa60bdfd119c57a9d20f02c92c6a3439089c51d68b0416545766d7721 +oid sha256:f7137779dcd762e322fe10230a5b585bb1511f7e3a18b7447b2589cf0dfa9508 size 28112 diff --git a/tests/test_corpora/swe-fsv/gold_export/cwb.vrt/fsv.vrt b/tests/test_corpora/swe-fsv/gold_export/cwb.vrt/fsv.vrt index 6c5ef07d..b57f3264 100644 --- a/tests/test_corpora/swe-fsv/gold_export/cwb.vrt/fsv.vrt +++ b/tests/test_corpora/swe-fsv/gold_export/cwb.vrt/fsv.vrt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0d7318d7c08e9b2d7dcf0f73bbcb68be39ac35ad77f0740e33c5505f7a5aed3d +oid sha256:6723e6b35b735c69766d39a26c243da67f52818caab16431c69f4575ccb8cc77 size 20155 diff --git a/tests/test_corpora/treetagger-nld/gold_export/cwb.vrt/document1.vrt b/tests/test_corpora/treetagger-nld/gold_export/cwb.vrt/document1.vrt index 43cc736a..33703aa2 100644 --- a/tests/test_corpora/treetagger-nld/gold_export/cwb.vrt/document1.vrt +++ b/tests/test_corpora/treetagger-nld/gold_export/cwb.vrt/document1.vrt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2d89d9a0c64152e79cef0963e5daad04e4b49f40f613164ecba1e6a8af4eda23 +oid sha256:8667450744789fd78e5085951d01640ae7195490b2300eac7b8bfad04f5f10d0 size 3224 diff --git a/tests/test_corpora/treetagger-nld/gold_export/cwb.vrt_scrambled/document1.vrt b/tests/test_corpora/treetagger-nld/gold_export/cwb.vrt_scrambled/document1.vrt index 8ef15229..ae51969a 100644 --- a/tests/test_corpora/treetagger-nld/gold_export/cwb.vrt_scrambled/document1.vrt +++ b/tests/test_corpora/treetagger-nld/gold_export/cwb.vrt_scrambled/document1.vrt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8ec22d40912c6d2297c24fcfef4d0331abc8ba73fa58eb76bf4a589d8ad5d3ff +oid sha256:5d8252f8ec099e33079b772fbf040b41ab727ea159d66e87535966dad9719e2f size 3224 From 8fd2b54e424485d71427e88b23be2812104ecbcf Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 10 Mar 2023 12:14:38 +0100 Subject: [PATCH 099/175] Change Exports -> Exporters in run-rule list for consistency --- sparv/core/Snakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sparv/core/Snakefile b/sparv/core/Snakefile index bfb0cd3a..48315963 100644 --- a/sparv/core/Snakefile +++ b/sparv/core/Snakefile @@ -263,7 +263,7 @@ rule list_targets: table.add_column(no_wrap=True) table.add_column() - table.add_row("[b]Exports[/b]") + table.add_row("[b]Exporters[/b]") for target, desc, _lang in sorted(snake_storage.export_targets): table.add_row(" " + target, desc) table.add_row() From e0587272745f555a7f4034331d744cf7b7be3b85 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 10 Mar 2023 18:00:08 +0100 Subject: [PATCH 100/175] Add support for autocompletion in bash --- CHANGELOG.md | 1 + setup.py | 1 + sparv/__main__.py | 82 ++++++++++++++++++++++++++++++++++++++++---- sparv/core/Snakefile | 32 +++++++++++++++++ 4 files changed, 109 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 443cbf64..b88a5589 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Added +- Added support for tab autocompletion in bash. - Added new `misc:inherit` annotator for inheriting attributes. - Added `korp.wordpicture_no_sentences` setting to disable generation of Word Picture sentences table. - `util.mysql_wrapper` can now execute SQL queries remotely over SSH. diff --git a/setup.py b/setup.py index d3d037b3..beb9efa5 100644 --- a/setup.py +++ b/setup.py @@ -39,6 +39,7 @@ def get_readme(readme_path): python_requires=">=3.8,<3.11", install_requires=[ "appdirs==1.4.4", + "argcomplete", "docx2python==1.27.1", "nltk==3.6.7", "protobuf~=3.19.0", # Used by Stanza; see https://github.com/spraakbanken/sparv-pipeline/issues/161 diff --git a/sparv/__main__.py b/sparv/__main__.py index a2e86ce2..e436a4cd 100644 --- a/sparv/__main__.py +++ b/sparv/__main__.py @@ -1,9 +1,11 @@ """Main Sparv executable.""" - import argparse import sys from pathlib import Path +# PYTHON_ARGCOMPLETE_OK +import argcomplete + from sparv import __version__ # Check Python version @@ -49,6 +51,38 @@ def _format_action(self, action): return result +class Completer: + """Reads and returns cached autocompletion data.""" + + def __init__(self, completion_type): + self.type = completion_type + + def __call__(self, parsed_args, **kwargs): + cache_file = (Path(parsed_args.dir) / ".cache") if parsed_args.dir else Path(".cache") + + if cache_file.is_file(): + import pickle + + with open(cache_file, "rb") as cache: + cache_data = pickle.load(cache) + else: + cache_data = {} + + return cache_data.get(self.type, []) + + +class SortedCompletionFinder(argcomplete.CompletionFinder): + """Custom CompletionFinder that sorts the completions. + + We use this instead of letting bash sort the completions, to sort flags separately. + """ + + def filter_completions(self, completions: list): + completions = super().filter_completions(completions) + completions.sort() + return completions + + def main(): """Run Sparv Pipeline (main entry point for Sparv).""" @@ -88,6 +122,7 @@ def main(): " create-file Create specified file(s)", " run-module Run annotator module independently", " preload Preload annotators and models", + " autocomplete Enable tab completion in bash", "", "See 'sparv -h' for help with a specific command", "For full documentation, visit https://spraakbanken.gu.se/sparv/docs/" @@ -98,15 +133,21 @@ def main(): # Annotate run_parser = subparsers.add_parser("run", description="Annotate a corpus and generate export files.") - run_parser.add_argument("output", nargs="*", default=[], help="The type of output format to generate") + run_parser.add_argument( + "output", nargs="*", default=[], help="The type of output format to generate", + ).completer = Completer("export") run_parser.add_argument("-l", "--list", action="store_true", help="List available output formats") install_parser = subparsers.add_parser("install", description="Install a corpus.") - install_parser.add_argument("type", nargs="*", default=[], help="The type of installation to perform") + install_parser.add_argument( + "type", nargs="*", default=[], help="The type of installation to perform" + ).completer = Completer("install") install_parser.add_argument("-l", "--list", action="store_true", help="List installations to be made") uninstall_parser = subparsers.add_parser("uninstall", description="Uninstall a corpus.") - uninstall_parser.add_argument("type", nargs="*", default=[], help="The type of uninstallation to perform") + uninstall_parser.add_argument( + "type", nargs="*", default=[], help="The type of uninstallation to perform" + ).completer = Completer("uninstall") uninstall_parser.add_argument("-l", "--list", action="store_true", help="List uninstallations to be made") clean_parser = subparsers.add_parser("clean", description="Remove output directories (by default only the " @@ -144,7 +185,9 @@ def main(): description=("Download and build the Sparv models. This is optional, as " "models will be downloaded and built automatically the first " "time they are needed.")) - models_parser.add_argument("model", nargs="*", default=[], help="The model(s) to be built") + models_parser.add_argument( + "model", nargs="*", default=[], help="The model(s) to be built" + ).completer = Completer("model") models_parser.add_argument("-l", "--list", action="store_true", help="List available models") models_parser.add_argument("--language", help="Language (ISO 639-3) if different from current corpus language") models_parser.add_argument("--all", action="store_true", help="Build all models for the current language") @@ -157,7 +200,7 @@ def main(): runrule_parser = subparsers.add_parser("run-rule", description="Run specified rule(s) for creating annotations.") runrule_parser.add_argument("targets", nargs="*", default=["list"], - help="Annotation(s) to create") + help="Annotation(s) to create").completer = Completer("annotate") runrule_parser.add_argument("-l", "--list", action="store_true", help="List available rules") runrule_parser.add_argument("-w", "--wildcards", nargs="*", metavar="WILDCARD", help="Supply values for wildcards using the format 'name=value'") @@ -174,6 +217,9 @@ def main(): preloader_parser.add_argument("-j", "--processes", help="Number of processes to use", default=1, type=int) preloader_parser.add_argument("-l", "--list", action="store_true", help="List annotators available for preloading") + autocomplete_parser = subparsers.add_parser("autocomplete", description="Enable tab completion in bash") + autocomplete_parser.add_argument("--enable", action="store_true", help="Output script to be sourced in bash") + # Add common arguments for subparser in [run_parser, runrule_parser]: subparser.add_argument("-f", "--file", nargs="+", default=[], help="Only annotate specified input file(s)") @@ -210,6 +256,9 @@ def main(): print("No rule to make target") sys.exit(1) + # Handle autocompletion + SortedCompletionFinder()(parser) + # Parse arguments. We allow unknown arguments for the "run-module" command which is handled separately. args, unknown_args = parser.parse_known_args(args=None if sys.argv[1:] else ["--help"]) @@ -262,7 +311,7 @@ def main(): print(f"{e.strerror}: {e.filename!r}") sys.exit(1) - if args.command not in ("build-models", "languages"): + if args.command not in ("autocomplete", "build-models", "languages"): if not config_exists: print(f"No config file ({paths.config_file}) found in working directory.") sys.exit(1) @@ -412,6 +461,25 @@ def main(): "targets": snakemake_args["targets"], "threads": args.cores}) + elif args.command == "autocomplete": + if args.enable: + print( + argcomplete.shellcode( + ["sparv"], complete_arguments=["-o nospace", "-o default", "-o bashdefault", "-o nosort"] + ) + ) + else: + print( + "To enable tab autocompletion for Sparv in bash, source the output of the 'sparv autocomplete --enable'" + " command in your shell by running the following:\n\n" + ' eval "$(sparv autocomplete --enable)"\n\n' + "To enable permanently, add the above line to ~/.bashrc by running the following in your terminal:\n\n" + " echo 'eval \"$(sparv autocomplete --enable)\"' >> ~/.bashrc\n\n" + "Note: Autocompletion of some arguments, such as available exporters, will not be available until a " + "Sparv command (e.g. 'sparv run --list') has been run at least once for the current corpus." + ) + sys.exit(0) + if simple_target: # Force Snakemake to use threads to prevent unnecessary processes for simple targets snakemake_args["force_use_threads"] = True diff --git a/sparv/core/Snakefile b/sparv/core/Snakefile index 48315963..8f2a7881 100644 --- a/sparv/core/Snakefile +++ b/sparv/core/Snakefile @@ -143,6 +143,35 @@ def make_all_files_rule(rule_storage: snake_utils.RuleStorage) -> None: for f in this_sm_rule.input: this_sm_rule.dependencies[f] = sm_rule + +def update_autocompletion_cache(): + """If needed, update the cache of available targets (exporters/installers/rules etc.) used by tab autocompletion.""" + import pickle + + # Create cache + new_cache = dict.fromkeys(("import", "export", "install", "uninstall", "model")) + for target_type in new_cache: + new_cache[target_type] = sorted([t[0] for t in getattr(snake_storage,f"{target_type}_targets")]) + + new_cache["annotate"] = [] + for target_type in ("named", "custom"): + new_cache["annotate"].extend([t[0] for t in getattr(snake_storage,f"{target_type}_targets")]) + new_cache["annotate"].sort() + + # Create or update cache file if needed + cache_file = Path(".cache") + + if cache_file.is_file(): + with open(cache_file, "rb") as cache_in: + current_cache = pickle.load(cache_in) + else: + current_cache = {} + + if not current_cache == new_cache: + with open(cache_file, "wb") as cache_out: + pickle.dump(new_cache, cache_out) + + # Init the storage for some essential variables involving all rules snake_storage = snake_utils.SnakeStorage() @@ -186,6 +215,9 @@ for key in registry.annotation_sources: # Load modules and create automatic rules make_rules(config_missing) +# Update autocompletion cache +update_autocompletion_cache() + # Validate config usage in modules sparv_config.validate_module_config() From ebfb16ee9eadb5d40de272330b0f172f18e52410 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Wed, 15 Mar 2023 14:41:33 +0100 Subject: [PATCH 101/175] Use SafeLoader instead of FullLoader for YAML files; try using the LibYAML version for speed --- sparv/core/config.py | 7 ++++++- sparv/core/paths.py | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/sparv/core/config.py b/sparv/core/config.py index 912857dd..ed1cf252 100644 --- a/sparv/core/config.py +++ b/sparv/core/config.py @@ -9,6 +9,11 @@ import yaml import yaml.scanner +try: + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader + from sparv.core import paths, registry from sparv.core.misc import SparvErrorMessage, get_logger @@ -48,7 +53,7 @@ def read_yaml(yaml_file): """Read YAML file and handle errors.""" try: with open(yaml_file, encoding="utf-8") as f: - data = yaml.load(f, Loader=yaml.FullLoader) + data = yaml.load(f, Loader=SafeLoader) except yaml.parser.ParserError as e: raise SparvErrorMessage("Could not parse the configuration file:\n" + str(e)) except yaml.scanner.ScannerError as e: diff --git a/sparv/core/paths.py b/sparv/core/paths.py index fdd3bec3..2b936d00 100644 --- a/sparv/core/paths.py +++ b/sparv/core/paths.py @@ -6,6 +6,11 @@ import appdirs import yaml +try: + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader + def read_sparv_config(): """Get Sparv data path from config file.""" @@ -13,7 +18,7 @@ def read_sparv_config(): if sparv_config_file.is_file(): try: with open(sparv_config_file, encoding="utf-8") as f: - data = yaml.load(f, Loader=yaml.FullLoader) + data = yaml.load(f, Loader=SafeLoader) except: data = {} return data From e9c82ab801574e0e8287b4d13ca9578d2cfe176d Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Wed, 15 Mar 2023 17:38:03 +0100 Subject: [PATCH 102/175] Move YAML dumping to a util function --- docs/developers-guide/utilities.md | 11 ++++++++ sparv/api/util/misc.py | 40 ++++++++++++++++++++++++++++++ sparv/core/config.py | 30 ---------------------- sparv/core/setup.py | 3 ++- sparv/core/snake_prints.py | 3 ++- sparv/core/wizard.py | 3 ++- sparv/modules/korp/config.py | 31 ++--------------------- 7 files changed, 59 insertions(+), 62 deletions(-) diff --git a/docs/developers-guide/utilities.md b/docs/developers-guide/utilities.md index 0a0784c7..9065d3c6 100644 --- a/docs/developers-guide/utilities.md +++ b/docs/developers-guide/utilities.md @@ -292,6 +292,17 @@ Take an iterable object and return a set in the format used by Corpus Workbench. - `encoding`: Encoding of `values`. Default: "UTF-8" +### dump_yaml() +Convert a dict to a YAML document string. + +**Arguments:** + +- `data`: The data to be dumped. +- `resolve_alias`: Will replace aliases with their anchor's content if set to True. +- `sort_keys`: Whether to sort the keys alphabetically. +- `indent`: Number of spaces used for indentation. + + ### indent_xml() Add pretty-print indentation to an XML tree. diff --git a/sparv/api/util/misc.py b/sparv/api/util/misc.py index aa8f2383..f97fc1b9 100644 --- a/sparv/api/util/misc.py +++ b/sparv/api/util/misc.py @@ -5,6 +5,7 @@ from typing import Optional, Union import pycountry +import yaml from sparv.api import get_logger from sparv.api.classes import Model @@ -13,6 +14,45 @@ logger = get_logger(__name__) +def dump_yaml(data: dict, resolve_alias: bool = False, sort_keys: bool = False, indent: int = 2) -> str: + """Convert a dict to a YAML document string. + + Args: + data: The data to be dumped. + resolve_alias: Will replace aliases with their anchor's content if set to True. + sort_keys: Whether to sort the keys alphabetically. + indent: Number of spaces used for indentation. + """ + + class IndentDumper(yaml.SafeDumper): + """Customized YAML dumper that indents lists.""" + + def increase_indent(self, flow=False, indentless=False): + """Force indentation.""" + return super(IndentDumper, self).increase_indent(flow) + + def str_representer(dumper, data): + """Custom string representer for prettier multiline strings.""" + if "\n" in data: # Check for multiline string + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|") + return dumper.represent_scalar("tag:yaml.org,2002:str", data) + + def obj_representer(dumper, data): + """Custom representer to cast subclasses of str to strings.""" + return dumper.represent_scalar("tag:yaml.org,2002:str", str(data)) + + yaml.representer.SafeRepresenter.add_representer(str, str_representer) + yaml.representer.SafeRepresenter.add_multi_representer(str, obj_representer) + + if resolve_alias: + # Resolve aliases and replace them with their anchors' contents + yaml.SafeDumper.ignore_aliases = lambda *args: True + + return yaml.dump( + data, sort_keys=sort_keys, allow_unicode=True, Dumper=IndentDumper, indent=indent, default_flow_style=False + ) + + # TODO: Split into two functions: one for Sparv-internal lists of values, and one used by the CWB module to create the # CWB-specific set format. def cwbset(values, delimiter="|", affix="|", sort=False, maxlength=4095, encoding="UTF-8"): diff --git a/sparv/core/config.py b/sparv/core/config.py index ed1cf252..49384531 100644 --- a/sparv/core/config.py +++ b/sparv/core/config.py @@ -120,36 +120,6 @@ def handle_parents(cfg, current_dir=Path(".")) -> dict: function="config") -def dump_config(data, resolve_alias=False, sort_keys=False): - """Dump config YAML to string. - - Args: - data: The data to be dumped. - resolve_alias: Will replace aliases with their anchor's content if set to True. - sort_keys: Whether to sort the keys alphabetically. - """ - class IndentDumper(yaml.Dumper): - """Customized YAML dumper that indents lists.""" - - def increase_indent(self, flow=False, indentless=False): - """Force indentation.""" - return super(IndentDumper, self).increase_indent(flow) - - # Add custom string representer for prettier multiline strings - def str_representer(dumper, data): - if len(data.splitlines()) > 1: # Check for multiline string - return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|") - return dumper.represent_scalar("tag:yaml.org,2002:str", data) - yaml.add_representer(str, str_representer) - - if resolve_alias: - # Resolve aliases and replace them with their anchors' contents - yaml.Dumper.ignore_aliases = lambda *args: True - - return yaml.dump(data, sort_keys=sort_keys, allow_unicode=True, Dumper=IndentDumper, indent=4, - default_flow_style=False) - - def _get(name: str, config_dict=None): """Try to get value from config, raising an exception if key doesn't exist.""" config_dict = config_dict if config_dict is not None else config diff --git a/sparv/core/setup.py b/sparv/core/setup.py index d68b434b..594bebb8 100644 --- a/sparv/core/setup.py +++ b/sparv/core/setup.py @@ -12,6 +12,7 @@ from rich.prompt import Confirm from sparv import __version__ +from sparv.api.util.misc import dump_yaml from sparv.core import config, paths from sparv.core.console import console @@ -154,7 +155,7 @@ def run(sparv_datadir: Optional[str] = None): paths.sparv_config_file.parent.mkdir(parents=True, exist_ok=True) with open(paths.sparv_config_file, "w", encoding="utf-8") as f: - f.write(config.dump_config(config_dict)) + f.write(dump_yaml(config_dict)) copy_resource_files(path) diff --git a/sparv/core/snake_prints.py b/sparv/core/snake_prints.py index 6e5e283c..cbd09ac6 100644 --- a/sparv/core/snake_prints.py +++ b/sparv/core/snake_prints.py @@ -13,7 +13,8 @@ def prettyprint_yaml(in_dict): """Pretty-print YAML.""" from rich.syntax import Syntax - yaml_str = config.dump_config(in_dict, resolve_alias=True, sort_keys=True) + from sparv.api.util.misc import dump_yaml + yaml_str = dump_yaml(in_dict, resolve_alias=True, sort_keys=True) # Print syntax highlighted console.print(Syntax(yaml_str, "yaml", background_color="default")) diff --git a/sparv/core/wizard.py b/sparv/core/wizard.py index bb681da9..343ddd20 100644 --- a/sparv/core/wizard.py +++ b/sparv/core/wizard.py @@ -13,6 +13,7 @@ from questionary import prompt from sparv.api import SourceStructureParser, Wildcard +from sparv.api.util.misc import dump_yaml from sparv.core import registry, paths, config, snake_utils from sparv.core.console import console @@ -189,7 +190,7 @@ def get_module_wizard(self, module_wizard: Tuple[Callable, list, bool]) -> List[ def save_config(self): """Save config to YAML file.""" with open("config.yaml", mode="w", encoding="utf-8") as out_file: - out_file.write(config.dump_config({k: v for k, v in self.corpus_config.items() if not k.startswith("_")})) + out_file.write(dump_yaml({k: v for k, v in self.corpus_config.items() if not k.startswith("_")})) print("Your corpus configuration has been saved as 'config.yaml'.") def load_config(self): diff --git a/sparv/modules/korp/config.py b/sparv/modules/korp/config.py index 85235604..ff9665d7 100644 --- a/sparv/modules/korp/config.py +++ b/sparv/modules/korp/config.py @@ -6,8 +6,6 @@ from pathlib import Path from typing import Dict, List, Optional, Union -import yaml - from sparv.api import ( AnnotationName, Config, @@ -165,7 +163,7 @@ def config( out: YAML file to create. """ config_dict = { - "id": str(id), + "id": id, "lang": language, "mode": modes, } @@ -299,7 +297,7 @@ def config( "# This file was automatically generated by Sparv. Do not make changes directly to this file as they will " "get overwritten.\n" ) - out_yaml.write(dict_to_yaml(config_dict)) + out_yaml.write(util.misc.dump_yaml(config_dict)) def build_annotations( @@ -462,28 +460,3 @@ def uninstall_config( util.install.uninstall_path(corpus_file, host=remote_host) install_marker.remove() marker.write() - - -def dict_to_yaml(data): - """Convert dict to YAML string. - - Args: - data: The dict to be converted. - """ - - class IndentDumper(yaml.Dumper): - """Customized YAML dumper that indents lists.""" - - def increase_indent(self, flow=False, indentless=False): - """Force indentation.""" - return super(IndentDumper, self).increase_indent(flow) - - # Add custom string representer for prettier multiline strings - def str_representer(dumper, data): - if len(data.splitlines()) > 1: # Check for multiline string - return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|") - return dumper.represent_scalar("tag:yaml.org,2002:str", data) - - yaml.add_representer(str, str_representer) - - return yaml.dump(data, sort_keys=False, allow_unicode=True, Dumper=IndentDumper, indent=4, default_flow_style=False) From 5cc885634c0498551374a1e374bfe8d77b830028 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Thu, 16 Mar 2023 15:43:25 +0100 Subject: [PATCH 103/175] Improve automatic selection between multiple annotations by also checking classes --- sparv/core/registry.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sparv/core/registry.py b/sparv/core/registry.py index f7fdefd9..78f49b53 100644 --- a/sparv/core/registry.py +++ b/sparv/core/registry.py @@ -558,8 +558,9 @@ def expand_classes(s: str, parents: Set[str]) -> Tuple[Optional[str], Optional[s rest.append(unknown) if is_annotation and len(strings) > 1: - # If multiple alternative annotations, return the first one that is explicitly used, or the last - if string in explicit_annotations: + # If multiple alternative annotations, return the first one that is explicitly used as an export annotation, + # or referred to by a class in the config. As a fallback use the last annotation. + if string in explicit_annotations or string in annotation_classes["config_classes"].values(): break return string, rest From da3a2ef13a9fa9351099d8c9823d170bf5a514dc Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 17 Mar 2023 16:20:44 +0100 Subject: [PATCH 104/175] Use a shared autocompletion cache instead of one per corpus --- sparv/__main__.py | 86 ++++++++++++++++++++++++++++---------------- sparv/core/Snakefile | 53 +++++++++++++++++---------- sparv/core/paths.py | 1 + 3 files changed, 91 insertions(+), 49 deletions(-) diff --git a/sparv/__main__.py b/sparv/__main__.py index e436a4cd..1032c3b0 100644 --- a/sparv/__main__.py +++ b/sparv/__main__.py @@ -58,17 +58,36 @@ def __init__(self, completion_type): self.type = completion_type def __call__(self, parsed_args, **kwargs): - cache_file = (Path(parsed_args.dir) / ".cache") if parsed_args.dir else Path(".cache") - - if cache_file.is_file(): - import pickle - - with open(cache_file, "rb") as cache: - cache_data = pickle.load(cache) - else: - cache_data = {} - - return cache_data.get(self.type, []) + # Read config file to find corpus language + config_file = Path(parsed_args.dir or Path.cwd(), "config.yaml") + cache_data = {} + + if config_file.is_file(): + import appdirs + import yaml + try: + from yaml import CSafeLoader as SafeLoader + except ImportError: + from yaml import SafeLoader + + with open(config_file, encoding="utf-8") as f: + data = yaml.load(f, Loader=SafeLoader) + language = data.get("metadata", {}).get("language") + + cache_file = Path(appdirs.user_config_dir("sparv"), "autocomplete") + + if cache_file.is_file(): + import pickle + try: + with open(cache_file, "rb") as cache: + cache_data = pickle.load(cache) + if not language: + language = cache_data.get("default_language") + cache_data = cache_data.get(language, {}) + except EOFError: # Cache placeholder created but not yet populated + pass + + return cache_data.get(self.type, []) class SortedCompletionFinder(argcomplete.CompletionFinder): @@ -267,6 +286,32 @@ def main(): from sparv.core import run run.main(unknown_args, log_level=args.log) sys.exit() + elif args.command == "autocomplete": + if args.enable: + import appdirs + try: + # Create empty autocomplete cache if it doesn't exist + # The cache contents will only be populated if this file exists + Path(appdirs.user_config_dir("sparv"), "autocomplete").touch() + except FileNotFoundError: + pass + print( + argcomplete.shellcode( + ["sparv"], complete_arguments=["-o nospace", "-o default", "-o bashdefault", "-o nosort"] + ) + ) + else: + print( + "To enable tab autocompletion for Sparv in bash, source the output of the 'sparv autocomplete --enable'" + " command in your shell by running the following:\n\n" + ' eval "$(sparv autocomplete --enable)"\n\n' + "To enable permanently, add the above line to ~/.bashrc by running the following in your terminal:\n\n" + " echo 'eval \"$(sparv autocomplete --enable)\"' >> ~/.bashrc\n\n" + "Note: Autocompletion of some arguments, such as available exporters, will not be available until some " + "part of the Sparv pipeline (e.g. 'sparv run') has been run at least once since enabling " + "autocompletion." + ) + sys.exit(0) else: import snakemake from snakemake.logging import logger @@ -461,25 +506,6 @@ def main(): "targets": snakemake_args["targets"], "threads": args.cores}) - elif args.command == "autocomplete": - if args.enable: - print( - argcomplete.shellcode( - ["sparv"], complete_arguments=["-o nospace", "-o default", "-o bashdefault", "-o nosort"] - ) - ) - else: - print( - "To enable tab autocompletion for Sparv in bash, source the output of the 'sparv autocomplete --enable'" - " command in your shell by running the following:\n\n" - ' eval "$(sparv autocomplete --enable)"\n\n' - "To enable permanently, add the above line to ~/.bashrc by running the following in your terminal:\n\n" - " echo 'eval \"$(sparv autocomplete --enable)\"' >> ~/.bashrc\n\n" - "Note: Autocompletion of some arguments, such as available exporters, will not be available until a " - "Sparv command (e.g. 'sparv run --list') has been run at least once for the current corpus." - ) - sys.exit(0) - if simple_target: # Force Snakemake to use threads to prevent unnecessary processes for simple targets snakemake_args["force_use_threads"] = True diff --git a/sparv/core/Snakefile b/sparv/core/Snakefile index 8f2a7881..4f9fa5a8 100644 --- a/sparv/core/Snakefile +++ b/sparv/core/Snakefile @@ -146,30 +146,45 @@ def make_all_files_rule(rule_storage: snake_utils.RuleStorage) -> None: def update_autocompletion_cache(): """If needed, update the cache of available targets (exporters/installers/rules etc.) used by tab autocompletion.""" - import pickle - # Create cache - new_cache = dict.fromkeys(("import", "export", "install", "uninstall", "model")) - for target_type in new_cache: - new_cache[target_type] = sorted([t[0] for t in getattr(snake_storage,f"{target_type}_targets")]) + # Only update cache if it exists + if paths.autocomplete_cache.is_file(): + import pickle - new_cache["annotate"] = [] - for target_type in ("named", "custom"): - new_cache["annotate"].extend([t[0] for t in getattr(snake_storage,f"{target_type}_targets")]) - new_cache["annotate"].sort() + # Collect data for cache + new_cache = dict.fromkeys(("import", "export", "install", "uninstall", "model")) + for target_type in new_cache: + new_cache[target_type] = sorted( + [t[0] for t in getattr(snake_storage, f"{target_type}_targets") if not t[0].startswith("custom.")] + ) - # Create or update cache file if needed - cache_file = Path(".cache") + new_cache["annotate"] = [] + for target_type in ("named", "custom"): + new_cache["annotate"].extend( + [t[0] for t in getattr(snake_storage, f"{target_type}_targets") if not t[0].startswith("custom.")] + ) + new_cache["annotate"].sort() - if cache_file.is_file(): - with open(cache_file, "rb") as cache_in: - current_cache = pickle.load(cache_in) - else: - current_cache = {} + # Create or update cache file if needed + cache_file = paths.autocomplete_cache + language = sparv_config.get("metadata.language") - if not current_cache == new_cache: - with open(cache_file, "wb") as cache_out: - pickle.dump(new_cache, cache_out) + try: + with open(cache_file, "rb") as cache_in: + current_cache = pickle.load(cache_in) + except: + current_cache = {} + + default_language = sparv_config._config_default.get("metadata", {}).get("language") + + if not (current_cache.get(language) == new_cache and current_cache.get("default_language") == default_language): + current_cache[language] = new_cache + current_cache["default_language"] = default_language + try: + with open(cache_file, "wb") as cache_out: + pickle.dump(current_cache, cache_out) + except: + pass # Init the storage for some essential variables involving all rules diff --git a/sparv/core/paths.py b/sparv/core/paths.py index 2b936d00..778a99be 100644 --- a/sparv/core/paths.py +++ b/sparv/core/paths.py @@ -47,6 +47,7 @@ def get_data_path(subpath: Union[str, Path] = "") -> Optional[Path]: # Config file containing path to Sparv data dir sparv_config_file = Path(appdirs.user_config_dir("sparv"), "config.yaml") +autocomplete_cache = Path(appdirs.user_config_dir("sparv"), "autocomplete") # Package-internal paths modules_dir = "modules" From 78b7512e5cac802bc0e5f33b4adbb4b7ab4bf819 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 17 Mar 2023 17:05:42 +0100 Subject: [PATCH 105/175] Add --enable-old flag for tab autocompletion in older bash versions --- sparv/__main__.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/sparv/__main__.py b/sparv/__main__.py index 1032c3b0..b6d2a7a9 100644 --- a/sparv/__main__.py +++ b/sparv/__main__.py @@ -238,6 +238,8 @@ def main(): autocomplete_parser = subparsers.add_parser("autocomplete", description="Enable tab completion in bash") autocomplete_parser.add_argument("--enable", action="store_true", help="Output script to be sourced in bash") + autocomplete_parser.add_argument("--enable-old", action="store_true", + help="Output script to be sourced in bash, for bash version 4.3 and below") # Add common arguments for subparser in [run_parser, runrule_parser]: @@ -287,7 +289,7 @@ def main(): run.main(unknown_args, log_level=args.log) sys.exit() elif args.command == "autocomplete": - if args.enable: + if args.enable or args.enable_old: import appdirs try: # Create empty autocomplete cache if it doesn't exist @@ -295,11 +297,10 @@ def main(): Path(appdirs.user_config_dir("sparv"), "autocomplete").touch() except FileNotFoundError: pass - print( - argcomplete.shellcode( - ["sparv"], complete_arguments=["-o nospace", "-o default", "-o bashdefault", "-o nosort"] - ) - ) + complete_arguments = ["-o nospace", "-o default", "-o bashdefault"] + if args.enable: + complete_arguments.append("-o nosort") + print(argcomplete.shellcode(["sparv"], complete_arguments=complete_arguments)) else: print( "To enable tab autocompletion for Sparv in bash, source the output of the 'sparv autocomplete --enable'" @@ -307,6 +308,7 @@ def main(): ' eval "$(sparv autocomplete --enable)"\n\n' "To enable permanently, add the above line to ~/.bashrc by running the following in your terminal:\n\n" " echo 'eval \"$(sparv autocomplete --enable)\"' >> ~/.bashrc\n\n" + "For bash version 4.3 and below, use the flag '--enable-old' instead.\n\n" "Note: Autocompletion of some arguments, such as available exporters, will not be available until some " "part of the Sparv pipeline (e.g. 'sparv run') has been run at least once since enabling " "autocompletion." From 1efd608667b23fcd2cb16903e45f94d8b9f00932 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Mon, 27 Mar 2023 16:27:30 +0200 Subject: [PATCH 106/175] Deduplicate export annotations before they get sent to modules --- sparv/core/snake_utils.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sparv/core/snake_utils.py b/sparv/core/snake_utils.py index cc9ec8b4..93f8cf20 100644 --- a/sparv/core/snake_utils.py +++ b/sparv/core/snake_utils.py @@ -379,27 +379,27 @@ def rule_helper(rule: RuleStorage, config: dict, storage: SnakeStorage, config_m annotation_type = Annotation if param_type in ( ExportAnnotations, ExportAnnotationNames) else AnnotationAllSourceFiles plain_annotations = set() - possible_plain_annotations = [] - for i, (export_annotation_name, export_name) in enumerate(export_annotations): + possible_plain_annotations = {} + full_annotations = {} # Using a dict for deduplication (parse_annotation_list's deduping isn't enough) + for export_annotation_name, export_name in export_annotations: annotation = annotation_type(export_annotation_name) rule.configs.update(registry.find_config_variables(annotation.name)) rule.classes.update(registry.find_classes(annotation.name)) rule.missing_config.update(annotation.expand_variables(rule.full_name)) - export_annotations[i] = (annotation, export_name) + full_annotations[annotation] = export_name plain_name, attr = annotation.split() if not attr: plain_annotations.add(plain_name) else: - if plain_name not in possible_plain_annotations: - possible_plain_annotations.append(plain_name) + possible_plain_annotations[plain_name] = None # Add plain annotations where needed for a in possible_plain_annotations: if a not in plain_annotations: - export_annotations.append((annotation_type(a), None)) + full_annotations[annotation_type(a)] = None items = [] - for annotation, export_name in export_annotations: + for annotation, export_name in full_annotations.items(): if param_value.is_input: if param_type == ExportAnnotationsAllSourceFiles: rule.inputs.extend( From 56e3f14444873eba4bbab8cfb9cc2a9a168164b1 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 31 Mar 2023 14:05:14 +0200 Subject: [PATCH 107/175] Upgrade Snakemake --- setup.py | 2 +- sparv/__main__.py | 5 +++- sparv/core/log_handler.py | 55 ++++++++++++++++++++++----------------- 3 files changed, 36 insertions(+), 26 deletions(-) diff --git a/setup.py b/setup.py index beb9efa5..1b165ed9 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ def get_readme(readme_path): "PyYAML==6.0", "questionary==1.10.0", "rich==11.0.0", - "snakemake==6.3.0", + "snakemake==7.25.0", "stanza==1.4.0", "torch>=1.9.1", # Used by Stanza; see https://github.com/spraakbanken/sparv-pipeline/issues/82 "typing-inspect==0.8.0" diff --git a/sparv/__main__.py b/sparv/__main__.py index b6d2a7a9..e574fb11 100644 --- a/sparv/__main__.py +++ b/sparv/__main__.py @@ -369,7 +369,10 @@ def main(): f"from a directory that has a config file ({paths.config_file}).") sys.exit(1) - snakemake_args = {"workdir": args.dir} + snakemake_args = { + "workdir": args.dir, + "rerun_triggers": "mtime" # Only rerun rules based on file modification times + } config = {"run_by_sparv": True} simple_target = False log_level = "" diff --git a/sparv/core/log_handler.py b/sparv/core/log_handler.py index 103ff735..b8bd80d8 100644 --- a/sparv/core/log_handler.py +++ b/sparv/core/log_handler.py @@ -457,12 +457,13 @@ def missing_annotations_or_files(source, files): if level == "run_info": # Parse list of jobs do to and total job count - lines = msg["msg"].splitlines()[2:] - total_jobs = lines[-1].strip() + lines = msg["msg"].splitlines()[3:] + total_jobs = lines[-1].split()[1] for j in lines[:-1]: - _, count, job = j.split("\t") + job, count, _, _ = j.split() + if ":" in job and not "::" in job: + job = job + "*" # Differentiate entrypoints from actual rules in the list self.jobs[job.replace("::", ":")] = int(count) - self.jobs_max_len = max(map(len, self.jobs)) if self.use_progressbar and not self.bar_started: @@ -513,21 +514,18 @@ def missing_annotations_or_files(source, files): self.progress.remove_task(this_job["task"]) self.job_ids.pop((this_job["name"], this_job["file"]), None) self.current_jobs.pop(msg["jobid"], None) - if level == "job_error": + if level == "job_error" and msg.get("msg"): self.messages["unhandled_error"].append(msg) elif level == "info": - if self.pass_through or msg["msg"] == "Nothing to be done.": - self.info(msg["msg"]) - - elif level == "error": if self.pass_through: - self.messages["unhandled_error"].append(msg) - return - handled = False + self.info(msg["msg"]) + elif msg["msg"].startswith("Nothing to be done"): + self.info("Nothing to be done.") - # SparvErrorMessage exception from pipeline core + elif level == "debug": if "SparvErrorMessage" in msg["msg"]: + # SparvErrorMessage exception from pipeline core # Parse error message message = re.search( r"{}([^\n]*)\n([^\n]*)\n(.*?){}".format(SparvErrorMessage.start_marker, @@ -537,15 +535,21 @@ def missing_annotations_or_files(source, files): module, function, error_message = message.groups() error_source = ":".join((module, function)) if module and function else None self.messages["error"].append((error_source, error_message)) - handled = True + self.handled_error = True + elif "exit status 123" in msg["msg"]: + # Exit status 123 means a Sparv module caused an exception, either a SparvErrorMessage exception, or + # an unexpected exception. Either way, the error message has already been logged, so it doesn't need to + # be printed again. + self.handled_error = True - # Exit status 123 means a Sparv module raised a SparvErrorMessage exception - # The error message has already been logged so doesn't need to be printed again - elif "exit status 123" in msg["msg"] or ("SystemExit" in msg["msg"] and "123" in msg["msg"]): - handled = True + elif level == "error": + if self.pass_through: + self.messages["unhandled_error"].append(msg) + return + handled = False # Errors due to missing config variables or binaries leading to missing input files - elif "MissingInputException" in msg["msg"]: + if "MissingInputException" in msg["msg"]: msg_contents = re.search(r" for rule (\S+):\n(.+)", msg["msg"]) rule_name, filelist = msg_contents.groups() rule_name = rule_name.replace("::", ":") @@ -597,7 +601,8 @@ def missing_annotations_or_files(source, files): elif level in ("warning", "job_error"): # Save other errors and warnings for later - self.messages["unhandled_error"].append(msg) + if msg.get("msg"): + self.messages["unhandled_error"].append(msg) elif level == "dag_debug" and "job" in msg: # Create regular expressions for searching for missing config variables or binaries @@ -673,17 +678,19 @@ def stop(self): if self.log_level and logging._nameToLevel[self.log_level.upper()] > logging.DEBUG: errmsg[0] += " To display further details about this error, rerun Sparv with the " \ "'--log debug' argument.\n" - if "msg" in error: + if error.get("msg"): + # Show only a summary of the error + # Parsing is based on the format in format_error() in snakemake/exceptions.py error_lines = error["msg"].splitlines() - if " in line " in error_lines[0]: - errmsg.append(error_lines[0].split(" in line ")[0] + ":") + if " in file " in error_lines[0]: + errmsg.append(error_lines[0].split(" in file ")[0] + ":") for line in error_lines[1:]: if line.startswith(" File "): break errmsg.append(line) else: errmsg.append("") - errmsg.append(error.get("msg", "An unknown error occurred.")) + errmsg.append(error.get("msg") or "An unknown error occurred.") self.error("\n".join(errmsg)) else: spacer = "" From 1f5d53be2a63db345145f0d29ce1595db80904a3 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 31 Mar 2023 14:31:56 +0200 Subject: [PATCH 108/175] Add support for Python 3.11 --- .github/workflows/ci.yml | 2 +- CHANGELOG.md | 2 +- README.md | 2 +- docs/user-manual/installation-and-setup.md | 2 +- docs/user-manual/quick-start.md | 2 +- setup.py | 2 +- sparv/__main__.py | 4 ++-- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3109b455..84f4ba9b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,7 +10,7 @@ jobs: max-parallel: 4 fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10", "3.11"] os: [ubuntu-latest] task: - name: Run tests diff --git a/CHANGELOG.md b/CHANGELOG.md index b88a5589..8b033f6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,7 +33,7 @@ ### Changed -- Added support for Python 3.10. +- Added support for Python 3.10 and 3.11. - Dropped support for Python 3.6 and 3.7. - `AnnotationAllSourceFiles` now have the same methods as `Annotation`. - The util function `install_mysql` can now install locally as well as to a remote server. diff --git a/README.md b/README.md index bf5affdd..b4900a0d 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ If you have any questions, problems or suggestions please contact [!NOTE] > Most of Sparv's features should work in a Windows environment as well, but since we don't do any testing on Windows diff --git a/docs/user-manual/quick-start.md b/docs/user-manual/quick-start.md index 62381849..c4885feb 100644 --- a/docs/user-manual/quick-start.md +++ b/docs/user-manual/quick-start.md @@ -13,7 +13,7 @@ user guide, please refer to the full documentation. ## Installation -Begin by making sure that you have [Python 3.8](https://python.org/) or newer (up to 3.10) installed by running the following +Begin by making sure that you have [Python 3.8](https://python.org/) or newer installed by running the following in your terminal: ``` python3 --version diff --git a/setup.py b/setup.py index 1b165ed9..08cfd5f2 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ def get_readme(readme_path): license="MIT", packages=setuptools.find_namespace_packages(include=["sparv", "sparv.*"]), zip_safe=False, - python_requires=">=3.8,<3.11", + python_requires=">=3.8", install_requires=[ "appdirs==1.4.4", "argcomplete", diff --git a/sparv/__main__.py b/sparv/__main__.py index e574fb11..8df65f4f 100644 --- a/sparv/__main__.py +++ b/sparv/__main__.py @@ -9,8 +9,8 @@ from sparv import __version__ # Check Python version -if sys.version_info < (3, 8, 0) or sys.version_info >= (3, 11, 0): - print("Python 3.8–3.10 is required.") +if sys.version_info < (3, 8, 0): + print("Python 3.8 or newer is required.") sys.exit(1) From 22f3b382c60046175c3050b945081db99b08cc11 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 31 Mar 2023 15:18:57 +0200 Subject: [PATCH 109/175] Update dependencies --- setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 08cfd5f2..58c7234e 100644 --- a/setup.py +++ b/setup.py @@ -39,17 +39,17 @@ def get_readme(readme_path): python_requires=">=3.8", install_requires=[ "appdirs==1.4.4", - "argcomplete", + "argcomplete==3.0.5", "docx2python==1.27.1", - "nltk==3.6.7", + "nltk==3.8.1", "protobuf~=3.19.0", # Used by Stanza; see https://github.com/spraakbanken/sparv-pipeline/issues/161 "pycountry==22.3.5", "python-dateutil==2.8.2", "PyYAML==6.0", "questionary==1.10.0", - "rich==11.0.0", + "rich==13.3.3", "snakemake==7.25.0", - "stanza==1.4.0", + "stanza==1.5.0", "torch>=1.9.1", # Used by Stanza; see https://github.com/spraakbanken/sparv-pipeline/issues/82 "typing-inspect==0.8.0" ], From 8c491f3eea2fe7a179310a8510005b113e4682f3 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Mon, 3 Apr 2023 10:52:45 +0200 Subject: [PATCH 110/175] Fix handling of Snakemake errors --- sparv/core/log_handler.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sparv/core/log_handler.py b/sparv/core/log_handler.py index b8bd80d8..80ea0527 100644 --- a/sparv/core/log_handler.py +++ b/sparv/core/log_handler.py @@ -550,7 +550,7 @@ def missing_annotations_or_files(source, files): # Errors due to missing config variables or binaries leading to missing input files if "MissingInputException" in msg["msg"]: - msg_contents = re.search(r" for rule (\S+):\n(.+)", msg["msg"]) + msg_contents = re.search(r" for rule (\S+):\n.*affected files:\n(.+)", msg["msg"], flags=re.DOTALL) rule_name, filelist = msg_contents.groups() rule_name = rule_name.replace("::", ":") if self.missing_configs_re.search(filelist): @@ -565,8 +565,7 @@ def missing_annotations_or_files(source, files): # Missing output files elif "MissingOutputException" in msg["msg"]: - msg_contents = re.search(r"Missing files after .*?:\n(.+)\nThis might be due to", msg["msg"], - flags=re.DOTALL) + msg_contents = re.search(r"Missing files after .*?:\n(.+)", msg["msg"], flags=re.DOTALL) missing_files = "\n • ".join(msg_contents.group(1).strip().splitlines()) message = f"The following output files were expected but are missing:\n" \ f" • {missing_files}\n" + missing_annotations_msg From d21e81c93a7990f6931e24052581b7c1f75b6c16 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Mon, 3 Apr 2023 14:58:10 +0200 Subject: [PATCH 111/175] Fix error messages not getting written to log file when using the --log debug flag (fixes #168) --- CHANGELOG.md | 2 ++ sparv/core/run_snake.py | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b033f6e..9b599ea6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,8 @@ - Fixed quoting of paths in `util.system.rsync`. - It's no longer possible to create an infinite loop of classes referring to each other. - Elapsed time exceeding 24 hours no longer gets cut off in the `--stats` output. +- Fixed bug where error messages were not getting written to the log file when the `--log debug` + flag was used. ## [5.1.0] - 2022-11-03 diff --git a/sparv/core/run_snake.py b/sparv/core/run_snake.py index 205fa58e..6228bcf6 100644 --- a/sparv/core/run_snake.py +++ b/sparv/core/run_snake.py @@ -139,10 +139,10 @@ def flush(): # something went wrong. exit_with_error_message(e.message, "sparv.modules." + module_name) except Exception as e: - errmsg = f"An error occurred while executing {module_name}:{f_name}:" + errmsg = f"An error occurred while executing {module_name}:{f_name}:\n\n {type(e).__name__}: {e}" if logger.level > logging.DEBUG: - errmsg += f"\n\n {type(e).__name__}: {e}\n\n" \ - "To display further details when errors occur, run Sparv with the '--log debug' argument." + errmsg += f"\n\nTo display further details when errors occur, run Sparv with the '--log debug' or " \ + "'--log-to-file debug' arguments." logger.error(errmsg) logger.debug(traceback.format_exc()) sys.exit(123) From 7dac37bc9861258426a4c1903de57c146cef1fa1 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Tue, 4 Apr 2023 14:29:21 +0200 Subject: [PATCH 112/175] Better span sub-positions in XML importer --- sparv/modules/xml_import/xml_import.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sparv/modules/xml_import/xml_import.py b/sparv/modules/xml_import/xml_import.py index f8c3ffea..223c5c38 100644 --- a/sparv/modules/xml_import/xml_import.py +++ b/sparv/modules/xml_import/xml_import.py @@ -335,10 +335,10 @@ def iter_tree(element: etree.Element, start_pos: int = 0, start_subpos: int = 0) child_tail = None for child in element: if not element_length: - start_subpos += 1 + child_start_subpos = start_subpos + 1 else: - start_subpos = 0 - child_length, child_tail, end_subpos = iter_tree(child, start_pos + element_length, start_subpos) + child_start_subpos = 0 + child_length, child_tail, end_subpos = iter_tree(child, start_pos + element_length, child_start_subpos) element_length += child_length + child_tail end_pos = start_pos + element_length if child_tail == 0: From 11ca42ca3b7cc9ba60eeb62f794dd0c54a9c9269 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Tue, 4 Apr 2023 18:01:57 +0200 Subject: [PATCH 113/175] Remove --rerun-incomplete and --mark-complete flags, and always rerun automatically instead --- CHANGELOG.md | 1 + sparv/__main__.py | 11 ++--------- sparv/core/log_handler.py | 10 ---------- 3 files changed, 3 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b599ea6..3263d04b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,7 @@ of lists that need parsing and expanding. - The classes `AllSourceFilenames`, `ExportAnnotations`, `ExportAnnotationsAllSourceFiles` and `ExportAnnotationNames` are now immutable iterables instead of lists. +- Removed the flags `--rerun-incomplete` and `--mark-complete`, as Sparv will now always rerun incomplete files. ### Fixed diff --git a/sparv/__main__.py b/sparv/__main__.py index 8df65f4f..c45d4158 100644 --- a/sparv/__main__.py +++ b/sparv/__main__.py @@ -269,8 +269,6 @@ def main(): # Add extra arguments to 'run' that we want to come last run_parser.add_argument("--unlock", action="store_true", help="Unlock the working directory") - run_parser.add_argument("--mark-complete", nargs="+", metavar="FILE", help="Mark output files as complete") - run_parser.add_argument("--rerun-incomplete", action="store_true", help="Rerun incomplete output files") # Backward compatibility if len(sys.argv) > 1 and sys.argv[1] == "make": @@ -371,7 +369,8 @@ def main(): snakemake_args = { "workdir": args.dir, - "rerun_triggers": "mtime" # Only rerun rules based on file modification times + "rerun_triggers": ["mtime"], # Only rerun rules based on file modification times + "force_incomplete": True # Always rerun incomplete files } config = {"run_by_sparv": True} simple_target = False @@ -433,12 +432,6 @@ def main(): snakemake_args["unlock"] = args.unlock simple_target = True pass_through = True - if args.mark_complete: - snakemake_args["cleanup_metadata"] = args.mark_complete - simple_target = True - pass_through = True - elif args.rerun_incomplete: - snakemake_args["force_incomplete"] = True if args.list: snakemake_args["targets"] = ["list_exports"] elif args.output: diff --git a/sparv/core/log_handler.py b/sparv/core/log_handler.py index 80ea0527..91543e1b 100644 --- a/sparv/core/log_handler.py +++ b/sparv/core/log_handler.py @@ -581,16 +581,6 @@ def missing_annotations_or_files(source, files): "directory, run 'sparv run --unlock' to remove the lock." self.messages["error"].append((None, message)) handled = True - elif "IncompleteFilesException:" in msg["msg"]: - msg_contents = re.search(r"Incomplete files:\n(.+)", msg["msg"], flags=re.DOTALL) - incomplete_files = "\n • ".join(msg_contents.group(1).strip().splitlines()) - message = "The files below seem to be incomplete. If you are sure that certain files are not " \ - "incomplete, mark them as complete with 'sparv run --mark-complete '.\n" \ - "To re-generate the files instead, rerun your command with the --rerun-incomplete flag.\n" \ - "Incomplete files:\n" \ - f" • {incomplete_files}" - self.messages["error"].append((None, message)) - handled = True # Unhandled errors if not handled: From 7939b0bcbf5f5b58bf73fd904505c28caaa78e52 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Tue, 4 Apr 2023 18:03:30 +0200 Subject: [PATCH 114/175] Make OutputMarker write current timestamp by default --- sparv/api/classes.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sparv/api/classes.py b/sparv/api/classes.py index 106ce6bb..322de213 100644 --- a/sparv/api/classes.py +++ b/sparv/api/classes.py @@ -4,6 +4,7 @@ import os import pathlib import pickle +import time import urllib.request import zipfile from abc import ABC, abstractmethod @@ -526,7 +527,8 @@ class OutputMarker(OutputCommonData): """A class for creating a marker, indicating that something has run.""" def write(self, value="", append: bool = False): - super().write(value) + # Write current timestamp, as Snakemake also compares checksums for small files, not just modified time + super().write(value or str(time.time())) class Text: From 6a124867c55af70ae3787b0cc79a9baa713aa058 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Wed, 12 Apr 2023 17:59:33 +0200 Subject: [PATCH 115/175] Rewrite calculate_element_hierarchy to handle circular hierarchy between annotations --- sparv/api/util/export.py | 101 +++++++++++++++++++-------------------- 1 file changed, 49 insertions(+), 52 deletions(-) diff --git a/sparv/api/util/export.py b/sparv/api/util/export.py index 3dd5807e..c38c7bad 100644 --- a/sparv/api/util/export.py +++ b/sparv/api/util/export.py @@ -92,33 +92,38 @@ def __lt__(self, other_span): 2. end position (larger indices first) 3. the calculated element hierarchy """ - def get_sort_key(span, sub_positions=False, empty_span=False): + def get_sort_key(span, hierarchy, sub_positions=False, empty_span=False): """Return a sort key for span which makes span comparison possible.""" - hierarchy_index = elem_hierarchy.index(span.name) if span.name in elem_hierarchy else -1 if empty_span: if sub_positions: - return (span.start, span.start_sub), hierarchy_index, (span.end, span.end_sub) + return (span.start, span.start_sub), hierarchy, (span.end, span.end_sub) else: - return span.start, hierarchy_index, span.end + return span.start, hierarchy, span.end else: if sub_positions: - return (span.start, span.start_sub), (-span.end, -span.end_sub), hierarchy_index + return (span.start, span.start_sub), (-span.end, -span.end_sub), hierarchy else: - return span.start, -span.end, hierarchy_index + return span.start, -span.end, hierarchy + + if self.name in elem_hierarchy and other_span.name in elem_hierarchy[self.name]: + self_hierarchy = elem_hierarchy[self.name][other_span.name] + other_hierarchy = 0 if self_hierarchy == 1 else 1 + else: + self_hierarchy = other_hierarchy = -1 # Sort empty spans according to hierarchy or put them first if (self.start, self.start_sub) == (self.end, self.end_sub) or ( other_span.start, other_span.start_sub) == (other_span.end, other_span.end_sub): - sort_key1 = get_sort_key(self, empty_span=True) - sort_key2 = get_sort_key(other_span, empty_span=True) + sort_key1 = get_sort_key(self, self_hierarchy, empty_span=True) + sort_key2 = get_sort_key(other_span, other_hierarchy, empty_span=True) # Both spans have sub positions - elif self.start_sub and other_span.start_sub: - sort_key1 = get_sort_key(self, sub_positions=True) - sort_key2 = get_sort_key(other_span, sub_positions=True) + elif self.start_sub is not False and other_span.start_sub is not False: + sort_key1 = get_sort_key(self, self_hierarchy, sub_positions=True) + sort_key2 = get_sort_key(other_span, other_hierarchy, sub_positions=True) # At least one of the spans does not have sub positions else: - sort_key1 = get_sort_key(self) - sort_key2 = get_sort_key(other_span) + sort_key1 = get_sort_key(self, self_hierarchy) + sort_key2 = get_sort_key(other_span, other_hierarchy) return sort_key1 < sort_key2 @@ -160,7 +165,7 @@ def get_sort_key(span, sub_positions=False, empty_span=False): if span.name in elem_hierarchy: for i, (instruction, s) in enumerate(spans_dict[span.start]): if instruction == "close": - if s.name in elem_hierarchy and elem_hierarchy.index(s.name) < elem_hierarchy.index(span.name): + if s.name in elem_hierarchy[span.name] and elem_hierarchy[span.name][s.name] == 1: insert_index = i break spans_dict[span.start].insert(insert_index, ("open", span)) @@ -232,59 +237,51 @@ def calculate_element_hierarchy(source_file, spans_list): """Calculate the hierarchy for spans with identical start and end positions. If two spans A and B have identical start and end positions, go through all occurrences of A and B - and check which element is most often parent to the other. That one will be first. + and check which element is most often parent to the other. """ # Find elements with identical spans span_duplicates = defaultdict(set) - start_positions = defaultdict(set) - end_positions = defaultdict(set) - empty_span_starts = set() + startend_positions = defaultdict(set) + empty_span_starts = defaultdict(set) for span in spans_list: + # We don't include sub-positions here as we still need to compare spans with sub-positions with spans without span_duplicates[(span.start, span.end)].add(span.name) - start_positions[span.start].add(span.name) - end_positions[span.end].add(span.name) - if span.start == span.end: - empty_span_starts.add(span.start) - span_duplicates = [v for v in span_duplicates.values() if len(v) > 1] - # Add empty spans and spans with identical start positions - for span_start in empty_span_starts: - span_duplicates.append(start_positions[span_start]) - span_duplicates.append(end_positions[span_start]) + if (span.start, span.start_sub) == (span.end, span.end_sub): + empty_span_starts[span.name].add(span.start) + else: + startend_positions[span.start].add(span.name) + startend_positions[span.end].add(span.name) + span_duplicates = set(tuple(sorted(v)) for v in span_duplicates.values() if len(v) > 1) - # Flatten structure - unclear_spans = set([elem for elem_set in span_duplicates for elem in elem_set]) + # Add empty spans and spans with identical start positions + empty_spans = set() + for empty_span, span_starts in empty_span_starts.items(): + for span_start in span_starts: + for a in startend_positions[span_start]: + empty_spans.add(tuple(sorted((empty_span, a)))) # Get pairs of relations that need to be ordered - relation_pairs = list(combinations(unclear_spans, r=2)) - # Order each pair into [parent, children] - ordered_pairs = set() + relation_pairs = set() + for collisions in span_duplicates: + for combination in combinations(collisions, r=2): + relation_pairs.add(combination) + relation_pairs.update(empty_spans) + + hierarchy = defaultdict(dict) + + # Calculate parent-child relation for every pair for a, b in relation_pairs: a_annot = Annotation(a, source_file=source_file) b_annot = Annotation(b, source_file=source_file) a_parent = len([i for i in (b_annot.get_parents(a_annot)) if i is not None]) b_parent = len([i for i in (a_annot.get_parents(b_annot)) if i is not None]) if a_parent > b_parent: - ordered_pairs.add((a, b)) + hierarchy[a][b] = 0 + hierarchy[b][a] = 1 elif a_parent < b_parent: - ordered_pairs.add((b, a)) - - hierarchy = [] - error_msg = ("Something went wrong while sorting annotation elements. Could there be circular relations? " - "The following elements could not be sorted: ") - # Loop until all unclear_spans are processed - while unclear_spans: - size = len(unclear_spans) - for span in unclear_spans.copy(): - # Span is never a child in ordered_pairs, then it is first in the hierarchy - if not any([b == span for _a, b in ordered_pairs]): - hierarchy.append(span) - unclear_spans.remove(span) - # Remove pairs from ordered_pairs where span is the parent - for pair in ordered_pairs.copy(): - if pair[0] == span: - ordered_pairs.remove(pair) - # Check that unclear_spans is getting smaller, otherwise there might be circularity - assert len(unclear_spans) < size, error_msg + " ".join(unclear_spans) + hierarchy[b][a] = 0 + hierarchy[a][b] = 1 + return hierarchy From a83dfa52c4ff75e232bdd879b0ba27b28438a720 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Thu, 13 Apr 2023 15:36:09 +0200 Subject: [PATCH 116/175] Also trigger rerun based on changes to the set of input files (e.g. removal of source files) --- sparv/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sparv/__main__.py b/sparv/__main__.py index c45d4158..fb335951 100644 --- a/sparv/__main__.py +++ b/sparv/__main__.py @@ -369,7 +369,7 @@ def main(): snakemake_args = { "workdir": args.dir, - "rerun_triggers": ["mtime"], # Only rerun rules based on file modification times + "rerun_triggers": ["mtime", "input"], # Rerun based on file modification times and changes to the set of input files "force_incomplete": True # Always rerun incomplete files } config = {"run_by_sparv": True} From cb98d272ee770f8f2ea8570ec68dce5b2d86d222 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Thu, 20 Apr 2023 13:29:35 +0200 Subject: [PATCH 117/175] Replace setup.py with pyproject.toml --- .gitignore | 1 - MANIFEST.in | 2 ++ pyproject.toml | 52 +++++++++++++++++++++++++++++++++++- setup.py | 71 -------------------------------------------------- 4 files changed, 53 insertions(+), 73 deletions(-) create mode 100644 MANIFEST.in delete mode 100644 setup.py diff --git a/.gitignore b/.gitignore index 6135fa83..33ae1d19 100644 --- a/.gitignore +++ b/.gitignore @@ -30,7 +30,6 @@ build/ dist/ *.egg-info/ *.egg -MANIFEST* # Unit test / coverage reports .pytest_cache/ diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..e61df937 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +prune tests +exclude MANIFEST.in diff --git a/pyproject.toml b/pyproject.toml index e43b7fb7..16786dd0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,57 @@ [build-system] -requires = ["setuptools >= 40.1.0", "wheel"] +requires = ["setuptools>=64.0.0", "wheel"] build-backend = "setuptools.build_meta" +[project] +name = "sparv-pipeline" +dynamic = ["version"] +description = "Språkbanken's text analysis tool" +readme = "README.md" +requires-python = ">=3.8" +license.text = "MIT License" +authors = [ + { name = "Språkbanken", email = "sb-info@svenska.gu.se" } +] +dependencies = [ + "appdirs==1.4.4", + "argcomplete==3.0.5", + "docx2python==1.27.1", + "nltk==3.8.1", + "protobuf~=3.19.0", # Used by Stanza; see https://github.com/spraakbanken/sparv-pipeline/issues/161 + "pycountry==22.3.5", + "python-dateutil==2.8.2", + "PyYAML==6.0", + "questionary==1.10.0", + "rich==13.3.3", + "snakemake==7.25.0", + "stanza==1.5.0", + "torch>=1.9.1", # Used by Stanza; see https://github.com/spraakbanken/sparv-pipeline/issues/82 + "typing-inspect==0.8.0", +] + +[project.optional-dependencies] +dev = [ + "pandocfilters==1.5.0", + "pytest", + "pytest-sugar>=0.9.6", +] + +[project.urls] +Homepage = "https://github.com/spraakbanken/sparv-pipeline/" +Documentation = "https://spraakbanken.gu.se/sparv/" + +[project.scripts] +sparv = "sparv.__main__:main" + +[tool.setuptools.dynamic] +version = {attr = "sparv.__version__"} + +[tool.setuptools.packages] +find.include = ["sparv", "sparv.*"] + +[tool.setuptools.package-data] +"sparv" = ["core/Snakefile", "resources/config/**"] + [tool.black] line-length = 120 target-version = ["py38", "py39", "py310"] diff --git a/setup.py b/setup.py deleted file mode 100644 index 58c7234e..00000000 --- a/setup.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Install script for the Sparv Pipeline.""" - -import os.path - -import setuptools - - -def get_version(rel_path): - """Get version number from package.""" - here = os.path.abspath(os.path.dirname(__file__)) - with open(os.path.join(here, rel_path), encoding="utf-8") as f: - for line in f: - if line.startswith("__version__"): - delim = '"' if '"' in line else "'" - return line.split(delim)[1] - else: - raise RuntimeError("Unable to find version string.") - - -def get_readme(readme_path): - """Get the contents of the README file.""" - here = os.path.abspath(os.path.dirname(__file__)) - with open(os.path.join(here, readme_path), encoding="utf-8") as f: - return f.read() - - -setuptools.setup( - name="sparv-pipeline", - version=get_version("sparv/__init__.py"), - description="Språkbanken's text analysis tool", - long_description=get_readme("README.md"), - long_description_content_type="text/markdown", - url="https://github.com/spraakbanken/sparv-pipeline/", - author="Språkbanken", - author_email="sb-info@svenska.gu.se", - license="MIT", - packages=setuptools.find_namespace_packages(include=["sparv", "sparv.*"]), - zip_safe=False, - python_requires=">=3.8", - install_requires=[ - "appdirs==1.4.4", - "argcomplete==3.0.5", - "docx2python==1.27.1", - "nltk==3.8.1", - "protobuf~=3.19.0", # Used by Stanza; see https://github.com/spraakbanken/sparv-pipeline/issues/161 - "pycountry==22.3.5", - "python-dateutil==2.8.2", - "PyYAML==6.0", - "questionary==1.10.0", - "rich==13.3.3", - "snakemake==7.25.0", - "stanza==1.5.0", - "torch>=1.9.1", # Used by Stanza; see https://github.com/spraakbanken/sparv-pipeline/issues/82 - "typing-inspect==0.8.0" - ], - extras_require={ - "dev": [ - "pandocfilters==1.5.0", - "pytest", - "pytest-sugar>=0.9.6" - ] - }, - entry_points={ - "console_scripts": [ - "sparv=sparv.__main__:main" - ] - }, - package_data={ - "sparv": ["core/Snakefile", "resources/config/*", "resources/config/presets/*"] - } -) From 18026bbac9b95bd52000cdd3550a015728979246 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Thu, 20 Apr 2023 17:28:12 +0200 Subject: [PATCH 118/175] Use hatch instead of setuptools --- MANIFEST.in | 2 -- pyproject.toml | 16 ++++++---------- 2 files changed, 6 insertions(+), 12 deletions(-) delete mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index e61df937..00000000 --- a/MANIFEST.in +++ /dev/null @@ -1,2 +0,0 @@ -prune tests -exclude MANIFEST.in diff --git a/pyproject.toml b/pyproject.toml index 16786dd0..1ab357b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [build-system] -requires = ["setuptools>=64.0.0", "wheel"] -build-backend = "setuptools.build_meta" +requires = ["hatchling"] +build-backend = "hatchling.build" [project] name = "sparv-pipeline" @@ -43,14 +43,10 @@ Documentation = "https://spraakbanken.gu.se/sparv/" [project.scripts] sparv = "sparv.__main__:main" -[tool.setuptools.dynamic] -version = {attr = "sparv.__version__"} - -[tool.setuptools.packages] -find.include = ["sparv", "sparv.*"] - -[tool.setuptools.package-data] -"sparv" = ["core/Snakefile", "resources/config/**"] +[tool.hatch] +version.path = "sparv/__init__.py" +build.include = ["/sparv"] +publish.index.disable = true # Require confirmation to publish [tool.black] line-length = 120 From e7d582457fa28ce1fafdc04acb65af355fdb3b87 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 21 Apr 2023 15:28:31 +0200 Subject: [PATCH 119/175] Update readme and changelog --- CHANGELOG.md | 2 ++ README.md | 3 +-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3263d04b..1ddd04a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,8 @@ - The classes `AllSourceFilenames`, `ExportAnnotations`, `ExportAnnotationsAllSourceFiles` and `ExportAnnotationNames` are now immutable iterables instead of lists. - Removed the flags `--rerun-incomplete` and `--mark-complete`, as Sparv will now always rerun incomplete files. +- Sparv will now recognize when source files have been deleted and trigger the necessary reruns. Previously, only + additions and modifications were recognized. ### Fixed diff --git a/README.md b/README.md index b4900a0d..601ffc55 100644 --- a/README.md +++ b/README.md @@ -44,8 +44,7 @@ Before cloning the repository with [git](https://git-scm.com/downloads) make sur Storage](https://git-lfs.github.com/) installed (`apt install git-lfs`). Some files will not be downloaded correctly otherwise. -We recommend that you set up a virtual environment and install the dependencies (including the dev dependencies) listed -in `setup.py`: +Install the dependencies, including the dev dependencies. We recommend that you first set up a virtual environment: ``` python3 -m venv venv From 9778a0b2ea90985a162a807832a905d70001b856 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Tue, 25 Apr 2023 16:13:06 +0200 Subject: [PATCH 120/175] Remove reference to setup.py from workflow --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 84f4ba9b..fe29a9bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,7 @@ jobs: uses: actions/cache@v3.2.3 with: path: venv - key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/pyproject.toml') }}-${{ hashFiles('**/setup.py') }}-${{ hashFiles('.github/workflows/ci.yml') }} + key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/pyproject.toml') }}-${{ hashFiles('.github/workflows/ci.yml') }} # Create virtual environment and install dependencies if cache does not exist - name: Create venv and install dependencies From 5b7b7f91d25e11a4e46afba6d45d7348041f7ac2 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Tue, 25 Apr 2023 16:50:48 +0200 Subject: [PATCH 121/175] Update documentation for writing plugins --- .../developers-guide/writing-sparv-plugins.md | 51 ++++++++++--------- docs/docsify/index.html | 1 + 2 files changed, 27 insertions(+), 25 deletions(-) diff --git a/docs/developers-guide/writing-sparv-plugins.md b/docs/developers-guide/writing-sparv-plugins.md index 415e68df..1a42fbf3 100644 --- a/docs/developers-guide/writing-sparv-plugins.md +++ b/docs/developers-guide/writing-sparv-plugins.md @@ -22,43 +22,44 @@ sparv-sbx-uppercase/ │ ├── uppercase.py │ └── __init__.py ├── LICENSE -├── README.md -└── setup.py +├── pyproject.toml +└── README.md ``` In the above example the `sbx_uppercase` directory is a Sparv module containing the [module code](#module-code) in -`uppercase.py` and the mandatory [init file](#init-file) `__init__.py`. The [setup file](#setup-file) `setup.py` in the -root directory is needed in order to install the plugin. +`uppercase.py` and the mandatory [init file](#init-file) `__init__.py`. The [project file](#pyprojecttoml) +`pyproject.toml` in the root directory is needed in order to install the plugin. The readme and license files are not strictly necessary for the plugin to work, but we strongly recommend that you include these if you want to publish your plugin. -## Setup File -The `setup.py` is needed in order to install a plugin and connect it to the Sparv Pipeline. Here is a minimal example of -a setup file (taken from the [Sparv plugin template](https://github.com/spraakbanken/sparv-plugin-template)): -```python -import setuptools - -setuptools.setup( - name="sparv-sbx-uppercase", - version="0.1", - description="Uppercase converter (example plug-in for Sparv)", - license="MIT", - packages=["sbx_uppercase"], - python_requires=">=3.6", - install_requires=["sparv-pipeline>=4,<5"], - entry_points={"sparv.plugin": ["sbx_uppercase = sbx_uppercase"]} -) +## pyproject.toml +The `pyproject.toml` file is needed in order to install a plugin and connect it to the Sparv Pipeline. Here is a minimal +example of a project file (taken from the [Sparv plugin template](https://github.com/spraakbanken/sparv-plugin-template)): +```toml +[project] +name = "sparv-sbx-uppercase" +version = "0.1.0" +description = "Uppercase converter (example plug-in for Sparv)" +readme = "README.md" +license.text = "MIT License" +dependencies = [ + "sparv-pipeline~=5.0" +] +entry-points."sparv.plugin" = { sbx_uppercase = "sbx_uppercase" } ``` -Make sure to include the name of your module (i.e. the directory containing the Sparv code) in `packages`. You also need -to make sure that there is a `sparv.plugin` entry point in `entry_points` that points to your module. +Make sure that there is a `sparv.plugin` entry point in `project.entry-points` that points to your module (the directory +containing the code). It is also a good idea to add `sparv-pipeline` to the list of dependencies, specifying which major +version of Sparv the plugin is developed for, as it might not be compatible with future major versions. +`"sparv-pipeline~=5.0"` under `dependencies` means that the plugin is compatible with any version of Sparv 5, but not +Sparv 6. -We strongly encourage you to also include the fields `author` and `author_email`. +We strongly encourage you to also include the `project.authors` field. -For more information about Python setup scripts check the [distutils -documentation](https://docs.python.org/3/distutils/setupscript.html). +For more information about `pyproject.toml` check the [Python Packaging User Guide +](https://packaging.python.org/en/latest/specifications/declaring-project-metadata/). ## Init File diff --git a/docs/docsify/index.html b/docs/docsify/index.html index e279152d..87a2ac5c 100644 --- a/docs/docsify/index.html +++ b/docs/docsify/index.html @@ -115,5 +115,6 @@ + From 3f2d68e49ab6499f1f41d232ac743dce76a8df1b Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Wed, 3 May 2023 16:02:59 +0200 Subject: [PATCH 122/175] Add --json-log flag to enable JSON format for logging --- CHANGELOG.md | 1 + pyproject.toml | 1 + sparv/__main__.py | 15 +++++-- sparv/core/log_handler.py | 90 ++++++++++++++++++++++++++++----------- 4 files changed, 80 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ddd04a5..5cf0b3b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ - Added `HeaderAnnotations` and `HeaderAnnotationsAllSourceFiles` classes. - Added `korp.keep_undefined_annotations` setting, to include even undefined annotations in the Korp config. - Added `dateformat.pre_regex` setting. +- Added `--json-log` flag to enable JSON format for logging. ### Changed diff --git a/pyproject.toml b/pyproject.toml index 1ab357b2..fedb9828 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,7 @@ dependencies = [ "protobuf~=3.19.0", # Used by Stanza; see https://github.com/spraakbanken/sparv-pipeline/issues/161 "pycountry==22.3.5", "python-dateutil==2.8.2", + "python-json-logger==2.0.7", "PyYAML==6.0", "questionary==1.10.0", "rich==13.3.3", diff --git a/sparv/__main__.py b/sparv/__main__.py index fb335951..c3a42b7a 100644 --- a/sparv/__main__.py +++ b/sparv/__main__.py @@ -261,6 +261,7 @@ def main(): "specified, 'info' if LOGLEVEL is not specified)", nargs="?", choices=["debug", "info", "warning", "error", "critical"]) subparser.add_argument("--stats", action="store_true", help="Show summary of time spent per annotator") + subparser.add_argument("--json-log", action="store_true", help="Use JSON format for logging") subparser.add_argument("--debug", action="store_true", help="Show debug messages") subparser.add_argument("--socket", help="Path to socket file created by the 'preload' command") subparser.add_argument("--force-preloader", action="store_true", @@ -510,9 +511,17 @@ def main(): # Disable Snakemake's default log handler and use our own logger.log_handler = [] - progress = log_handler.LogHandler(progressbar=not simple_target, log_level=log_level, log_file_level=log_file_level, - simple=simple_mode, stats=stats, pass_through=pass_through, dry_run=dry_run, - keep_going=keep_going) + progress = log_handler.LogHandler( + progressbar=not (simple_target or args.json_log), + log_level=log_level, + log_file_level=log_file_level, + simple=simple_mode, + stats=stats, + pass_through=pass_through, + dry_run=dry_run, + keep_going=keep_going, + json=args.json_log, + ) snakemake_args["log_handler"] = [progress.log_handler] config["log_server"] = progress.log_server diff --git a/sparv/core/log_handler.py b/sparv/core/log_handler.py index 91543e1b..62c49efe 100644 --- a/sparv/core/log_handler.py +++ b/sparv/core/log_handler.py @@ -9,13 +9,14 @@ import struct import threading import time -from collections import defaultdict, OrderedDict +from collections import OrderedDict, defaultdict from datetime import timedelta from pathlib import Path from typing import Optional import rich.box as box import rich.progress as progress +from pythonjsonlogger import jsonlogger from rich.control import Control, ControlType from rich.logging import RichHandler from rich.table import Table @@ -55,6 +56,10 @@ def _log_progress(self, progress=None, advance=None, total=None): PROGRESS = 90 logging.addLevelName(PROGRESS, "PROGRESS") +# Add logging level used for final messages when logging in JSON format, always displayed +FINAL = 80 +logging.addLevelName(FINAL, "FINAL") + def _export_dirs(self, dirs): """Send list of export dirs to log handler.""" @@ -111,7 +116,7 @@ def __init__(self, count_dict, *args, **kwargs): def emit(self, record): """Increment level counter for each log message.""" - if record.levelno < PROGRESS: + if record.levelno < FINAL: self.levelcount[record.levelname] += 1 @@ -239,7 +244,7 @@ class LogHandler: icon = "\U0001f426" def __init__(self, progressbar=True, log_level=None, log_file_level=None, simple=False, stats=False, - pass_through=False, dry_run=False, keep_going=False): + pass_through=False, dry_run=False, keep_going=False, json=False): """Initialize log handler. Args: @@ -251,12 +256,14 @@ def __init__(self, progressbar=True, log_level=None, log_file_level=None, simple pass_through: Let Snakemake's log messages pass through uninterrupted. dry_run: Set to True to print summary about jobs. keep_going: Set to True if the keepgoing flag is enabled for Snakemake. + json: Set to True to enable JSON output. """ self.use_progressbar = progressbar and console.is_terminal self.simple = simple or not console.is_terminal self.pass_through = pass_through self.dry_run = dry_run self.keep_going = keep_going + self.json = json self.log_level = log_level self.log_file_level = log_file_level self.log_filename = None @@ -307,12 +314,34 @@ def setup_loggers(self): internal_filter = InternalFilter() progress_internal_filter = ProgressInternalFilter() + # Set logger to use the lowest selected log level, but never higher than warning (we still want to count warnings) + self.logger.setLevel( + min( + logging.WARNING, getattr(logging, self.log_level.upper()), getattr(logging, self.log_file_level.upper()) + ) + ) + # stdout logger - stream_handler = ModifiedRichHandler(enable_link_path=False, rich_tracebacks=True, console=console) + if self.json: + stream_handler = logging.StreamHandler() + else: + stream_handler = ModifiedRichHandler(enable_link_path=False, rich_tracebacks=True, console=console) + stream_handler.setLevel(self.log_level.upper()) stream_handler.addFilter(internal_filter) - log_format = "%(message)s" if stream_handler.level > logging.DEBUG else "(%(process)d) - %(message)s" - stream_handler.setFormatter(logging.Formatter(log_format, datefmt=TIME_FORMAT)) + + if self.json: + stream_formatter = json_formatter = jsonlogger.JsonFormatter(LOG_FORMAT_DEBUG, rename_fields={ + "asctime": "time", + "levelname": "level" + }) + else: + stream_formatter = logging.Formatter( + "%(message)s" if stream_handler.level > logging.DEBUG else "(%(process)d) - %(message)s", + datefmt=TIME_FORMAT + ) + + stream_handler.setFormatter(stream_formatter) self.logger.addHandler(stream_handler) # File logger @@ -321,8 +350,13 @@ def setup_loggers(self): encoding="UTF-8", delay=True) file_handler.setLevel(self.log_file_level.upper()) file_handler.addFilter(progress_internal_filter) - log_format = LOG_FORMAT if file_handler.level > logging.DEBUG else LOG_FORMAT_DEBUG - file_handler.setFormatter(logging.Formatter(log_format)) + + if self.json: + file_formatter = json_formatter + else: + file_formatter = logging.Formatter(LOG_FORMAT if file_handler.level > logging.DEBUG else LOG_FORMAT_DEBUG) + + file_handler.setFormatter(file_formatter) self.logger.addHandler(file_handler) # Level counter @@ -354,7 +388,7 @@ def setup_bar(self): self.progress.start() self.bar = self.progress.add_task(self.icon, start=False, total=0, text="[dim]Preparing...[/dim]") - # Logging needs to be set up after the bar, to make use if its print hook + # Logging needs to be set up after the bar, to make use of its print hook self.setup_loggers() def start_bar(self, total: int): @@ -363,20 +397,26 @@ def start_bar(self, total: int): self.progress.start_task(self.bar) self.bar_started = True - @staticmethod - def info(msg): + def info(self, msg): """Print info message.""" - console.print(Text(msg, style="green")) + if self.json: + self.logger.log(FINAL, msg) + else: + console.print(Text(msg, style="green")) - @staticmethod - def warning(msg): + def warning(self, msg): """Print warning message.""" - console.print(Text(msg, style="yellow")) + if self.json: + self.logger.log(FINAL, msg) + else: + console.print(Text(msg, style="yellow")) - @staticmethod - def error(msg): + def error(self, msg): """Print error message.""" - console.print(Text(msg, style="red")) + if self.json: + self.logger.log(FINAL, msg) + else: + console.print(Text(msg, style="red")) def log_handler(self, msg): """Log handler for Snakemake displaying a progress bar.""" @@ -476,8 +516,9 @@ def missing_annotations_or_files(source, files): # Advance progress self.progress.advance(self.bar) - # Print regular updates if output is not a terminal (i.e. doesn't support the progress bar) - elif self.logger and not console.is_terminal: + # Print regular progress updates if output is not a terminal (i.e. doesn't support the progress bar) or + # output format is JSON + elif self.logger and (self.json or not console.is_terminal): percentage = (100 * msg["done"]) // msg["total"] if percentage > self.last_percentage: self.last_percentage = percentage @@ -646,12 +687,13 @@ def stop(self): if self.handled_error: # Print any collected core error messages if self.messages["error"]: - self.error("Sparv exited with the following error message{}:".format( - "s" if len(self.messages) > 1 else "")) + errmsg = ["Sparv exited with the following error message{}:".format( + "s" if len(self.messages) > 1 else "")] for message in self.messages["error"]: error_source, msg = message error_source = f"[{error_source}]\n" if error_source else "" - self.error(f"\n{error_source}{msg}") + errmsg.append(f"\n{error_source}{msg}") + self.error("\n".join(errmsg)) else: # Errors from modules have already been logged, so notify user if self.log_filename: @@ -739,7 +781,7 @@ def cleanup(): def setup_logging(log_server, log_level: str = "warning", log_file_level: str = "warning", file=None, job=None): """Set up logging with socket handler.""" - # Use the lowest log level, but never higher than warning + # Set logger to use the lowest selected log level, but never higher than warning (we still want to count warnings) log_level = min(logging.WARNING, getattr(logging, log_level.upper()), getattr(logging, log_file_level.upper())) socket_logger = logging.getLogger("sparv") socket_logger.setLevel(log_level) From a1063c1b0574d8090340ed83c3e68f51a0bff8ee Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Wed, 3 May 2023 17:08:32 +0200 Subject: [PATCH 123/175] Remove 'critical' from log levels as we're not using it --- sparv/__main__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sparv/__main__.py b/sparv/__main__.py index c3a42b7a..dd5cf3fe 100644 --- a/sparv/__main__.py +++ b/sparv/__main__.py @@ -255,11 +255,11 @@ def main(): subparser.add_argument("--log", metavar="LOGLEVEL", const="info", help="Set the log level (default: 'warning' if --log is not specified, " "'info' if LOGLEVEL is not specified)", - nargs="?", choices=["debug", "info", "warning", "error", "critical"]) + nargs="?", choices=["debug", "info", "warning", "error"]) subparser.add_argument("--log-to-file", metavar="LOGLEVEL", const="info", help="Set log level for logging to file (default: 'warning' if --log-to-file is not " "specified, 'info' if LOGLEVEL is not specified)", - nargs="?", choices=["debug", "info", "warning", "error", "critical"]) + nargs="?", choices=["debug", "info", "warning", "error"]) subparser.add_argument("--stats", action="store_true", help="Show summary of time spent per annotator") subparser.add_argument("--json-log", action="store_true", help="Use JSON format for logging") subparser.add_argument("--debug", action="store_true", help="Show debug messages") From dcd60435b0f3f70d58b43243a769e0eb9552ee0a Mon Sep 17 00:00:00 2001 From: Anne Schumacher Date: Wed, 3 May 2023 17:17:39 +0200 Subject: [PATCH 124/175] add module for PDF import --- pyproject.toml | 1 + sparv/modules/pdf_import/__init__.py | 3 ++ sparv/modules/pdf_import/pdf_import.py | 71 ++++++++++++++++++++++++++ 3 files changed, 75 insertions(+) create mode 100644 sparv/modules/pdf_import/__init__.py create mode 100644 sparv/modules/pdf_import/pdf_import.py diff --git a/pyproject.toml b/pyproject.toml index fedb9828..907a5085 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ dependencies = [ "argcomplete==3.0.5", "docx2python==1.27.1", "nltk==3.8.1", + "pdfplumber==0.9.0", "protobuf~=3.19.0", # Used by Stanza; see https://github.com/spraakbanken/sparv-pipeline/issues/161 "pycountry==22.3.5", "python-dateutil==2.8.2", diff --git a/sparv/modules/pdf_import/__init__.py b/sparv/modules/pdf_import/__init__.py new file mode 100644 index 00000000..551689f6 --- /dev/null +++ b/sparv/modules/pdf_import/__init__.py @@ -0,0 +1,3 @@ +"""Import of pdf source files.""" + +from . import pdf_import diff --git a/sparv/modules/pdf_import/pdf_import.py b/sparv/modules/pdf_import/pdf_import.py new file mode 100644 index 00000000..d6e07a88 --- /dev/null +++ b/sparv/modules/pdf_import/pdf_import.py @@ -0,0 +1,71 @@ +"""Import module for pdf source files.""" + +import re +import unicodedata +from typing import Optional + +import pdfplumber + +from sparv.api import Config, Output, Source, SourceFilename, SourceStructure, Text, importer, util + + +@importer("pdf import", file_extension="pdf", outputs=["text", "page:number"], text_annotation="text", config=[ + Config("pdf_import.prefix", description="Optional prefix to add to annotation names."), + Config("pdf_import.keep_control_chars", False, description="Set to True if control characters should not be " + "removed from the text."), + Config("pdf_import.normalize", "NFC", description="Normalize input using any of the following forms: " + "'NFC', 'NFKC', 'NFD', and 'NFKD'.") +]) +def parse(source_file: SourceFilename = SourceFilename(), + source_dir: Source = Source(), + prefix: Optional[str] = Config("pdf_import.prefix"), + keep_control_chars: bool = Config("pdf_import.keep_control_chars"), + normalize: str = Config("pdf_import.normalize")) -> None: + """Parse pdf file as input to the Sparv Pipeline and keep page information. + + Args: + source_file: The source filename. + source_dir: The source directory. + prefix: Optional prefix for output annotation. + keep_control_chars: Set to True to keep control characters in the text. + normalize: Normalize input text using any of the following forms: 'NFC', 'NFKC', 'NFD', and 'NFKD'. + 'NFC' is used by default. + """ + source_file_path = source_dir.get_path(source_file, ".pdf") + + # Extract text from PDF pages + texts = [] + pages = [] + start_position = 0 + with pdfplumber.open(source_file_path) as pdf: + for n, p in enumerate(pdf.pages): + pagetext = p.extract_text(layout=True, y_density=12) + # Keep track of the page indentation + min_indent = min(map(len, re.findall(r"^ +", pagetext, flags=re.MULTILINE))) + if not keep_control_chars: + pagetext = util.misc.remove_control_characters(pagetext) + if normalize: + pagetext = unicodedata.normalize(normalize, pagetext) + # Remove indentation and trailing whitespaces from every line + pagetext = "\n".join(line[min_indent:].strip() for line in pagetext.split("\n")) + texts.append(pagetext) + + # Create page span + if n + 1 == len(pdf.pages): + end_position = start_position + len(pagetext) + else: + end_position = start_position + len(pagetext) + 1 + pages.append((start_position, end_position)) + start_position = end_position + + # Write page spans + Output("{}.page".format(prefix) if prefix else "page", source_file=source_file).write(pages) + Output("{}.page:number".format(prefix) if prefix else "page:number", source_file=source_file).write(list(str(i) for i in range(1, len(pages) + 1))) + + text = "\n".join(texts) + Text(source_file).write(text) + + # Make up a text annotation surrounding the whole file + text_annotation = "{}.text".format(prefix) if prefix else "text" + Output(text_annotation, source_file=source_file).write([(0, len(text))]) + SourceStructure(source_file).write([text_annotation, "page", "page:number"]) From 4a76ba3a8658a3d8ef497975cea830ec4080e953 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Thu, 4 May 2023 16:23:45 +0200 Subject: [PATCH 125/175] Fix crash for commands where --json-log is not an option --- sparv/__main__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sparv/__main__.py b/sparv/__main__.py index dd5cf3fe..3a99f94f 100644 --- a/sparv/__main__.py +++ b/sparv/__main__.py @@ -377,6 +377,7 @@ def main(): simple_target = False log_level = "" log_file_level = "" + json_log = False simple_mode = False stats = False pass_through = False @@ -485,6 +486,7 @@ def main(): log_level = args.log or "warning" log_file_level = args.log_to_file or "warning" + json_log = args.json_log simple_mode = args.simple socket = args.socket @@ -512,7 +514,7 @@ def main(): # Disable Snakemake's default log handler and use our own logger.log_handler = [] progress = log_handler.LogHandler( - progressbar=not (simple_target or args.json_log), + progressbar=not (simple_target or json_log), log_level=log_level, log_file_level=log_file_level, simple=simple_mode, @@ -520,7 +522,7 @@ def main(): pass_through=pass_through, dry_run=dry_run, keep_going=keep_going, - json=args.json_log, + json=json_log, ) snakemake_args["log_handler"] = [progress.log_handler] From f5f035f04bcdb17d04ddf94b87445fd05c618379 Mon Sep 17 00:00:00 2001 From: Anne Schumacher Date: Thu, 11 May 2023 11:42:28 +0200 Subject: [PATCH 126/175] crash more gracefully when encountering recursion error in stanza (fixes #166) --- sparv/modules/stanza/stanza_utils.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sparv/modules/stanza/stanza_utils.py b/sparv/modules/stanza/stanza_utils.py index a25558e7..7dae8221 100644 --- a/sparv/modules/stanza/stanza_utils.py +++ b/sparv/modules/stanza/stanza_utils.py @@ -17,6 +17,11 @@ def run_stanza(nlp, document, batch_size, max_sentence_length: int = 0, max_toke """Run Stanza and handle possible errors.""" try: doc = nlp(document) + except RecursionError: + msg = "Stanza encountered a recursion error. To prevent this from happening you can try to limit " \ + "the Stanza batch size by setting the 'stanza.batch_size' config variable to something lower " \ + f"(current value: {batch_size}).\n" + raise SparvErrorMessage(msg) except RuntimeError as e: gpu_error = "CUDA out of memory" in str(e) cpu_error = "DefaultCPUAllocator: can't allocate memory" in str(e) From 057419d805df8a707f018fd00fe4eecb678894e0 Mon Sep 17 00:00:00 2001 From: Anne Schumacher Date: Fri, 12 May 2023 17:53:11 +0200 Subject: [PATCH 127/175] make PDF import crash gracefully when encountering textless documents --- sparv/modules/pdf_import/pdf_import.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/sparv/modules/pdf_import/pdf_import.py b/sparv/modules/pdf_import/pdf_import.py index d6e07a88..9e4ecf80 100644 --- a/sparv/modules/pdf_import/pdf_import.py +++ b/sparv/modules/pdf_import/pdf_import.py @@ -6,7 +6,7 @@ import pdfplumber -from sparv.api import Config, Output, Source, SourceFilename, SourceStructure, Text, importer, util +from sparv.api import Config, Output, Source, SourceFilename, SourceStructure, SparvErrorMessage, Text, importer, util @importer("pdf import", file_extension="pdf", outputs=["text", "page:number"], text_annotation="text", config=[ @@ -41,7 +41,7 @@ def parse(source_file: SourceFilename = SourceFilename(), for n, p in enumerate(pdf.pages): pagetext = p.extract_text(layout=True, y_density=12) # Keep track of the page indentation - min_indent = min(map(len, re.findall(r"^ +", pagetext, flags=re.MULTILINE))) + min_indent = min(map(len, re.findall(r"^ +", pagetext, flags=re.MULTILINE)), default=0) if not keep_control_chars: pagetext = util.misc.remove_control_characters(pagetext) if normalize: @@ -58,6 +58,12 @@ def parse(source_file: SourceFilename = SourceFilename(), pages.append((start_position, end_position)) start_position = end_position + # Check if any text was found in the PDF + if not "".join(text.strip() for text in texts): + raise SparvErrorMessage(f"No text was found in the file '{source_file}.pdf'! This file cannot be processed " \ + "with Sparv. Please make sure that every PDF input file contains machine readable " \ + "text.") + # Write page spans Output("{}.page".format(prefix) if prefix else "page", source_file=source_file).write(pages) Output("{}.page:number".format(prefix) if prefix else "page:number", source_file=source_file).write(list(str(i) for i in range(1, len(pages) + 1))) From 9188251ac9531744cc7285d0aecc8230194c2dba Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Tue, 16 May 2023 17:19:45 +0200 Subject: [PATCH 128/175] Replace whitespace with underscore in XML export names --- CHANGELOG.md | 1 + sparv/modules/xml_export/preserved_format.py | 1 + sparv/modules/xml_export/pretty.py | 1 + sparv/modules/xml_export/scrambled.py | 1 + sparv/modules/xml_export/xml_utils.py | 14 ++++++++++++++ 5 files changed, 18 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5cf0b3b1..488d5c6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,7 @@ - Removed the flags `--rerun-incomplete` and `--mark-complete`, as Sparv will now always rerun incomplete files. - Sparv will now recognize when source files have been deleted and trigger the necessary reruns. Previously, only additions and modifications were recognized. +- Whitespace is now replaced with underscore in XML element and attribute names during XML export. ### Fixed diff --git a/sparv/modules/xml_export/preserved_format.py b/sparv/modules/xml_export/preserved_format.py index 7dc105cc..826920de 100644 --- a/sparv/modules/xml_export/preserved_format.py +++ b/sparv/modules/xml_export/preserved_format.py @@ -72,6 +72,7 @@ def preserved_format(source_file: SourceFilename = SourceFilename(), xml_mode=True) h_annotations, h_export_names = util.export.get_header_names(header_annotations, xml_namespaces) export_names.update(h_export_names) + xml_utils.replace_whitespace_in_names(export_names) span_positions, annotation_dict = util.export.gather_annotations(annotation_list, export_names, h_annotations, source_file=source_file, flatten=False, split_overlaps=True) sorted_positions = [(pos, span[0], span[1]) for pos, spans in sorted(span_positions.items()) for span in spans] diff --git a/sparv/modules/xml_export/pretty.py b/sparv/modules/xml_export/pretty.py index a64fc696..43057061 100644 --- a/sparv/modules/xml_export/pretty.py +++ b/sparv/modules/xml_export/pretty.py @@ -94,6 +94,7 @@ def pretty(source_file: SourceFilename = SourceFilename(), "source text. Make sure to add to the list of export annotations.") h_annotations, h_export_names = util.export.get_header_names(header_annotations, xml_namespaces) export_names.update(h_export_names) + xml_utils.replace_whitespace_in_names(export_names) span_positions, annotation_dict = util.export.gather_annotations(annotation_list, export_names, h_annotations, source_file=source_file, split_overlaps=True) xmlstr = xml_utils.make_pretty_xml(span_positions, annotation_dict, export_names, token_name, word_annotation, diff --git a/sparv/modules/xml_export/scrambled.py b/sparv/modules/xml_export/scrambled.py index e4f09da7..c83f3afa 100644 --- a/sparv/modules/xml_export/scrambled.py +++ b/sparv/modules/xml_export/scrambled.py @@ -66,6 +66,7 @@ def scrambled(source_file: SourceFilename = SourceFilename(), if chunk not in annotation_list: raise SparvErrorMessage( "The annotation used for scrambling ({}) needs to be included in the output.".format(chunk)) + xml_utils.replace_whitespace_in_names(export_names) span_positions, annotation_dict = util.export.gather_annotations(annotation_list, export_names, source_file=source_file, split_overlaps=True) diff --git a/sparv/modules/xml_export/xml_utils.py b/sparv/modules/xml_export/xml_utils.py index b8a9061b..d9f3f342 100644 --- a/sparv/modules/xml_export/xml_utils.py +++ b/sparv/modules/xml_export/xml_utils.py @@ -137,6 +137,20 @@ def add_attrs(node, annotation, annotation_dict, export_names, index, include_em node.set(export_name, attrib_values[index]) +def replace_whitespace_in_names(export_names: dict) -> None: + """Replace whitespace with underscore in export names. + + Args: + export_names: Dictionary with export names. + """ + for n in export_names: + if " " in export_names[n]: + new_name = export_names[n].replace(" ", "_") + logger.warning("Whitespace is not permitted in XML element and attribute names. " + f"{export_names[n]!r} will be renamed to {new_name!r}.") + export_names[n] = new_name + + def combine(corpus, out, source_files, xml_input, version_info_file=None): """Combine xml_files into one single file.""" xml_files = [xml_input.replace("{file}", file) for file in source_files] From 114701bd2e87b40b14ff4fe5a990df65a05722d0 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Wed, 17 May 2023 14:38:55 +0200 Subject: [PATCH 129/175] Replace all invalid characters in XML export names; also replace in other related exports --- CHANGELOG.md | 3 +- sparv/modules/cwb/cwb.py | 4 ++ sparv/modules/korp/config.py | 3 ++ sparv/modules/xml_export/preserved_format.py | 2 +- sparv/modules/xml_export/pretty.py | 2 +- sparv/modules/xml_export/scrambled.py | 2 +- sparv/modules/xml_export/xml_utils.py | 46 +++++++++++++++++--- 7 files changed, 51 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 488d5c6d..a4a2193c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,7 +50,8 @@ - Removed the flags `--rerun-incomplete` and `--mark-complete`, as Sparv will now always rerun incomplete files. - Sparv will now recognize when source files have been deleted and trigger the necessary reruns. Previously, only additions and modifications were recognized. -- Whitespace is now replaced with underscore in XML element and attribute names during XML export. +- Illegal characters are now replaced with underscore in XML element and attribute names during XML export. This also + applies to CWB and Korp config exports. ### Fixed diff --git a/sparv/modules/cwb/cwb.py b/sparv/modules/cwb/cwb.py index b3553275..6c6da6ba 100644 --- a/sparv/modules/cwb/cwb.py +++ b/sparv/modules/cwb/cwb.py @@ -10,6 +10,7 @@ from sparv.api import (AllSourceFilenames, Annotation, AnnotationAllSourceFiles, Config, Corpus, SourceFilename, Export, ExportAnnotations, ExportAnnotationNames, ExportInput, SourceAnnotations, SourceAnnotationsAllSourceFiles, SparvErrorMessage, exporter, get_logger, util) +from sparv.modules.xml_export import xml_utils logger = get_logger(__name__) @@ -61,6 +62,7 @@ def vrt(source_file: SourceFilename = SourceFilename(), if token not in annotation_list: logger.warning("The 'cwb:vrt' export requires the annotation for the output to include " "the source text. Make sure to add to the list of export annotations.") + xml_utils.replace_invalid_chars_in_names(export_names) span_positions, annotation_dict = util.export.gather_annotations(annotation_list, export_names, source_file=source_file) vrt_data = create_vrt(span_positions, token.name, word_annotation, all_token_attributes, annotation_dict, @@ -101,6 +103,7 @@ def vrt_scrambled(source_file: SourceFilename = SourceFilename(), if chunk not in annotation_list: raise SparvErrorMessage( "The annotation used for scrambling ({}) needs to be included in the output.".format(chunk)) + xml_utils.replace_invalid_chars_in_names(export_names) span_positions, annotation_dict = util.export.gather_annotations(annotation_list, export_names, source_file=source_file, split_overlaps=True) logger.progress() @@ -209,6 +212,7 @@ def cwb_encode(corpus, annotations, source_annotations, source_files, words, vrt annotations, source_annotations, token_name=token_name, remove_namespaces=remove_namespaces, sparv_namespace=sparv_namespace, source_namespace=source_namespace, keep_struct_names=True) + xml_utils.replace_invalid_chars_in_names(export_names) # Sort token attributes (but keep word first) to be in the same order as the VRT columns token_attributes = token_attributes[:1] + sorted(token_attributes[1:]) diff --git a/sparv/modules/korp/config.py b/sparv/modules/korp/config.py index ff9665d7..17c8aff0 100644 --- a/sparv/modules/korp/config.py +++ b/sparv/modules/korp/config.py @@ -23,6 +23,7 @@ util, ) from sparv.modules.cwb.cwb import cwb_escape +from sparv.modules.xml_export import xml_utils logger = get_logger(__name__) @@ -199,6 +200,8 @@ def config( keep_struct_names=True, ) + xml_utils.replace_invalid_chars_in_names(export_names) + # Context and within if not within and not context: # Figure out based on available annotations and scrambling diff --git a/sparv/modules/xml_export/preserved_format.py b/sparv/modules/xml_export/preserved_format.py index 826920de..b9ef41f7 100644 --- a/sparv/modules/xml_export/preserved_format.py +++ b/sparv/modules/xml_export/preserved_format.py @@ -72,7 +72,7 @@ def preserved_format(source_file: SourceFilename = SourceFilename(), xml_mode=True) h_annotations, h_export_names = util.export.get_header_names(header_annotations, xml_namespaces) export_names.update(h_export_names) - xml_utils.replace_whitespace_in_names(export_names) + xml_utils.replace_invalid_chars_in_names(export_names) span_positions, annotation_dict = util.export.gather_annotations(annotation_list, export_names, h_annotations, source_file=source_file, flatten=False, split_overlaps=True) sorted_positions = [(pos, span[0], span[1]) for pos, spans in sorted(span_positions.items()) for span in spans] diff --git a/sparv/modules/xml_export/pretty.py b/sparv/modules/xml_export/pretty.py index 43057061..8cdc46ea 100644 --- a/sparv/modules/xml_export/pretty.py +++ b/sparv/modules/xml_export/pretty.py @@ -94,7 +94,7 @@ def pretty(source_file: SourceFilename = SourceFilename(), "source text. Make sure to add to the list of export annotations.") h_annotations, h_export_names = util.export.get_header_names(header_annotations, xml_namespaces) export_names.update(h_export_names) - xml_utils.replace_whitespace_in_names(export_names) + xml_utils.replace_invalid_chars_in_names(export_names) span_positions, annotation_dict = util.export.gather_annotations(annotation_list, export_names, h_annotations, source_file=source_file, split_overlaps=True) xmlstr = xml_utils.make_pretty_xml(span_positions, annotation_dict, export_names, token_name, word_annotation, diff --git a/sparv/modules/xml_export/scrambled.py b/sparv/modules/xml_export/scrambled.py index c83f3afa..885102d3 100644 --- a/sparv/modules/xml_export/scrambled.py +++ b/sparv/modules/xml_export/scrambled.py @@ -66,7 +66,7 @@ def scrambled(source_file: SourceFilename = SourceFilename(), if chunk not in annotation_list: raise SparvErrorMessage( "The annotation used for scrambling ({}) needs to be included in the output.".format(chunk)) - xml_utils.replace_whitespace_in_names(export_names) + xml_utils.replace_invalid_chars_in_names(export_names) span_positions, annotation_dict = util.export.gather_annotations(annotation_list, export_names, source_file=source_file, split_overlaps=True) diff --git a/sparv/modules/xml_export/xml_utils.py b/sparv/modules/xml_export/xml_utils.py index d9f3f342..03a7f86e 100644 --- a/sparv/modules/xml_export/xml_utils.py +++ b/sparv/modules/xml_export/xml_utils.py @@ -137,18 +137,50 @@ def add_attrs(node, annotation, annotation_dict, export_names, index, include_em node.set(export_name, attrib_values[index]) -def replace_whitespace_in_names(export_names: dict) -> None: - """Replace whitespace with underscore in export names. +def replace_invalid_chars_in_names(export_names: dict) -> None: + """Replace invalid characters with underscore in export names. Args: export_names: Dictionary with export names. """ + # https://www.w3.org/TR/REC-xml/#NT-NameStartChar + start_chars = [ + ":", + "A-Z", + "_", + "a-z", + "\xC0-\xD6", + "\xD8-\xF6", + "\xF8-\u02FF", + "\u0370-\u037D", + "\u037F-\u1FFF", + "\u200C-\u200D", + "\u2070-\u218F", + "\u2C00-\u2FEF", + "\u3001-\uD7FF", + "\uF900-\uFDCF", + "\uFDF0-\uFFFD", + "\u10000-\uEFFFF" + ] + chars = ["-", ".", "0-9", "\xB7", "\u0300-\u036F", "\u203F-\u2040"] + + name_start_char = re.compile(r"[^{}]".format("".join(start_chars))) + name_char = re.compile(r"[^{}{}]".format("".join(chars), "".join(start_chars))) + namespace_split = re.compile(r"^({[^}]+})?(.+)") + for n in export_names: - if " " in export_names[n]: - new_name = export_names[n].replace(" ", "_") - logger.warning("Whitespace is not permitted in XML element and attribute names. " - f"{export_names[n]!r} will be renamed to {new_name!r}.") - export_names[n] = new_name + namespace, name = namespace_split.match(export_names[n]).groups() + original_name = name + + if name_start_char.match(name[0]): + name = name_start_char.sub("_", name[0]) + name[1:] + + if name_char.search(name[1:]): + name = name[0] + name_char.sub("_", name[1:]) + + if name != original_name: + logger.warning(f"The name {original_name!r} contains invalid characters and will be renamed to {name!r}.") + export_names[n] = (namespace or "") + name def combine(corpus, out, source_files, xml_input, version_info_file=None): From ec45c0c411f9b62b24fad6667c98c12b693cfcdd Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Wed, 31 May 2023 18:00:06 +0200 Subject: [PATCH 130/175] Add support for restricting a whole module to one or more languages --- CHANGELOG.md | 1 + .../developers-guide/writing-sparv-plugins.md | 4 +++ sparv/core/registry.py | 36 +++++++++++++------ sparv/core/run_snake.py | 8 ++--- 4 files changed, 35 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a4a2193c..175f80a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ - Added `korp.keep_undefined_annotations` setting, to include even undefined annotations in the Korp config. - Added `dateformat.pre_regex` setting. - Added `--json-log` flag to enable JSON format for logging. +- Added support for restricting a whole module to one or more languages by using the `__language__` variable. ### Changed diff --git a/docs/developers-guide/writing-sparv-plugins.md b/docs/developers-guide/writing-sparv-plugins.md index 1a42fbf3..8282edd5 100644 --- a/docs/developers-guide/writing-sparv-plugins.md +++ b/docs/developers-guide/writing-sparv-plugins.md @@ -201,6 +201,7 @@ def uppercase(word: Annotation = Annotation(""), ... ``` + ## Languages and varieties It is possible to restrict the use of an annotator, exporter, installer or modelbuilder to one or more specific language(s). This is done by passing a list of ISO 639-3 language codes to the optional `language` parameter in the @@ -214,6 +215,9 @@ Sparv functions are only available for use if one of their languages match the l file](user-manual/corpus-configuration.md). If no language codes are provided in the decorator, the function is available for any corpus. +You may also restrict a whole module, instead of just parts of a module, to one or more languages, by assigning a list +of language codes to the `__language__` variable in the module's `__init__.py` file. + Sparv also supports language varieties which is useful when you want to write Sparv functions for a specific variety of a language. For instance, Sparv has some built-in annotators that are restricted to corpora with historical Swedish from the 1800's. They are marked with the language code `swe-1800`, where `swe` is the ISO 639-3 code for Swedish and `1800` diff --git a/sparv/core/registry.py b/sparv/core/registry.py index 78f49b53..169ba350 100644 --- a/sparv/core/registry.py +++ b/sparv/core/registry.py @@ -55,6 +55,8 @@ def add_language(self, lang: str) -> str: self[lang] = lang return self[lang] +# Dictionary with annotators that will be added to Sparv unless they are excluded (e.g. due to incompatible language) +_potential_annotators = defaultdict(list) # All loaded Sparv modules with their functions (possibly limited by the selected language) modules: Dict[str, Module] = {} @@ -115,7 +117,7 @@ def find_modules(no_import: bool = False, find_custom: bool = False) -> list: module_names.append(module.name) if not no_import: m = importlib.import_module(".".join((path, module.name))) - add_module_metadata(m, module.name) + add_module_to_registry(m, module.name) if find_custom: custom_annotators = [a.get("annotator", "").split(":")[0] for a in sparv_config.get("custom_annotations", [])] @@ -136,7 +138,7 @@ def find_modules(no_import: bool = False, find_custom: bool = False) -> list: except Exception as e: raise SparvErrorMessage(f"Module '{module_name}' cannot be imported due to an error in file " f"'{module_path}': {e}") - add_module_metadata(m, module_name) + add_module_to_registry(m, module_name) # Search for installed plugins for entry_point in iter_entry_points("sparv.plugin"): @@ -146,19 +148,35 @@ def find_modules(no_import: bool = False, find_custom: bool = False) -> list: console.print(f"[red]:warning-emoji: The plugin {entry_point.dist} could not be loaded. " f"It requires {e.req}, but the current installed version is {e.dist}.\n") continue - add_module_metadata(m, entry_point.name) + add_module_to_registry(m, entry_point.name) module_names.append(entry_point.name) return module_names -def add_module_metadata(module, module_name): - """Add module metadata.""" +def add_module_to_registry(module, module_name) -> None: + """Add module and its annotators to registry.""" + if hasattr(module, "__language__"): + # Add to set of supported languages... + for lang in module.__language__: + languages.add_language(lang) + # ...but skip modules for other languages than the one specified in the config + if sparv_config.get("metadata.language") and not check_language( + sparv_config.get("metadata.language"), module.__language__, sparv_config.get("metadata.variety")): + del _potential_annotators[module_name] + return if hasattr(module, "__config__"): for cfg in module.__config__: handle_config(cfg, module_name) - if module_name in modules: - modules[module_name].description = getattr(module, "__description__", module.__doc__) + + # Add module to registry + modules[module_name] = Module(module_name) + modules[module_name].description = getattr(module, "__description__", module.__doc__) + + # Register annotators with Sparv + for a in _potential_annotators[module_name]: + _add_to_registry(a) + del _potential_annotators[module_name] def wizard(config_keys: List[str], source_structure: bool = False): @@ -198,7 +216,7 @@ def _annotator(description: str, a_type: Annotator, name: Optional[str] = None, def decorator(f): """Add wrapped function to registry.""" module_name = _get_module_name(f.__module__) - _add_to_registry({ + _potential_annotators[module_name].append({ "module_name": module_name, "description": description, "function": f, @@ -400,8 +418,6 @@ def _add_to_registry(annotator): raise SparvErrorMessage(f"'{rule_name}' creates no OutputMarker, which is required by all installers and " "uninstallers.") - if module_name not in modules: - modules[module_name] = Module(module_name) if f_name in modules[module_name].functions: print("Annotator function '{}' collides with other function with same name in module '{}'.".format(f_name, module_name)) diff --git a/sparv/core/run_snake.py b/sparv/core/run_snake.py index 6228bcf6..85c372b0 100644 --- a/sparv/core/run_snake.py +++ b/sparv/core/run_snake.py @@ -88,8 +88,8 @@ def flush(): name = module_name[len(custom_name) + 1:] module_path = paths.corpus_dir.resolve() / f"{name}.py" spec = importlib.util.spec_from_file_location(module_name, module_path) - m = importlib.util.module_from_spec(spec) - spec.loader.exec_module(m) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) else: try: # Try to import standard Sparv module @@ -99,11 +99,11 @@ def flush(): entry_points = dict((e.name, e) for e in iter_entry_points(f"sparv.{plugin_name}")) entry_point = entry_points.get(module_name) if entry_point: - entry_point.load() + module = entry_point.load() else: exit_with_error_message( f"Couldn't load plugin '{module_name}'. Please make sure it was installed correctly.", "sparv") - + registry.add_module_to_registry(module, module_name) # Get function name and parameters f_name = snakemake.params.f_name From 4c47d2173f5abbbe1b3f9daead6a57a08dd897b2 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 2 Jun 2023 18:23:07 +0200 Subject: [PATCH 131/175] Simplify language checking; not specifying a corpus language now excludes all language specific annotators --- CHANGELOG.md | 1 + sparv/core/Snakefile | 14 ++++++++++---- sparv/core/registry.py | 29 +++++++++++++++++++++-------- sparv/core/snake_utils.py | 10 ++++++---- 4 files changed, 38 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 175f80a4..8cda0f4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,7 @@ additions and modifications were recognized. - Illegal characters are now replaced with underscore in XML element and attribute names during XML export. This also applies to CWB and Korp config exports. +- Not specifying a corpus language now excludes all language specific annotators. ### Fixed diff --git a/sparv/core/Snakefile b/sparv/core/Snakefile index 4f9fa5a8..7b770f62 100644 --- a/sparv/core/Snakefile +++ b/sparv/core/Snakefile @@ -358,8 +358,11 @@ rule list_exports: table.add_column(no_wrap=True) table.add_column() for target, desc, language in selected_exports: - if not language or registry.check_language(sparv_config.get("metadata.language"), language, - sparv_config.get("metadata.variety")): + if registry.check_language( + sparv_config.get("metadata.language"), + language, + sparv_config.get("metadata.variety") + ): table.add_row(target, desc) console.print(table) @@ -370,8 +373,11 @@ rule list_exports: table.add_column(no_wrap=True) table.add_column() for target, desc, language in other_exports: - if not language or registry.check_language(sparv_config.get("metadata.language"), language, - sparv_config.get("metadata.variety")): + if registry.check_language( + sparv_config.get("metadata.language"), + language, + sparv_config.get("metadata.variety") + ): table.add_row(target, desc) console.print(table) diff --git a/sparv/core/registry.py b/sparv/core/registry.py index 169ba350..608a067b 100644 --- a/sparv/core/registry.py +++ b/sparv/core/registry.py @@ -161,8 +161,9 @@ def add_module_to_registry(module, module_name) -> None: for lang in module.__language__: languages.add_language(lang) # ...but skip modules for other languages than the one specified in the config - if sparv_config.get("metadata.language") and not check_language( - sparv_config.get("metadata.language"), module.__language__, sparv_config.get("metadata.variety")): + if not check_language( + sparv_config.get("metadata.language"), module.__language__, sparv_config.get("metadata.variety") + ): del _potential_annotators[module_name] return if hasattr(module, "__config__"): @@ -331,8 +332,9 @@ def _add_to_registry(annotator): for lang in annotator["language"]: languages.add_language(lang) # ... but skip annotators for other languages than the one specified in the config - if sparv_config.get("metadata.language") and not check_language( - sparv_config.get("metadata.language"), annotator["language"], sparv_config.get("metadata.variety")): + if not check_language( + sparv_config.get("metadata.language"), annotator["language"], sparv_config.get("metadata.variety") + ): return # Add config variables to config @@ -388,10 +390,11 @@ def _add_to_registry(annotator): all_module_classes[language][cls].append(cls_target) # Only add classes for relevant languages - if not annotator["language"] or ( - annotator["language"] and sparv_config.get("metadata.language") - and check_language(sparv_config.get("metadata.language"), annotator["language"], - sparv_config.get("metadata.variety"))): + if check_language( + sparv_config.get("metadata.language"), + annotator["language"], + sparv_config.get("metadata.variety") + ): if cls_target not in annotation_classes["module_classes"][cls]: annotation_classes["module_classes"][cls].append(cls_target) @@ -608,7 +611,17 @@ def check_language(corpus_lang: str, langs: List[str], corpus_lang_suffix: Optio """Check if corpus language is among a list of languages. Any suffix on corpus_lang will be ignored. + + If langs is empty, always return True. + If corpus_lang is "__all__", always return True. """ + if not langs or corpus_lang == "__all__": + return True + + if not isinstance(corpus_lang, str): + return False + if corpus_lang_suffix: corpus_lang = corpus_lang + "-" + corpus_lang_suffix + return corpus_lang in langs or corpus_lang.split("-")[0] in langs diff --git a/sparv/core/snake_utils.py b/sparv/core/snake_utils.py index 93f8cf20..be08d22b 100644 --- a/sparv/core/snake_utils.py +++ b/sparv/core/snake_utils.py @@ -144,9 +144,11 @@ def rule_helper(rule: RuleStorage, config: dict, storage: SnakeStorage, config_m return False # Skip any annotator that is not available for the selected corpus language - if rule.annotator_info["language"] and sparv_config.get("metadata.language") and \ - not registry.check_language(sparv_config.get("metadata.language"), rule.annotator_info["language"], - sparv_config.get("metadata.variety")): + if not registry.check_language( + sparv_config.get("metadata.language"), + rule.annotator_info["language"], + sparv_config.get("metadata.variety") + ): return False # Get this function's parameters @@ -750,7 +752,7 @@ def load_config(snakemake_config): config_missing = True # Some commands may override the corpus language - if snakemake_config.get("language"): + if "language" in snakemake_config: sparv_config.set_value("metadata.language", snakemake_config["language"]) return config_missing From 995765708ad5a748d10289f264b1eb577eae51c2 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Thu, 29 Jun 2023 16:13:26 +0200 Subject: [PATCH 132/175] Don't check language in run_snake --- sparv/core/registry.py | 11 ++++++----- sparv/core/run_snake.py | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/sparv/core/registry.py b/sparv/core/registry.py index 608a067b..4b1d31dc 100644 --- a/sparv/core/registry.py +++ b/sparv/core/registry.py @@ -5,6 +5,7 @@ import re from collections import defaultdict from enum import Enum +from types import ModuleType from typing import Callable, Dict, List, Optional, Set, Tuple, Type, TypeVar import typing_inspect @@ -154,9 +155,9 @@ def find_modules(no_import: bool = False, find_custom: bool = False) -> list: return module_names -def add_module_to_registry(module, module_name) -> None: +def add_module_to_registry(module: ModuleType, module_name: str, skip_language_check: bool = False) -> None: """Add module and its annotators to registry.""" - if hasattr(module, "__language__"): + if not skip_language_check and hasattr(module, "__language__"): # Add to set of supported languages... for lang in module.__language__: languages.add_language(lang) @@ -176,7 +177,7 @@ def add_module_to_registry(module, module_name) -> None: # Register annotators with Sparv for a in _potential_annotators[module_name]: - _add_to_registry(a) + _add_to_registry(a, skip_language_check=skip_language_check) del _potential_annotators[module_name] @@ -321,13 +322,13 @@ def modelbuilder(description: str, name: Optional[str] = None, config: Optional[ language=language, order=order) -def _add_to_registry(annotator): +def _add_to_registry(annotator: dict, skip_language_check: bool = False): """Add function to annotator registry. Used by annotator.""" module_name = annotator["module_name"] f_name = annotator["function"].__name__ if not annotator["name"] else annotator["name"] rule_name = f"{module_name}:{f_name}" - if annotator["language"]: + if not skip_language_check and annotator["language"]: # Add to set of supported languages... for lang in annotator["language"]: languages.add_language(lang) diff --git a/sparv/core/run_snake.py b/sparv/core/run_snake.py index 85c372b0..b552d3f8 100644 --- a/sparv/core/run_snake.py +++ b/sparv/core/run_snake.py @@ -103,7 +103,7 @@ def flush(): else: exit_with_error_message( f"Couldn't load plugin '{module_name}'. Please make sure it was installed correctly.", "sparv") - registry.add_module_to_registry(module, module_name) + registry.add_module_to_registry(module, module_name, skip_language_check=True) # Get function name and parameters f_name = snakemake.params.f_name From be1d4e5671c32e06c90ff020e0deb0f1f2744924 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 4 Aug 2023 16:15:45 +0200 Subject: [PATCH 133/175] Add JSON schema generation for validating config files --- CHANGELOG.md | 1 + sparv/__main__.py | 11 +- sparv/api/classes.py | 23 ++- sparv/core/Snakefile | 29 ++-- sparv/core/config.py | 36 +++-- sparv/core/registry.py | 29 +++- sparv/core/schema.py | 347 +++++++++++++++++++++++++++++++++++++++++ 7 files changed, 443 insertions(+), 33 deletions(-) create mode 100644 sparv/core/schema.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cda0f4b..c7531507 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ - Added `dateformat.pre_regex` setting. - Added `--json-log` flag to enable JSON format for logging. - Added support for restricting a whole module to one or more languages by using the `__language__` variable. +- Running `sparv schema` will now generate a JSON schema which can be used to validate corpus config files. ### Changed diff --git a/sparv/__main__.py b/sparv/__main__.py index 3a99f94f..6b043008 100644 --- a/sparv/__main__.py +++ b/sparv/__main__.py @@ -142,6 +142,7 @@ def main(): " run-module Run annotator module independently", " preload Preload annotators and models", " autocomplete Enable tab completion in bash", + " schema Print a JSON schema for the Sparv config format", "", "See 'sparv -h' for help with a specific command", "For full documentation, visit https://spraakbanken.gu.se/sparv/docs/" @@ -241,6 +242,8 @@ def main(): autocomplete_parser.add_argument("--enable-old", action="store_true", help="Output script to be sourced in bash, for bash version 4.3 and below") + subparsers.add_parser("schema", description="Print a JSON schema for the Sparv config format") + # Add common arguments for subparser in [run_parser, runrule_parser]: subparser.add_argument("-f", "--file", nargs="+", default=[], help="Only annotate specified input file(s)") @@ -357,7 +360,7 @@ def main(): print(f"{e.strerror}: {e.filename!r}") sys.exit(1) - if args.command not in ("autocomplete", "build-models", "languages"): + if args.command not in ("autocomplete", "build-models", "languages", "schema"): if not config_exists: print(f"No config file ({paths.config_file}) found in working directory.") sys.exit(1) @@ -384,7 +387,7 @@ def main(): dry_run = False keep_going = False - if args.command in ("modules", "config", "files", "clean", "presets", "classes", "languages", "preload"): + if args.command in ("modules", "config", "files", "clean", "presets", "classes", "languages", "preload", "schema"): snakemake_args["targets"] = [args.command] simple_target = True if args.command == "clean": @@ -408,6 +411,10 @@ def main(): config["targets"] = ["preload"] if args.list: snakemake_args["targets"] = ["preload_list"] + elif args.command == "schema": + config["targets"] = ["schema"] + # For the schema we include modules from all languages + config["language"] = "__all__" elif args.command in ("run", "run-rule", "create-file", "install", "uninstall", "build-models"): try: diff --git a/sparv/api/classes.py b/sparv/api/classes.py index 322de213..8ab307d7 100644 --- a/sparv/api/classes.py +++ b/sparv/api/classes.py @@ -8,7 +8,7 @@ import urllib.request import zipfile from abc import ABC, abstractmethod -from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple, Union +from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple, Union import sparv.core from sparv.core import io @@ -635,10 +635,29 @@ class Config(str): def __new__(cls, name: str, *args, **kwargs): return super().__new__(cls, name) - def __init__(self, name: str, default: Any = None, description: Optional[str] = None): + def __init__( + self, + name: str, + default: Any = None, + description: Optional[str] = None, + datatype=None, + choices: Optional[Union[Iterable, Callable]] = None, + pattern: Optional[str] = None, + min: Optional[float] = None, + max: Optional[float] = None, + const: Optional[Any] = None, + conditions: Optional[List["Config"]] = None + ): self.name = name self.default = default self.description = description + self.datatype = datatype + self.choices = choices + self.pattern = pattern + self.min = min + self.max = max + self.const = const + self.conditions = conditions or [] class Wildcard(str): diff --git a/sparv/core/Snakefile b/sparv/core/Snakefile index 7b770f62..e46b49ad 100644 --- a/sparv/core/Snakefile +++ b/sparv/core/Snakefile @@ -52,17 +52,18 @@ def make_rules(config_missing: bool) -> None: annotator = registry.modules[module_name].functions[f_name] make_rule(module_name, f_name, annotator, config_missing, custom_rule_obj) - # Check and set rule orders - ordered_rules = snake_utils.check_ruleorder(snake_storage) - for rule1, rule2 in ordered_rules: - # ruleorder: rule1.rule_name > rule2.rule_name - workflow.ruleorder(rule1.rule_name, rule2.rule_name) - # Print ordered rules when in debug mode - if config.get("debug") and ordered_rules: - console.print("\n\n\n[b]ORDERED RULES:[/b]") + # Check and set rule orders (but not when language is set to __all__ for schema purposes) + if not config.get("language") == "__all__": + ordered_rules = snake_utils.check_ruleorder(snake_storage) for rule1, rule2 in ordered_rules: - print(" • {} > {}".format(rule1.rule_name, rule2.rule_name)) - print() + # ruleorder: rule1.rule_name > rule2.rule_name + workflow.ruleorder(rule1.rule_name, rule2.rule_name) + # Print ordered rules when in debug mode + if config.get("debug") and ordered_rules: + console.print("\n\n\n[b]ORDERED RULES:[/b]") + for rule1, rule2 in ordered_rules: + print(" • {} > {}".format(rule1.rule_name, rule2.rule_name)) + print() def make_rule(module_name: str, f_name: str, annotator_info: dict, config_missing: bool = False, @@ -579,3 +580,11 @@ rule preload_list: table.add_row(f"{module}:{a}", registry.modules[module].functions[a]["description"]) console.print(table) + + +rule schema: + run: + import json + from sparv.core import schema + json_schema = schema.build_json_schema(sparv_config.config_structure) + print(json.dumps(json_schema, indent=2)) diff --git a/sparv/core/config.py b/sparv/core/config.py index 49384531..36c9c7b8 100644 --- a/sparv/core/config.py +++ b/sparv/core/config.py @@ -16,6 +16,7 @@ from sparv.core import paths, registry from sparv.core.misc import SparvErrorMessage, get_logger +from sparv.api.classes import Config logger = get_logger(__name__) @@ -31,13 +32,13 @@ # Dict with info about config structure, prepopulated with some module-independent keys config_structure = { - "classes": {"_source": "core"}, - "custom_annotations": {"_source": "core"}, - "install": {"_source": "core"}, - PARENT: {"_source": "core"}, - MAX_THREADS: {"_source": "core"}, - "preload": {"_source": "core"}, - "uninstall": {"_source": "core"} + "classes": {"_source": "core", "_cfg": Config("uninstall", datatype=dict)}, + "custom_annotations": {"_source": "core", "_cfg": Config("uninstall", datatype=list)}, + "install": {"_source": "core", "_cfg": Config("uninstall", datatype=list)}, + PARENT: {"_source": "core", "_cfg": Config("uninstall", datatype=str)}, + MAX_THREADS: {"_source": "core", "_cfg": Config("uninstall", datatype=int)}, + "preload": {"_source": "core", "_cfg": Config("uninstall", datatype=list)}, + "uninstall": {"_source": "core", "_cfg": Config("uninstall", datatype=list)} } config_usage = defaultdict(set) # For each config key, a list of annotators using that key @@ -183,22 +184,25 @@ def _merge_dicts(d: dict, default: dict): return d -def add_to_structure(name, default=None, description=None, annotator: Optional[str] = None): +def add_to_structure(cfg: Config, annotator: Optional[str] = None): """Add config variable to config structure.""" - set_value(name, - {"_default": default, - "_description": description, - "_source": "module"}, - config_dict=config_structure - ) + set_value( + cfg.name, + { + "_cfg": cfg, + "_source": "module" + }, + config_dict=config_structure + ) if annotator: - add_config_usage(name, annotator) + add_config_usage(cfg.name, annotator) def get_config_description(name): """Get description for config key.""" - return _get(name, config_structure).get("_description") + cfg = _get(name, config_structure).get("_cfg") + return cfg.description if cfg else None def add_config_usage(config_key, annotator): diff --git a/sparv/core/registry.py b/sparv/core/registry.py index 4b1d31dc..077ee9a2 100644 --- a/sparv/core/registry.py +++ b/sparv/core/registry.py @@ -40,6 +40,7 @@ def __init__(self, name): self.name = name self.functions: Dict[str, dict] = {} self.description = None + self.language = None class LanguageRegistry(dict): @@ -174,9 +175,13 @@ def add_module_to_registry(module: ModuleType, module_name: str, skip_language_c # Add module to registry modules[module_name] = Module(module_name) modules[module_name].description = getattr(module, "__description__", module.__doc__) + modules[module_name].language = getattr(module, "__language__", None) # Register annotators with Sparv for a in _potential_annotators[module_name]: + # Set annotator language to same as module, unless overridden + if hasattr(module, "__language__") and not a["language"]: + a["language"] = module.__language__ _add_to_registry(a, skip_language_check=skip_language_check) del _potential_annotators[module_name] @@ -341,7 +346,7 @@ def _add_to_registry(annotator: dict, skip_language_check: bool = False): # Add config variables to config if annotator["config"]: for c in annotator["config"]: - handle_config(c, module_name, rule_name) + handle_config(c, module_name, rule_name, annotator["language"]) # Handle text annotation for selected importer if annotator["type"] == Annotator.importer and rule_name == sparv_config.get("import.importer"): @@ -448,7 +453,12 @@ def find_implicit_classes() -> None: annotation_classes["implicit_classes"][cls] = annotation -def handle_config(cfg, module_name, rule_name: Optional[str] = None) -> None: +def handle_config( + cfg: Config, + module_name: str, + rule_name: Optional[str] = None, + language: Optional[List[str]] = None +) -> None: """Handle Config instances.""" if not cfg.name.startswith(module_name + "."): raise SparvErrorMessage(f"Config option '{cfg.name}' in module '{module_name}' doesn't include module " @@ -464,7 +474,20 @@ def handle_config(cfg, module_name, rule_name: Optional[str] = None) -> None: f"The config variable '{cfg.name}' in '{rule_name or module_name}' has already been declared.") if cfg.default is not None: sparv_config.set_default(cfg.name, cfg.default) - sparv_config.add_to_structure(cfg.name, cfg.default, description=cfg.description, annotator=rule_name) + if language: + langcodes = [] + suffixes = [] + for lang in language: + langcode, _, suffix = lang.partition("-") + langcodes.append(langcode) + suffixes.append(suffix) + suffixes = set(suffixes) + if len(suffixes) == 1 and next(iter(suffixes)) == "": + suffixes = [] + cfg.conditions.append(Config("metadata.language", datatype=str, choices=langcodes)) + if suffixes: + cfg.conditions.append(Config("metadata.variety", datatype=str, choices=suffixes)) + sparv_config.add_to_structure(cfg, annotator=rule_name) if not cfg.description: raise SparvErrorMessage(f"Missing description for configuration key '{cfg.name}' in module '{module_name}'.") diff --git a/sparv/core/schema.py b/sparv/core/schema.py new file mode 100644 index 00000000..2f147171 --- /dev/null +++ b/sparv/core/schema.py @@ -0,0 +1,347 @@ +""" +Functions for creating and validating JSON schemas. +""" + +import itertools +import json +import re +from collections import defaultdict +from typing import DefaultDict, List, Optional, Tuple, Union + +import typing_inspect +from sparv.api import Config, SparvErrorMessage +from sparv.core import registry + +NO_COND = ((), ()) + +class BaseProperty: + def __init__(self, prop_type: Optional[str], allow_null: Optional[bool] = False, **kwargs): + self.schema = { + "type": prop_type if not allow_null else [prop_type, "null"], + **kwargs + } if prop_type else kwargs + +class Any(BaseProperty): + def __init__(self, **kwargs): + super().__init__(None, **kwargs) + +class String(BaseProperty): + def __init__(self, pattern=None, choices=None, min=None, max=None, allow_null=False, **kwargs): + if pattern: + kwargs["pattern"] = pattern + if choices: + if callable(choices): + choices = choices() + kwargs["enum"] = list(choices) + if min: + kwargs["min"] = min + if max: + kwargs["max"] = max + super().__init__("string", allow_null, **kwargs) + +class Integer(BaseProperty): + def __init__(self, **kwargs): + super().__init__("integer", **kwargs) + +class Number(BaseProperty): + def __init__(self, **kwargs): + super().__init__("number", **kwargs) + +class Boolean(BaseProperty): + def __init__(self, **kwargs): + super().__init__("boolean", **kwargs) + +class Null(BaseProperty): + def __init__(self, **kwargs): + super().__init__("null", **kwargs) + +class Array(BaseProperty): + def __init__(self, items: Optional[List[Union[String, Integer, Number, "Object"]]] = None, **kwargs): + super().__init__("array", **kwargs) + +class Object: + def __init__(self, additional_properties: bool = True, description: Optional[str] = None, **kwargs): + if not additional_properties: + kwargs["additionalProperties"] = False + if description: + kwargs["description"] = description + self.obj_schema = {"type": "object", **kwargs} + self.properties = {} + self.required = [] + self.allof: DefaultDict[Tuple[Tuple[Object, ...], Tuple[Object, ...]], list] = defaultdict(list) + + def __hash__(self): + return hash(json.dumps(self.schema, sort_keys=True)) + + def __eq__(self, other): + if other is None: + return False + return json.dumps(self.schema, sort_keys=True) == json.dumps(other.schema, sort_keys=True) + + def __lt__(self, other): + if other is None: + return False + return json.dumps(self.schema, sort_keys=True) < json.dumps(other.schema, sort_keys=True) + + def add_property( + self, + name: str, + prop_obj: Union[List, Union[String, Integer, Number, "Object", Any]], + required: bool = False, + condition: Optional[Tuple[Tuple["Object", ...], Tuple["Object", ...]]] = None + ): + if condition and not condition == NO_COND: + self.allof[condition].append((name, prop_obj)) + else: + self.properties[name] = prop_obj + if required: + self.required.append(name) + return self + + @property + def schema(self): + prop_schemas = {} + for name, prop_obj in self.properties.items(): + if isinstance(prop_obj, list): + combined_schema = {"type": []} + for obj in prop_obj: + obj_schema = obj.schema + for key in obj_schema: + if key == "type": + combined_schema["type"].append(obj_schema[key]) + else: + combined_schema[key] = obj_schema[key] + prop_schemas[name] = combined_schema + else: + prop_schemas[name] = prop_obj.schema + self.obj_schema["properties"] = prop_schemas + if self.required: + self.obj_schema["required"] = self.required + if self.allof: + conditionals = [] + for condition in self.allof: + pos_conds, neg_conds = condition + if len(pos_conds) + len(neg_conds) > 1: + cond_schema = { + "allOf": [c.schema for c in pos_conds if not c is None] + [ # noqa + {"not": c.schema} + for c in neg_conds + ] + } + else: + cond_schema = pos_conds[0].schema + + conditionals.append( + { + "if": cond_schema, + "then": { + "properties": {name: prop_obj.schema for name, prop_obj in self.allof[condition]} + } + } + ) + self.obj_schema["allOf"] = conditionals + return self.obj_schema + +class JsonSchema(Object): + + def __init__(self): + super().__init__(**{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://spraakbanken.gu.se/sparv/schema.json", + "type": "object", + "properties": {}, + "required": [], + "unevaluatedProperties": False + }) + + def to_json(self): + return json.dumps(self.schema, indent=2) + + +def build_json_schema(config_structure: dict) -> dict: + """Build a JSON schema based on Sparv's config structure.""" + schema = JsonSchema() + + def handle_object( + structure: dict, + parent_obj: Optional[Object] = None, + parent_name: Optional[str] = None, + is_condition: Optional[bool] = False + ) -> DefaultDict[Tuple[Tuple[Optional[Object], ...], Tuple[Object, ...]], list]: + """Handle dictionary which will become an object in the JSON schema. + + Return a dictionary with conditionals as keys and lists of children to each conditional as values. + """ + conditionals: DefaultDict[Tuple[Tuple[Optional[Object], ...], Tuple[Object, ...]], list] = defaultdict(list) + + for key in structure: + if not structure[key].get("_source"): # Not a leaf, has children + description = None + if parent_name is None and key in registry.modules: + # This is a module + description = registry.modules[key].description + child_obj = Object(additional_properties=is_condition, description=description) + children = handle_object(structure[key], parent_name=key, is_condition=is_condition) + + if len(children) == 1: + # Same or no condition for all children + cond = next(iter(children.keys())) + for subkey, prop in children[cond]: + child_obj.add_property(subkey, prop, required=is_condition) + + conditionals[cond].append((key, child_obj)) + else: + no_cond = children.get(NO_COND) + conds = [c for c in children if c != NO_COND] + combinations = list( + itertools.chain.from_iterable( + [itertools.combinations(conds, i + 1) for i in range(len(conds))] + ) + ) + + if no_cond: + combinations.insert(0, ()) + + for combination in combinations: + if no_cond: + combination = (NO_COND,) + combination + + child_obj = Object(additional_properties=False, description=description) + + for cond in combination: + for subkey, prop in children[cond]: + child_obj.add_property(subkey, prop, required=is_condition) + positive_conds = tuple(set(cc for c in combination for cc in c[0] or (None,))) + negative_conds = tuple( + set(cc for c in conds if c != NO_COND for cc in c[0] if cc not in positive_conds) + ) + + if not set(positive_conds).intersection(set(negative_conds)): + conditionals[(positive_conds, negative_conds)].append((key, child_obj)) + + elif "_cfg" in structure[key]: # A leaf + try: + prop, condition = handle_property(structure[key]["_cfg"]) + except ValueError: + full_key = f"{parent_name}.{key}" if parent_name else key + raise ValueError(f"Unsupported datatype for '{full_key}': '{structure[key]['_cfg'].datatype}'") + + conditionals[(condition, ())].append((key, prop)) + + else: + full_key = f"{parent_name}.{key}" if parent_name else key + raise SparvErrorMessage(f"Unknown error while handling config variable {full_key!r}.") + + if parent_obj: + # Either this is the root, or we're constructing a condition object + for condition in conditionals: + for key, prop in conditionals[condition]: + parent_obj.add_property(key, prop, condition=condition, required=is_condition) + + return conditionals + + def handle_property( + cfg: Config + ) -> Tuple[Union[BaseProperty, List[BaseProperty]], Tuple[Object, ...]]: + """ + Handle a property and its conditions. + + Args: + cfg: A Config object + + Returns: + A tuple with two values. The first is either a datatype object or a list of datatype objects, and the + second is a tuple of conditions (possible empty). + """ + kwargs = {} + if cfg.description: + kwargs["description"] = cfg.description + if cfg.default is not None: + kwargs["default"] = cfg.default + if cfg.const is not None: + kwargs["const"] = cfg.const + + # Datatype is either a single type or a union of types + if typing_inspect.is_union_type(cfg.datatype): + cfg_datatypes = typing_inspect.get_args(cfg.datatype) + else: + cfg_datatypes = [cfg.datatype] + + datatypes = [] + + for cfg_datatype in cfg_datatypes: + if cfg_datatype is str: + datatype = String( + pattern=cfg.pattern, + choices=cfg.choices, + min=cfg.min, + max=cfg.max, + **kwargs + ) + elif cfg_datatype is int: + datatype = Integer(**kwargs) + elif cfg_datatype is float: + datatype = Number(**kwargs) + elif cfg_datatype is bool: + datatype = Boolean(**kwargs) + elif cfg_datatype is type(None): + datatype = Null(**kwargs) + elif cfg_datatype is list: + datatype = Array(**kwargs) + elif cfg_datatype is dict: + datatype = Object(**kwargs) + elif cfg_datatype is None: + datatype = Any(**kwargs) + else: + raise ValueError() + datatypes.append(datatype) + + if cfg.conditions: + conditions = set() + for condition_cfg in cfg.conditions: + cond_structure = {} + prev = cond_structure + for part in condition_cfg.name.split(".")[:-1]: + prev.setdefault(part, {}) + prev = prev[part] + prev[condition_cfg.name.split(".")[-1]] = { + "_cfg": condition_cfg, + "_source": "condition" + } + + condition = Object() + handle_object(cond_structure, condition, is_condition=True) + conditions.add(condition) + conditions = tuple(sorted(conditions)) + else: + conditions = () + + return datatypes if len(datatypes) > 1 else datatypes[0], conditions + + handle_object(config_structure, schema) + + return schema.schema + + +def validate(cfg: dict, schema: dict) -> None: + """Validate a Sparv config using JSON schema.""" + import jsonschema + + try: + jsonschema.validate(cfg, schema) + except jsonschema.ValidationError as e: + msg = ["There was a problem trying to parse the corpus config file.\n"] + + if e.validator == "unevaluatedProperties": + # This only happens for unexpected keys at the root level + prop = re.search(r"'(.+)' was unexpected", e.message) + if prop: + msg.append(f"Unexpected property at root level: {prop.group(1)!r}") + else: + msg.append(e.message) + else: + if e.absolute_path: + msg.append(f"Offending config path: {'.'.join(e.absolute_path)}") + msg.append(e.message) + + raise SparvErrorMessage("\n".join(msg)) From 9f3df55c02f9ea5f48a5a7ea2417fe52051f4cb4 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 8 Sep 2023 15:32:22 +0200 Subject: [PATCH 134/175] Add support for list element validation in JSON schema --- sparv/core/schema.py | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/sparv/core/schema.py b/sparv/core/schema.py index 2f147171..d2944946 100644 --- a/sparv/core/schema.py +++ b/sparv/core/schema.py @@ -6,7 +6,7 @@ import json import re from collections import defaultdict -from typing import DefaultDict, List, Optional, Tuple, Union +from typing import DefaultDict, List, Optional, Tuple, Type, Union import typing_inspect from sparv.api import Config, SparvErrorMessage @@ -56,7 +56,13 @@ def __init__(self, **kwargs): super().__init__("null", **kwargs) class Array(BaseProperty): - def __init__(self, items: Optional[List[Union[String, Integer, Number, "Object"]]] = None, **kwargs): + def __init__( + self, + items: Optional[Type[Union[String, Integer, Number, Boolean, Null, Any, "Array", "Object"]]] = None, + **kwargs + ): + if items: + kwargs["items"] = items().schema super().__init__("array", **kwargs) class Object: @@ -158,6 +164,21 @@ def to_json(self): return json.dumps(self.schema, indent=2) +def get_class_from_type(t): + """Get JSON schema class from Python type.""" + types = { + str: String, + int: Integer, + float: Number, + bool: Boolean, + type(None): Null, + list: Array, + dict: Object, + None: Any + } + return types[t] + + def build_json_schema(config_structure: dict) -> dict: """Build a JSON schema based on Sparv's config structure.""" schema = JsonSchema() @@ -286,7 +307,10 @@ def handle_property( datatype = Boolean(**kwargs) elif cfg_datatype is type(None): datatype = Null(**kwargs) - elif cfg_datatype is list: + elif cfg_datatype is list or typing_inspect.get_origin(cfg_datatype) is list: + args = typing_inspect.get_args(cfg_datatype) + if args: + kwargs["items"] = get_class_from_type(args[0]) datatype = Array(**kwargs) elif cfg_datatype is dict: datatype = Object(**kwargs) From a5b9c0f32bfb6a9bc9d52ba0dad9fe8df063457f Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 8 Sep 2023 14:02:57 +0200 Subject: [PATCH 135/175] Use JSON schema for automatic config validation --- pyproject.toml | 1 + sparv/core/Snakefile | 5 ++++- sparv/core/config.py | 19 ------------------- sparv/core/schema.py | 27 ++++++++++++++++++--------- 4 files changed, 23 insertions(+), 29 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 907a5085..68c7553e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ dependencies = [ "appdirs==1.4.4", "argcomplete==3.0.5", "docx2python==1.27.1", + "jsonschema==4.19.0", "nltk==3.8.1", "pdfplumber==0.9.0", "protobuf~=3.19.0", # Used by Stanza; see https://github.com/spraakbanken/sparv-pipeline/issues/161 diff --git a/sparv/core/Snakefile b/sparv/core/Snakefile index e46b49ad..bb11fb34 100644 --- a/sparv/core/Snakefile +++ b/sparv/core/Snakefile @@ -238,7 +238,10 @@ update_autocompletion_cache() sparv_config.validate_module_config() # Validate config -sparv_config.validate_config() +if not "schema" in selected_targets: + from sparv.core import schema + json_schema = schema.build_json_schema(sparv_config.config_structure) + schema.validate(sparv_config.config, json_schema) # Get reverse_config_usage dict for look-ups reverse_config_usage = snake_utils.get_reverse_config_usage() diff --git a/sparv/core/config.py b/sparv/core/config.py index 36c9c7b8..7b1b3199 100644 --- a/sparv/core/config.py +++ b/sparv/core/config.py @@ -223,25 +223,6 @@ def validate_module_config(): "are" if len(annotators) > 1 else "is", config_key), "sparv", "config") -def validate_config(config_dict=None, structure=None, parent=""): - """Make sure the corpus config doesn't contain invalid keys.""" - config_dict = config_dict or config - structure = structure or config_structure - for key in config_dict: - path = (parent + "." + key) if parent else key - if key not in structure: - if not parent: - raise SparvErrorMessage(f"Unknown key in config file: '{path}'. No module with that name found.", - module="sparv", function="config") - else: - module_name = parent.split(".", 1)[0] - raise SparvErrorMessage(f"Unknown key in config file: '{path}'. The module '{module_name}' " - f"doesn't have an option with that name.", - module="sparv", function="config") - elif not structure[key].get("_source"): - validate_config(config_dict[key], structure[key], path) - - def load_presets(lang, lang_variety): """Read presets files and return dictionaries with all available presets annotations and preset classes.""" global presets diff --git a/sparv/core/schema.py b/sparv/core/schema.py index d2944946..c4c984b4 100644 --- a/sparv/core/schema.py +++ b/sparv/core/schema.py @@ -351,21 +351,30 @@ def validate(cfg: dict, schema: dict) -> None: """Validate a Sparv config using JSON schema.""" import jsonschema + def build_path_string(path): + parts = [] + for part in path: + if isinstance(part, str): + parts.append(part) + elif isinstance(part, int): + parts[-1] += f"[{part}]" + return ".".join(parts) + try: jsonschema.validate(cfg, schema) except jsonschema.ValidationError as e: msg = ["There was a problem trying to parse the corpus config file.\n"] - if e.validator == "unevaluatedProperties": - # This only happens for unexpected keys at the root level - prop = re.search(r"'(.+)' was unexpected", e.message) - if prop: - msg.append(f"Unexpected property at root level: {prop.group(1)!r}") - else: - msg.append(e.message) + # Rephrase messages about unexpected keys + unknown_key = re.search(r"properties are not allowed \('(.+)' was unexpected", e.message) + if unknown_key: + full_path = ".".join(list(e.absolute_path) + [unknown_key.group(1)]) + msg.append(f"Unexpected key in config file: {full_path!r}") else: - if e.absolute_path: - msg.append(f"Offending config path: {'.'.join(e.absolute_path)}") msg.append(e.message) + if e.absolute_path: + msg.append(f"Offending config path: {build_path_string(e.absolute_path)}") + if "description" in e.schema: + msg.append(f"Description of config key: {e.schema['description']}") raise SparvErrorMessage("\n".join(msg)) From d6b7938644018efda904dc8adaf5f732e006413a Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 8 Sep 2023 17:37:54 +0200 Subject: [PATCH 136/175] Fix datatype for threads setting --- sparv/core/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sparv/core/config.py b/sparv/core/config.py index 7b1b3199..ec9cd476 100644 --- a/sparv/core/config.py +++ b/sparv/core/config.py @@ -36,7 +36,7 @@ "custom_annotations": {"_source": "core", "_cfg": Config("uninstall", datatype=list)}, "install": {"_source": "core", "_cfg": Config("uninstall", datatype=list)}, PARENT: {"_source": "core", "_cfg": Config("uninstall", datatype=str)}, - MAX_THREADS: {"_source": "core", "_cfg": Config("uninstall", datatype=int)}, + MAX_THREADS: {"_source": "core", "_cfg": Config("uninstall", datatype=dict)}, "preload": {"_source": "core", "_cfg": Config("uninstall", datatype=list)}, "uninstall": {"_source": "core", "_cfg": Config("uninstall", datatype=list)} } From 3e2e77042ce8f8f8ea28030c88c3eced0ec1fb6b Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Thu, 14 Sep 2023 11:26:27 +0200 Subject: [PATCH 137/175] Print message when Sparv was stopped by a TERM signal --- sparv/core/log_handler.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sparv/core/log_handler.py b/sparv/core/log_handler.py index 62c49efe..b2fed65c 100644 --- a/sparv/core/log_handler.py +++ b/sparv/core/log_handler.py @@ -281,6 +281,7 @@ def __init__(self, progressbar=True, log_level=None, log_file_level=None, simple self.stats = stats self.stats_data = defaultdict(float) self.logger = None + self.terminated = False # Progress bar related variables self.progress: Optional[progress.Progress] = None @@ -563,6 +564,10 @@ def missing_annotations_or_files(source, files): self.info(msg["msg"]) elif msg["msg"].startswith("Nothing to be done"): self.info("Nothing to be done.") + elif msg["msg"].startswith("Will exit after finishing currently running jobs"): + if not self.terminated: + self.logger.log(logging.INFO, "Will exit after finishing currently running jobs") + self.terminated = True elif level == "debug": if "SparvErrorMessage" in msg["msg"]: @@ -766,6 +771,9 @@ def stop(self): table.add_row(str(sum(self.jobs.values())), "Total number of tasks") console.print(table) + if self.terminated: + self.info(f"{spacer}Sparv was stopped by a TERM signal") + @staticmethod def cleanup(): """Remove Snakemake log files.""" From 5c51d2cb96bd27f5105189cde48150527b04c2aa Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Thu, 14 Sep 2023 14:06:10 +0200 Subject: [PATCH 138/175] Add validation of object value types in JSON schema --- sparv/core/schema.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/sparv/core/schema.py b/sparv/core/schema.py index c4c984b4..d2669487 100644 --- a/sparv/core/schema.py +++ b/sparv/core/schema.py @@ -66,9 +66,9 @@ def __init__( super().__init__("array", **kwargs) class Object: - def __init__(self, additional_properties: bool = True, description: Optional[str] = None, **kwargs): - if not additional_properties: - kwargs["additionalProperties"] = False + def __init__(self, additional_properties: Union[dict, bool] = True, description: Optional[str] = None, **kwargs): + if additional_properties is False or isinstance(additional_properties, dict): + kwargs["additionalProperties"] = additional_properties if description: kwargs["description"] = description self.obj_schema = {"type": "object", **kwargs} @@ -312,7 +312,10 @@ def handle_property( if args: kwargs["items"] = get_class_from_type(args[0]) datatype = Array(**kwargs) - elif cfg_datatype is dict: + elif cfg_datatype is dict or typing_inspect.get_origin(cfg_datatype) is dict: + args = typing_inspect.get_args(cfg_datatype) + if args: + kwargs["additionalProperties"] = get_class_from_type(args[1])().schema datatype = Object(**kwargs) elif cfg_datatype is None: datatype = Any(**kwargs) From 630af0db007d03861c77ec4b1421814def6ca408 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 15 Sep 2023 14:19:07 +0200 Subject: [PATCH 139/175] Add validation of core config values --- sparv/core/config.py | 14 ++++----- sparv/core_modules/export/__init__.py | 39 +++++++++++++++++++------ sparv/core_modules/import/__init__.py | 19 +++++++----- sparv/core_modules/metadata/__init__.py | 23 +++++++++++---- sparv/core_modules/sparv/__init__.py | 9 ++++-- 5 files changed, 73 insertions(+), 31 deletions(-) diff --git a/sparv/core/config.py b/sparv/core/config.py index ec9cd476..e494f846 100644 --- a/sparv/core/config.py +++ b/sparv/core/config.py @@ -4,7 +4,7 @@ from collections import defaultdict from functools import reduce from pathlib import Path -from typing import Any, Optional +from typing import Any, Dict, Optional import yaml import yaml.scanner @@ -32,12 +32,12 @@ # Dict with info about config structure, prepopulated with some module-independent keys config_structure = { - "classes": {"_source": "core", "_cfg": Config("uninstall", datatype=dict)}, - "custom_annotations": {"_source": "core", "_cfg": Config("uninstall", datatype=list)}, - "install": {"_source": "core", "_cfg": Config("uninstall", datatype=list)}, - PARENT: {"_source": "core", "_cfg": Config("uninstall", datatype=str)}, - MAX_THREADS: {"_source": "core", "_cfg": Config("uninstall", datatype=dict)}, - "preload": {"_source": "core", "_cfg": Config("uninstall", datatype=list)}, + "classes": {"_source": "core", "_cfg": Config("classes", datatype=dict)}, + "custom_annotations": {"_source": "core", "_cfg": Config("custom_annotations", datatype=list)}, + "install": {"_source": "core", "_cfg": Config("install", datatype=list)}, + PARENT: {"_source": "core", "_cfg": Config(PARENT, datatype=str)}, + MAX_THREADS: {"_source": "core", "_cfg": Config(MAX_THREADS, datatype=Dict[str, int])}, + "preload": {"_source": "core", "_cfg": Config("preload", datatype=list)}, "uninstall": {"_source": "core", "_cfg": Config("uninstall", datatype=list)} } diff --git a/sparv/core_modules/export/__init__.py b/sparv/core_modules/export/__init__.py index e99b354f..a4180181 100644 --- a/sparv/core_modules/export/__init__.py +++ b/sparv/core_modules/export/__init__.py @@ -1,15 +1,36 @@ +from typing import Optional + from sparv.api import Config, SourceStructureParser, wizard __config__ = [ - Config("export.default", description="List of exporters to use by default"), - Config("export.annotations", description="List of automatic annotations to include in export"), - Config("export.source_annotations", description="List of annotations and attributes from the source to include"), - Config("export.word", description="Annotation to use as token text in export"), - Config("export.remove_module_namespaces", - description="Remove module name prefixes from annotation names in export"), - Config("export.sparv_namespace", description="Prefix to add to the names of all automatically created annotations"), - Config("export.source_namespace", description="Prefix to add to the names of all annotations from source"), - Config("export.scramble_on", description="What annotation to use as the smallest unit when scrambling") + Config("export.default", description="List of exporters to use by default", datatype=list), + Config("export.annotations", description="List of automatic annotations to include in export", datatype=list), + Config( + "export.source_annotations", + description="List of annotations and attributes from the source to include", + datatype=list + ), + Config("export.word", description="Annotation to use as token text in export", datatype=str), + Config( + "export.remove_module_namespaces", + description="Remove module name prefixes from annotation names in export", + datatype=bool + ), + Config( + "export.sparv_namespace", + description="Prefix to add to the names of all automatically created annotations", + datatype=Optional[str] + ), + Config( + "export.source_namespace", + description="Prefix to add to the names of all annotations from source", + datatype=Optional[str] + ), + Config( + "export.scramble_on", + description="What annotation to use as the smallest unit when scrambling", + datatype=str + ) ] diff --git a/sparv/core_modules/import/__init__.py b/sparv/core_modules/import/__init__.py index 39b61388..456fbb4d 100644 --- a/sparv/core_modules/import/__init__.py +++ b/sparv/core_modules/import/__init__.py @@ -4,13 +4,18 @@ from sparv.core import paths, registry __config__ = [ - Config("import.text_annotation", description="Annotation representing a text"), - Config("import.source_dir", paths.source_dir, description="Directory containing corpus source files"), - Config("import.importer", description="Name of importer to use"), - Config("import.keep_control_chars", False, description="Set to True to keep control characters"), - Config("import.normalize", "NFC", description="Normalize input using any of the following forms: " - "'NFC', 'NFKC', 'NFD', and 'NFKD'"), - Config("import.encoding", "UTF-8", description="Encoding of source files"), + Config("import.text_annotation", description="Annotation representing a text", datatype=str), + Config("import.source_dir", paths.source_dir, description="Directory containing corpus source files", datatype=str), + Config("import.importer", description="Name of importer to use", datatype=str), + Config("import.keep_control_chars", False, description="Set to True to keep control characters", datatype=bool), + Config( + "import.normalize", + "NFC", + description="Normalize input using any of the following forms: 'NFC', 'NFKC', 'NFD', and 'NFKD'", + datatype=str, + choices=["NFC", "NFKC", "NFD", "NFKD"] + ), + Config("import.encoding", "UTF-8", description="Encoding of source files", datatype=str), ] diff --git a/sparv/core_modules/metadata/__init__.py b/sparv/core_modules/metadata/__init__.py index 1eb5252d..9a91246a 100644 --- a/sparv/core_modules/metadata/__init__.py +++ b/sparv/core_modules/metadata/__init__.py @@ -1,16 +1,27 @@ """General metadata about corpus.""" import re +from typing import Dict, Union from sparv.api import Config, wizard from sparv.core import registry __config__ = [ - Config("metadata.id", description="Machine name of corpus (a-z, 0-9, -)"), - Config("metadata.name", description="Human readable name of corpus"), - Config("metadata.language", description="Language of source files (ISO 639-3)"), - Config("metadata.variety", description="Language variety of source files (if applicable)"), - Config("metadata.description", description="Description of corpus"), - Config("metadata.short_description", description="Short description of corpus (one line)") + Config("metadata.id", description="Machine name of corpus (a-z, 0-9, -)", datatype=str, pattern=r"^[a-z0-9-]+$"), + Config("metadata.name", description="Human readable name of corpus", datatype=Dict[str, str]), + Config( + "metadata.language", + description="Language of source files (ISO 639-3)", + datatype=Union[str, None], + choices=lambda: sorted(set(l.split("-")[0] for l in registry.languages)) + [None] + ), + Config( + "metadata.variety", + description="Language variety of source files (if applicable)", + datatype=str, + choices=lambda: sorted(set(l.split("-")[1] for l in registry.languages if "-" in l)) + ), + Config("metadata.description", description="Description of corpus", datatype=Dict[str, str]), + Config("metadata.short_description", description="Short description of corpus (one line)", datatype=Dict[str, str]) ] diff --git a/sparv/core_modules/sparv/__init__.py b/sparv/core_modules/sparv/__init__.py index 45213031..72c77a90 100644 --- a/sparv/core_modules/sparv/__init__.py +++ b/sparv/core_modules/sparv/__init__.py @@ -4,6 +4,11 @@ from sparv.core.io import compression __config__ = [ - Config("sparv.compression", default=compression, - description="Compression to use for files in work-dir ('none', 'gzip', 'bzip2' or 'lzma'. Default: gzip)") + Config( + "sparv.compression", + default=compression, + description="Compression to use for files in work-dir ('none', 'gzip', 'bzip2' or 'lzma'. Default: 'gzip')", + datatype=str, + choices=("none", "gzip", "bzip2", "lzma") + ) ] From 0961241d2fc741079753af9e2105aa2a893e0888 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 15 Sep 2023 15:16:57 +0200 Subject: [PATCH 140/175] Add validation of SALDO config values --- sparv/modules/saldo/__init__.py | 2 + sparv/modules/saldo/compound.py | 41 ++++++++++--- sparv/modules/saldo/nst_comp_model.py | 4 +- sparv/modules/saldo/saldo.py | 86 ++++++++++++++++++++------- sparv/modules/saldo/saldo_model.py | 6 +- sparv/modules/saldo/stats_model.py | 4 +- 6 files changed, 107 insertions(+), 36 deletions(-) diff --git a/sparv/modules/saldo/__init__.py b/sparv/modules/saldo/__init__.py index 8540a505..4025505a 100644 --- a/sparv/modules/saldo/__init__.py +++ b/sparv/modules/saldo/__init__.py @@ -1,3 +1,5 @@ """SALDO-related annotations.""" from . import compound, nst_comp_model, saldo, saldo_model, stats_model + +__language__ = ["swe"] diff --git a/sparv/modules/saldo/compound.py b/sparv/modules/saldo/compound.py index 8892d7e4..dbcec5cd 100644 --- a/sparv/modules/saldo/compound.py +++ b/sparv/modules/saldo/compound.py @@ -31,13 +31,36 @@ def preloader(saldo_comp_model, stats_model): return SaldoCompLexicon(saldo_comp_model.path), StatsLexicon(stats_model.path) -@annotator("Compound analysis", name="compound", language=["swe"], config=[ - Config("saldo.comp_model", default="saldo/saldo.compound.pickle", description="Path to SALDO compound model"), - Config("saldo.comp_nst_model", default="saldo/nst_comp_pos.pickle", - description="Path to NST part of speech compound model"), - Config("saldo.comp_stats_model", default="saldo/stats.pickle", description="Path to statistics model"), - Config("saldo.comp_use_source", default=True, description="Also use source text as lexicon for compound analysis") -], preloader=preloader, preloader_params=["saldo_comp_model", "stats_model"], preloader_target="preloaded_models") +@annotator( + "Compound analysis", + name="compound", + config=[ + Config( + "saldo.comp_model", + default="saldo/saldo.compound.pickle", + description="Path to SALDO compound model", + datatype=str, + ), + Config( + "saldo.comp_nst_model", + default="saldo/nst_comp_pos.pickle", + description="Path to NST part of speech compound model", + datatype=str, + ), + Config( + "saldo.comp_stats_model", default="saldo/stats.pickle", description="Path to statistics model", datatype=str + ), + Config( + "saldo.comp_use_source", + default=True, + description="Also use source text as lexicon for compound analysis", + datatype=bool, + ), + ], + preloader=preloader, + preloader_params=["saldo_comp_model", "stats_model"], + preloader_target="preloaded_models", +) def annotate(out_complemgrams: Output = Output(":saldo.complemgram", description="Compound analysis using lemgrams"), out_compwf: Output = Output(":saldo.compwf", description="Compound analysis using wordforms"), @@ -142,13 +165,13 @@ def annotate(out_complemgrams: Output = Output(":saldo.complemgram", logger.progress() -@modelbuilder("SALDO compound model", language=["swe"], order=1) +@modelbuilder("SALDO compound model", order=1) def download_saldo_comp(out: ModelOutput = ModelOutput("saldo/saldo.compound.pickle")): """Download SALDO compound model from sparv-models repo.""" out.download("https://github.com/spraakbanken/sparv-models/raw/master/saldo/saldo.compound.pickle") -@modelbuilder("SALDO compound model", language=["swe"], order=2) +@modelbuilder("SALDO compound model", order=2) def build_saldo_comp(out: ModelOutput = ModelOutput("saldo/saldo.compound.pickle"), saldom: Model = Model("saldo/saldom.xml")): """Extract compound info from saldom.xml and save as a pickle file.""" diff --git a/sparv/modules/saldo/nst_comp_model.py b/sparv/modules/saldo/nst_comp_model.py index 5f34c89b..2e03573d 100644 --- a/sparv/modules/saldo/nst_comp_model.py +++ b/sparv/modules/saldo/nst_comp_model.py @@ -10,13 +10,13 @@ logger = get_logger(__name__) -@modelbuilder("Compound POS model", language=["swe"], order=1) +@modelbuilder("Compound POS model", order=1) def download_nst_comp(out: ModelOutput = ModelOutput("saldo/nst_comp_pos.pickle")): """Download compound POS model from sparv-models repo.""" out.download("https://github.com/spraakbanken/sparv-models/raw/master/saldo/nst_comp_pos.pickle") -@modelbuilder("Compound POS model", language=["swe"], order=2) +@modelbuilder("Compound POS model", order=2) def build_nst_comp(out: ModelOutput = ModelOutput("saldo/nst_comp_pos.pickle"), nst_lexicon: Model = Model("saldo/nst_utf8.txt")): """Download NST lexicon and convert it to a compound POS model. diff --git a/sparv/modules/saldo/saldo.py b/sparv/modules/saldo/saldo.py index 1ff3b39f..d946f040 100644 --- a/sparv/modules/saldo/saldo.py +++ b/sparv/modules/saldo/saldo.py @@ -21,26 +21,72 @@ def preloader(models): return {m.path.stem: SaldoLexicon(m.path) for m in models} -@annotator("SALDO annotations", language=["swe"], config=[ - Config("saldo.model", default="saldo/saldo.pickle", description="Path to SALDO model"), - Config("saldo.delimiter", default=util.constants.DELIM, description="Character to put between ambiguous results"), - Config("saldo.affix", default=util.constants.AFFIX, description="Character to put before and after sets of results"), - Config("saldo.precision", description="Format string for appending precision to each value (e.g. ':%.3f')"), - Config("saldo.precision_filter", default="max", - description="Precision filter with possible values 'max' (only use the most probable annotations), " - "'first' (only use the single most probable annotation), 'none' (use all annotations)"), - Config("saldo.min_precision", default=0.66, - description="Only use annotations with a probability score greater than or equal to this. " - "0.25: part-of-speech does not match, 0.5: part-of-speech is missing, 0.66: part-of-speech " - "matches, 0.75: morphosyntactic descriptor matches"), - Config("saldo.skip_multiword", default=False, description="Whether to disable annotation of multiword expressions"), - Config("saldo.max_mwe_gaps", default=1, description="Max amount of gaps allowed within a multiword expression"), - Config("saldo.allow_multiword_overlap", default=False, - description="Whether all multiword expressions may overlap with each other. " - "If set to False, some cleanup is done."), - Config("saldo.word_separator", - description="Character used to split the values of 'word' into several word variations"), -], preloader=preloader, preloader_params=["models"], preloader_target="models_preloaded") +@annotator( + "SALDO annotations", + config=[ + Config("saldo.model", default="saldo/saldo.pickle", description="Path to SALDO model", datatype=str), + Config( + "saldo.delimiter", + default=util.constants.DELIM, + description="Character to put between ambiguous results", + datatype=str, + ), + Config( + "saldo.affix", + default=util.constants.AFFIX, + description="Character to put before and after sets of results", + datatype=str, + ), + Config( + "saldo.precision", + description="Format string for appending precision to each value (e.g. ':%.3f')", + datatype=str, + ), + Config( + "saldo.precision_filter", + default="max", + description="Precision filter with possible values 'max' (only use the most probable annotations), " + "'first' (only use the single most probable annotation), 'none' (use all annotations)", + datatype=str, + choices=("max", "first", "none"), + ), + Config( + "saldo.min_precision", + default=0.66, + description="Only use annotations with a probability score greater than or equal to this. " + "0.25: part-of-speech does not match, 0.5: part-of-speech is missing, 0.66: part-of-speech " + "matches, 0.75: morphosyntactic descriptor matches", + datatype=float, + ), + Config( + "saldo.skip_multiword", + default=False, + description="Whether to disable annotation of multiword expressions", + datatype=bool, + ), + Config( + "saldo.max_mwe_gaps", + default=1, + description="Max amount of gaps allowed within a multiword expression", + datatype=int, + ), + Config( + "saldo.allow_multiword_overlap", + default=False, + description="Whether all multiword expressions may overlap with each other. " + "If set to False, some cleanup is done.", + datatype=bool, + ), + Config( + "saldo.word_separator", + description="Character used to split the values of 'word' into several word variations", + datatype=str, + ), + ], + preloader=preloader, + preloader_params=["models"], + preloader_target="models_preloaded", +) def annotate(token: Annotation = Annotation(""), word: Annotation = Annotation(""), sentence: Annotation = Annotation(""), diff --git a/sparv/modules/saldo/saldo_model.py b/sparv/modules/saldo/saldo_model.py index b98d62e3..17aaa6f2 100644 --- a/sparv/modules/saldo/saldo_model.py +++ b/sparv/modules/saldo/saldo_model.py @@ -18,19 +18,19 @@ PART_DELIM3 = "^3" -@modelbuilder("SALDO morphology XML", language=["swe"]) +@modelbuilder("SALDO morphology XML") def download_saldo_xml(out: ModelOutput = ModelOutput("saldo/saldom.xml")): """Download SALDO morphology XML.""" out.download("https://svn.spraakdata.gu.se/sb-arkiv/pub/lexikon/saldom/saldom.xml") -@modelbuilder("SALDO morphology model", language=["swe"], order=1) +@modelbuilder("SALDO morphology model", order=1) def download_saldo_pickle(out: ModelOutput = ModelOutput("saldo/saldo.pickle")): """Download SALDO morphology model from sparv-models repo.""" out.download("https://github.com/spraakbanken/sparv-models/raw/master/saldo/saldo.pickle") -@modelbuilder("SALDO morphology model", language=["swe"], order=2) +@modelbuilder("SALDO morphology model", order=2) def build_saldo_pickle(out: ModelOutput = ModelOutput("saldo/saldo.pickle"), saldom: Model = Model("saldo/saldom.xml")): """Save SALDO morphology as a pickle file.""" diff --git a/sparv/modules/saldo/stats_model.py b/sparv/modules/saldo/stats_model.py index 72a792d4..382af775 100644 --- a/sparv/modules/saldo/stats_model.py +++ b/sparv/modules/saldo/stats_model.py @@ -13,13 +13,13 @@ MIN_FREQ = 4 -@modelbuilder("Korp statistic model", language=["swe"], order=1) +@modelbuilder("Korp statistic model", order=1) def download_korp_stats(out: ModelOutput = ModelOutput("saldo/stats.pickle")): """Download stats.pickle model.""" out.download("https://github.com/spraakbanken/sparv-models/raw/master/saldo/stats.pickle") -@modelbuilder("Korp statistic model", language=["swe"], order=2) +@modelbuilder("Korp statistic model", order=2) def build_korp_stats(out: ModelOutput = ModelOutput("saldo/stats.pickle"), _saldom: Model = Model("saldo/saldom.xml")): """Download Korp's word frequency file and convert it to a model.""" From 0bfa46436b2fa88a68e7beeac6a136647a5c043e Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 15 Sep 2023 17:09:17 +0200 Subject: [PATCH 141/175] Add config validation for more modules --- sparv/core_modules/import/__init__.py | 4 +- sparv/modules/csv_export/csv_export.py | 14 ++- sparv/modules/cwb/__init__.py | 6 +- sparv/modules/cwb/cwb.py | 23 ++-- sparv/modules/docx_import/docx_import.py | 19 +++- sparv/modules/korp/__init__.py | 34 ++++-- sparv/modules/korp/config.py | 20 +++- sparv/modules/korp/wordpicture.py | 8 +- sparv/modules/odt_import/odt_import.py | 19 +++- sparv/modules/pdf_import/pdf_import.py | 19 +++- sparv/modules/stanza/__init__.py | 110 +++++++++++++------ sparv/modules/stats_export/__init__.py | 26 +++-- sparv/modules/swener/swener.py | 2 +- sparv/modules/text_import/text_import.py | 50 ++++++--- sparv/modules/xml_export/preserved_format.py | 9 +- sparv/modules/xml_export/pretty.py | 65 +++++++---- sparv/modules/xml_export/scrambled.py | 6 +- sparv/modules/xml_import/xml_import.py | 74 +++++++++---- 18 files changed, 360 insertions(+), 148 deletions(-) diff --git a/sparv/core_modules/import/__init__.py b/sparv/core_modules/import/__init__.py index 456fbb4d..c215b9e4 100644 --- a/sparv/core_modules/import/__init__.py +++ b/sparv/core_modules/import/__init__.py @@ -10,10 +10,10 @@ Config("import.keep_control_chars", False, description="Set to True to keep control characters", datatype=bool), Config( "import.normalize", - "NFC", + default="NFC", description="Normalize input using any of the following forms: 'NFC', 'NFKC', 'NFD', and 'NFKD'", datatype=str, - choices=["NFC", "NFKC", "NFD", "NFKD"] + choices=("NFC", "NFKC", "NFD", "NFKD"), ), Config("import.encoding", "UTF-8", description="Encoding of source files", datatype=str), ] diff --git a/sparv/modules/csv_export/csv_export.py b/sparv/modules/csv_export/csv_export.py index f47a1ef3..eb2c5f42 100644 --- a/sparv/modules/csv_export/csv_export.py +++ b/sparv/modules/csv_export/csv_export.py @@ -1,6 +1,7 @@ """CSV file export.""" import os +from typing import List from sparv.api import (Annotation, Config, SourceFilename, Export, ExportAnnotations, SourceAnnotations, exporter, get_logger, util) @@ -9,11 +10,14 @@ @exporter("CSV export", config=[ - Config("csv_export.delimiter", default="\t", description="Delimiter separating fields."), - Config("csv_export.source_annotations", - description="List of annotations and attributes from the source data to include. Everything will be " - "included by default."), - Config("csv_export.annotations", description="Sparv annotations to include.") + Config("csv_export.delimiter", default="\t", description="Delimiter separating fields.", datatype=str), + Config( + "csv_export.source_annotations", + description="List of annotations and attributes from the source data to include. Everything will be " + "included by default.", + datatype=List[str], + ), + Config("csv_export.annotations", description="Sparv annotations to include.", datatype=List[str]) ]) def csv(source_file: SourceFilename = SourceFilename(), out: Export = Export("csv_export/{file}.csv"), diff --git a/sparv/modules/cwb/__init__.py b/sparv/modules/cwb/__init__.py index 1faed0b0..38bc7cf6 100644 --- a/sparv/modules/cwb/__init__.py +++ b/sparv/modules/cwb/__init__.py @@ -4,7 +4,7 @@ from . import cwb, info, install_corpus __config__ = [ - Config("cwb.remote_host", description="Remote host to install CWB files to"), - Config("cwb.remote_registry_dir", description="CWB registry path on remote host"), - Config("cwb.remote_data_dir", description="CWB datadir path on remote host") + Config("cwb.remote_host", description="Remote host to install CWB files to", datatype=str), + Config("cwb.remote_registry_dir", description="CWB registry path on remote host", datatype=str), + Config("cwb.remote_data_dir", description="CWB datadir path on remote host", datatype=str) ] diff --git a/sparv/modules/cwb/cwb.py b/sparv/modules/cwb/cwb.py index 6c6da6ba..d07371b7 100644 --- a/sparv/modules/cwb/cwb.py +++ b/sparv/modules/cwb/cwb.py @@ -5,7 +5,7 @@ from collections import OrderedDict from glob import glob from pathlib import Path -from typing import Optional +from typing import List, Optional from sparv.api import (AllSourceFilenames, Annotation, AnnotationAllSourceFiles, Config, Corpus, SourceFilename, Export, ExportAnnotations, ExportAnnotationNames, ExportInput, SourceAnnotations, @@ -18,10 +18,13 @@ @exporter("VRT export", config=[ - Config("cwb.source_annotations", - description="List of annotations and attributes from the source data to include. Everything will be " - "included by default."), - Config("cwb.annotations", description="Sparv annotations to include.") + Config( + "cwb.source_annotations", + description="List of annotations and attributes from the source data to include. Everything will be included " + "by default.", + datatype=List[str] + ), + Config("cwb.annotations", description="Sparv annotations to include.", datatype=List[str]) ]) def vrt(source_file: SourceFilename = SourceFilename(), out: Export = Export("cwb.vrt/{file}.vrt"), @@ -75,7 +78,7 @@ def vrt(source_file: SourceFilename = SourceFilename(), @exporter("Scrambled VRT export", config=[ - Config("cwb.scramble_on", description="Annotation to use for scrambling.") + Config("cwb.scramble_on", description="Annotation to use for scrambling.", datatype=str) ]) def vrt_scrambled(source_file: SourceFilename = SourceFilename(), out: Export = Export("cwb.vrt_scrambled/{file}.vrt"), @@ -141,10 +144,10 @@ def vrt_scrambled(source_file: SourceFilename = SourceFilename(), @exporter("CWB encode", order=2, config=[ - Config("cwb.bin_path", default="", description="Path to directory containing the CWB executables"), - Config("cwb.encoding", default="utf8", description="Encoding to use"), - Config("cwb.skip_compression", False, description="Whether to skip compression"), - Config("cwb.skip_validation", False, description="Whether to skip validation") + Config("cwb.bin_path", default="", description="Path to directory containing the CWB executables", datatype=str), + Config("cwb.encoding", default="utf8", description="Encoding to use", datatype=str), + Config("cwb.skip_compression", default=False, description="Whether to skip compression", datatype=bool), + Config("cwb.skip_validation", default=False, description="Whether to skip validation", datatype=bool) ]) def encode(corpus: Corpus = Corpus(), annotations: ExportAnnotationNames = ExportAnnotationNames("cwb.annotations"), diff --git a/sparv/modules/docx_import/docx_import.py b/sparv/modules/docx_import/docx_import.py index f30a229e..d5b47ecc 100644 --- a/sparv/modules/docx_import/docx_import.py +++ b/sparv/modules/docx_import/docx_import.py @@ -10,11 +10,20 @@ @importer("docx import", file_extension="docx", outputs=["text"], text_annotation="text", config=[ - Config("docx_import.prefix", description="Optional prefix to add to annotation names."), - Config("docx_import.keep_control_chars", False, description="Set to True if control characters should not be " - "removed from the text."), - Config("docx_import.normalize", "NFC", description="Normalize input using any of the following forms: " - "'NFC', 'NFKC', 'NFD', and 'NFKD'.") + Config("docx_import.prefix", description="Optional prefix to add to annotation names.", datatype=str), + Config( + "docx_import.keep_control_chars", + default=False, + description="Set to True if control characters should not be removed from the text.", + datatype=bool, + ), + Config( + "docx_import.normalize", + default="NFC", + description="Normalize input using any of the following forms: 'NFC', 'NFKC', 'NFD', and 'NFKD'.", + datatype=str, + choices=("NFC", "NFKC", "NFD", "NFKD"), + ) ]) def parse(source_file: SourceFilename = SourceFilename(), source_dir: Source = Source(), diff --git a/sparv/modules/korp/__init__.py b/sparv/modules/korp/__init__.py index 7f5d753d..005bd05d 100644 --- a/sparv/modules/korp/__init__.py +++ b/sparv/modules/korp/__init__.py @@ -1,15 +1,33 @@ """Korp-related annotators, exporters and installers.""" +from typing import List from sparv.api import Config from . import config, lemgram_index, wordpicture, timespan __config__ = [ - Config("korp.remote_host", description="Remote host to install to. Leave blank to install locally."), - Config("korp.mysql_dbname", description="Name of database where Korp data will be stored"), - Config("korp.modes", default=[{"name": "default"}], - description="The Korp modes in which the corpus will be published"), - Config("korp.protected", False, description="Whether this corpus should have limited access or not"), - Config("korp.config_dir", description="Path on remote host where Korp corpus configuration files are stored"), - Config("korp.wordpicture_table", default="relations", - description="Prefix used for Word Picture database table names") + Config("korp.remote_host", description="Remote host to install to. Leave blank to install locally.", datatype=str), + Config("korp.mysql_dbname", description="Name of database where Korp data will be stored", datatype=str), + Config( + "korp.modes", + default=[{"name": "default"}], + description="The Korp modes in which the corpus will be published", + datatype=List[dict], + ), + Config( + "korp.protected", + default=False, + description="Whether this corpus should have limited access or not", + datatype=bool, + ), + Config( + "korp.config_dir", + description="Path on remote host where Korp corpus configuration files are stored", + datatype=str, + ), + Config( + "korp.wordpicture_table", + default="relations", + description="Prefix used for Word Picture database table names", + datatype=str, + ), ] diff --git a/sparv/modules/korp/config.py b/sparv/modules/korp/config.py index 17c8aff0..c84ed234 100644 --- a/sparv/modules/korp/config.py +++ b/sparv/modules/korp/config.py @@ -56,39 +56,49 @@ @exporter( "Create Korp config file for the corpus.", config=[ - Config("korp.annotations", description="Sparv annotations to include. Leave blank to use cwb.annotations."), + Config( + "korp.annotations", + description="Sparv annotations to include. Leave blank to use cwb.annotations.", + datatype=List[str], + ), Config( "korp.source_annotations", description="List of annotations and attributes from the source data to include. Leave blank to use " "cwb.source_annotations.", + datatype=List[str], ), Config( "korp.annotation_definitions", description="Frontend definitions of annotations in 'annotations' and 'source_annotations'. Classes and " "config keys are currently not supported.", + datatype=dict, ), Config( "korp.context", - description="Contexts to use in Korp, from smaller to bigger. " "Leave blank to detect automatically.", + description="Contexts to use in Korp, from smaller to bigger. Leave blank to detect automatically.", + datatype=List[str], ), Config( "korp.within", description="Search boundaries to use in Korp, from smaller to bigger. " "Leave blank to detect automatically.", + datatype=List[str], ), - Config("korp.custom_annotations", description="Custom Korp-annotations."), + Config("korp.custom_annotations", description="Custom Korp-annotations.", datatype=List[dict]), Config("korp.morphology", description="Morphologies"), - Config("korp.reading_mode", description="Reading mode configuration"), - Config("korp.filters", description="List of annotations to use for filtering in Korp"), + Config("korp.reading_mode", description="Reading mode configuration", datatype=dict), + Config("korp.filters", description="List of annotations to use for filtering in Korp", datatype=List[str]), Config( "korp.hidden_annotations", description="List of annotations not to include in corpus config", default=HIDDEN_ANNOTATIONS, + datatype=List[str], ), Config( "korp.keep_undefined_annotations", description="Include all annotations in config, even those without an annotation definition/preset.", default=False, + datatype=bool, ), ], ) diff --git a/sparv/modules/korp/wordpicture.py b/sparv/modules/korp/wordpicture.py index 5321aebf..e3a350be 100644 --- a/sparv/modules/korp/wordpicture.py +++ b/sparv/modules/korp/wordpicture.py @@ -341,8 +341,12 @@ def mi_lex(rel, x_rel_y, x_rel, rel_y): @exporter("Word Picture SQL for use in Korp", language=["swe"], config=[ - Config("korp.wordpicture_no_sentences", default=False, - description="Set to 'true' to skip generating sentences table.") + Config( + "korp.wordpicture_no_sentences", + default=False, + description="Set to 'true' to skip generating sentences table.", + datatype=bool + ) ]) def wordpicture_sql( corpus: Corpus = Corpus(), diff --git a/sparv/modules/odt_import/odt_import.py b/sparv/modules/odt_import/odt_import.py index 44eac8d7..448b9f0b 100644 --- a/sparv/modules/odt_import/odt_import.py +++ b/sparv/modules/odt_import/odt_import.py @@ -11,11 +11,20 @@ @importer("odt import", file_extension="odt", outputs=["text"], text_annotation="text", config=[ - Config("odt_import.prefix", description="Optional prefix to add to annotation names."), - Config("odt_import.keep_control_chars", False, description="Set to True if control characters should not be " - "removed from the text."), - Config("odt_import.normalize", "NFC", description="Normalize input using any of the following forms: " - "'NFC', 'NFKC', 'NFD', and 'NFKD'.") + Config("odt_import.prefix", description="Optional prefix to add to annotation names.", datatype=str), + Config( + "odt_import.keep_control_chars", + default=False, + description="Set to True if control characters should not be removed from the text.", + datatype=bool + ), + Config( + "odt_import.normalize", + default="NFC", + description="Normalize input using any of the following forms: 'NFC', 'NFKC', 'NFD', and 'NFKD'.", + datatype=str, + choices=("NFC", "NFKC", "NFD", "NFKD"), + ) ]) def parse(source_file: SourceFilename = SourceFilename(), source_dir: Source = Source(), diff --git a/sparv/modules/pdf_import/pdf_import.py b/sparv/modules/pdf_import/pdf_import.py index 9e4ecf80..86732afa 100644 --- a/sparv/modules/pdf_import/pdf_import.py +++ b/sparv/modules/pdf_import/pdf_import.py @@ -10,11 +10,20 @@ @importer("pdf import", file_extension="pdf", outputs=["text", "page:number"], text_annotation="text", config=[ - Config("pdf_import.prefix", description="Optional prefix to add to annotation names."), - Config("pdf_import.keep_control_chars", False, description="Set to True if control characters should not be " - "removed from the text."), - Config("pdf_import.normalize", "NFC", description="Normalize input using any of the following forms: " - "'NFC', 'NFKC', 'NFD', and 'NFKD'.") + Config("pdf_import.prefix", description="Optional prefix to add to annotation names.", datatype=str), + Config( + "pdf_import.keep_control_chars", + default=False, + description="Set to True if control characters should not be removed from the text.", + datatype=bool, + ), + Config( + "pdf_import.normalize", + default="NFC", + description="Normalize input using any of the following forms: 'NFC', 'NFKC', 'NFD', and 'NFKD'.", + datatype=str, + choices=("NFC", "NFKC", "NFD", "NFKD"), + ) ]) def parse(source_file: SourceFilename = SourceFilename(), source_dir: Source = Source(), diff --git a/sparv/modules/stanza/__init__.py b/sparv/modules/stanza/__init__.py index 3501fee9..e1489100 100644 --- a/sparv/modules/stanza/__init__.py +++ b/sparv/modules/stanza/__init__.py @@ -5,36 +5,84 @@ from . import models, stanza, stanza_swe __config__ = [ - Config("stanza.resources_file", default="stanza/[metadata.language]/resources.json", - description="Stanza resources file"), - Config("stanza.use_gpu", default=True, description="Use GPU instead of CPU if available"), - Config("stanza.batch_size", default=5000, - description="Limit Stanza batch size. Sentences with a token count exceeding this value will be excluded " - "from analysis."), - Config("stanza.max_sentence_length", default=250, - description="Max length (in number of tokens) of sentences that will get dependence annotations (set to 0 " - "for no limit)"), - Config("stanza.cpu_fallback", default=False, - description="Fall back to CPU for sentences exceeding the max_sentence_length, instead of " - "excluding them from dependence parsing. Only usable with use_gpu enabled."), - Config("stanza.max_token_length", default=0, - description="Max number of characters per token. Any sentence containing a token exceeding this limit will " - "be excluded from analysis. Disabled by default."), - Config("stanza.sentence_chunk", default="", - description="Text chunk (annotation) to use as input when segmenting sentences (not used for Swedish)"), - Config("stanza.sentence_annotation", - description="Optional existing sentence segmentation annotation (not used for Swedish)"), - Config("stanza.token_annotation", description="Optional existing token annotation (not used for Swedish)"), - + Config( + "stanza.resources_file", + default="stanza/[metadata.language]/resources.json", + description="Stanza resources file", + datatype=str, + ), + Config("stanza.use_gpu", default=True, description="Use GPU instead of CPU if available", datatype=bool), + Config( + "stanza.batch_size", + default=5000, + description="Limit Stanza batch size. Sentences with a token count exceeding this value will be excluded " + "from analysis.", + datatype=int, + ), + Config( + "stanza.max_sentence_length", + default=250, + description="Max length (in number of tokens) of sentences that will get dependence annotations (set to 0 " + "for no limit)", + datatype=int, + ), + Config( + "stanza.cpu_fallback", + default=False, + description="Fall back to CPU for sentences exceeding the max_sentence_length, instead of " + "excluding them from dependence parsing. Only usable with use_gpu enabled.", + datatype=bool, + ), + Config( + "stanza.max_token_length", + default=0, + description="Max number of characters per token. Any sentence containing a token exceeding this limit will " + "be excluded from analysis. Disabled by default.", + datatype=int, + ), + Config( + "stanza.sentence_chunk", + default="", + description="Text chunk (annotation) to use as input when segmenting sentences (not used for Swedish)", + datatype=str, + ), + Config( + "stanza.sentence_annotation", + description="Optional existing sentence segmentation annotation (not used for Swedish)", + datatype=str, + ), + Config( + "stanza.token_annotation", description="Optional existing token annotation (not used for Swedish)", datatype=str + ), # Config for Swedish - Config("stanza.swe_lem_model", default="stanza/swe/lem/sv_suc_lemmatizer.pt", - description="Stanza lemmatisation model for Swedish"), - Config("stanza.swe_pos_model", default="stanza/swe/pos/sv_talbanken_tagger.pt", - description="Stanza POS model for Swedish"), - Config("stanza.swe_pretrain_pos_model", default="stanza/swe/sv_talbanken.pretrain.pt", - description="Stanza pretrain POS model for Swedish"), - Config("stanza.swe_dep_model", default="stanza/swe/dep/sv_talbanken_parser.pt", - description="Stanza dependency model for Swedish"), - Config("stanza.swe_pretrain_dep_model", default="stanza/swe/sv_talbanken.pretrain.pt", - description="Stanza pretrain dependency model for Swedish"), + Config( + "stanza.swe_lem_model", + default="stanza/swe/lem/sv_suc_lemmatizer.pt", + description="Stanza lemmatisation model for Swedish", + datatype=str, + ), + Config( + "stanza.swe_pos_model", + default="stanza/swe/pos/sv_talbanken_tagger.pt", + description="Stanza POS model for Swedish", + datatype=str, + ), + Config( + "stanza.swe_pretrain_pos_model", + default="stanza/swe/sv_talbanken.pretrain.pt", + description="Stanza pretrain POS model for Swedish", + datatype=str, + ), + Config( + "stanza.swe_dep_model", + default="stanza/swe/dep/sv_talbanken_parser.pt", + description="Stanza dependency model for Swedish", + datatype=str, + ), + Config( + "stanza.swe_pretrain_dep_model", + default="stanza/swe/sv_talbanken.pretrain.pt", + description="Stanza pretrain dependency model for Swedish", + datatype=str, + ), ] diff --git a/sparv/modules/stats_export/__init__.py b/sparv/modules/stats_export/__init__.py index 70a584ab..dc9bef3a 100644 --- a/sparv/modules/stats_export/__init__.py +++ b/sparv/modules/stats_export/__init__.py @@ -1,17 +1,25 @@ """Word frequency list generation.""" +from typing import List from sparv.api import Config from . import stats_export, sbx_stats __config__ = [ - Config("stats_export.annotations", description="Sparv annotations to include."), - Config("stats_export.source_annotations", - description="List of annotations and attributes from the source data to include. None will be included by " - "default."), - Config("stats_export.delimiter", default="\t", description="Delimiter separating columns"), - Config("stats_export.cutoff", default=1, - description="The minimum frequency a word must have in order to be included in the result"), - Config("stats_export.remote_host", description="Remote host to install to"), - Config("stats_export.remote_dir", description="Path on remote host to install to") + Config("stats_export.annotations", description="Sparv annotations to include.", datatype=List[str]), + Config( + "stats_export.source_annotations", + description="List of annotations and attributes from the source data to include. None will be included by " + "default.", + datatype=List[str], + ), + Config("stats_export.delimiter", default="\t", description="Delimiter separating columns", datatype=str), + Config( + "stats_export.cutoff", + default=1, + description="The minimum frequency a word must have in order to be included in the result", + datatype=int, + ), + Config("stats_export.remote_host", description="Remote host to install to", datatype=str), + Config("stats_export.remote_dir", description="Path on remote host to install to", datatype=str), ] diff --git a/sparv/modules/swener/swener.py b/sparv/modules/swener/swener.py index d1d4b1e8..2f5c553a 100644 --- a/sparv/modules/swener/swener.py +++ b/sparv/modules/swener/swener.py @@ -14,7 +14,7 @@ @annotator("Named entity tagging with SweNER", language=["swe"], - config=[Config("swener.binary", default="hfst-swener", description="SweNER executable")]) + config=[Config("swener.binary", default="hfst-swener", description="SweNER executable", datatype=str)]) def annotate(out_ne: Output = Output("swener.ne", cls="named_entity", description="Named entity segments from SweNER"), out_ne_ex: Output = Output("swener.ne:swener.ex", description="Named entity expressions from SweNER"), out_ne_type: Output = Output("swener.ne:swener.type", cls="named_entity:type", diff --git a/sparv/modules/text_import/text_import.py b/sparv/modules/text_import/text_import.py index 6cbd0321..d49eca88 100644 --- a/sparv/modules/text_import/text_import.py +++ b/sparv/modules/text_import/text_import.py @@ -6,20 +6,42 @@ from sparv.api import Config, SourceFilename, Output, Source, SourceStructure, Text, importer, util -@importer("TXT import", file_extension="txt", outputs=["text"], text_annotation="text", config=[ - Config("text_import.prefix", description="Optional prefix to add to annotation names."), - Config("text_import.encoding", util.constants.UTF8, description="Encoding of source file. Defaults to UTF-8."), - Config("text_import.keep_control_chars", False, description="Set to True if control characters should not be " - "removed from the text."), - Config("text_import.normalize", "NFC", description="Normalize input using any of the following forms: " - "'NFC', 'NFKC', 'NFD', and 'NFKD'.") -]) -def parse(source_file: SourceFilename = SourceFilename(), - source_dir: Source = Source(), - prefix: Optional[str] = Config("text_import.prefix"), - encoding: str = Config("text_import.encoding"), - keep_control_chars: bool = Config("text_import.keep_control_chars"), - normalize: str = Config("text_import.normalize")) -> None: +@importer( + "TXT import", + file_extension="txt", + outputs=["text"], + text_annotation="text", + config=[ + Config("text_import.prefix", description="Optional prefix to add to annotation names.", datatype=str), + Config( + "text_import.encoding", + util.constants.UTF8, + description="Encoding of source file. Defaults to UTF-8.", + datatype=str, + ), + Config( + "text_import.keep_control_chars", + False, + description="Set to True if control characters should not be removed from the text.", + datatype=bool, + ), + Config( + "text_import.normalize", + default="NFC", + description="Normalize input using any of the following forms: 'NFC', 'NFKC', 'NFD', and 'NFKD'.", + datatype=str, + choices=("NFC", "NFKC", "NFD", "NFKD"), + ), + ], +) +def parse( + source_file: SourceFilename = SourceFilename(), + source_dir: Source = Source(), + prefix: Optional[str] = Config("text_import.prefix"), + encoding: str = Config("text_import.encoding"), + keep_control_chars: bool = Config("text_import.keep_control_chars"), + normalize: str = Config("text_import.normalize"), +) -> None: """Parse plain text file as input to the Sparv Pipeline. Args: diff --git a/sparv/modules/xml_export/preserved_format.py b/sparv/modules/xml_export/preserved_format.py index b9ef41f7..a81921ce 100644 --- a/sparv/modules/xml_export/preserved_format.py +++ b/sparv/modules/xml_export/preserved_format.py @@ -24,8 +24,13 @@ @exporter("XML export preserving whitespaces from source file", config=[ - Config("xml_export.filename_formatted", default="{file}_export.xml", - description="Filename pattern for resulting XML files, with '{file}' representing the source name.") + Config( + "xml_export.filename_formatted", + default="{file}_export.xml", + description="Filename pattern for resulting XML files, with '{file}' representing the source name.", + datatype=str, + pattern=r".*\{file\}.*", + ) ]) def preserved_format(source_file: SourceFilename = SourceFilename(), text: Text = Text(), diff --git a/sparv/modules/xml_export/pretty.py b/sparv/modules/xml_export/pretty.py index 8cdc46ea..b7e2834d 100644 --- a/sparv/modules/xml_export/pretty.py +++ b/sparv/modules/xml_export/pretty.py @@ -1,7 +1,7 @@ """Export annotated corpus data to pretty-printed xml.""" import os -from typing import Optional +from typing import List, Optional from sparv.api import ( AllSourceFilenames, @@ -30,16 +30,31 @@ @exporter("XML export with one token element per line", config=[ - Config("xml_export.filename", default="{file}_export.xml", - description="Filename pattern for resulting XML files, with '{file}' representing the source name."), - Config("xml_export.annotations", description="Sparv annotations to include."), - Config("xml_export.source_annotations", - description="List of annotations and attributes from the source data to include. Everything will be " - "included by default."), - Config("xml_export.header_annotations", - description="List of headers from the source data to include. All headers will be included by default."), - Config("xml_export.include_empty_attributes", False, - description="Whether to include attributes even when they are empty.") + Config( + "xml_export.filename", + default="{file}_export.xml", + description="Filename pattern for resulting XML files, with '{file}' representing the source name.", + datatype=str, + pattern=r".*\{file\}.*" + ), + Config("xml_export.annotations", description="Sparv annotations to include.", datatype=List[str]), + Config( + "xml_export.source_annotations", + description="List of annotations and attributes from the source data to include. Everything will be included " + "by default.", + datatype=List[str] + ), + Config( + "xml_export.header_annotations", + description="List of headers from the source data to include. All headers will be included by default.", + datatype=List[str], + ), + Config( + "xml_export.include_empty_attributes", + default=False, + description="Whether to include attributes even when they are empty.", + datatype=bool, + ) ]) def pretty(source_file: SourceFilename = SourceFilename(), fileid: AnnotationData = AnnotationData(""), @@ -107,10 +122,18 @@ def pretty(source_file: SourceFilename = SourceFilename(), @exporter("Combined XML export (all results in one file)", config=[ - Config("xml_export.filename_combined", default="[metadata.id].xml", - description="Filename of resulting combined XML."), - Config("xml_export.include_version_info", default=True, - description="Whether to include annotation version info in the combined XML.") + Config( + "xml_export.filename_combined", + default="[metadata.id].xml", + description="Filename of resulting combined XML.", + datatype=str, + ), + Config( + "xml_export.include_version_info", + default=True, + description="Whether to include annotation version info in the combined XML.", + datatype=bool, + ) ]) def combined(corpus: Corpus = Corpus(), out: Export = Export("xml_export.combined/[xml_export.filename_combined]"), @@ -126,8 +149,12 @@ def combined(corpus: Corpus = Corpus(), @exporter("Compressed combined XML export", config=[ - Config("xml_export.filename_compressed", default="[metadata.id].xml.bz2", - description="Filename of resulting compressed combined XML.") + Config( + "xml_export.filename_compressed", + default="[metadata.id].xml.bz2", + description="Filename of resulting compressed combined XML.", + datatype=str, + ) ]) def compressed(out: Export = Export("xml_export.combined/[xml_export.filename_compressed]"), xmlfile: ExportInput = ExportInput("xml_export.combined/[xml_export.filename_combined]")): @@ -136,8 +163,8 @@ def compressed(out: Export = Export("xml_export.combined/[xml_export.filename_co @installer("Copy compressed XML to a target path, optionally on a remote host", config=[ - Config("xml_export.export_host", description="Remote host to copy XML export to"), - Config("xml_export.export_path", description="Target path to copy XML export to") + Config("xml_export.export_host", description="Remote host to copy XML export to", datatype=str), + Config("xml_export.export_path", description="Target path to copy XML export to", datatype=str) ], uninstaller="xml_export:uninstall") def install( corpus: Corpus = Corpus(), diff --git a/sparv/modules/xml_export/scrambled.py b/sparv/modules/xml_export/scrambled.py index 885102d3..df2d68f6 100644 --- a/sparv/modules/xml_export/scrambled.py +++ b/sparv/modules/xml_export/scrambled.py @@ -30,7 +30,7 @@ @exporter("Scrambled XML export", config=[ - Config("xml_export.scramble_on", description="Annotation to use for scrambling.") + Config("xml_export.scramble_on", description="Annotation to use for scrambling.", datatype=str) ]) def scrambled(source_file: SourceFilename = SourceFilename(), fileid: AnnotationData = AnnotationData(""), @@ -110,8 +110,8 @@ def compressed_scrambled(out: Export = Export("xml_export.combined_scrambled/[me @installer("Copy compressed scrambled XML to a target path, optionally on a remote host", config=[ - Config("xml_export.export_scrambled_host", description="Remote host to copy scrambled XML export to"), - Config("xml_export.export_scrambled_path", description="Target path to copy scrambled XML export to") + Config("xml_export.export_scrambled_host", description="Remote host to copy scrambled XML export to", datatype=str), + Config("xml_export.export_scrambled_path", description="Target path to copy scrambled XML export to", datatype=str) ], uninstaller="xml_export:uninstall_scrambled") def install_scrambled( corpus: Corpus = Corpus(), diff --git a/sparv/modules/xml_import/xml_import.py b/sparv/modules/xml_import/xml_import.py index 223c5c38..d8dcde73 100644 --- a/sparv/modules/xml_import/xml_import.py +++ b/sparv/modules/xml_import/xml_import.py @@ -44,25 +44,61 @@ def get_annotations(self, corpus_config: dict) -> List[str]: self.annotations = sorted(elements) return self.annotations - -@importer("XML import", file_extension="xml", outputs=Config("xml_import.elements", []), config=[ - Config("xml_import.elements", [], description="List of elements and attributes in source file. Only needed for " - "renaming or when used as input to other annotations, as everything " - "is parsed whether listed or not."), - Config("xml_import.skip", [], description="Elements and attributes to skip. " - "Use elementname:@contents to skip contents as well."), - Config("xml_import.header_elements", [], description="Elements containing header metadata. Contents will not be " - "included in corpus text."), - Config("xml_import.header_data", [], description="List of header elements and attributes from which to extract " - "metadata."), - Config("xml_import.prefix", description="Optional prefix to add to annotation names."), - Config("xml_import.remove_namespaces", False, description="Remove XML namespaces upon import."), - Config("xml_import.encoding", util.constants.UTF8, description="Encoding of source file. Defaults to UTF-8."), - Config("xml_import.keep_control_chars", False, description="Set to True if control characters should not be " - "removed from the text."), - Config("xml_import.normalize", "NFC", description="Normalize input using any of the following forms: " - "'NFC', 'NFKC', 'NFD', and 'NFKD'.") -], structure=XMLStructure) +@importer( + "XML import", + file_extension="xml", + outputs=Config("xml_import.elements", [], datatype=List[str]), + config=[ + Config( + "xml_import.elements", + [], + description="List of elements and attributes in source file. Only needed for " + "renaming or when used as input to other annotations, as everything " + "is parsed whether listed or not.", + datatype=List[str], + ), + Config( + "xml_import.skip", + [], + description="Elements and attributes to skip. Use elementname:@contents to skip contents as well.", + datatype=List[str], + ), + Config( + "xml_import.header_elements", + [], + description="Elements containing header metadata. Contents will not be included in corpus text.", + datatype=List[str], + ), + Config( + "xml_import.header_data", + [], + description="List of header elements and attributes from which to extract metadata.", + datatype=List[str], + ), + Config("xml_import.prefix", description="Optional prefix to add to annotation names.", datatype=str), + Config("xml_import.remove_namespaces", False, description="Remove XML namespaces upon import.", datatype=bool), + Config( + "xml_import.encoding", + util.constants.UTF8, + description="Encoding of source file. Defaults to UTF-8.", + datatype=str, + ), + Config( + "xml_import.keep_control_chars", + False, + description="Set to True if control characters should not be removed from the text.", + datatype=bool, + ), + Config( + "xml_import.normalize", + default="NFC", + description="Normalize input using any of the following forms: 'NFC', 'NFKC', 'NFD', and 'NFKD'.", + datatype=str, + choices=("NFC", "NFKC", "NFD", "NFKD"), + ), + ], + structure=XMLStructure, +) def parse(filename: SourceFilename = SourceFilename(), source_dir: Source = Source(), elements: list = Config("xml_import.elements"), From e6d633238ca8bab4bbb0ea1746199f0590ff0d1e Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 15 Sep 2023 17:11:02 +0200 Subject: [PATCH 142/175] Remove unnecessary Null values from default config --- sparv/resources/config/config_default.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sparv/resources/config/config_default.yaml b/sparv/resources/config/config_default.yaml index 9b25635c..7e9ad8fb 100644 --- a/sparv/resources/config/config_default.yaml +++ b/sparv/resources/config/config_default.yaml @@ -40,9 +40,9 @@ export: # Set to false if module name spaces should be kept in the export (e.g. 'pos' will be called 'hunpos.pos') remove_module_namespaces: true # A string representing the name space to be added to all annotations created by Sparv - sparv_namespace: Null + sparv_namespace: # A string representing the name space to be added to all annotations present in the source - source_namespace: Null + source_namespace: # Common settings for XML exporters # xml_export: From 92722385cec94bcd05ee745f60b9eadb2231800c Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 15 Sep 2023 17:26:23 +0200 Subject: [PATCH 143/175] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c7531507..ea287128 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ - Added `--json-log` flag to enable JSON format for logging. - Added support for restricting a whole module to one or more languages by using the `__language__` variable. - Running `sparv schema` will now generate a JSON schema which can be used to validate corpus config files. +- More strict config validation, including validation of config values and data types. ### Changed From def32fc29ee01f9e282bf5404ad6b2c06acf8c56 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Wed, 20 Sep 2023 14:42:28 +0200 Subject: [PATCH 144/175] Fix progress reporting for tasks not associated with source files --- sparv/core/log_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sparv/core/log_handler.py b/sparv/core/log_handler.py index b2fed65c..fe976b88 100644 --- a/sparv/core/log_handler.py +++ b/sparv/core/log_handler.py @@ -161,7 +161,7 @@ def emit(self, record): if record.msg == "export_dirs": self.export_dirs_list.update(record.export_dirs) elif record.msg == "progress": - job_id = self.job_ids.get((record.job, record.file)) + job_id = self.job_ids.get((record.job, record.file or "")) if job_id is not None: try: if not self.jobs[job_id]["task"]: From 67dcdbf48ae21e4dc59d8e0a2ad4b3dbe094b17d Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Wed, 20 Sep 2023 14:46:19 +0200 Subject: [PATCH 145/175] Add progress bars for downloading models --- sparv/api/classes.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sparv/api/classes.py b/sparv/api/classes.py index 8ab307d7..41874836 100644 --- a/sparv/api/classes.py +++ b/sparv/api/classes.py @@ -734,10 +734,13 @@ def read_pickle(self): def download(self, url: str): """Download file from url and save to modeldir.""" + def log_progress(block_num, block_size, total_size): + if total_size > 0: + logger.progress(progress=block_num * block_size, total=total_size) os.makedirs(self.path.parent, exist_ok=True) logger.debug("Downloading from: %s", url) try: - urllib.request.urlretrieve(url, self.path) + urllib.request.urlretrieve(url, self.path, reporthook=log_progress) logger.info("Successfully downloaded %s", self.name) except Exception as e: logger.error("Download of %s from %s failed", self.name, url) From 3af936663e09b9936e1fdbbf43628ead54fea7fc Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 29 Sep 2023 11:22:05 +0200 Subject: [PATCH 146/175] Upgrade Stanza (resolves #166) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 68c7553e..e31e4aae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ dependencies = [ "questionary==1.10.0", "rich==13.3.3", "snakemake==7.25.0", - "stanza==1.5.0", + "stanza==1.5.1", "torch>=1.9.1", # Used by Stanza; see https://github.com/spraakbanken/sparv-pipeline/issues/82 "typing-inspect==0.8.0", ] From 196b41543f09a6a935d6b5be0a6d87db7044868f Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Mon, 2 Oct 2023 16:27:19 +0200 Subject: [PATCH 147/175] Add priority parameter to decorators --- CHANGELOG.md | 1 + docs/developers-guide/sparv-decorators.md | 10 + sparv/core/Snakefile | 1 + sparv/core/registry.py | 212 ++++++++++++++++------ sparv/core/snake_utils.py | 1 + 5 files changed, 165 insertions(+), 60 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea287128..db067b69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ - Added support for restricting a whole module to one or more languages by using the `__language__` variable. - Running `sparv schema` will now generate a JSON schema which can be used to validate corpus config files. - More strict config validation, including validation of config values and data types. +- Most Sparv decorators now have a `priority` parameter, to control the order in which functions are run. ### Changed diff --git a/docs/developers-guide/sparv-decorators.md b/docs/developers-guide/sparv-decorators.md index 8316e4c9..f4dceeba 100644 --- a/docs/developers-guide/sparv-decorators.md +++ b/docs/developers-guide/sparv-decorators.md @@ -17,6 +17,8 @@ tokens, sentences, parts of speeches etc.) and outputs one or more new annotatio - `description`: Description of the annotator. Used for displaying help texts in the CLI. - `config`: List of Config instances defining config options for the annotator. - `language`: List of supported languages. If no list is supplied all languages are supported. +- `priority`: Functions with higher priority will be preferred when scheduling which functions to run. The default + priority is 0. - `order`: If several annotators have the same output, this integer value will help decide which to try to use first. A lower number indicates higher priority. - `wildcards`: List of wildcards used in the annotator function's arguments. @@ -92,6 +94,8 @@ files into one output file. - `name`: Optional name to use instead of the function name. - `config`: List of Config instances defining config options for the exporter. - `language`: List of supported languages. If no list is supplied all languages are supported. +- `priority`: Functions with higher priority will be preferred when scheduling which functions to run. The default + priority is 0. - `order`: If several exporters have the same output, this integer value will help decide which to try to use first. A lower number indicates higher priority. - `abstract`: Set to True if this exporter has no output. @@ -129,6 +133,8 @@ It is recommended that an installer removes any related uninstaller's marker, to - `name`: Optional name to use instead of the function name. - `config`: List of Config instances defining config options for the installer. - `language`: List of supported languages. If no list is supplied all languages are supported. +- `priority`: Functions with higher priority will be preferred when scheduling which functions to run. The default + priority is 0. - `uninstaller`: Name of related uninstaller. **Example:** @@ -161,6 +167,8 @@ It is recommended that an uninstaller removes any related installer's marker, to - `name`: Optional name to use instead of the function name. - `config`: List of Config instances defining config options for the uninstaller. - `language`: List of supported languages. If no list is supplied all languages are supported. +- `priority`: Functions with higher priority will be preferred when scheduling which functions to run. The default + priority is 0. **Example:** ```python @@ -188,6 +196,8 @@ your system it will be available for any corpus. - `name`: Optional name to use instead of the function name. - `config`: List of Config instances defining config options for the installer. - `language`: List of supported languages. If no list is supplied all languages are supported. +- `priority`: Functions with higher priority will be preferred when scheduling which functions to run. The default + priority is 0. - `order`: If several modelbuilders have the same output, this integer value will help decide which to try to use first. A lower number indicates higher priority. diff --git a/sparv/core/Snakefile b/sparv/core/Snakefile index bb11fb34..b57624a8 100644 --- a/sparv/core/Snakefile +++ b/sparv/core/Snakefile @@ -103,6 +103,7 @@ def make_rule(module_name: str, f_name: str, annotator_info: dict, config_missin force_preloader=config.get("force_preloader", False), compression=sparv_config.get("sparv.compression") resources: **resources + priority: rule_storage.priority # We use "script" instead of "run" since with "run" the whole Snakefile would have to be reloaded for every # single job, due to how Snakemake creates processes for run-jobs. script: "run_snake.py" diff --git a/sparv/core/registry.py b/sparv/core/registry.py index 077ee9a2..deedcfa3 100644 --- a/sparv/core/registry.py +++ b/sparv/core/registry.py @@ -212,54 +212,94 @@ def _get_module_name(module_string: str) -> str: return module_name -def _annotator(description: str, a_type: Annotator, name: Optional[str] = None, file_extension: Optional[str] = None, - outputs=(), text_annotation=None, structure=None, language: Optional[List[str]] = None, - config: Optional[List[Config]] = None, order: Optional[int] = None, abstract: bool = False, - wildcards: Optional[List[Wildcard]] = None, preloader: Optional[Callable] = None, - preloader_params: Optional[List[str]] = None, preloader_target: Optional[str] = None, - preloader_cleanup: Optional[Callable] = None, preloader_shared: bool = True, - uninstaller: Optional[str] = None): +def _annotator( + description: str, + a_type: Annotator, + name: Optional[str] = None, + file_extension: Optional[str] = None, + outputs=(), + text_annotation=None, + structure=None, + language: Optional[List[str]] = None, + config: Optional[List[Config]] = None, + priority: Optional[int] = None, + order: Optional[int] = None, + abstract: bool = False, + wildcards: Optional[List[Wildcard]] = None, + preloader: Optional[Callable] = None, + preloader_params: Optional[List[str]] = None, + preloader_target: Optional[str] = None, + preloader_cleanup: Optional[Callable] = None, + preloader_shared: bool = True, + uninstaller: Optional[str] = None, +): """Return a decorator for annotator functions, adding them to annotator registry.""" + def decorator(f): """Add wrapped function to registry.""" module_name = _get_module_name(f.__module__) - _potential_annotators[module_name].append({ - "module_name": module_name, - "description": description, - "function": f, - "name": name, - "type": a_type, - "file_extension": file_extension, - "outputs": outputs, - "text_annotation": text_annotation, - "structure": structure, - "language": language, - "config": config, - "order": order, - "abstract": abstract, - "wildcards": wildcards, - "preloader": preloader, - "preloader_params": preloader_params, - "preloader_target": preloader_target, - "preloader_cleanup": preloader_cleanup, - "preloader_shared": preloader_shared, - "uninstaller": uninstaller - }) + _potential_annotators[module_name].append( + { + "module_name": module_name, + "description": description, + "function": f, + "name": name, + "type": a_type, + "file_extension": file_extension, + "outputs": outputs, + "text_annotation": text_annotation, + "structure": structure, + "language": language, + "config": config, + "priority": priority, + "order": order, + "abstract": abstract, + "wildcards": wildcards, + "preloader": preloader, + "preloader_params": preloader_params, + "preloader_target": preloader_target, + "preloader_cleanup": preloader_cleanup, + "preloader_shared": preloader_shared, + "uninstaller": uninstaller, + } + ) return f return decorator -def annotator(description: str, name: Optional[str] = None, language: Optional[List[str]] = None, - config: Optional[List[Config]] = None, order: Optional[int] = None, - wildcards: Optional[List[Wildcard]] = None, preloader: Optional[Callable] = None, - preloader_params: Optional[List[str]] = None, preloader_target: Optional[str] = None, - preloader_cleanup: Optional[Callable] = None, preloader_shared: bool = True): + +def annotator( + description: str, + name: Optional[str] = None, + language: Optional[List[str]] = None, + config: Optional[List[Config]] = None, + priority: Optional[int] = None, + order: Optional[int] = None, + wildcards: Optional[List[Wildcard]] = None, + preloader: Optional[Callable] = None, + preloader_params: Optional[List[str]] = None, + preloader_target: Optional[str] = None, + preloader_cleanup: Optional[Callable] = None, + preloader_shared: bool = True, +): """Return a decorator for annotator functions, adding them to the annotator registry.""" - return _annotator(description=description, a_type=Annotator.annotator, name=name, language=language, - config=config, order=order, wildcards=wildcards, preloader=preloader, - preloader_params=preloader_params, preloader_target=preloader_target, - preloader_cleanup=preloader_cleanup, preloader_shared=preloader_shared) + return _annotator( + description=description, + a_type=Annotator.annotator, + name=name, + language=language, + config=config, + priority=priority, + order=order, + wildcards=wildcards, + preloader=preloader, + preloader_params=preloader_params, + preloader_target=preloader_target, + preloader_cleanup=preloader_cleanup, + preloader_shared=preloader_shared, + ) + def importer(description: str, file_extension: str, name: Optional[str] = None, outputs=None, @@ -287,8 +327,15 @@ def importer(description: str, file_extension: str, name: Optional[str] = None, outputs=outputs, text_annotation=text_annotation, structure=structure, config=config) -def exporter(description: str, name: Optional[str] = None, config: Optional[List[Config]] = None, - language: Optional[List[str]] = None, order: Optional[int] = None, abstract: bool = False): +def exporter( + description: str, + name: Optional[str] = None, + config: Optional[List[Config]] = None, + language: Optional[List[str]] = None, + priority: Optional[int] = None, + order: Optional[int] = None, + abstract: bool = False, +): """Return a decorator for exporter functions. Args: @@ -302,29 +349,74 @@ def exporter(description: str, name: Optional[str] = None, config: Optional[List Returns: A decorator """ - return _annotator(description=description, a_type=Annotator.exporter, name=name, config=config, language=language, - order=order, abstract=abstract) - - -def installer(description: str, name: Optional[str] = None, config: Optional[List[Config]] = None, - language: Optional[List[str]] = None, uninstaller: Optional[str] = None): + return _annotator( + description=description, + a_type=Annotator.exporter, + name=name, + config=config, + language=language, + priority=priority, + order=order, + abstract=abstract, + ) + + +def installer( + description: str, + name: Optional[str] = None, + config: Optional[List[Config]] = None, + language: Optional[List[str]] = None, + priority: Optional[int] = None, + uninstaller: Optional[str] = None, +): """Return a decorator for installer functions.""" - return _annotator(description=description, a_type=Annotator.installer, name=name, config=config, language=language, - uninstaller=uninstaller) - - -def uninstaller(description: str, name: Optional[str] = None, config: Optional[List[Config]] = None, - language: Optional[List[str]] = None): + return _annotator( + description=description, + a_type=Annotator.installer, + name=name, + config=config, + language=language, + priority=priority, + uninstaller=uninstaller, + ) + + +def uninstaller( + description: str, + name: Optional[str] = None, + config: Optional[List[Config]] = None, + language: Optional[List[str]] = None, + priority: Optional[int] = None, +): """Return a decorator for uninstaller functions.""" - return _annotator(description=description, a_type=Annotator.uninstaller, name=name, config=config, - language=language) - - -def modelbuilder(description: str, name: Optional[str] = None, config: Optional[List[Config]] = None, - language: Optional[List[str]] = None, order: Optional[int] = None): + return _annotator( + description=description, + a_type=Annotator.uninstaller, + name=name, + config=config, + language=language, + priority=priority, + ) + + +def modelbuilder( + description: str, + name: Optional[str] = None, + config: Optional[List[Config]] = None, + language: Optional[List[str]] = None, + priority: Optional[int] = None, + order: Optional[int] = None, +): """Return a decorator for modelbuilder functions.""" - return _annotator(description=description, a_type=Annotator.modelbuilder, name=name, config=config, - language=language, order=order) + return _annotator( + description=description, + a_type=Annotator.modelbuilder, + name=name, + config=config, + language=language, + priority=priority, + order=order, + ) def _add_to_registry(annotator: dict, skip_language_check: bool = False): diff --git a/sparv/core/snake_utils.py b/sparv/core/snake_utils.py index be08d22b..6920229b 100644 --- a/sparv/core/snake_utils.py +++ b/sparv/core/snake_utils.py @@ -120,6 +120,7 @@ def __init__(self, module_name, f_name, annotator_info): self.description = annotator_info["description"] self.file_extension = annotator_info["file_extension"] self.import_outputs = annotator_info["outputs"] + self.priority: int = annotator_info["priority"] or 0 self.order = annotator_info["order"] self.abstract = annotator_info["abstract"] self.wildcards = annotator_info["wildcards"] # Information about the wildcards used From 1ceacf0bc849ae2a87cc9b1a641f677c5b0988d5 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Mon, 2 Oct 2023 16:37:49 +0200 Subject: [PATCH 148/175] Make sure the Korp config is installed last Otherwise the corpus will be visible to Korp before it is searchable. --- sparv/modules/korp/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sparv/modules/korp/config.py b/sparv/modules/korp/config.py index c84ed234..a2f8a410 100644 --- a/sparv/modules/korp/config.py +++ b/sparv/modules/korp/config.py @@ -441,7 +441,7 @@ def build_description(description, short_description) -> Union[Dict[str, str], s return lang_dict.get(None) or lang_dict -@installer("Install Korp corpus configuration file.", uninstaller="korp:uninstall_config") +@installer("Install Korp corpus configuration file.", uninstaller="korp:uninstall_config", priority=-1) def install_config( remote_host: Optional[str] = Config("korp.remote_host"), config_dir: str = Config("korp.config_dir"), From ccd06ef759818e26f6add8395ad8ef38ee1f492b Mon Sep 17 00:00:00 2001 From: Anne Schumacher Date: Tue, 3 Oct 2023 15:34:38 +0200 Subject: [PATCH 149/175] raise human-friendly Sparv errors when parsing of odt or docx fails --- sparv/modules/docx_import/docx_import.py | 7 +++++-- sparv/modules/odt_import/odt_import.py | 8 ++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/sparv/modules/docx_import/docx_import.py b/sparv/modules/docx_import/docx_import.py index d5b47ecc..e1f300c3 100644 --- a/sparv/modules/docx_import/docx_import.py +++ b/sparv/modules/docx_import/docx_import.py @@ -6,7 +6,7 @@ from docx2python import docx2python from docx2python.iterators import iter_at_depth -from sparv.api import Config, SourceFilename, Output, Source, SourceStructure, Text, importer, util +from sparv.api import Config, SourceFilename, Output, Source, SourceStructure, Text, importer, util, SparvErrorMessage @importer("docx import", file_extension="docx", outputs=["text"], text_annotation="text", config=[ @@ -41,7 +41,10 @@ def parse(source_file: SourceFilename = SourceFilename(), 'NFC' is used by default. """ source_file_path = source_dir.get_path(source_file, ".docx") - d = docx2python(source_file_path) + try: + d = docx2python(source_file_path) + except Exception as e: + raise SparvErrorMessage(f"Failed to parse docx file '{source_file}'. {type(e).__name__}: {e}") # Extract all text from the body, ignoring headers and footers text = "\n\n".join(iter_at_depth(d.body, 4)) diff --git a/sparv/modules/odt_import/odt_import.py b/sparv/modules/odt_import/odt_import.py index 448b9f0b..dca2f748 100644 --- a/sparv/modules/odt_import/odt_import.py +++ b/sparv/modules/odt_import/odt_import.py @@ -5,7 +5,8 @@ import zipfile from typing import Optional -from sparv.api import Config, SourceFilename, Output, Source, SourceStructure, Text, get_logger, importer, util +from sparv.api import (Config, Output, Source, SourceFilename, SourceStructure, SparvErrorMessage, Text, get_logger, + importer, util) logger = get_logger(__name__) @@ -44,7 +45,10 @@ def parse(source_file: SourceFilename = SourceFilename(), source_file_path = str(source_dir.get_path(source_file, ".odt")) # Parse odt and extract all text content - text = OdtParser(source_file_path).text + try: + text = OdtParser(source_file_path).text + except Exception as e: + raise SparvErrorMessage(f"Failed to parse odt file '{source_file}'. {type(e).__name__}: {e}") if not keep_control_chars: text = util.misc.remove_control_characters(text) From 53172fb9cffd8d7e9f99a8d2c19c2116f0cec505 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Thu, 5 Oct 2023 14:38:05 +0200 Subject: [PATCH 150/175] Add links to changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index db067b69..0d12a7c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -299,3 +299,9 @@ - Improved code modularity - Increased independence between modules and language models - This facilitates adding new annotation modules and import/export formats. + +[5.1.0]: https://github.com/spraakbanken/sparv-pipeline/releases/tag/v5.1.0 +[5.0.0]: https://github.com/spraakbanken/sparv-pipeline/releases/tag/v5.0.0 +[4.1.1]: https://github.com/spraakbanken/sparv-pipeline/releases/tag/v4.1.1 +[4.1.0]: https://github.com/spraakbanken/sparv-pipeline/releases/tag/v4.1.0 +[4.0.0]: https://github.com/spraakbanken/sparv-pipeline/releases/tag/v4.0.0 From ade36aafb3d113fd13a48dc15e70b3839be96366 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Wed, 11 Oct 2023 15:41:37 +0200 Subject: [PATCH 151/175] Fix typos --- CHANGELOG.md | 2 +- docs/developers-guide/config-parameters.md | 2 +- docs/developers-guide/general-concepts.md | 4 ++-- docs/developers-guide/sparv-classes.md | 6 +++--- docs/developers-guide/sparv-decorators.md | 4 ++-- docs/developers-guide/utilities.md | 6 +++--- docs/developers-guide/writing-sparv-plugins.md | 2 +- docs/docsify/sparv-pipeline.md | 2 +- docs/user-manual/available-analyses.md | 6 +++--- docs/user-manual/corpus-configuration.md | 2 +- sparv/api/util/tagsets/pos_to_upos.py | 4 ++-- 11 files changed, 20 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d12a7c0..e96eb169 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -270,7 +270,7 @@ - New plugin system facilitates installation of Sparv plugins (like FreeLing). - New format for corpus config files - - The new format is yaml which is easier to write and more human readable than makefiles. + - The new format is yaml which is easier to write and more human-readable than makefiles. - There is a command-line wizard which helps you create corpus config files. - You no longer have to specify XML elements and attributes that should be kept from the original files. The XML parser now parses all existing elements and their attributes by default. Their original names will be kept and diff --git a/docs/developers-guide/config-parameters.md b/docs/developers-guide/config-parameters.md index 39e8bcff..cd46f479 100644 --- a/docs/developers-guide/config-parameters.md +++ b/docs/developers-guide/config-parameters.md @@ -38,7 +38,7 @@ the decorator belonging to that same function, but the declaration may be done i Sparv function, or even a different module. Please note that it is mandatory to set a description for each declared config parameter. These descriptions are -displayed to the user when lising modules with the `sparv modules` command. +displayed to the user when listing modules with the `sparv modules` command. ## Config hierarchy diff --git a/docs/developers-guide/general-concepts.md b/docs/developers-guide/general-concepts.md index ac565ced..08806239 100644 --- a/docs/developers-guide/general-concepts.md +++ b/docs/developers-guide/general-concepts.md @@ -5,7 +5,7 @@ provided in the following chapters. The Sparv Pipeline is made up of a core and different modules. The modules contain Sparv functions that serve different purposes like reading and parsing source files, building or downloading models, producing annotations and producing output files that contain the source text and annotations. All of these modules (i.e. the -code inside the `sparv/modules` directory) are replacable. A Sparv function is decorated with a special +code inside the `sparv/modules` directory) are replaceable. A Sparv function is decorated with a special [decorator](developers-guide/sparv-decorators) that tells Sparv what purpose it serves. A function's parameters hold information about what input is needed in order to run the function and what output is produced by it. The Sparv core automatically finds all decorated functions, scans their parameters and builds a registry of what modules are available @@ -34,7 +34,7 @@ Some Sparv functions may require annotations from other functions before they ca expressed in the function arguments. By using special [Sparv classes](developers-guide/sparv-classes) as default arguments in a function's signature the central Sparv registry can automatically keep track of what annotations can be produced by what function and in what order things need to be run. These dependencies can either be described in a -module specific manner or in a more abstact way. For example, an annotator producing word base forms (or lemmas) may +module specific manner or in a more abstract way. For example, an annotator producing word base forms (or lemmas) may depend on a part-of-speech annotation with a specific tagset and therefore this annotator might define that its input needs to be an annotation produced by a specific module. A part-of-speech tagger on the other hand usually needs word segments as input, and it probably does not matter exactly what module produces these segments. In this case the diff --git a/docs/developers-guide/sparv-classes.md b/docs/developers-guide/sparv-classes.md index b5c4ec67..66bb3c91 100644 --- a/docs/developers-guide/sparv-classes.md +++ b/docs/developers-guide/sparv-classes.md @@ -52,7 +52,7 @@ annotation is needed as input for a function, e.g. `Annotation("")`. ## AnnotationAllSourceFiles Regular annotation but the source filename must be specified for all actions. Use as input to an annotator function to -require the specificed annotation for every source file in the corpus. +require the specified annotation for every source file in the corpus. **Arguments:** @@ -144,7 +144,7 @@ This class represents an annotation holding arbitrary data, i.e. data that is no ## AnnotationDataAllSourceFiles Like [`AnnotationData`](#annotationdata), this class is used for annotations holding arbitrary data but the source file -must be specified for all actions. Use as input to an annotator to require the specificed annotation for every source +must be specified for all actions. Use as input to an annotator to require the specified annotation for every source file in the corpus. **Arguments:** @@ -281,7 +281,7 @@ List of header annotation names for a given source file. ## Language -In instance of this class holds information about the luanguage of the corpus. This information is retrieved from the +In instance of this class holds information about the language of the corpus. This information is retrieved from the corpus configuration and is specified as ISO 639-3 code. diff --git a/docs/developers-guide/sparv-decorators.md b/docs/developers-guide/sparv-decorators.md index f4dceeba..37643bd7 100644 --- a/docs/developers-guide/sparv-decorators.md +++ b/docs/developers-guide/sparv-decorators.md @@ -185,9 +185,9 @@ def uninstall(corpus: Corpus = Corpus(), ``` ## @modelbuilder -A function decorated with `@modelbuilder` is used to setup a model used by other Sparv components (typically +A function decorated with `@modelbuilder` is used to set up a model used by other Sparv components (typically annotators). Setting up a model could for example mean downloading a file, unzipping it, converting it into a different -format and saving it in Sparv's data directory. A model is usually not specific to one corpus. Once a model is setup on +format and saving it in Sparv's data directory. A model is usually not specific to one corpus. Once a model is set up on your system it will be available for any corpus. **Arguments:** diff --git a/docs/developers-guide/utilities.md b/docs/developers-guide/utilities.md index 9065d3c6..1830efff 100644 --- a/docs/developers-guide/utilities.md +++ b/docs/developers-guide/utilities.md @@ -154,7 +154,7 @@ Call a binary with `arguments` and `stdin` and return a pair `(stdout, stderr)`. is returned as the stderr component. Defaults to `False`. - `use_shell`: Don't use this unless you really have to! If set to `True` the binary will be executed through the shell. Defaults to `False`. Is automatically set to `True` when using `raw_command`. -- `allow_error`: If set to `False` an exeption is raised if stderr is not empty and stderr and stdout will be logged. +- `allow_error`: If set to `False` an exception is raised if stderr is not empty and stderr and stdout will be logged. Defaults to `False`. - `return_command`: If set to `True` the process is returned. Defaults to `False`. @@ -238,7 +238,7 @@ Dictionary containing mappings (dictionaries) for of part-of-speech tag mappings ### pos_to_upos() -Map POS tags to Universal Depenendy POS tags. This only works if there is a conversion function in +Map POS tags to Universal Dependency POS tags. This only works if there is a conversion function in `util.tagsets.pos_to_upos` for the given language and tagset. **Arguments:** @@ -390,7 +390,7 @@ Exception (class) used to notify users of errors in a friendly way without displ in the [Writing Sparv Plugins](developers-guide/writing-sparv-plugins#error-messages) section. > [!NOTE] -> Only the `message` argument should be used when raisning this exception in a Sparv module. +> Only the `message` argument should be used when raising this exception in a Sparv module. **Arguments:** diff --git a/docs/developers-guide/writing-sparv-plugins.md b/docs/developers-guide/writing-sparv-plugins.md index 8282edd5..3dc96a00 100644 --- a/docs/developers-guide/writing-sparv-plugins.md +++ b/docs/developers-guide/writing-sparv-plugins.md @@ -332,7 +332,7 @@ This annotator uses a model. It also has an extra argument called `model_preload an already loaded model. In the decorator we point out the preloader function using the `preloader` parameter. `preloader_params` is a list of parameter names from the annotator, which the preloader needs as arguments. In this case it's only one: the `model` parameter. `preloader_target` points to a single parameter name of the annotator, which -is the one that will recieve the return value from the preloader function. +is the one that will receive the return value from the preloader function. When using the `sparv preload` command with this annotator, the preloader function will be run only once, and every time the annotator is used, it will get the preloaded model via the `model_preloaded` parameter. diff --git a/docs/docsify/sparv-pipeline.md b/docs/docsify/sparv-pipeline.md index 406a72a3..2d286ca5 100644 --- a/docs/docsify/sparv-pipeline.md +++ b/docs/docsify/sparv-pipeline.md @@ -26,4 +26,4 @@ Cite Sparv: *[Martin Hammarstedt, Anne Schumacher, Lars Borin, Markus Forsberg ( > >

--> > -> Depending on your [notification settings](https://github.com/settings/notifications) you will now recieve notifications about new Sparv releases on GitHub's website, via email or on your phone. +> Depending on your [notification settings](https://github.com/settings/notifications) you will now receive notifications about new Sparv releases on GitHub's website, via email or on your phone. diff --git a/docs/user-manual/available-analyses.md b/docs/user-manual/available-analyses.md index c16107ae..1adcf2ac 100644 --- a/docs/user-manual/available-analyses.md +++ b/docs/user-manual/available-analyses.md @@ -10,7 +10,7 @@ whitespace information) are not listed here. > configuration](user-manual/corpus-configuration.md#export-options)). Please observe that the annotations usually have > shorter names in the corpus exports. > ->**Annotators** are the names of the annotation functions (including their module names) which are used for procuding +>**Annotators** are the names of the annotation functions (including their module names) which are used for producing >the annotations. They can be run directly with the `sparv run-rule [annotator]` command. In most cases this is not >necessary though, due to the fact that the annotation functions producing the annotations listed in the corpus config >file are executed automatically when running `sparv run`. @@ -121,7 +121,7 @@ preset](user-manual/corpus-configuration.md#annotation-presets) called `SWE_DEFA |:---|:-----------| |**Description** | SALDO IDs from the `:saldo.sense`-attribute are enriched with likelihoods. |**Tool** | [Sparv wsd](https://github.com/spraakbanken/sparv-wsd) -|**Dokumentation**| [Running the Koala word sense disambiguators](https://github.com/spraakbanken/sparv-wsd/blob/master/README.pdf) +|**Documentation**| [Running the Koala word sense disambiguators](https://github.com/spraakbanken/sparv-wsd/blob/master/README.pdf) |**Model** | - [ALL_512_128_w10_A2_140403_ctx1.bin](https://github.com/spraakbanken/sparv-wsd/blob/master/models/scouse/ALL_512_128_w10_A2_140403_ctx1.bin)
- [lem_cbow0_s512_w10_NEW2_ctx.bin](https://github.com/spraakbanken/sparv-wsd/blob/master/models/scouse/lem_cbow0_s512_w10_NEW2_ctx.bin) |**Annotations** | - `:wsd.sense` (identifies senses in SALDO along with their likelihoods) |**Annotators** | `wsd:annotate` @@ -188,7 +188,7 @@ for Swedish from the 1800's: |:---|:-----------| |**Description** | Sentence segments are analysed to enrich tokens with part-of-speech tags and morphosyntactic information. |**Tool** | [Hunpos](https://code.google.com/archive/p/hunpos/) -|**Model** | - [suc3_suc-tags_default-setting_utf8.model](https://github.com/spraakbanken/sparv-models/blob/master/hunpos/suc3_suc-tags_default-setting_utf8.model?raw=true) trained on [SUC 3.0](https://spraakbanken.gu.se/resurser/suc3)
- a word list along with the words' morphosyntactig information generated from the [Dalin morphology](https://spraakbanken.gu.se/resurser/dalinm) and the [Swedberg morphology](https://spraakbanken.gu.se/resurser/swedbergm) +|**Model** | - [suc3_suc-tags_default-setting_utf8.model](https://github.com/spraakbanken/sparv-models/blob/master/hunpos/suc3_suc-tags_default-setting_utf8.model?raw=true) trained on [SUC 3.0](https://spraakbanken.gu.se/resurser/suc3)
- a word list along with the words' morphosyntactic information generated from the [Dalin morphology](https://spraakbanken.gu.se/resurser/dalinm) and the [Swedberg morphology](https://spraakbanken.gu.se/resurser/swedbergm) |**Tagset** | [SUC MSD tags](https://spraakbanken.gu.se/korp/markup/msdtags.html) |**Annotations** | - `:hunpos.msd` (morphosyntactic tag)
- `:hunpos.pos` (part-of-speech tag |**Annotators** | - `hunpos:msdtag_hist`
- `hunpos:postag` diff --git a/docs/user-manual/corpus-configuration.md b/docs/user-manual/corpus-configuration.md index 88dfc050..4c94eac1 100644 --- a/docs/user-manual/corpus-configuration.md +++ b/docs/user-manual/corpus-configuration.md @@ -87,7 +87,7 @@ The `metadata` section of your corpus config contains metadata about your corpus The `import` section of your corpus config is used to give Sparv some information about your input files (i.e. your corpus). -- `import.source_dir` defines the location of your input files and it defaults to `source`. Sparv will check the +- `import.source_dir` defines the location of your input files, and it defaults to `source`. Sparv will check the source directory recursively for valid input files to process. - `import.importer` is used to tell Sparv which importer to use when processing your source files. The setting you diff --git a/sparv/api/util/tagsets/pos_to_upos.py b/sparv/api/util/tagsets/pos_to_upos.py index 7f2724e9..282388bb 100644 --- a/sparv/api/util/tagsets/pos_to_upos.py +++ b/sparv/api/util/tagsets/pos_to_upos.py @@ -1,4 +1,4 @@ -"""Map different POS (or MSD) tags to simple Universal Depenendy POS (UPOS) tags. +"""Map different POS (or MSD) tags to simple Universal Dependency POS (UPOS) tags. http://universaldependencies.org/u/pos/all.html """ @@ -27,7 +27,7 @@ def pos_to_upos(pos, lang, tagset): - """Map POS tags to Universal Depenendy POS tags.""" + """Map POS tags to Universal Dependency POS tags.""" if (lang, tagset) in CONVERTERS: lang_convert = CONVERTERS[(lang, tagset)] return lang_convert(pos) From 294145f31d653a4442542266d6fd2d3a9d79ceb6 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Wed, 18 Oct 2023 11:07:13 +0200 Subject: [PATCH 152/175] Show source file name also for unhandled errors --- sparv/core/run_snake.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sparv/core/run_snake.py b/sparv/core/run_snake.py index b552d3f8..5c8fa8df 100644 --- a/sparv/core/run_snake.py +++ b/sparv/core/run_snake.py @@ -139,7 +139,9 @@ def flush(): # something went wrong. exit_with_error_message(e.message, "sparv.modules." + module_name) except Exception as e: - errmsg = f"An error occurred while executing {module_name}:{f_name}:\n\n {type(e).__name__}: {e}" + current_file = f" for the file {snakemake.params.source_file!r}" if snakemake.params.source_file else "" + errmsg = f"An error occurred while executing {module_name}:{f_name}{current_file}:" \ + f"\n\n {type(e).__name__}: {e}" if logger.level > logging.DEBUG: errmsg += f"\n\nTo display further details when errors occur, run Sparv with the '--log debug' or " \ "'--log-to-file debug' arguments." From a4275d77da42d8a16c6e7dfa07cb2c5f5bcdbb0f Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Wed, 18 Oct 2023 11:27:22 +0200 Subject: [PATCH 153/175] Raise error when dateformat.pre_regex doesn't match --- sparv/modules/dateformat/dateformat.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sparv/modules/dateformat/dateformat.py b/sparv/modules/dateformat/dateformat.py index b7fb831f..aa44e5d2 100644 --- a/sparv/modules/dateformat/dateformat.py +++ b/sparv/modules/dateformat/dateformat.py @@ -228,6 +228,8 @@ def get_date_length(informat): if pre_regex: matches = re.match(pre_regex, val) + if not matches: + raise SparvErrorMessage(f"dateformat.pre_regex did not match the value {val!r}") val = [v for v in matches.groups() if v][0] if not val: # If the regex doesn't match, treat as no date From 1b1ecc585ef6a4e13419011291de4b6121d8b8b9 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Tue, 24 Oct 2023 12:00:56 +0200 Subject: [PATCH 154/175] Don't handle localhost differently during installations User may want to actually connect to localhost --- sparv/api/util/install.py | 3 +-- sparv/modules/korp/config.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/sparv/api/util/install.py b/sparv/api/util/install.py index b1882b31..8c68c2ac 100644 --- a/sparv/api/util/install.py +++ b/sparv/api/util/install.py @@ -34,7 +34,6 @@ def install_mysql(host: Optional[str], db_name: str, sqlfile: Union[str, List[st db_name: Name of the database. sqlfile: Path to a SQL file, or list of paths. """ - local = host is None or host == "localhost" if isinstance(sqlfile, str): sqlfile = [sqlfile] @@ -49,7 +48,7 @@ def install_mysql(host: Optional[str], db_name: str, sqlfile: Union[str, List[st logger.info("Skipping empty file: %s (%d/%d)", f, file_count, file_total) else: logger.info(f"Installing MySQL database: {db_name}, source: {f} ({file_count}/{file_total})") - if local: + if not host: subprocess.check_call( f"cat {shlex.quote(f)} | mysql {shlex.quote(db_name)}", shell=True ) diff --git a/sparv/modules/korp/config.py b/sparv/modules/korp/config.py index a2f8a410..830f84b3 100644 --- a/sparv/modules/korp/config.py +++ b/sparv/modules/korp/config.py @@ -392,7 +392,7 @@ def build_annotations( def get_presets(remote_host, config_dir): """Get list of presets from file system.""" presets = {} - if remote_host and remote_host != "localhost": + if remote_host: remote_path = shlex.quote(f"{config_dir}/attributes/") cmd = ["ssh", remote_host, f"find {remote_path}"] else: From 3f241a05129432d9355c69111fee78ebea7f4231 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 27 Oct 2023 15:20:26 +0200 Subject: [PATCH 155/175] Don't automatically add plain annotations when all connected attributes are omitted --- sparv/core/misc.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/sparv/core/misc.py b/sparv/core/misc.py index be510940..bf04a4ed 100644 --- a/sparv/core/misc.py +++ b/sparv/core/misc.py @@ -62,7 +62,7 @@ def parse_annotation_list(annotation_names: Optional[Iterable[str]], all_annotat possible_plain_annotations = set() omit_annotations = set() include_rest = False - plain_to_atts = defaultdict(list) + plain_to_atts = defaultdict(set) result: OrderedDict = OrderedDict() for a in annotation_names: @@ -77,11 +77,11 @@ def parse_annotation_list(annotation_names: Optional[Iterable[str]], all_annotat plain_name, attr = Annotation(name).split() else: plain_name, attr = None, None - result.pop(name, None) + result.pop(name, None) # Remove any previous occurrence first, to keep the order result[name] = export_name or None if attr: possible_plain_annotations.add(plain_name) - plain_to_atts[plain_name].append(name) + plain_to_atts[plain_name].add(name) else: plain_annotations.add(name) @@ -91,17 +91,25 @@ def parse_annotation_list(annotation_names: Optional[Iterable[str]], all_annotat # Add all_annotations to result if required if include_rest and all_annotations: - for a in [a for a in all_annotations if not a in omit_annotations]: - if a not in result: + for a in all_annotations: + if a not in result and a not in omit_annotations: result[a] = None - plain_name, _ = Annotation(a).split() - plain_to_atts[plain_name].append(a) + plain_name, attr = Annotation(a).split() + if attr: + plain_to_atts[plain_name].add(a) plain_annotations.add(plain_name) # Add annotations names without attributes to result if required if add_plain_annotations: - for a in sorted(possible_plain_annotations.difference(plain_annotations)): - if a not in result: + new_plain_annotations = possible_plain_annotations.difference(plain_annotations) + if omit_annotations: + # Don't add new plain annotation if all connected attributes have been omitted + for annotation in omit_annotations: + plain_name, _ = Annotation(annotation).split() + plain_to_atts[plain_name].discard(annotation) + + for a in sorted(new_plain_annotations): + if a not in result and plain_to_atts[a]: result[a] = None # Remove any exclusions from final list From 256b1bf8a0fc07a16afba630227d06c1dc06d708 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 27 Oct 2023 17:10:10 +0200 Subject: [PATCH 156/175] =?UTF-8?q?Remove=20single=20comma=20that=20preven?= =?UTF-8?q?ted=20Stanza=20from=20using=20GPU=20=F0=9F=98=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sparv/modules/stanza/stanza_swe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sparv/modules/stanza/stanza_swe.py b/sparv/modules/stanza/stanza_swe.py index 51e94b9e..8aa08c1a 100644 --- a/sparv/modules/stanza/stanza_swe.py +++ b/sparv/modules/stanza/stanza_swe.py @@ -133,7 +133,7 @@ def annotate_swe( logger.debug(f"Running dependency parsing and POS-taggning on {len(sentences)} sentences" f" (using {'GPU' if use_gpu and not fallback else 'CPU'}).") nlp_args["processors"] = "tokenize,pos,lemma,depparse" # Comma-separated list of processors to use - nlp_args["use_gpu"] = use_gpu and not fallback, + nlp_args["use_gpu"] = use_gpu and not fallback nlp = stanza.Pipeline(**nlp_args) else: From 7f8c804f29d8ac41550d4eae840517cd8335164e Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 27 Oct 2023 17:19:41 +0200 Subject: [PATCH 157/175] Update changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e96eb169..9a68c1d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,8 @@ - Illegal characters are now replaced with underscore in XML element and attribute names during XML export. This also applies to CWB and Korp config exports. - Not specifying a corpus language now excludes all language specific annotators. +- When an unhandled exception occurs, the relevant source document will be displayed in the log. +- `localhost` as an installation target is no longer handled as if host was omitted. ### Fixed @@ -69,6 +71,7 @@ - Elapsed time exceeding 24 hours no longer gets cut off in the `--stats` output. - Fixed bug where error messages were not getting written to the log file when the `--log debug` flag was used. +- Fixed bug that prevented Stanza from using GPU. ## [5.1.0] - 2022-11-03 From 0aa0828b35f205e4b548d07a8117ec36564918de Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Mon, 30 Oct 2023 15:35:26 +0100 Subject: [PATCH 158/175] Use lazy formatting in some logging --- sparv/api/util/export.py | 18 +++++++++---- sparv/api/util/install.py | 2 +- sparv/api/util/system.py | 4 +-- sparv/core/config.py | 2 +- sparv/core/io.py | 20 ++++++++++----- sparv/modules/cwb/install_corpus.py | 4 +-- sparv/modules/hist/models.py | 2 +- sparv/modules/korp/config.py | 18 ++++++++++--- sparv/modules/misc/number.py | 4 +-- .../phrase_structure/phrase_structure.py | 25 ++++++++++++++++--- sparv/modules/stanza/models.py | 2 +- sparv/modules/stanza/stanza_swe.py | 9 ++++--- sparv/modules/stats_export/sbx_stats.py | 4 +-- sparv/modules/stats_export/stats_export.py | 2 +- 14 files changed, 81 insertions(+), 35 deletions(-) diff --git a/sparv/api/util/export.py b/sparv/api/util/export.py index c38c7bad..bee5632a 100644 --- a/sparv/api/util/export.py +++ b/sparv/api/util/export.py @@ -515,15 +515,23 @@ def _check_name_collision(export_names, source_annotations): source_annot = annots[0] if annots[0].name in source_names else annots[1] new_name = SPARV_DEFAULT_NAMESPACE + "." + export_names[sparv_annot.name] export_names[sparv_annot.name] = new_name - logger.info("Changing name of automatic annotation '{}' to '{}' due to collision with '{}'.".format( - sparv_annot.name, new_name, source_annot.name)) + logger.info( + "Changing name of automatic annotation '%s' to '%s' due to collision with '%s'.", + sparv_annot.name, + new_name, + source_annot.name + ) # Warn the user if we cannot resolve collisions automatically else: annots_string = "\n".join([f"{a.name} ({'source' if a.name in source_names else 'sparv'} annotation)" for a in annots]) - logger.warning("The following annotations are exported with the same name ({}) and might overwrite " - "each other: \n\n{}\n\nIf you want to keep all of these annotations you can change " - "their export names.".format(attr, annots_string)) + logger.warning( + "The following annotations are exported with the same name (%s) and might overwrite " + "each other: \n\n%s\n\nIf you want to keep all of these annotations you can change " + "their export names.", + attr, + annots_string + ) return export_names ################################################################################ diff --git a/sparv/api/util/install.py b/sparv/api/util/install.py index 8c68c2ac..06d5e16a 100644 --- a/sparv/api/util/install.py +++ b/sparv/api/util/install.py @@ -47,7 +47,7 @@ def install_mysql(host: Optional[str], db_name: str, sqlfile: Union[str, List[st elif os.path.getsize(f) < 10: logger.info("Skipping empty file: %s (%d/%d)", f, file_count, file_total) else: - logger.info(f"Installing MySQL database: {db_name}, source: {f} ({file_count}/{file_total})") + logger.info("Installing MySQL database: %s, source: %s (%d/%d)", db_name, f, file_count, file_total) if not host: subprocess.check_call( f"cat {shlex.quote(f)} | mysql {shlex.quote(db_name)}", shell=True diff --git a/sparv/api/util/system.py b/sparv/api/util/system.py index 7ad75c93..758f3d75 100644 --- a/sparv/api/util/system.py +++ b/sparv/api/util/system.py @@ -159,10 +159,10 @@ def rsync(local: Union[str, Path], host: Optional[str], remote: Union[str, Path] remote_dir = os.path.dirname(remote) if os.path.isdir(local): - logger.info(f"Copying directory: {local} => {host + ':' if host else ''}{remote}") + logger.info("Copying directory: %s => %s%s", local, host + ":" if host else "", remote) args = ["--recursive", "--delete", f"{local}/"] else: - logger.info(f"Copying file: {local} => {host + ':' if host else ''}{remote}") + logger.info("Copying file: %s => %s%s", local, host + ":" if host else "", remote) args = [local] if host: diff --git a/sparv/core/config.py b/sparv/core/config.py index e494f846..974ef4bb 100644 --- a/sparv/core/config.py +++ b/sparv/core/config.py @@ -78,7 +78,7 @@ def load_config(config_file: Optional[str], config_dict: Optional[dict] = None) if DEFAULT_CONFIG.is_file(): _config_default = read_yaml(DEFAULT_CONFIG) else: - logger.warning("Default config file is missing: {}".format(DEFAULT_CONFIG)) + logger.warning("Default config file is missing: %s", DEFAULT_CONFIG) _config_default = {} if config_file: diff --git a/sparv/core/io.py b/sparv/core/io.py index 0dc36146..167e3100 100644 --- a/sparv/core/io.py +++ b/sparv/core/io.py @@ -111,7 +111,7 @@ def _write_single_annotation(source_file: str, annotation: str, values, append: ctr += 1 # Update file modification time even if nothing was written os.utime(file_path, None) - logger.info(f"Wrote {ctr} items: {source_file + '/' if source_file else ''}{annotation}") + logger.info("Wrote %d items: %s%s", ctr, source_file + "/" if source_file else "", annotation) def get_annotation_size(source_file: str, annotation: BaseAnnotation): @@ -215,7 +215,7 @@ def _read_single_annotation(source_file: str, annotation: str, with_annotation_n if isinstance(e, OSError) and str(e) != "Invalid data stream": raise e raise_format_error(ann_file) - logger.debug(f"Read {ctr} items: {source_file + '/' if source_file else ''}{annotation}") + logger.debug("Read %d items: %s%s", ctr, source_file + "/" if source_file else "", annotation) def write_data(source_file: Optional[str], name: Union[BaseAnnotation, str], value: str, append: bool = False): @@ -228,8 +228,12 @@ def write_data(source_file: Optional[str], name: Union[BaseAnnotation, str], val f.write(value) # Update file modification time even if nothing was written os.utime(file_path, None) - logger.info(f"Wrote {len(value)} bytes: {source_file + '/' if source_file else ''}" - f"{name.name if isinstance(name, BaseAnnotation) else name}") + logger.info( + "Wrote %d bytes: %s%s", + len(value), + source_file + "/" if source_file else "", + name.name if isinstance(name, BaseAnnotation) else name + ) def read_data(source_file: Optional[str], name: Union[BaseAnnotation, str]): @@ -244,8 +248,12 @@ def read_data(source_file: Optional[str], name: Union[BaseAnnotation, str]): raise e raise_format_error(file_path) - logger.debug(f"Read {len(data)} bytes: {source_file + '/' if source_file else ''}" - f"{name.name if isinstance(name, BaseAnnotation) else name}") + logger.debug( + "Read %d bytes: %s%s", + len(data), + source_file + "/" if source_file else "", + name.name if isinstance(name, BaseAnnotation) else name + ) return data diff --git a/sparv/modules/cwb/install_corpus.py b/sparv/modules/cwb/install_corpus.py index e97f5579..d9cc9bdc 100644 --- a/sparv/modules/cwb/install_corpus.py +++ b/sparv/modules/cwb/install_corpus.py @@ -70,11 +70,11 @@ def uninstall_corpus( assert corpus and data_dir and registry_dir # Already checked by Sparv, but just to be sure registry_file = Path(registry_dir) / corpus - logger.info(f"Removing CWB registry file from {host + ':' if host else ''}{registry_file}") + logger.info("Removing CWB registry file from %s%s", host + ":" if host else "", registry_file) util.install.uninstall_path(registry_file, host=host) corpus_dir = Path(data_dir) / corpus - logger.info(f"Removing CWB data from {host + ':' if host else ''}{corpus_dir}") + logger.info("Removing CWB data from %s%s", host + ":" if host else "", corpus_dir) util.install.uninstall_path(corpus_dir, host=host) install_marker.remove() diff --git a/sparv/modules/hist/models.py b/sparv/modules/hist/models.py index 3a99d891..69e3ece6 100644 --- a/sparv/modules/hist/models.py +++ b/sparv/modules/hist/models.py @@ -166,7 +166,7 @@ def read_lmf(xml, annotation_elements=("writtenForm", "lemgram"), verbose=True, "katt", "doktor"] util.misc.test_lexicon(lexicon, testwords) - logger.info(f"OK, read {len(lexicon)} entries") + logger.info("OK, read %d entries", len(lexicon)) return lexicon diff --git a/sparv/modules/korp/config.py b/sparv/modules/korp/config.py index 830f84b3..b4d54578 100644 --- a/sparv/modules/korp/config.py +++ b/sparv/modules/korp/config.py @@ -342,7 +342,7 @@ def build_annotations( and annotation.name not in annotation_definitions and not export_name in include ): - logger.debug(f"Skipping annotation {annotation.name!r}") + logger.debug("Skipping annotation '%s'", annotation.name) continue export_name_cwb = cwb_escape(export_name.replace(":", "_")) is_token = annotation.annotation_name == token.name @@ -397,7 +397,11 @@ def get_presets(remote_host, config_dir): cmd = ["ssh", remote_host, f"find {remote_path}"] else: cmd = ["find", f"{config_dir}/attributes/"] - logger.debug(f"Getting Korp annotation presets from {remote_host}:{config_dir}") + logger.debug( + "Getting Korp annotation presets from %s%s", + remote_host + ":" if remote_host else "", + config_dir + ) s = subprocess.run(cmd, capture_output=True, encoding="utf-8") if s.returncode == 0: for p in s.stdout.splitlines(): @@ -451,7 +455,11 @@ def install_config( ): """Install Korp corpus configuration file.""" corpus_dir = Path(config_dir) / "corpora" - logger.info(f"Installing Korp corpus configuration file to {remote_host}:{corpus_dir}") + logger.info( + "Installing Korp corpus configuration file to %s%s", + remote_host + ":" if remote_host else "", + corpus_dir + ) util.install.install_path(config_file, remote_host, corpus_dir) uninstall_marker.remove() marker.write() @@ -468,7 +476,9 @@ def uninstall_config( """Uninstall Korp corpus configuration file.""" corpus_file = Path(config_dir) / "corpora" / f"{corpus_id}.yaml" logger.info( - f"Uninstalling Korp corpus configuration file from {remote_host + ':' if remote_host else ''}{corpus_file}" + "Uninstalling Korp corpus configuration file from %s%s", + remote_host + ":" if remote_host else "", + corpus_file ) util.install.uninstall_path(corpus_file, host=remote_host) install_marker.remove() diff --git a/sparv/modules/misc/number.py b/sparv/modules/misc/number.py index 540ed5db..6bb40552 100644 --- a/sparv/modules/misc/number.py +++ b/sparv/modules/misc/number.py @@ -152,7 +152,7 @@ def count_chunks(out: OutputCommonData = OutputCommonData("misc.{annotation}_cou pass if chunk_count == 0: - logger.info(f"No {chunk.name} chunks found in corpus") + logger.info("No %s chunks found in corpus", chunk.name) # Write chunk count data out.write(str(chunk_count)) @@ -163,7 +163,7 @@ def count_chunks(out: OutputCommonData = OutputCommonData("misc.{annotation}_cou def count_zero_chunks(out: OutputCommonData = OutputCommonData("misc.{annotation}_count"), _files: AllSourceFilenames = AllSourceFilenames()): """Create chunk count file for non-existent 'annotation' chunks.""" - logger.info(f"No {out.name[5:-6]} chunks found in corpus") + logger.info("No %s chunks found in corpus", out.name[5:-6]) out.write("0") diff --git a/sparv/modules/phrase_structure/phrase_structure.py b/sparv/modules/phrase_structure/phrase_structure.py index 5e8b291c..5325cc73 100644 --- a/sparv/modules/phrase_structure/phrase_structure.py +++ b/sparv/modules/phrase_structure/phrase_structure.py @@ -54,24 +54,41 @@ def get_token_span(index): if not child[0].startswith("WORD:"): start_pos = get_token_span(s[position])[0] open_elem_stack.append(child + (start_pos,)) - logger.debug(f" {s[position]}") + logger.debug( + " %s", + child[0], + child[1], + s[position] + ) else: # Close nodes while open_elem_stack[-1][2] == child[2]: start_pos = open_elem_stack[-1][3] end_pos = get_token_span(s[position - 1])[1] nodes.append(((start_pos, end_pos), open_elem_stack[-1][0], open_elem_stack[-1][1])) - logger.debug(f" {start_pos}-{end_pos}") + logger.debug( + " %d-%d", + open_elem_stack[-1][0], + open_elem_stack[-1][1], + start_pos, + end_pos + ) open_elem_stack.pop() position += 1 - logger.debug(f" {child[0][5:]}") + logger.debug(" %s", child[0][5:]) # Close remaining open nodes end_pos = get_token_span(s[-1])[1] for elem in reversed(open_elem_stack): start_pos = elem[3] nodes.append(((start_pos, end_pos), elem[0], elem[1])) - logger.debug(f" {start_pos}-{end_pos}") + logger.debug( + " %d-%d", + elem[0], + elem[1], + start_pos, + end_pos + ) # Sort nodes sorted_nodes = sorted(nodes) diff --git a/sparv/modules/stanza/models.py b/sparv/modules/stanza/models.py index 2270f27f..96cf4fb2 100644 --- a/sparv/modules/stanza/models.py +++ b/sparv/modules/stanza/models.py @@ -70,7 +70,7 @@ def get_model(lang: Language = Language(), import stanza lang_name = util.misc.get_language_name_by_part3(lang) or lang stanza_lang = util.misc.get_language_part1_by_part3(lang) - logger.info(f"Downloading Stanza language model for {lang_name}") + logger.info("Downloading Stanza language model for %s", lang_name) stanza.download(lang=stanza_lang, model_dir=str(resources_file.path.parent), verbose=False, logging_level=logging.WARNING) zip_file = Model(f"stanza/{lang}/{stanza_lang}/default.zip") diff --git a/sparv/modules/stanza/stanza_swe.py b/sparv/modules/stanza/stanza_swe.py index 8aa08c1a..0e171219 100644 --- a/sparv/modules/stanza/stanza_swe.py +++ b/sparv/modules/stanza/stanza_swe.py @@ -130,14 +130,17 @@ def annotate_swe( # Init Stanza pipeline if dep or fallback: - logger.debug(f"Running dependency parsing and POS-taggning on {len(sentences)} sentences" - f" (using {'GPU' if use_gpu and not fallback else 'CPU'}).") + logger.debug( + "Running dependency parsing and POS-taggning on %d sentences (using %s).", + len(sentences), + "GPU" if use_gpu and not fallback else "CPU" + ) nlp_args["processors"] = "tokenize,pos,lemma,depparse" # Comma-separated list of processors to use nlp_args["use_gpu"] = use_gpu and not fallback nlp = stanza.Pipeline(**nlp_args) else: - logger.debug(f"Running POS-taggning on {len(sentences)} sentences.") + logger.debug("Running POS-taggning on %d sentences.", len(sentences)) nlp_args["processors"] = "tokenize,pos" # Comma-separated list of processors to use nlp_args["use_gpu"] = use_gpu nlp = stanza.Pipeline(**nlp_args) diff --git a/sparv/modules/stats_export/sbx_stats.py b/sparv/modules/stats_export/sbx_stats.py index 4689bf84..8eb36478 100644 --- a/sparv/modules/stats_export/sbx_stats.py +++ b/sparv/modules/stats_export/sbx_stats.py @@ -264,7 +264,7 @@ def uninstall_sbx_freq_list( ): """Uninstall SBX word frequency list.""" remote_file = os.path.join(remote_dir, f"stats_{corpus_id}.csv") - logger.info(f"Removing SBX word frequency file {host + ':' if host else ''}{remote_file}") + logger.info("Removing SBX word frequency file %s%s", host + ":" if host else "", remote_file) util.install.uninstall_path(remote_file, host) install_marker.remove() marker.write() @@ -280,7 +280,7 @@ def uninstall_sbx_freq_list_date( ): """Uninstall SBX word frequency list with dates.""" remote_file = os.path.join(remote_dir, f"stats_{corpus_id}.csv") - logger.info(f"Removing SBX word frequency with dates file {host + ':' if host else ''}{remote_file}") + logger.info("Removing SBX word frequency with dates file %s%s", host + ":" if host else "", remote_file) util.install.uninstall_path(remote_file, host) install_marker.remove() marker.write() diff --git a/sparv/modules/stats_export/stats_export.py b/sparv/modules/stats_export/stats_export.py index cf6e9eb1..ab4c0ec3 100644 --- a/sparv/modules/stats_export/stats_export.py +++ b/sparv/modules/stats_export/stats_export.py @@ -120,7 +120,7 @@ def uninstall_freq_list( ): """Uninstall word frequency list.""" remote_file = os.path.join(remote_dir, f"stats_{corpus_id}.csv") - logger.info(f"Removing word frequency file {host + ':' if host else ''}{remote_file}") + logger.info("Removing word frequency file %s%s", host + ":" if host else "", remote_file) util.install.uninstall_path(remote_file, host) install_marker.remove() marker.write() From 49585e576f67b0d21b41fd8fe641b5acd836fdfb Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Mon, 6 Nov 2023 16:06:41 +0100 Subject: [PATCH 159/175] Extend run-rule autocompletion to include everything --- sparv/__main__.py | 4 ++++ sparv/core/Snakefile | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/sparv/__main__.py b/sparv/__main__.py index 6b043008..8bfe3b3b 100644 --- a/sparv/__main__.py +++ b/sparv/__main__.py @@ -87,6 +87,10 @@ def __call__(self, parsed_args, **kwargs): except EOFError: # Cache placeholder created but not yet populated pass + # run-rule includes everything + if self.type == "annotate": + return [v for t in cache_data.values() for v in t] + return cache_data.get(self.type, []) diff --git a/sparv/core/Snakefile b/sparv/core/Snakefile index b57624a8..e24565d3 100644 --- a/sparv/core/Snakefile +++ b/sparv/core/Snakefile @@ -154,14 +154,14 @@ def update_autocompletion_cache(): import pickle # Collect data for cache - new_cache = dict.fromkeys(("import", "export", "install", "uninstall", "model")) + new_cache = dict.fromkeys(("export", "install", "uninstall", "model")) for target_type in new_cache: new_cache[target_type] = sorted( [t[0] for t in getattr(snake_storage, f"{target_type}_targets") if not t[0].startswith("custom.")] ) new_cache["annotate"] = [] - for target_type in ("named", "custom"): + for target_type in ("named", "custom", "import"): new_cache["annotate"].extend( [t[0] for t in getattr(snake_storage, f"{target_type}_targets") if not t[0].startswith("custom.")] ) From 67f2a9dcf577fabd96de0d6845fe6c2237addf54 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Mon, 6 Nov 2023 16:09:28 +0100 Subject: [PATCH 160/175] Fix crash when exporting scrambled XML without any text --- CHANGELOG.md | 1 + sparv/modules/xml_export/scrambled.py | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a68c1d2..c646dd07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,6 +72,7 @@ - Fixed bug where error messages were not getting written to the log file when the `--log debug` flag was used. - Fixed bug that prevented Stanza from using GPU. +- Fixed crash when exporting scrambled XML without any text. ## [5.1.0] - 2022-11-03 diff --git a/sparv/modules/xml_export/scrambled.py b/sparv/modules/xml_export/scrambled.py index df2d68f6..c2c4ba99 100644 --- a/sparv/modules/xml_export/scrambled.py +++ b/sparv/modules/xml_export/scrambled.py @@ -73,6 +73,14 @@ def scrambled(source_file: SourceFilename = SourceFilename(), # Reorder chunks new_span_positions = util.export.scramble_spans(span_positions, chunk.name, chunk_order) + # If the scrambled document contains no text, export a document containing just the root node and nothing else (we + # need to produce a file, and an empty file would be invalid XML). + # Alternatively, we could export the original (span_positions), but then any text outside the scramble_on chunks + # would be included, unscrambled, and we don't want to risk that. + if not new_span_positions: + logger.warning(f"{source_file!r} contains no text after scrambling (using the annotation {chunk.name!r})") + new_span_positions = [span_positions[0], span_positions[-1]] + # Construct XML string xmlstr = xml_utils.make_pretty_xml(new_span_positions, annotation_dict, export_names, token.name, word_annotation, fileid_annotation, include_empty_attributes, sparv_namespace, xml_namespaces) From 254b4d3dddac7f4d38b932c128647f1282bbb2c9 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Thu, 9 Nov 2023 11:37:43 +0100 Subject: [PATCH 161/175] Change to the new logo --- docs/docsify/_coverpage.md | 2 +- docs/docsify/index.html | 2 +- docs/images/sparv.png | Bin 59040 -> 0 bytes docs/images/sparv_detailed.png | Bin 0 -> 9243 bytes docs/md2pdf/make_pdf.sh | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 docs/images/sparv.png create mode 100644 docs/images/sparv_detailed.png diff --git a/docs/docsify/_coverpage.md b/docs/docsify/_coverpage.md index aff138c4..2d235052 100644 --- a/docs/docsify/_coverpage.md +++ b/docs/docsify/_coverpage.md @@ -1,4 +1,4 @@ - + # Sparv Pipeline Documentation diff --git a/docs/docsify/index.html b/docs/docsify/index.html index 87a2ac5c..c9fddc44 100644 --- a/docs/docsify/index.html +++ b/docs/docsify/index.html @@ -9,7 +9,7 @@ name="viewport" content="width=device-width, user-scalable=no, initial-scale=1.0, maximum-scale=1.0, minimum-scale=1.0" /> - + 98=JWr84xMfWOf0zrqKfIpuY zkAk0wu1kkP5FNjZnT4N)p@9n4(_PZu(bK_6GRWNv{2qcXY6N-NW8IwmE;u+jyLhOf zRvKDR7hD|GQ0DT6(uQ7IPPbjIhWI#{hFmklhPYvs9Z?$UOc#SxzyR(}e)boF-0yh! zssyQ_{*9{w{!aU`6zamiNBrE>P*;MS+`U{bh+VKSG`^ta>Em=kUQ$6)8l}#3;i8Wt zPQ?V){@+o+ztm8-{rtRCq@)4^10@5|lAb=!QZmZQ%2Lv@QnIoV;0X!eU=Kh0APEm& zKH3ofGY0JBi}i8w@^kU@xIi1z-oew~PYs2VqJ2c_f4>6eE%pB=4*v1~m+s^N*6RZn z7G^du1wqq_y6|PQpwXpo!P#S&G{t@2o~8+s_|gSc#)`rVmRHSJcpWJhZW0>k3$9+B zi=h@wuFX;Z`#Td%>A0EwMVIqZA7cR?clYoWmXQ!c2tjw*+ujs*eBd{@HSQY`X@(g% zM#s@_=^r^>nO=ED<(uwSZTa%ny4qH5C=u+4ow?P1%Szt7kn_Fo2_b*EBqjWBr=jUKn1i8>)Lzqt`j_^<}xlke+`+4a%`{Y}cPZcLG z9}2;X=EDe()q1_C20h9sve zMEfrHx@4;7%cw7=PR&>J^qdadS`okXs*}K|JbQcOrBmF~V~oI|p@x=OZ#OrwgXO@M z&~7Mrxo&oO>$9m5_hk3Oq#o{H4i|^u@vCqLapVVSCl)z*4t6|*PQ|nPuD*7~!6Zj2 zJ^eG1C?c7_nBJ1t^xr-PlZdx~@%U(BeB@n5U77Gx2U)f|Zruq6X?UVTf+Mb1Iavsv zlELk#DQ)>4{}G`#5GrZW_}>rxQ<7sfYa;72-KmpF8snp*qwI2YZ#f`{GOJcmQ-iS) zgNKF&xzpYUW#eRpZe`}eI8S*p=i|8`j2czO`TcJCA?*ACNi_MUvQwH?-o}LO6^o;X zM|pS7d_2f?S#7AYR=-wsVU)))kmW7a;CdR<_Fq=-<>6~nJ@0pIZZx^KXtCC9&188- zoWIP;E=~v8jeQ%aXxhBBzggQN!L$GS!A2p^-rUC_uA!l!^HQjl0;G|Bav(ftDnY>} z&auy09T;8@%q1yc5*x-8KVshHGbd`b>uH!qGM+@(;}8fAoGdC%h9gT2)w5@ny?Bzm z{CKXXqM}0R=)RNh40Tl~+v{FUP0bmEh)wLjjY-qHb?cS|rmU$cyd-YwGl@hmWRUS& z^5LqV0I3FfAWCYsh6SGtE3;ZG-YyDuJ;%=Fl>JTY<6YB_8X9P_@5488MU}Xu_)hEG z+l3*Kivvb#{vF=}C#;eG?OIgX-F<$mMh_v204s$=Q3lJBjg8w9o?YjIr;=BEVCN2` z8>Z$1_2Jrh%L!CGi_|TfB@3pcGz)(&$vxA*sGmoVb|d)^eS6fgUpc!X%C*5CpGYJz zQ<29{iF*}IaMpMK8(7;qgPC6A%a<=f&q>#W$Y;)K-Y;wJ3ZBqNvn0XVAuio~np4g5 zl}mAWslcRk(H0}sWJ$#Kb;o^gXPu5c`WCOZ_l}%TGdQddK~abX9hTCdjR(mt&N&+o zErh8rkrRcm=@rLF!Gh9T*%cBm9&i~aM)NWe>F)_P6!^%aPtJU8uk#;Yq9j^dw*Q+|lqY6h~BAuNpI(lyGqjqrTiP=uVBho3@k&OS_>MW~KzcFOVBOduSYS}0Hf0oxY z{Q2{gPhGZk(2tmDk?Vm0TJLOUv45*bJ*Q^02&G(tGVRuNyyO% z&%!eMj5o{xGr`xx&5jjcKJv_!F?CYRF=Ry-L%6c+hngVZhW!HqHa%^lMgL6&x;}rq zeJwdoC@;)s=F@$)ePg`03~`XX$t(w^?|D;1BB6f5(M500uj>l=_$ugDj8r*&H2QGY z$J?#E+|7-A{L}S^cE=!SQqed!f^vqzu*Xp{0hh;~9GLd{sHQ(hz-d|c?`Fz~Z$EN& zmyd^sPc;8}{dM0sB0TMhnO!E=zwqJrv~fw+Wp}}x4{=>GT~+#HqjZKmx$_uFk2kvW zO6j#Wv1*f(rf_tQ?+I*}UWO1YeLyHo zVq?req#viPL)5_KtSmc1UO}N*wRm15{;D_E7PVAyNr@txrFIdWD?K#J-9>NrzxXTfnL?WGX?ow+yXXZ0n(rluN?}CG%X^Jy4(yX-*%MGyDhDihL|9yL?`W2J42`u8=Op;|k~gq{}{ z(I_*5YHxSUg7%dJT*CkxG9*L%-%K;xV5?8`+S7F zx8uJkEunt3N)i;J0kUpG;s@VWA{1|!o3}`KFFz3P|1vy$TF*`7`lBd43Jl9=8#YGv z_UUkM>?WR5+`qFtR=2;uyP~7jYiDf-?2an%*(eTJ_~; zoq+T6_jfOHsAh4Kq=j=kX)ULX&ta_B!^A)%!2z zF=mynpr5CLm1Tmxv2F3NF8w&IF4^;{X{+oMHKhMc9`ss`8XUUVN~ zHlOvkF$H@bwHnHn0X}Bg5)s~sF+b3v)E$VPVA7MyDQA{Gk3?tTFQySz9~*wHJxG=o z*suTLasqrSQ$F-H?Tyk3(vE?MnRc`uGKtMXMw@d+>v+$e7~&=hKBlO0W%5P(`LX@6w6X^`Gvg0~vSFh|=hMJlly*UiRny}duv9y|+gDK*Ht;^22e2(u$ zB&V~4;;qd;ek&>IpFWWpKAmKRq-`{(pWlqw+u6vkAT7wwjzw;4Y~-69tMU!vN39j8 z93*;%O?rDq3}lV}-eutV8$3UG&#P|sGy+slRU>5QUH`?QnEK32ePW%Op{ZXaeWQtz zgWxQe@~m+2`~vlvZe)Y^B!!z+hA4jX{@oJ~?%>$FsGAN^zC>r4LH9IjeSu@!kdNwb(Wy=WQ_cEDp+>l*j7|;zz`#fb^CKX@*o}!)mOXz#+9?e>=$12N0ju|q`8b#3E3WzeuUa0z z?U%;j^~#Tt31WXZ?kFCHq>pyzs(qy$Rdi!r?Fi0n44(BxS+ij|8C)eI4h|@kIRhqr}_)_u=D^ zG>)=%lZ>nD&-{=?7#YT-#ghY&vvjfdwsPgP9g=4)rglqmB6ixQKc9va%=*KNfACueB8(?OhXKW&4ru9Vu_7ScE~R}gRG z;$BM4QZ#y~vHxEaQzkM9WH-}D@+B?nZ?Y&?$rDV~f4mcZL0V@-2_U!>hd6J&b}wP0 zOx(0IPcL=VgpVN%GcWl)<>0zyuDoe)P?NW04|dr=pclCUYE0r(^; zGb04r41RL`y0seC{;A<(URiVF1(AzYQC)D+0S?E+%Z@&{2}e+RHcS}L<*h94j^qT6 zZ~uu}>iolG{Tmqttai0-y@5np=svc-!7p26EPNR8HZwTNok!N+gRM5@5gCrCF#SgjQHYqjViSsFH~cR<>gY>gWBtVO;)J}6(?|*H0l!D zc%1}l?L{{?w}yClGhYE7lwrpC)QO_UrF!9{l{e817xm8o%fX|mGnhY9K}-{tijZtlPVkIXb0$kNu>AF6F` z2f#|FCja&3yS^p26O+%Lo+X~yMbIaNy?KUUy|1;8CXwP8Lsw#mwtiH7g}{J_#DEd5 z^Rx)!x5)FVyu@JJRx)+F7I~SRO&RdE&qbUF-bf#9|713D4;AtCSsdqo_8nprN{()8@ zNZB@D1R0iRJN7>74lg>gpt4emeY5m%6v7eL7usv^XYa+=T8%1D7wFIzx6zzO+rCuX zKO*l3;dM&qpXTOfpIJrPCoC!)aSG%^Me{8P1jgKaxG*--Cj z)Fq6YTs6HTbQt2h)-AVcgTL)G`>S@M0eoPqIw|~4$)5P?6bL6OjA;)8%3oCM%OS#b zF}j$(XY>$X57jI#)K~pXj%rbD`J&NuW8LB(`r%rhB{u@v&+wL#7{OXLeW*!$K7w zF`%5L?wb$4b|03y{fLD~O-^ldsQ;ZCU)Eu~hHRJ4dKA5os|s|k=aQ`<9LgzD^7 z?B7(St+Iy|6>c{;x?(E5t|@_vv9*)7^qb9L!OuTVpF*w=C(M0EYT9x2GD7bv4;;C1 zr&iR;M5{a9ZKh(I<(i6XYqyeKeH2lpL1ezlNV|%%vhBxwJ3jd26CLP7t&GvSDAa8B zdQbAHUX{pB<=PwF6)Ou13*wPJZf*ckxW!j)e@<9rf}}5IW@UX_oBlW{Qj(jgY}fxR z;M6vpZ~Vz2i4}YCXp}umYR}0$r|u;g#CCE}B{wkK)5MlvIa%ZBy*da)LOUe=__G1wYRpLJ3W2DL4Z8s=5IVQkc*pv` ze%*z)*Z)DSPXJW@9N)J@QXwc-#B9c=Zh(6|khty68~qs1VFmVwbn)gb#rYgxe(DK~ zsnE+op{_&|;pcTVC9`k8Me7!+J->zmxtdvOKggFHhTUGv0CrTGDeqcpm5UZeSr{FI z?!Op|9+>Fk#`2#UnSpPReHA@MUQUnGyz4Zb2fJ| zFLZ1dZ*d6n-r_Us?gL&z!Po7Sej9OV(V@IDi5`WK|-O+`FxdWxd9RRwp>3w71d zqEEVpcU7j>zxZ?QKv$&(gCQGComEg(dXG;9PCjwSsrpQZz<6pVBFo)AxpMnXKzFfW zR<2n1_*2CnG?gHwBVh>3N2xuCbZjd>EJd|@nhc`4|bkNSiXLu zQRa8{VIO;C)<5$e6(dZw^+$~xn{Q1R{Mp*pe5-eoUJgui+yo;JoOawaVO6;!?1gge zb26FSXvi*g^ChgUpjp|7)UE8!S0=NHL#BsX&zq93 z;+9h3XrkE$9ir}HW_Wo?$?UO$KILzTw0*hyA!owhw79Xou_gJ+_}v~)=u7#fva?s7 zefi<8{XR@6D_mZ_p&1;?E2x6)_1Rr{w;|^3=G1?u({XcYB;s4WKgbm`a@U)>GLjN` zpRS*SHu1zOxPb9c2bAoL#GRqd&*a|mpkB4D6Zjpm$Avz7{Isxefe7mJfAE1sBKc4k zGqzcH5Bo*Xx8|CxG?kK+lWS7=ZNGuTLs{do2^eu&F1OutZ9l39c#Ehe=ClnA#(!b& z7RwJ~S`ZCAC)%~7b93dO#ZpzJ{)6%zxxgSk+M#*5_9yVy=Q5V5u@KBRo_r2y^D(X9 z<3d=EjC^Y<{qtijcE{#LmluB_7n~t<7p>Dnc=uKQ0^nLAAemgumUDB7jwh%y&2j{q7Iy zEXfWbvv>;6%8N1?tlqx=Gmr(E92Ds8o^BHLy|%H)t6k(^t~3Ou{#10?g5m%(BuwXV z8cjpek}vpEj;Bi{fs@?f%2hT5q+OWM+xc_ zLs;(QS0=hvc3GSPF5*UGq`9;6ET37n{CVH?9~n#xQQoIT^|`-4xxN?c>*1OpL$k2) zb@E)(X@twl(o)#7nwr7HcH@zQ&oor`d3aceo15Fk8Z&s{Poz2rE`33KqGN@$+NlMc z2*HrCMvbzD^2IKb{j;fWIe?Loin#8~X;l@KegK@42wQpI`i36>e*&d42F^C(P#`~9 zuBd2{lb28I67d3O0ckE-H^uR~^d#fQv|%~V@kVcXt$QHcVS9#K_Wiei zutg$CZmzhv5%c3QnvMvANHBi}F~n)j3lCRur~+Dlf-=yXGy^_q+Pqm)+0}m9_D*8m z*w;h8C8YClw9#2fXTwwQ?LotLADj=Fk)_r<61M-GnktqLcSQv(8G%DkY`AVWjLX}|5fBG)2baKfa& zBe5rvbqj9kz_&&nZjWhJ9gfmtORH#VQX4YESo8PfSPP$!KEevU>&h6xZy!b#{KTE0 z#q^WPu{E`|i=g>8ewBKVTwE5%NRfw-9(PE+2t?u*FC)D)WCjR9K|z%M?#^hA?}Jj& z;#N94B_jTfoZ)?f9|z(E!1Gq=;-Cn6YNAy{J42wJNLO0v?VeqyGyM*V!;G8w+bG2K zMWx6$SHQ&LxEe;ke%*L(wcJf#+=>7>E?fM?{VX(`J_TERrcZ_)I)k zlnC&q>I(uW8Ex6Yx_?A^@nAV0&9SGWKlq&f??_LbFTm)uX~E>jq_gpj>rd~q^&k*6 zuU;j5l{p52t*9&26-*4UIYL#Pk*8^vD2yU$+Y1^F7_lcO&PqIx_CE}zZA#zquNzCc zI`s`OpP;C06}E$6)LPr&f>edgO?89GKeu&&V?H%eLiJN9lx#wyYIKwr~OA)gs2I}^n)S0$1Fz0uJ0Rd6?FHr@V5b9IVtIEZ+A3GFNA$)-n8WWUgJXe zHOEM?3b!_;qflWivHgz1qC=+S^62Ydae(uER#?c#ZA32*UAmGm48ue%w(x8s&xzR` zjiZqCEot%zrX%t*m7k}5HbCf|Um78+0b8~TznYlp>M7?kA(wSeSGdQjGBOAx$pqHdysyh6;0tB4~ZVR9@NL4?`6dkImtZ3H!w}1d-eWUL93r-Jdm~+hyWE8%Xc~&dGmTTuE1>U zeMDpf^t7LN^8`W&C6X0hO0EqYw8%K612{PCqQ#1ailF|7zB-(JrytLm$%cMb?!cY$ zj)9)0$dFAji83MBeH(wn?umynF5VY6=r3I{j8DudS&T@_!wV7ioXAz+S%gCivEk9p)WHQBC{VQLoP&iduczIMc6cj;E4C>Khcz`ZM(} z?$NiF$_6f5>A(vaUvqp-nRpM=keQ;<5Y+)bg&`NzlROqai_o?^{HM39_?_(MMg!2+Yu%O;OOr2EmhYC&e5<*QGpag&VS) zlIN_OhO2qg%M+_ulbSjv9YBmQnu%HzCH!j9E$BV$VvwA^XV1xuj*owD?vYHK1P!+>nT0dcwbUaThAinV2);IlrYJJ-YB|Ab3`c<<(5=W*3ml(%)djW@@9k_0&7&CLd z|54&yE+`fQc9538od`k9>u*e8#0B8-T# z)1M4d#+*nxk4b46YIM*SecY*3es5k%&wW<)|DJOiUF^H@+#(-#cde#!X6E} z3=W!_7A*JQ{C@ij_)H*gD!+QPmkr>B-L!oo_05YEjz@4q(#(_0px3LL%KF8#C6BX@K#HRce7P8B@1U?apqXD2y1VH#ZLJi$9@^Rp z%68INmG5slhF3u=6DKq;y<>%1jic)w&`B?SZAlds5uqYxtcyYMZ@HmZzXTUIw*f23 zKr^#FGZ0Tu86seh&pyXdsHxoR(|`K!o_MFQOv}NKT-@Ek{t2zQ%#QKOXdB$6hnSsS zio0K)%FeO@#kp>1Xy_eR*R!v^Pp1J2f&?qxY^udgBV5WZdDUI)aoUTQ$PfZ zqV3y6vJ~8Rur5p7Yo@`*%1{*V^@Ym}OFV#o2yIZ<^zE9YREZG?Y(4~y(p9j(&hX5PK%fvHS7O1sK zNB@aj^jwbWJG7Kl&}w^q(CSZK?j)Hn0K8?&I>jLn)0l%JI*NdU5l1Am-&0vRsA+EA z#{-{Xp0{ZO30L~|DM_n2^;Lq=Z7qNQLVRP1*^BpmiGVG9PzY!PMGHpTXS^O$m%$0V zY-(-o4#Ry*@i1SiN{&QIRXs zlXFtj81ZjuQ`N*X@j*VzEs{bU7=g3*ob+fD5Y`*iLy#l;kjILEp{_p`*B4p~nmt=t zL8||6L97Cq!Wmrk^gOgV^0Phu8!d85rf21?M))^Z8r?k1RspJvcv&=|(=?SjzA-R9 z-f{bp(bU^kHfZ+_fLRK@GX_Su11_x#4%)z>?fxW}y0(SQ9Y@*b^WOfqs&Jq2;MrhQ z|7>3HO!#=+fEFVn=Q|iw?xt)*SU!7GC&MDCT;mgHa_H}}u`wCM07(CkH*Bp-Xm?N0 zuwDxuCrHEs0$9xnhfCHl)Eh8W7>n4o{lGIZ`#sL>I8-_OcYB>1F;AB&7~ZKIXp&hB z0yb2bS4vWu;j=YJsbo)P9eS)J`UI+ynq=s`IGar+8>t5i5gHA#au=%(L1stg;{k(w zlhIeT*1e$&BtKAtcZBObfAbvr9rEh+Yo?RKH*8u_o@_2E&pH6~XnktVphU{dlu!en zz3J)cIWfx@B;#}hs&u(Ed!+PP;;Ds#^i- zk8>tZVlcp;j{)o{s1Nd(v3;&--bcfR-)+Y?HkRE9qV&+xMn>|-X;!;}(9WgXXe}Wn z=u!-toDsBw>Kn?<&2=eS9oIYfihXkiYCV6h>a|aP;Bp3%Z|DYa{1_4)SuFY~;p)3r zJ3GGktLfr5Uk*>I(hP1s*-j;**}^uUlgi$0nOa8xA= z?B53Vdjv+@MbK78Vlb!zW8uGQG*QjFvw&%^0xA~+c=P9aukAw0)KxPv089lD-Y$&V z@QO$0^tEQN`YqSSjzl`h=6G#Qjf)m`lK0u@Ez%8#B?v+mVw*|Yh%R_WAHNfS9Gs)7 zm4DT3r<`HdQ8O_a7La1K3HWICs~8P0${A-a&}2Ndcdzb*swm?RoiCK;E{BiJFVWly)g^9c>>4%U*piOi2PT?pBR! z(ra$bX)wEAZwFi#7g(#=BV2F~>?ddsaf;Rn=kbON&y0`T+UoPqwosa%ytxdNA8D#@ zZ~9mJF0;%XW&mu95(SxaRUs~02lm^$cce0y?GmU{ZM^T?26esC(iiU}8J_FnHyi-j zkCb{WjLY_gKM#xym|E`qE?;&dEP(WHu>Cm4LRefActq=e{4j?Dzr1Sr=uPLQq7$OD zuKDIsV7vLUlH8CgVod=jNYH=B+01H`xW|VTQ}MUox*0qG)Mo3o4}%}NMwT?C?xEj; zjZo%8VbveZ)ik`j9s@;L^C(CtI``21Ez$U<59{R9MhTnM!{Hnv3fSAeF2C8^EYaKU zvzZqJ+96oV|K6^~!M1~xj~t0++4EUMrgeMHmm?xYq3vShr@h`mQTuNTU%W6}Tv%ZJ zgglTeb#Mn-phqMU{t*@c(Y=}c2S_k=8_Y+i^kWDjCFrhx3YO-aQ5*_r6S6C+fUUxM zH*Qhz_atjbjQ^82F3_a| z;`b!(fG@l4yR|Xw#rZ|UE~pi#xCrQ8<&F;IL9SWinbb2t zaK!1~Bj92?VT;&M45cO79-3wjcHExL&EiElN3$moxhsG&=}(Cs<|NLC>M4ZU0YaMx zAQec}`puj0p+rj+2#kGh+4RSOQ8Iiz1yCxh?>VcJ0PFd@sHh#0tN^rHsZ?A6h%iuP zR3g4}2}TvXeyu=b0dh)Db;M*8AHTw1KFtPnILyoms5`h5*ozNsrl+`{e_}v(80Eo^ z&e`6S%V$v$PzCMw-Y1}>qmL7G)(MOup`># z)FzZ?&E$WQz;L-$>BVifl*u(HLA4Pm3XQF;)pRiv`Y?YH3+ReMHtT{v$LwJ^>Eaq= zaT^__;mMOzyr|7_qtnTp{c`PfAW>BbB@jBzCN9CR8YUyX(R4^>2k&ydc;2vQa>7r& z86aqr87 zd$@GqKLx(CLCIQGfxtVLjwoPE$nOYvpft7#U<364QfK^Ne@hH;z09S9aey|H2XORY zf?PD`HRY|by@0HX-b9Gfn<|L7>F>Xh{5+%u!5ziN1Vijmx0zRaGI5!EqoE;OU1jmAs z9{R#?cCvxc`A}{1OU|-T=?y3$wZuU5+I^l|nMXBF&?faeL{CLnf(Sn6L znuA$x$>{MsvkwWbLgglc~Ko7Mp3ZNvTpHi~30)}q-!;V17 z&Iy_URuDA+F4KtD1q%=qN!ZxokukhJm&o$nNQ-W7CY_tAt|0 zRc%alH#=A$69OSWX?J6bkb3RInAbZNfFkXf#XQlDFxpEVTnl*Qosev{Aq=@z(^_~tDTH)+C;`MX45KP&O`Kk!>1 zAu`*L%rLWD#1UJBhb)90zIWt!1|HvC9K2HZiZ)NX-!eS<@RYXzL@CCU-*B0Cd71zB zbq5A4_U*&RIBiWoaoR3oHxsYHqqK_wGL$kI?K@}?`m}soWcWChftuAWL6m3m_V(X1 zntpFiW(62mgK&3Q0ObfjBQbywmT-L<%hgqI~AVP zhz5Ke#C${$!$dlR0z4@))zb%+qYlvb*|rnUE5U&IW}_EIh&S(Lv9uyhZEVl%?(|$p z)~cp0A~qYJB!iqhdRFF!pp#SFuK2&TL zYb)~lc#G_}YHS=kH+~NkBzAA4$tRhP0OzbOI0*AfK6>iyK5d{o2{%*OX$Jt=*U~IY zsW&Nr0d#?4g)A~%7Mf_LWoh3?bIuJ8l2gT0N7774Q@`U)#)2S9)c!`Au8$xZ8&3QA z>C~TV<(fNx*Mq@VWb9gZK7>5@w=OE+4Sc4ORcXtM0?R{U>3bL${voL;0#lZvCo9dx zzh6$qmsT|IE1^{-Vx#`OC~lS(O?+QByW@EkGOp8`y`NPyznks5du8w{O{LfRTN;JJ zMx@{wbyNU6_f|F!625(HnN zXm-^2FnAy#A)^a_9M$x)B&l zd?B-xnVA`%cRi7DopiS@wjlDszG^1~9S9@(b|(%Z?ie027MY>uET#h@$IZ8{Zfb0| zUz3*!(CL85vY0>EC*bG6g&|O8mOf@^%kpTvUUD#K`^ytPctO%uvIqG{8;MR$yY`K@ zEkFx8x!NpYc-L+J>3zw6=t6kB1(;XV!vqG#OIwS`q3KjwZR=AvW_x^nctB1Z zBLBp4qxfqJzkV4K7|%VUPv8jC^+nt2@?V=$anLRWk*6<96wokNGitXsH)$05Lp*c0 zDpc572XrMBR?QdxKCj;q8?S581q_z>y)7ZI$~!gD#|VnN##DpbLcMA50f z6^pqIE0T|g%xSiI7IXachm+=32$~kgsGi?N_yiN+K%pqTRv5nX`YwJhS?BzZoP7{q zhKHLsE$aXbGU#tf3AEL{gZS2-1%9!A z&?QomjfxEZySkzZcqZ)o6CjNzgpf$%%J#|bmLVa{n(g=SX$M@2vQhR!fMYbb4fk)H z%?(vwPJMQs=55B84YK)!*7#5bpl`V3chL6gTBsW{fXd5m6w;RUe?UdXx4~*a0L!q@J>JHd(Z(L=pseyp&Sq*$n}PhkdTNg z4Sx62K>(6|{~+qlf!EHh~sqjYU2iH2G4fmZ9LejTfHbZngPOY@}dB@lgvir-EE=1jZwe#P%kI<|LX|3Ogol(`f0d@CTG6-Ix=}Z-PW>h4Wk7vd$h8P&Ri{M zTzCM#fAtrX_E8@$@E+EKNAdXOxnJSkL=V6$F|!DwdZvALkM~J+FA1Zy02qcu?-8%8 zH0ck#+Dd7UcRMXl02f{_(8icP(moBe;kQ9ZS!WpY%2jY%yXTa+v2G}M=XkBO$Xpi zG%AkqTwi7Ge;;kHg@3=xz%<=)=}^i-7cRhV z^$%cxqL_5kNHSU7Do!517M8360JXL~SZ@A6t^V@QHM%9D8~z!&Ce8d+&4wql>aXYI zYoCx4FL<-hVcc>$&t(v)2@VO-Gywjz-$&rITEL~u=JhA4%R$9|R@F!;-(wkmpIu2d z@i+PCNg_K_m6ZN$O$HUhn^8We#m&I&+7AnD6%2Hzl+z%ub&vy(ih2hlAFe1@W3;ZCW zJuvPL3jPV6Yr9)Y0dfo$Juj#}CCafgT z|6ZB5z}9c$PxcL_#|EKaSZrd?2_`gL(uK{K(mZa3?=dX8xkZkq7i$O<1d~MROBzF8U9O0=YfJp90i(dVH7FGAX=#uWhxwHGP+1oTqN6r+f33XE+ z6R*I(n}Lh0YMZ4A=-%K&a7X@Ktz7hDGl+R-(GhQHS=3mz4Jf)Jmp2C5tWIho-$Zb9 zNPx`x#*ucOFnjI3QjO9ZxuMIt^ZsUH1nF)D>=&>fYKW1z*RPdc)YreRc)h7}c9Knw zLTYYSPi%kLqD4E9-si>R6II8hnm|xzO1BOeS?qaGYSCuFUnC?^URI`j4ZnW{qajFx zZM2^215prjP)-QJ!3iGkKYc3z4E&0pZrl#>Y=y8#(n*i?U^8>`+KlcH;v6Swp4*`jpUPb_Xq$LlZ}%oJcm6c^KW(5blGXvPn6d)?pHU)3DF1u7)xFoFvw$iP9sj`1J00hudks?Exm)nKi zJv>fZdEK?~AH=MJJ6DfE?1-&jSV+6>&zF^xJF}jQXQtJM-!Uddq~MjCu;c_7z!apV zmzDn(Udl|L^!O*xK3J*eVeAaOfw?>|%G*TV9C!`r#TI#tmKmXu+ePzl?}y$zoML&fu+5sQ_g@x%4t-LJ*ZoBvPM`J-+|qK5(1OWD*64Im9U^bDEmcSp#Z7`izw`c>U@X zo@4_0TnF*5>+4mzaO9P^m^NkH<}gm7J%h~)bpp$eE<|FVIhHH;rets7(-M4xHS4G<&O9eVi1|>4-c0fg0ZXwBYOBYqdu8S)Mx2rYzZPLLU?h{43 zHBa<13io;UDEIgC znSyxt$K(nJi!vBUE0~N6Ae~J)#`)yQa*Njb%6F4@i^1Qle7ohxMn<~CZan`|CM$x9 zXl&uaX#1V8{%sr0e0#Q(zO5b~%g3lm+sas)3{I6JBl)1en zhJgTL7gs08(!2Tqiij27i|LlLPO} z&2Q7Q*tc^iMtu3TSn%)akl*%ndW+nkwe3N`4Qo5x3Yv`0AR43HXi zduS)jg<606#v}Tz#J_Z3?XW(wG?LWt_bw>@&j0yI?(KO9^T-U6bFSWq|29@kLeTP# z-5*)BVK(Qrlo24(QB242P8#opGCpZF)PHi-W8S3Ucaad1Xk$C^DZQA-j9h0kg%p+l z@jbzVR)zrFf};HWYM>Ch2DGmM%)zCtPCT`a`@yqanSRqMK`>!aooy>G)Q$%2juTH@Y5==o1Uce0qMB zkmyV5xorRhflhYd@!7s*@vEP~9gqnft;S}Kw?2O#0X6hTAaUI(PaPnYPaudIPjNhQ z1%Zs{b_ai1_SZ+j$tztlyLr(C{MQOQAh{}wnO8hMF4buKU}vIT{RwvX+F%z@Q{SF^ z<-#dX2{cVOebQ{=7c7_|T1GmGNkugZf^UneS#S`D)gK)tg-`%kaX)cDi9W$n<%g~K z8glXL!z?U9O~%66M(~PIvf}aUk$kN6^PlOk6tj@(0S{xtG!&-2|) zaF(A}U8-BmUmYAA^bDk)K03%2n#_r>3ZS{p#aTVc5N*Qv2+%!Rh%C-~h0`;hFU$jt zMHSRblhP^jxwE*>!=J6*!GY3!&M|Y*02%|GCqGKeKVUKtL~+^&Ng*r5D;0~se(ho1 z+!i0LSk{S|iR+Z4%TF2r*#pb2KoSxnO_6@dV;|4wxu%A_0FjTYsc(pk76F zrO>_t1i^>04PQlX@BEp^#Dwin%22jck`+tOxU8CSwU2_b47wo7J(1BBByekR@Y~+_ zk5Tq;^DoSyQ(b^RbQ~mMe%w1QX^biW$V^L#vIP$&%>*t5)#BNS*KY~Kq7^xh`me`> zQ6PR)0gQtWwBXNueKM#RA+rW;?K*^x%Oyg6ettg1;i3jb$rgSnI9hgw!8rhM0IeTO zZ&LXQ=s^kVrY2|W@QuN+zALBauZBsLTCf%ZR^EJEuypL38mMWqHl!N+~L}sTEsEOYA*DYp=$p_NIfP zR_$FawPMw1?O9t;R8d6jQ58F4K36{9-}5~E%W?D|x$pbBuj?G|_v>WRS-Drd&abqH z0sYRMZy};ovo|Co!nS@V*N}6E^#Kd}*!cJ_(`P{NqMuWowknO5oA>7*4_4IR@X_I7T(9v1~13d%W)J%7DCMCZn zb&?b*k}2IrR3Ay5```rN{=$BUmg{O5!Kz5XtJwp5d6rKc_5xfE_csEh zRYqs@HO@wdWR4HpV;E@**Sism4-u!`yp|8#iPx!*riZ0PJ*e``l9qNa6hAJ>T{`}* z=fkD7DKt|G1JdK-@}g&0eEW5{&2sRAoSNI$LdCZVL4r>FsrH%%E-t5I24?EC zCLVDy`8d~rvg?h@^mfwoWRqS~PJXk2__e#b`SK;uJSYog`kWMZN|P$L`y zN-6M+Nd{6!TGpp<;)mvPBhl$rDE<*IqJ@U9VWU#Jg5{`RcAV~vanFTT@5p6*VnUqb z?UxBXn`^L8N=~%pDM+t?473Y_Km-dfdjpB4jM16+>m^DDJ3Cjj3ecp*h1>Qea=D}m zTeOD_Y%ExNrEA+Y+^^!qq;3)yh_*UXPBdd5u~!ZbN;5}b2EKI7hAdgm0>l;0e~x;A z3b)2Eiq;{DnPw$M4|$D+6J`W99`LE=)i4yhs6U^S?~WAFH?By})-ATFk*9gS6VIj! z+E>1()^c)k>Q?FleX031)Wh(*!*M{3+%-a&`R|?A;?fiG!#!(kL4l@FFla(CA{la8 zcvF$~AHnxXy71gwrlDecx>?vn78U#enwZa(_RLz%8@%|Bp9?{ukY0veJ85a@mK<#$ z=jkx-1&j}G&>dcz{qg9${W>ldU5e6|L*Ld@R|PACxULHI(DYT4C9pA^$H88}XP{O| z@>fzQR!O^tgS)wc)Y$@T?#JUlfARSp^m4eU%GC~9mXNd9vUDkRexIl?(RvQgy2}SX zKgt^lMbon>pEV@9CqzwYTvA!vj@~-nSj~f~nwSI{XZLhlMWu+Z#6$?{ zkg>cYqdv$kS;a>KI8%kYMk78k-|mCn`C}+YGCWKx5LEPCecoWKtJ()o_kWcdi6wfq z+}%f}410XXaz%&9p@jNLeC_~`-*o1jmExK6$LDhX;fB-!dW`!n3kJw3g`s{4 z?lG9CYoW4<(AX5?PMH8MqQ#F*8HzSE&&ht*)5oeDRQo@Mi9^1|sp*>Tvw;tHmfyUa zB%bfa=8#W_MODP-Jp1R*0~Q{f-ZTb^4C=|fs-YmO-{anlu<=6fq_YCF{pwbkt&{=B zfD#fch1q_;;F(gQ{m(4&I)5D^$QM5|0Il>;G&5sy9i-8tmC8wI#0MMr)warf2A6_L zd$y-MueARS=w|G>`N&6A2xp3ZlF#(jvNywfcK91#+=PvT9jIE1GepNz^~an^8tytW zTIDko3e-)JO0V?B6Y$L9fBwAD{JXJ6Fy_+hBZ?1!I*rr7#P|76I`};+VfJ}7(ZAuV zKWeSImrx2jI#QWM_JyCt=!xeekPbn?*&2+v4HP~)+IMX~zs|#C!)K}Eyd0-ff9WF! zs8`mHkB%-v*z(q#OFor6LZjot#S?mKqry3d%aQxD@s0<3gIMfj7Kovz-TEI6rF@H} zf{yg_w>(N7H)K!4_(3dif%LM6%OT{{WjoaDjkzaYx%0#NnFFXmE?dhOP`U3?$Y<4A zk?yA4R2lVn2vKGe-nb!L!gVMKB`9&dq=bx(jWhS79}0H$t9ibTZpB|dK9k{eQR0Vg zWabtgy{4|~j5MZ}cb=?&-XF7bsS~{AK7Z0S8+2!Xyf741&z|`)x@!c@0jJS`_-%rsW4lh8Gi-&k}X?2Z4C&yq8j@oW=tceL!8Oo7o^ky>OGEzCK9zY!PIp zM2AL)GGcJd33RgX_iduZf zxOn&vY$fu`Di90Uac9ENBiMen%zn{}?M_;Q=}#MEb}(HLIkc%^t&OVp#mH<4(2Yt%}U4X%lDn7Sh%P&9$C zm>d%`B)#`T&L|ka^<7iDLXyZPsCJqXo#_{QO9^2P+#B{dKgIuz0OYXykr=TtML@ui ztgqip0~_mSKzy@_kzP){R5`iffuunO#-vrzWe`mUxC&*WF{yV5SN|6aaCOF;wF@!t z{<6_k3K*bufRA%+f8YHYdLUmiA&rYju~M+G8tl*a92JgM@!%+T&h&l~$>)w5*Wbgm$;Z19t7G4;1DY>=>ifi$Twbo|_xf>Yy(j z^yQ?t%4X<*`RGU3-pTF|vlF=y=K2xU_gZUM zqTaD+3Gf(fb^Y#zcRE1ZKRoO37eZ1kXGgjhKN?0~;F!OG*TXlzRR!{G=C#!Exb;NJ z1s_8SkhIZw@U!QvW?IF#8m=7t%StjWy|-uXJL5Utt0=Pr$pY5X^sYlQRPt?x=7@K& zgzxqRxtG(@uCY8~7YvHUG%HSSWDAG*u^e#>enSe%MT14VJ zBk;4cmCsdaAxUOGmWno?xOCiA`9F$PMSlLWlFmQNWopYPg$Z`Bk`*8#*Qq(wwpc=f zH`mt)=X}qqdrY2v`pX^0`}EeKR4EXe3SaxAk5rC$R1}n&Rm?Qsm9nEJ78#zHU!N2n z#EIdN2$7PFZ&b#}$fB;tvr0T3xaw6DOAt^-+(rWBD%=e*oBg8o0z+^BabIsSFRZT* z(*>bZC1Cwji>73_V2VW)K)V=3YqzeUgJ$C**#y~KxT+K-Rs~{y|M6~O!vVQ$y7PA7V~9?60N6-laB4Es)$ut-1L-f zYrQfwM^JQZhMB!zT|7^KP)$wXiYQt8`}Y{&RA$dsc4&?L*761=49In7{P*iWoW*!= z;7=BN@oa;PjH)(6R^pcc)vrl8HL3F=TGZhq9Zo{wB68s1k7VK!0q}N zp=V}xuAzH+ujuSy^&?622g z1pb3w5~gGn)10i9`UR*^KBS3DWSlAq8)dTgW;i)&dg8+zAfHAgX0CKk^9+`{v6M_r zjPACIlPB=|OaEGObC33QQqop(FH1(V4_9c3%x<-b?>DqL{ zl{BSJ_j-ZGZ{nUFd<}tppU;()Za#hg1e&O#J2NOj1<|Pr>)r5UUU)Y!_3bntxOx(j z2nqHFbF9ij6;HG7Me5Eh39R)^T_Q3yhNxEbGkWl1F4&X)S1s8f*XViiPfS9^bo3EG zL16$fa_lccB zDcVg!Z?>oX<}dH+JZz+%d^{MKo11I(5TXt2ZpI?VpTmU{MJ4!2e^*qmSXb!ut7&u) zET6}R6AT9H7+{!Umw&6PpHhC==vgb`&22$G0>F}yz>&g{<$kU$iG}z-0}Yg(nT~!% zQ7R{&vrgKfroSujSiQMpA3DCTV?_8r>P* z%9)~|q4-59qZ;SH#PGsTWzL_9&ZHw7oWp*^6ueX5HVRBtSV0)G1nPP2MQ4mWZEt4{!LBa<@mW zdHypiTySuF8`qmN_pmCD7(6mN{#EFgATB1C8apvs8hTa3(?;m7BH}ZOLTK19g;mTE z02tqduU3z0!VAmtmv=h?$f+4=^|)?d1MMxbtK7O$p}l7gpN=l4olZ#MPtCm?3^a#e zOZ;@T&TgPoWbvrQ7X*q#>|?a%IbjHQFvuABl_>wo5%Yx7Kx|N&u4F5ovq8tyh{e@B z>D=yn*zs~FTU&9c`lz4ED4aCT>fWwRqdV@}oCr+A_rXreuWMb0yizr7uwZ-|t=UO$ z7W{Zp7lHWd>8U7LC9mxnC%{Y-Nz+#$G&9p;5+5$kFKe865Ui6bCbbguesBNO=ljcl zuewyJ2mjtjeEupTGHa7_B~-Qc>l^=0l?jxw6(@2nSdfqnNez`g zYm%gdAVFSKxt`L5r%mI|V@WLd>`j~FC23mWQ~7aP0Xi;)YiaQv&7Vzy6Ef4&zk{YP zEBc#c+=K3kw3&u~t9Rl`=TGrojfeB~Q_c+5~bXz?3}5tboHJq@W+$T zo#_?P1J6>&N$B9_W-t#kR*=cL%fC3-q%~?(L#ozgzT=MWDTk|bOMpyy-XpFCc1~`C zMHL3GjLGkqM$y01-p*jf(jO9{qoYY}$!M>P1#dySlP*5zL}u4kn%Q(NK>Il^CB69? zUShfO>&C{aLhAJK_|l|JcHm(o$RW(;PuudG^8Y81DhmISetPgx?v?~0nfbI(t3EcC z;7>3Ot?rDZZ6MmBfL8U=;C;$Nb%?xU?B(*`KXWuBO(@H``DC>Fp~@^Ow?C zLo0PPq(!REJ0gM|P}x)~Siz67&w6i4NEz6XdU#yljpkDh0;J7k>;?XBZi;xHNhaGb zW<#Q--$ZRNDP5HIxs%(dQVNQ2JS=YTyc z6O$R=lB2s1&?70owDj#iPM_YL5j<1QY?zD;7|NWAirM_MpD)%T;Xlzv>av9$52brx zWeAdyw=9<^A>_mELGvBn&NG!u6qn#J=Fb+xIf2gQXVs*px(I_MF#g|#xMI|xvKY0c zhhcjA+PLZHjZ^P$j*}zW%^!)7J75VZ1kzc?E6^NJs#v?du%f)^c=OdrpHttYQ>xxr z!P7D@% zPbAsPvo61?;W!u6posqSn3U{n{GNNhMwZ`4+Pd!;tSu^@m0TYAj%DM1Rm&H5P1r7V z1m64IWa-fUzEjgejMN9db2hM;BG5XDii)5$3lb&n&@EpPe`c62;1lTxlzt}p!OZS0 zk$Ws*f#n@M{m8HGGD(!p^_#u)O!&T=URGW{3(sFO!JKr_y{6K@5+7qr}2B-!`Y_<3kTt^-CZs?wkElhM^8C{f%c#JP z>ABS4kApMQlVkf@gzNu5U#)Cmt>`%`F_{JSdG@Dds+m0uyP>j_kXRm2DUtiYIbhl4 zFF5w0Y;$w4xV+hruYNVh)pK?>w`dEL1=N4aT0{!MJ28JJjvZckvQC5yLadM-=Y_O@Xb2h?Z%DI*)Dg4<6K(;e)>>^t& zNmEx;mAi2EYTPn2YgpRQbjWK^VnxkonLoHL1rhU5y0&&?=URS+JJI+O_09ZQ`c>NH z>)*UQO{9RTJQ#FGql>?*9vreu`5$bM|NT|^m**T0T&gET!4fbNT^c8)hYbH`JZw2} zV}tfDxhW?T7GAB6wx8rvTIG58;e-dI|1`Q$6M@AH%8U8n%)v@*`NfRG>J>--5)ySa zl-boYYelx3RgjV0uZ0rOx4P~gfZ|lcKoYbab)aRI{=s8qDCl6m)iK1Ja4q|s-e@lI zyCo|$UEfHG19^H8rWu_C;q*>lX8M8>3h!|Ohpj}&eulr_CjFuO>ME0GxgGHg?}G)a zMQCWsiKpvPTo+_oKpcE$16;%>nn`&N-(ohZvSwz??ElC9TW}^Y;FeaJb=H@64Anpe ze{^K1b9yl?D>wK2Q2wF1JKozprQNzsVw?>sCOMYvys>_CrQLeT&ji=7l06x4 ztoHO{Na9X_LNUCTzFqZulIo9G%nw;jHUbDvhQlh$^oB~)sY`$Xh47otz$Qwcyq9A# zkFy$3BR8T1fz18YS<0+c;Bl`%b8Xb;!E-$Y{pDJ)8PXIqf|b4{ZZ|U9O_`q6b7n_| zLP@^kmX%U>m4Z+KfEg>B-%qKUxCrO0$JKKjm+I>v?A>p5JS9L~IyhLbso7(XlPFrd zq0)69N)<;dM9rQFMET2MDg=Au<+n|@lK)S>f`nFH!3DY;R46i=SfjGN1{qBx0W7kI zRD}UhQh}U>YZK5V#J%CL^Pj#grbvG&DXDD6t*=YYM0T@Wc#nN?agBg`;3utcn(8JR zuqrJt{nF>dZEQN<9UALvx|zIm4S1Tye*Fr1EO*!#-l&RHtQRbRE4jt-xRC2fURfpQ z({MfSDRSg z-3%wBbTp2!^>vb3AZimw&p34(9 z_IIkV$2}*dHvzqy5jUoiGhW7*d_E1}Kbmf+a}@**mSbt9u}%HfDY2hX$t}^1u3Wv8c=J*?n}9{E7Xiof0rtfc5q@9(ijonQZ2m z2Xx+>Sg)9nx>SJyQ`Tjv%$_?!e^coeoZy@PPIp`RoA*Xkv)p3)dVSQ@s3wlYGWO1(27t9;;RgnEC zp&ax%u|=z>$K!n~uHj5(eJM>4NO(#$- z)xY!DD5>r*z?mL*x-j|Nhi@H&62%7|O&984%I(FCWj3I?^doXe#=l&U`IY@r^A#1G zbP~2kTEU~1+EXRYiJtKRW ztjegRQQ;f1g?h+nG<|q@r$jGGs97&9z!XuRN;qeS2FqcyrzfYRf*ovD3_zvO`FP+G z#UjWZjX_zTC$Rh*Ftup;akuhQdKK?i_5%k5z@Q$6sK$dr;=8S?am|-#bb4Zpgh&n! z#ja7ZzWJ~8y5I+P#6O^z5Ju?t!2LfeOT3Al4>qodlgQ=$JjE!X4liCs(^ZCmWV~7JCru}W4Nm7o23dmTyUY`;APjo zLF^++K*XViK*t(5mCrQ6H~@V3kytvvSm*Y1o%>9*7?uu5i{Vhjtkzf*x=Y(5iU-() zV-pac2PH(~jyB86oS|-rkgrJ%EMb&&x-n8H7*drOmPM-*M zDJkr6D0&q5vd4F(b_x|v(*tDDn|3=OTpObvbpahw0k*zFx0hSdXd(@qyZS^PZvu8aN;=hVry!D5=ISkLMP#^;%zXl z+XE}7TL=sM)MV;6}s*xTECBzXTLZu|IK zi`z2;?~tzAKG^s&gztj6qjJ@xV?4#2i@(2L8P7us*{^QiZIys-n-$R`q3uZyVSJ8i z#|+k4eku@dI?X^>!y@-oWLi9T=R5JE?V*hkV{hN2ieLlx z4=}s5#Ahmb^;?;9nt<5SF5&5scsm=39z9QK@g7SEGb0`Y+jXfuKT8z2a<<-9X}|sx zXDxJB6iS_Q+*>yo0dmQySeKU5qF34Bp78?j$RW=_#RUbmSCXT{jrHveN6t6dz#ix} zWjD4!w}Y0zS@D7O_c0f)iC{t*H}?O*o@IR-A5+ti=ylyU{4M)1q4#}ysT#Xlt;c9- z%_gTb>x1(~!!H3}87_m}*4Dsz!mhCB*<%cRm0|X#)ahRi6Fb~fOKiuWiJ;xcXOX36 zVNXi5n&)dr{ztBk1$Zg8u0~npGi1d$xHj3sD6GFCf?pv=^Wb35Smr*A<$BYvn}2{} zutjR)%lGi`)wwX{Wxx@EJfY%V7Xf7{HT*_`=D~Y+-TA{}{p8{VUNAs{6Wz=SbufuH z{t9k(bVJCNg})14Y+nyKz!B#VZrk$GUWg=Wxxl^Zy)u;xx`hJ8JpB#O07mHD7IUPV zxE81$=dGI=Ju{pAXjh+CozUh}7@kDIcHcn8Z8!?EA%;15Sbp^HA+erz!emxy9l>i- z?}51539LHD`v9zN%xCa7HMkX}m@f!5zSz2YJqgW{(d(l#gHo>i2oj5705ch%4cHyH zU|=(3VR(&O-;NoXLn-s;U50<`(!hZ_G8cUnIuA}rp|mI~Fvy2cABC^-zJ0C0Yk%(q zNBm&aQeV;t7!VTPlgwu#nK+6qw+GZD3ou9kus3UXqFw8SW>Z)rbYVAsk_ez>pMwzx zPdrUj_BID&ViYqnGJvlg3&i)_gs%<&W{|lB?jcqc$`RA{4vgS{KqA6-u_^Egj=}wF zyWGDk7D$b7wF|>Lw2Ao*iZc=V|CcbzXG(gmYe)3h-|5W@l{Me+Bqw=r@yYm?q}8eJyJas?T40D@e)Rub7wo<}(>g)eim)`Hy

!83nQ z@>*HVD_y7LHseqWm6PWl@MFkZBh6IG(|CguZZi4(yZw2j?z9?)f_k?`esMur^muR( z@DxCDken>b28gO15D2(u-6PMixqcSDc5)VzzlXOsm}4HfH5Zi)ut)Hpo&W`=*kL*s zwQD>p)wR&zRZze;wj)ro=xs9S@qW@_4ll_EWaM|-rZA}qOi8@y5ST78>5Geq;Jo?d z*`fu9TS2&oR8^@aZen})KZ()RW;0ser+9R4XsQ^NQSQ(0HvhV)gU);-1{%aDfW?4N97gRvOuK(l46EP;3)MuZySNU2Yq@0 zPAey09#MJ$sbW-{KQ3)`DF3K!6SNkSYZSh-$$bfNJxR|3h&Bs;NxbR<$2Tn47ga0yC>^o#d1!b`yBF%aPH5B&mOsm zCIFKIh3p<42!UpK!iqs@TFK7O{}EHtkxp$%@i9BhOzT?8tp#e5#t&L+jkH3lxzA`J zj2Kh_tFo-4LSKL`ONjhbjnnZ0TZ-{~S>sM$c=20v@`aOqKfpQRI9C4( zJBEFZ>~g~!>>>`Eo0~zk9`WA`r|#H>bh9`V7CTtwBnNx~DthsYfV_ZeRB!ZF@Q9_Z zGj}i;ACjrCjcdSdHaNhr*|!Ha`Lh+ixh<*G0nCNq8Bjm_h`L}OwSXY-$5W%@4YvRG7D>*mXMt{WBI^bl>WQzI19z?cjhR9)z< z3&xv!w_2z0Hk2x)Z)W8Mv=T73#{nWL_{%yk$Bq)Xy?I}?3wcRed6Z#;zM{toG}o#? z*N$1nd+?swwWMhvx+embjesdBdWcQ&yD1wwIHec;bHr5$$a`NvWA--nBIW4ss0Bx^9y?#ma#s6~Wx zXk@M`TQ^Z5y`82G-i&bF5~ttrdM^2HiRB3F9gfZ+F z+`Q2GG3IMrngozykW;gVhfmMR%S;T9?PEbpLWYHaS z96|UpEOfIB-DV^7Vc-a{cpXTq&XxB@>Lw1y{x$qp0M4^3B`T}p*9$D8Rw8>ki2fj} zqKNAYcU$QOld%5!V+9>`8G-S}HK6FgoXo@*`o+ubCm@h0H2sHJ7?J(-BvY!ygYQr& zj11JFG*bUeZg>f1=wC~UDp+nq{_nvPHGeb1W1h+ipWlU-p_ZG0B40&!0gT_c#0DvH zkBPQjyPPvB*-$??3yK>92y0?dV`-Z%gd+L#jATmejb*=4NvPsG3sSMRkW#Hd`ZN=cq13 zNvI%LwoR25=Rp~;WolyTSc^y~13e)`2WWD1%AIh&DF`SG87~|F)wcFgO&NkJO6JpT zUd8!OJs_!0i;6^3#4EXbd*l6}IqcB!vz^ls^KerNAQ~ozjcbvS<{|np1>#lx;e_zv zS}@3GWB(~br$qLEHD{NPr;RtBBVRN7gX459WGXh=4LEEHiLbOG;$%LOUo^ZYJhX2o zXB#|Vam(gkY|3Q~4gP_3Pe?zx9WsR!yVZ%kpH$GE;l-9`{W(>SSv{v4p6gVL&?47r zdnF_Qvw_{^$AI-M(>eH%9XeU-YW2AA-sy{+D^d1}_gq6Dah-Iu@5n+a9aAZ$1d(GP zI8fWyWKlo=_WjxP38UgeHz}&iWt_;(gI4(3*2H_X!pVA zRnN-Nu?drIZ1%W+YrYp#3MxNZlIQ*c+qHV#P9G1k4qxDkx3U30&0H?^oJBQ)Rjz`Z;?19#fsCv$1UB|Q{; zwwwCy?gjjecQypVeBLE?rX!cyfeBt}%HGJNhFKwRq7+ssN$F09!qE0I%BQuG>$nNlD|$HJ=wt-GqozSr)X=aqOfERby7g62rLFCS(&*OSu#w))Gq8rxy0 zHwf}N0K2YIV748h7)JTl(IoGGGoy0Q6mjm^>+2gQOc2Rtp7;N+W2KK7)ONbs2(+fQ ztZ`{5eMaQ03hFq5EGfm}E#;RL;3d@=d*5sQs0iyY7?=Yp(P3n;C&1SzR}#SFp{_n4 zm9-w5L`bESwO;;I$eBi@tbSJ;6#kK>$0Ofac^kQ7mNr8gC1h7m#H5O zzY^?vljO^n7RcEpuEVf-DT<&aA(*kRy|1VV-^I*gcl*s`BCZkLtnZ0&(&>QM>^$7`Vlu?JSwdJ$tN~2;v5F$P z`6VOo@Ov4QJcYOeD+3+@7tC3BTd4DFLqUzI8=-|J*r&oWx4J+e zdnW?Dm-5i)#DB9{s;RPKyS!FwVi1!cOSbeirE|?SZioJc@F~fL%%kRwKnv6pM5A|% z-u^7Wo3@xEFr7o+{XD_5{N&!7BM(N8m4$yT$Uf8Ww+?E@xZHcEWrxy6Wb%7GIB<(P zpQsD;=YaguiE)=j7#f(t)x*zt-U@`KMWpp&lm^~;%Z}~*1$(W02)XWxfX4UF$yiFD z`0z8cC0HR?!9lCHFM%|fZU{W~Oer_!N-_0eNu~vygBzuxfG219!2Oaq!u_tEmuz7%n)-rV(EGYZvGs=ivqf^OB4qFJf|k4&FSab?5gp=*+%2J ztfFE!4MKeQf@vUc2LkUP0j-_XF!ey^MCSVm#zD`M|FUv0Yu0KsGfkz);dpMl*z z{jL>nXq%q^6D8)V^rOw?W*0PlmOGg5CD_FYt%;nZ;7TCa*&pPHI6 z^rL+Pq8slb;`R2)A%r_p-UC41jJtREbru?r8_eKvkva1TQJ8qZltS`WHJ%lLNZbx8 zyEXX78ck1)lNKya|Em7mrG?*l;>#QtC?S3(OA?s)Rc>T}jm541USGdE>3HxURMw(U zbT`cPJr|N#yYFAw#gh%Y$Fq*ruQ1??(dq7k5)h^r9(KXzAYlll^a`JZ&7Z~r*(JbA zOUr|rWzPB9`nq539@`@jPT1 z1;Xar$Qp)SxRC!GsFyu? z<0A!|myb+fx?FrwtUv;a+2mN3!)%1mY`|=}LAi2=@#E25tgz~CU7|VlS5(=)VaONA z3zGVB6h+k}EnYP~8os%D=ma)d z8DRw{vizWL47Y=c!gnGa1Qp()wck1PJ+o>=?hs3(?rFmA*>xi+ zsd8xh9a(|OX2O6=U$@MGU}ufiv|Fzt$Bwd1+mEy7RYX}Fy2{lTot&-303 z={ztQQx-(ik(bKYx|AM&38}~hyau43<<%%JE?$eQaG)q?i(K*d$EBt<_Vo3f5CG~~ z@#=KrcJ<8%_CQ173;?DTV1gmf#_24ma0dRP5`$Vk@cS^pqz8+e|NBTgM=U8Oi`SVV zmmaaGz;cvCB@+LvH2i&1OfD-_MNaFE?^8t3LuhQp`YtO~l&w%P)4hlooiQu*eJxk# z*F1p$MoISSVDG{Umm+&!=2!`Vty_+dCfRP#B)DuxI;Fr@R8}!!RjI5Ik%DKsL*Rb6 zuH~iFo1M(ljtmdS0WBW$3#?i8+s9Cy=09+9)H#@iVY{P=&f1FihU;enEU$gd!j5hJTin^s$Kh;-lD&Y|fOnJKa?Ml9t zRO@5Q>C5bwr++@rFjecmLwS+>5859$>2P{hi>IHPFjwY_Zv=zO&2Fn2j11SU1Po&R zOPj!StF^U3iholx<1}CPm%Co)t##+CrOguoHv;va7ynlDTynkl*Jheb5AoW2(5B{^ z!-Kf<@=ss)wHuzMn}%OlmB>}NsU`$3K!-j5uI*i3KA*WD9%BZ{8BOC11T2{~zzYde zx>p{Z!wHd9650iEU(3cc6Od>N=05w%OVj;^aaW&&-{%h(E0seEpGm!^joB8lIl1)( zF<+Rvs_IH+2F*#h^y1ySL^QXj=VTw>Wg8!yvOHzj`^#V0j0&li&OB~qblh0KEgnHg zV0h}~f)|g$E0JJleA9(YRl{DK0rZ?(px07+dEmc~55b61hehLx!shEMn}!vZofq#{ z{UjxkF*lPlzUqs!|;uKZPfwy&1#x~fuReJ z?d;Ydx1N?@G^ut6cnnH>L}?v)czApP4I+EoFp^y@^^Y5D2A%F;+cX+hy_1u3{qms` zr39ID_Wb$hyEX1w<`*Qq#af4OX}g>u0iJxZvOU)^ zgj-4^;!lwjw^%Yip%?rqUrLqB{kzvg_7e5|kMms!5~<8eb+3DSBtcVpUlw-{q%v&T z5K5^b%nu;f6`#tSMmd1%w-Up;cSqgjU^#K%b-VW7Bn+R+@9>QXAsvQ~iHV-|A8s!ITN6k z(i`ZaqB+2QfAjVhBbsFEmlW=YH!bgb9`8!&B`^F1@V}pk|9h2!A-}xkOj7!rweq9& zlB(rJHYx+qFS%u{K<&-;`LuJ~97XF(p*`~%I+|{1gqt=YZE0)_c$}$GQ>+9%-rc&@ zoZUheQz!EI*`I%kvT`J;EYOLh1Gov{l%0w^TP4rg9#nJ;<>f=c=5k#`MC4K6kq7XM z-vU3z(Xi5APwv{F_XdJnv)ejF#jktDQvtCkd|jWtK4o<6{Ar#2SaFu~hhMTidw!EI zC{vOXPu{(>Tl1*rm`b`P)lJ|2n?mMfs~W(OM3S!mM;P?C>ad6YdAHeYghQ7y=3#i! zBz`-eF4)XI{bnS_1!dH}{ZOTI``p3VIcO6UJ-xMWTXaR;&62|vY%05x{4_A2cB!FO z_xG22?j6_GM#{Xg*@VYhD=YVwLvaOcQj(RE>eae0PON7vQPJEV-@Sims53TzIUz*_ z#iou^Z_&RLBR?c*g;ey2#MC`XQn_L+YBuq3P6!U44HGYOcC<_L1tybYFohcE>cLJR zI72r%=_SX@^-%ivC0c>uxn9a30KaPd1*Qu|x;bD6AM(jK7q3@w$B%z^C=p)~-#Lq4 z+TV}WMZ zd2U`)L;8S37!bTqMhOHs=@w%l)00U^_s4Y9axD2_kMB7j;&)pk6BPH-R=P1*RUS(n z2Nx3+b)KB0<>KQxe*St@5lC)l9vTg!%AMvjAF!Zpo$U?d{lzQ16DrK@)M$qq*4ORt zKpcbJD^N}O@JHfFcWBS~A%UaZz@gp z#JO|9&;5I?-D*PmHwGmZQ{gI+nKQqCA31t8Xk}a!yTcp3wkh#cSNA*6bLC!1JGFhN zl`g|0rT#EiF_x)xZLiV;Wz@7;*56-xX__iB6TpbS-3p#GM3tw46>2kn5{PP@4qBHK zdp>~i&%G-)kR_D@%Frr=#@ISs^KojLpic+c!wead@}otczZ#deO}s7kN^6!o(L7K0 zOZ3QQf@~*bjDgYd)mI}aNd@@+PhI9#!=k%j(bHK;n&=jl)Ye{UeO4sCc~+QbpGhk( zLjrfUw?D!$(rl-;oG>mbFlM<=mCtP)^f-)lM3fzTo07G#n5+0uCokW8A3Zaf?4>v- zq=AUj4BMIj6A=9(jyF_Xt$3f(TVc*}Dl12ZGQ{|qObiY4(6EFey&5Bm2>b9x?=|tD zqiTQ$*gcqXPWW(@bu@6?EVwB{#6h2vQDps**Jhjt86N|$e*phKJ+2_tsr4~k$ELis z;!MViR#B6IrUBdXS6*5*Z-v%aE;CaNUszq<6hU;L(Fl({Fo^fu*5Cj52rPv0-@onK z+?by>`F+*b(>vrye(3MC^~D0GfYt2Tq?ldMR)x-s$1J=)IcaBkb2V`9a3oKiW}%B< z@T=@Am+NT3m;XMlv>J`>dlE&qyTz3Y`R4s5iOj2Ty?e1bM3W#S6>o0+^ESR{*p9>sl}Idli1kl zH{%XDe99N7AeLhiE6K@B zM%OPl7%&i@Vij|Spw(NF)@(6n<1#3}Kh9rgV8HbrDem+>squ}MT``OfxlfOi`HX7? zNdB~ae``V;rp3%g1;k+Y_{hy65a?0+FUn2n?av8R-m`&X8y4+PzX*4wo{IYrkHt;E z%u^+B`UyxeViP1B*`POB0%%_bExhA#lxd_riMjBnjg(1gxnR`HAkH8Y5B}=(oA_{J z$)tA=aFMsTIcv9nJCr9y@;6T~vH$PV%>s|7X77;PUtzkz(lJwdgH*@?H!sev4WNx_ zHn-}R*Nd-+z8+&BBdIy*;HvXu_Du#SX$ zjju24{FGV>n(( z4}xeWCjq#(s>VLc|XwDQ5|g}RrE zhtxW8pY7VPahr1ivQvss%5OR-(hts|H`(VYFY0}+E@?aoaePtX;KQ#C)>2AJiemJ@ zfcw1o*)ak-SNY(l^tBX3M-ipv%Jf|z35X_<1@i+>vH$yy6un^f5P^u2Nysu~%FBoLY8{)Vib69Q}SkbIcq=MEyEh?*HMBj5;>+F)|_V>BwTh!XTalPqS-%Tv6c zWm=3-w9R+$^Db|pmwk_T($>@U_N`6jrOV&2l_{r#E=~LT^@QaN6Vuz$^O;icEDd1r z_fIrL$}MJ6kBM9%Wz_S+2zGx4-Q}2>isaGMnXV6BQG%$U5j2{6{O80W$H5seq{i0x zV1|agzknxoDixbU1`7Mu^n-yWhz~KDe_dl$p>D)mdfKDTAbK(%Beh9JLqj9V=KSAc z3iA8}Mh*6~o^oH@ocx-iciAqEyk;h-K|KtwGj8IYMe<@mU7eY9rR&cNC8yEzT2w^C< zqw!b!edCwq)!<7OU3>BMF{kt{Mi(N_gILtkrdgl-9!eD_wA=O3@aM=#<(+S=?R&|j zc2bwYd*5vb7J>hIFNYXx_w|N{PmSEm@Q_Bt>#;g=k{7cfkv{>l@QWW`$-Qk3Au%LMrpHZbx=lhw9(9gJ3j2Q``u`PbEV zM#3JXL`Szo=@y;y_4NOBz2tdSoUvOLBSuS0n{8b4!ov9Z8-@Sgg1z7e3Q<}l;V(($ zSvPw<#7XWz9$QaSaAU3#ALMtQM0 z5@^Xk{>JotEv*p-fq-B>JVPeXkK}QGMn*PU&KN*FIqUBT!NMr@uCS7M{ymHtd*d-UA8c}y|Mxw;t$27C zNjwLol@MBS&gGG}b6)JSK^->EYvcxs*>iW#B6t%^K zHII$7a$YX(^!gWb?T7KId=+Q^_a|*;)5SDN?rUoam?0!H-EYnC*Rvqy$t>=Xqyw2G zcncVHuGA*=Cp+yC2n&y^2mNtXS4?IaKEWFFxM{*ySDuH4$^$sN(jzdWs$nRjsqdFL zofW3pw=iWSCuf#=*SAPd#qSGNCrx5eO3cm9YN6_j!en6I{>zLjA@lR|qt86+3YmD( zADyw;NMs|vr1fuwlD3h<-|R}fpA-kiw4wGcI7cJhbbl?Lo=OL{(vJN}4OFupoeTWI zdtWb#VFjpFvUQeS1gmr?{EVp}-LCgiS0{SvCP@UGr^VmiWKC=8xPu2#is=F9!d-lM zi&1iqF?vC{!}uk5DE6q1cp7orK>y<`{^rA+XtjO8@)8rCrKKfp;Na{3{`OYjxIbJb zg!0EgTef3u)Gk#7^;PCTVLTi>p=gfv9xz%y zph(j*TnqpzOchLBC#KHXVn(Vc(hz-Fp<20mLeh`p#>;czyY7rng#W!ruBkkfTqC4!$&%_CMy+XMXOecUu>8adV$7KQ z%Mh?v;g`G~Q{cOx=%nO&M*d_p*QcH_-$2brx`WZcIN>I!)uX zJ~L)DinR}ewS}pAvEEcQd<08^1;`$wL_(*1)=FmkrMD{K9e?Yv^Q_`x4T2y;)(=Vw zRGu^JG-1g{K?@0d=0wF(jq(M?jS^qMb<-><5t(y8&U^dzt@Gn>I+_Rs_2A$)I4+V& zQuw8#Bj^A9v7umSp$7T+Ae0qIKr;UH#}*vCV2}#~lP{e9`x_7Lb*gUeFZ5c?F9OCWNtU-rafv*Q+>e^gP zH2+Pb6iPzed|eS@LqF&t#O+u_DZg|$?{cx>5>P=bMmVxn?0@E=aGfaNs8=d6Bk&t8 z;1yZsMEu-}yOh?#V&pA}7eDL4()h36)Le&{#h&ATc?Zyj5kjtKO@~rr!cwi=T_RM- zp>L9-bz*lRKLNPp#Flvx>+4+iK0RPDzK_r(6B)Q{pZi)Wo>s+ux1g;JUaU9y_qA!( zEJ7kpz;;R!TQi0J2ldj?B}j{qjz5hzyQe{G4n!ma@0XOeVVump+vDhRm& zE5D6hC_wqxoLS|OjL^{Jq}38xgEtuRutZ$CEyb(6_fry^Y85~C?;09{HArNT<<@pe z2csFu#LEd&DU^?|clj;gy&x&Ui){`@*Ae)10C!m1D0DwgJR&BRExHyxpOmbvdFK%N(zkEqI(2kmy&=Ie>d_A`I%a_*tp%qr~s>)L@{FMjA zmX$PWsX|Kue{UinGmE_mO^U7LTTPx7h|pi`WONjSf+P^|gD*0qaj107XxaT?74Un> z4bMbG;M%;MpL(oc^`AX{sykS-c0(WvJd#Fc+kdAL0~CBT5Q(iVU{#tJj0n>3kRYLA zamqIR-%9Vg<j>%Bo0aS*z3SL*E7K4xXHdT#{+2M-z%c+Epb zQGK%S9jBpD?mm0K57Hd(xP|wX{r0|LN#C&pq!u5pG${|T0NU}CPDtP&8JfOna8!;U z6QNRb9(SiKetH~`?fhp>M^|??<=L}0Em^Q~!u6Wy+vQs7u3u@goo~SV5@43Deftxxve?Xj z1J93S^Dzz7veh!1DzB8ZPr#|l8@VNp6HZ;s2Z|EW32)#h6{ANY?N|suKWe%e2M_+ z?lL4t@iljF*?%5hQYMBEE?qy8f-nnPBO;A91Kk0EhXCRvyI!^xv8vwhJ35u-jX)@K zmaWn^!Ku;%|05>Fp8ue%!UGsWO`kxns!K>n@CrN99qog%*a9SzaJveWlS&oY>Ng+< z?XM)Okl~{;$IB{O`@WQ8iQAOvio zt}kQaB#1%f6}^MWAO+XNrLVXI9|P;2r?-H~gI2x#?ex1#VNGyz0secZXyCU#A0AHr zzaJC{W<%@{cYUx6^mf#!?-p~t{>Vv>`|I0|QhqH)00prdHO4S0)`|-Z$;fVQuGvgb z6K_q*$6vp0`1^md`g`?%PmH|;S?H=Qd$eFnuiWB+no6VeV!wpw?#u>c+rd|5wvUYxmf>l>k72`#~X)9q610#c+CvudQ+NT^t3kf5mZe!QlKQAvY ze-{xRKFnhF?`hfCTIV&d#C`uha!IBKb?J-r+T67ulGwYDWqipo^$X`>xjy!-xw)!} zo#OPv?oR67mv?eBu1tzE+)|S98kpmD)^i>Hx&>O_*MHxFyu`sj@$s9UY-MnqO;LM; zOnF=yB^OOyS!MBh0bby2!J&3ir5thN{UdNW!}|IzZiN1fK|n{}ohPtC6SeNlpPrt& z&|kl>NI0RLqlgx^K8*0l#GQA4VCu}jiJ`(s8xgD>S}5s_-_>iTnscs&a10ezIAWWg~7D*J0x#_ zHmK{p^^Pq>*mM0!FAKg~_`Y`oC=bXXG31tEh*S24POHZ%32q~^q_n#>5S^z_?T6wX z0;1P-2rcxJp|w)AYVnEw93C7a!JdgnsH@!(cvi`s4$!VNhkqZJkP_$_(G`uZdp|Wb zM*~X^SV?Ybui1FyeBi5RXtx~#=8{{4b{mOE{;!o8`wX4j#ccf{mPTKXtCRv$?<25U4N`V zzkst=V}cZt#TBsKsXyuEN{xB(;sx8-&cD!c(Db*JsTYk(x%Qf!@m_-rGe7d1RX)+prBPQ|_zWtS7pcJ@@ZHDl;CaUEtEu3|?Q%Nq^fgNBfNfNGEX)gc17zr4Uea+3i z-Pb=gy|J>bvB(b-6=!By@BVzR5Q%id&ovDZ=R(rAG1-f>C{&>VFHCoD{+Dicf89p) zL>sW~aemtI_jdU^0cS&U$bVm0M08R<;*uV263RXiQZ>)^x39(0#AzKormJQb$n+r{XMKa5j_y|vQn z{#_Ce(UA;joMP0_a#k~eX7xY;pY;0*M54nLk+pp2&b!YFQa)`8?UsD;gz@Z`ZfEgNLQp3;`?KCL3ubbk)G#&Hh4H1uk zz}BfmRBjO#ou6)t;EuBDQRfT~p=1~q3>?fFy#rX@fJvL}S$iVpt0?$AN$%e0kQ-d3 zYmRNF8SHi$nnm$PSs7;O_on-MazfIiy}nMdqQt7Is)Sv4tZOXSeqX5N4{3XEr6CVq zJZ7=YV|#nE_NZ~UU%H^IOvh4x?(fO0>pugR4E&v48oE9l#>*i>h34|QEZwd+Waj*2 ztLEh0(XjWDa~-jO?1f)=Y7rqdLUpj6=Z{1bIO{)*^2f{hc%+xd_3uk|aP(egV(4R2 z23?m>xw4F@KSh+10#jEaUUBRVmUu^5W~=0VG7b{-Y`X*;T!%T4;>x-Uo+|OLrP3(9 z!#XIGm6h$LfVb&?o{N7&lr8IzgUXHvq|C!P$N?!Vndz2&4=7VkbKek=CJ-ukcvMYs zpz`m1mY0XjoW6Yif1BW-mW z8rzT@z3Ct4HOx;cWJ{39EFjf#THvn#&sQ7G^JkMw9h#_?`!*tLP7}Z}`v6a<`M% z>o$3&Cmb5~RpZX5OlfsNsSIB>y{^XCCiy0?E4ZX#t2yOs)4?;}`W377uvT!b-rTJ? z@sr1l9Wn?Bc=X$Q`8>1DZzt_yX2ue*rBI4Xmjlu(fx`OabO`nNSl=Rjh8zwShoFp{ zKTL$&Z%F)C>DZcgZ|rI!;gu;Odgzr`hBL;H;k@>~eyw-_ZUak>Ky5?-(?-Y!3&387 zR%Vj9;%?k9!hjRV1Y;_4@8g@k4B+aWduWdSC@)vSJjx#>l#nuhG8q5hQZDK@Z=g{Y z9lp*1i7_QD6|BWH1kESokMM}Zpye!gDP}~$gr$yQr?<9_sX%KX*wV*XflW^f^bd~2 zV@?;D`bgwtV@L=EDs{~sy|}VXt@x^+0+HHq2?FXLr{V7PZNFY;oazxuykIq9UFlJc z5*v+cs#XNtImzn$RyjwuXTiU>nbagN|90F9EvBv?=sSs1Y823E;vi7_p#5smw?K%Q z>tbXykD5Qs*BzLka$0k`JW`mO{qQ<4BM5nf^U1sz7{yNsU^}e)>DR@7-9n$?ia$p${FANa#=~`i->(#MN zq)eVqR)nJEkTrJw@*qeo{w-jS>C~AUK%Hh$l$iCs^GGM%Kw8`InKo3fG zp`J9oG?aQ^Ks+LGV)5$}k1Kut(3`M_PMEUHRZlj9k>N4PvB&9-_ypAOfQX0hE@P(a z>09It_w4-KGB~PrHDqZ?@d?5EBPMVNU%@`lxcInJdXWoL72#X8q}d7G2i4YD6l(4- z;c&R`0aJDCu35kzA@GY^y1~S!?px-p=JWHW-Ve`0zG7H8FB#5!*!|HvaOn#O+|~Vk z>jq9vC6JkDF~`F|04SDweZ@}5hI5OsE+GQmtbdjF%=vc=y?-Y?5Km{V%{xmzT$=60J$9dh>1TL);Q5i}tFPZBYbL6FIkQ~iu3wKV0V5;*;u@P$Tl&kO zFl-t1lEng`7*p@Drd}+iT8|mhLm)DbZRdiBw4QzQJ2HBr-OGPQ`SRtM&B6%15a#af-qqby2 z4OSZYH)@Iyu5~vDd5SOfBa`KQXQz|^rP*2Zo8c=Q0rFKyw zbVZL#wb1sX4Pyqi;I{3qol*rJDg`X9dXx@I`yUSQU@Ke_#-U8o7g zm#h?MsipObmho@iyPqAkds%EK4dc6gZ;Wt(d$XHvQnAG0S6|7-=l0+z zHbEq&!qk~pz?UDg*z$+Bu&e-g`<0rCQXWI2pF_Us)*Ar+)@Wf> zh#9m?my|g;C#I2vn3`gK;)JcB5bD>xHVc4(*k;d7(7uSi zt>cVW^YEzj>k2c3rQ+6`LJRmw2IGLG*uh$93kFz0-yO5J$0JWD7WaX5v|r!IWA&%5 zf0Woh$Ef8#x7}Q2@b&2L1ch73@iBIZ&~NdanZHK!zl0k%w1{>EV8P0qcd|Bz1zvcD za@qjuRK~4mcb-9!7gNK-d$7&ssk&c{`tjbS6PsoA_2Yd>LL704@q7lc-x}^d3D{}m z_^8U+7+%?hoCXFxPQ?IR24jbBO>4L_#&&!X-A~cbR)3yVH^pqweNg!Uwm$);jf;s} zHMr>Y$|6XNkc~}?o9&D{R*BI}`#axIx zs0%HD$r7NFUYQ3?OXpC&HWgau5t!9C*Gx_*bzf!q*zM~JZ4s={&L0v&guGr-0VKS; zz%6SIbdpqz5)*dot&D(a;#x8q0lb~QPyiiJgxdWaX=52FTS{ap#nEnt_W4Fp=@QZv zR|HL;bYDBlPxE};^@gHT*6SALKW|m>P^#DJN}r z*W9Gp@hKKkGjJEx6~(+p&-d(FG>^cA9nF!waPoS;>Je7YuqR~aCUR7~x^hwyunWY2 zw*1ZY=B_qW*V&(Ljsc=C0M0!bk<9@df&p;s z9~eYe?;7v!CO+6N2NJ*UXx;C!!aARPH$Yy7UDDw*_(B4^pvby~l;YwmzYkEl@p3Rs ziw4Xnc|a$UyA{+A*f#wkYM4ZhWZJF`-@U>xgtDO`Syb{CHxJJd7Bp8Ry^{Z`NOpdG znkc_4^aD=%kb0@l92z+f_FqD}htt-%GIz>*{lJ9Z^ndWMEWxEGc@eHl6y35>0F4y|`Jq(DhJsokrtwt0{5?k>c-Z86b!V6&+^uPsu@ z_JTp<=dZarn@bMDwGE5BsBa~0sbuWIfERaDL|}&dS+yU!TwFR(57eJT_+Y$#U_UPjF(>Y>v$4r{ zd6a8}?QfAylZA2i_xA3x2Tw2N4?X5LtyuE6e&|%kdOxEEkId(J6%-fm0rE#g?~fl} z-*itEI(L$V8mGtp?8qQ{=3`1S?(MO#BMa|)JK`s>WUyI!S<<#&#(8&J&a?I!-p z7gd*>6o@rsH69;3G{DAy#J?!4?(*i~z>~1-ta)kq`5`C7=Kv64Flu|eB3LggCl{}F z8&!a&7??+=o=jY)prG;yXkEP~>@@JNfcJUTHwp%(O6)tdLD>yH+4LhZ+ZciuFJCzG zD_<~wn>>I=g8Lx=2~6D3eq`F{wRCK_c`u1Z1WUMXb~zaqXZ(FT3%+@o^d6d_s}DR{ zRzFOHCb+XD<}K+_l1oYbZ}jDN{qFKe8q6OW0JORxrBeBISLIjHz88l({Y0PX6 z@W)0of(bjTnB$XDN3=CQcxjX-Otnk$^YNcc(w%t-r$yDR@wvq%=dy)Al+`uA;5;^O zcmE7XZfEOu@)1#Ts|C_YV82hnY5#lGo&Xsy%&%Ao+sT&BduWA`AZx~fr392!;DbNvc$_Jh|=uWH~eZC?5uQnpS zssIvFiw1I_Kvl8fwS0GpCEW}?Fx3l$gOx=mp)5Y!n|58Q5&bf|fq38&_=@-A8`axWU2Mn5#0`+SKcpzxy#T1a@XdEa8GG>oA!Hj zG}t^;&3ki0Ra+Oh?nq=uBXCu$=k_6CFMe>X$!+^T(w09!rg+i0)oi9;w~M8D>9a#lW!bj)!sKYfH#%+S`_BFs9u z9xTPVVl!^zwkBvjJ%{5Ft*D8IW&~^?1HQ(NCm%_AI#Z+?Gdp2UPw75!m3DQ6O}rf| z(9I4J9#q%iTP8*;Ny}b3oW%E}L8~eKCtb+~LYNwZb;u2i`Z$f4(!0DCYge zp3?Ae=lJ+D9zH% zOVFmC)f-zSFsG;^@oQZX3F6`{wb`SQBw@#qhnFDkJWj(_S=!8uq2fIuyQn{f-Y!Scf`$lz4cz_*h*Qb(`F+lGD_fWaH|Y zF-bRA-Qs0dL0wrP0k@Ssy55O#%zA1XAlG>R1H4Rw>M2@FIy)mTXSEsdj^yt9qb+*K z?>RfZWJsha@&`+rAk4Bpdo-@XNA-lHc11ab@UkCOyEL(so|`q7mnIT^j{<-~YQ8Jl zOZo}CcBs@C5Kr;kYLI3+0S=CRurN!l7~CYzhnZ?cdq?kTm+X0 zg?}?q0kUP=`}5J-l~gykuhFH&P!KgRK|t&Vqf6xZH0Y}?x(!oti|_RG_2+^1n&n4t zZy=ki%w76b*d;cfQK5myS-?%6LIC2M7k;>2F%Xp?V4lv;^RM~HAM)ojIS`k2kYm01 zBA5HKHp~O;)ba~x7?=klZ7{=x%j>MqdG8eZi2>in;n7k3O>`+3%t)1bP0fmzj32H2 zFl#Ei2>`1LjoMQ;`HRfZxnCbY@?09u(<}Vh=_%fZx`!NK5wmFRR!O{fd6_l){w0M0 zK0fAKN$R4(aZGhU$kop%3-AZDxK@p@BX9MO}wdJNP`>uYc zJoMw8B_qOsQ0f4$ZWwOQA%g%9%G7rsQ^Z@oz}_`5SX|N> z(4g6sy!-jH5$KEH@;q~B%lH!(%D9QZG#w*7H@=_-$Ki5ZsK&rmH%eC~CBtdkC9KTu z(Du0~`K$-6w?OBqu9=F88wXrX`Bxpi)LvP;#MXTDluB5zwzX&V4tsDg@*&Is@&)2* z0@<4N6llcuAO?U`Z3yaWKuA>ia$!Pt+X590`1nX5AvCV90^K-Pv(#T16Il zPEf+|6^gBVHDBws8|NI}7Xp9-$fO?1cf)4od**>@>9wAor@?-KR1=}CbAf&BIqFE?; zx{g#*wjY|_A`n3$>(KJpH+QgS&r6TTJUdky&N6%cKF?_~2)Ikp zS6B9~Abk!!<9}xOD&2-cN95VVwO-%{eaBm3(Y}VxeJ}BIbMLJSO9$aw4PW;eMNgvK zc`dRgziOt9DhrDkT)(7X;B}K+`oazy=s6C(ma%I#VKl1+d8_(4**L5q-N2k{3D`S5eVwWZ39f34M(%T(urJR@Mq21CjHr`GzuEM@Br)T@%h%++w>)S^5aR%; zbJ(18Hr$=S6E<#=eCh1$?21{ffd0y5)~nvHGypZxq@UKoZgKv!bIio;!knB1hn#u^ z!(AXAlFxoW0M?(1v@lKmH?5z)rzCx7YAm+gMfEX@cX`+^BRP$O(B>dAngh%7lRBUZ ztn{T=KE76$CIg1HZVi8e%jf2N5zDK9m_n!fIL|abS*aP$ktil67tuUDR6dHwvKs&r zt=RL_E=>h9{R7oylM>~H2@$Fxit2R&Qq*US%=sTs`PjVY6+@-J?2(4>%W?8OR}+|z zS!oO{p&3r+0{wMpU~i76lC516l=n*erui7W*AH}cX=^w+4Vt+NKf*CeMsK1r4Zx!GYGw-t63DY>w2`Rw3$J*Meqg z+sF)MQN9CpFR_jzuC}{-j)wPzJ9+E&gYsOChYxa*_3sh3yo1_|jEpriBDS1=2W13* zae%4U^O2Ss_A9kdtRJLmUGcC?=~{oL-qm^-uc;XUAfMWnN|sgSf~63~*Tq(s7yyZ=ey7NPh15#MNNzTz4B!Czvv9c}JtcfT$*XB!2dKdeGGz8EhaOG^OgQ<%8DT>agm%uz^E59^CVmmUpkun{ zxIHP0>&zRaU#*V~_wN&EJ&vVQwobyy5wO+Z4)_Inmc>a08`QP)nO{rMt8NIs^1K~f zg!I9xzQGYx5=jh7CbG`mbS1O2BYslnWRiKjdz_coGIc&X_;;YCUHJdHz?W=p`n+kI zYG6SIG0mb#BA<)1GyX<^7V`;BG`n^Fuc4Lu&0vdr4Xf*njzAUq`hq->H4JH$_0G-r!4i2+dC#`~Ws%33tU!P0TafntM=J`LnHXgK%FD|W zS5s4OSr2kiCwu(%NSzlbN7@f8MJJx2cz*sIo-~QtsF(lyP75xI29L|Xlv>5VP+A0ZhU zas0u8MEz$D0lb|l$t{3Z+`Mc+^>{|-xg1j~gqi~KDmX+?;Y~Ydlu8*+Wq~}`b8iPq zn+lf;hX%VJPehiKTugXN`JUdgV>$$s5--tUM#NE3oz3O6s0In_lexpi%q|w+)pt}f zG3!kH_}WfGpso(%yVZ2K>D6+@y@UdlewwcH%%qXLwcBxM!IYU#4+u>^uxX0@CB;a;_9sqT|n9q9~c2VHK8u+IIZy*rPPj*H@B87K7oG8|s6< z==)S-aYHjbH#)@2kAOvjYVZ?zJOUnMYWQh+X5CRKFshw=vGOZd2(NI;di3c|ov}-k zXezkEq0BKEEpNlL9(DaWr{ZIB>PyP%{&Q)T<6(|V znyAA>RTfay2Fx{Gj7mhkM~q2^qRxqT_@%w1#hg>0UZ6y_rd+XUr|-4`15Zfh>Sse3 z5Az&Lhv6!b3~>kN*bd3f$bS?9YE{#I#5nH8=ji=HKZB6LG@UI#VcpR9)8dC#W~IfPkm8F^26Dg?{EeaWcd_?GexbhKOmdE4!LmEkvjYN%P#$h`p}x{c;Im7zGd4b@xb%b zn-DcMwexU;Kz8Uhh6oB@OGzmmZ+_iXZ7lpf39!(90@r;d!1!9R$<3|Ft#0@`u3JwxB;4h5J+VfFhZ^ZI<6r&ZGN58jE(G55H^|-jdw>*RAmzw6b#X*a&L#9 zZi;wm0=Rc$AVEEVH}Lw{z3Bk{TLJq}LS0Qg^csW|-%S4M=Vt=B^+2m2r`Nr)IN`WZ~;t5kj` zj|>mLI(Yc!p1)6Ujx@mtmww@lL7Eutg$#IqYLpxIgopEe6dvrgOB()6pN{W!E5y; zA1Dap(xz^oDB|nVygd)$Qqszxfr<`Q`#VwzkKn>84{GZb8ZanS-pyu0Pd((OWDllp zMZF7qVS<=~yn$2h55P#+_`T&j)B464jL?J~I}jX(9wyX-Eb9S^ zNM!%X+b=$=#2Xv%@o1Tk=_XC%TGT@}U|2vNgm)+1JKkSoWfAf_j6goSraU+c{Wt+2 z;b}M$!HO-*$a#qhtxrdl5EF5~v9kFPR*&KF zX;j9A3M5I;@@*inS8(r=Kj?3-_$$k+s0HTd0^Ga|H;Hx?$-L1HE{V?fVI6g`vD9w` zcj;eod)8;l{)H_rj>)S@hS%b1yV}J>f6`5J~98y_HcxCxH5Hx!RE%~ znf%?$fk5+mqM1)&;9~sa;+EVM+pk2x0HVbFV)7esc4%7le~TbbMktX9`S(-apE_?t z&NXq-cxxpOcp$;5W)I#rHTT!BIBYf|3}Fxt6D>e)7H;Kp5r_n{>FlHY>V?h4{kmfa zltQQimw@Z|#7`az!UhRi2Ru}I@C5)1wcw9GBB5kF2uwb5ke!kY7Jg!Q+qlG^QzXz~ ztjrR)xpmOoy}|b^F7=7wB64E<#e;$Tg=>&A_o;gM(pyrMZ6Z{9BVX^UP0!*4*b!dN zs2O&pgnOp-m1lU7IhYVu-+<*!VC^+EBYiFYd#kxNK9P78Oek;Bq$MD-{(~wa89Z@v z^XOLYSlm0+g|NKp>i28EOE1oj0hl^JHXoIuq(DT|RHVQX<|h;m-Ac7GyiK#B32+FG zBv@Qme|&g1E`*BtnV|dOc47sAzQfCxCV(c1ZTXTG4a!(7|2X6o9&#K8ZRD;FGc@_f z>Xg?f{p-~1;ZREGz$du8v*U}`HzCd~SQfLPUy9%oPVsO5Kinz)ERy~xW5Kzg)o^4c z4URmXBnV{^ULCWFdNd8#55Dz3rq)TfYVafpFqu{Y&}rad7ppb}@}d<*BTvCgfo>i@z|UKf(Q2kA}x;sU5-=92Db3`t2QIv;U1> zH5)(YncjA@hP}|@4K_Bas%jz!^Uc4+xpkWf&r%+Nj4$jmaUehrQMo0$xM6&;!U7kJ zv`3}!e0&o(OpT0;jsdOVtWyrva5HKjEaC>Fp805Vy{vZ?zSOwo^Gu{GGyVXxzJ{xaOS7c4($pXJC>O6*H1mo(sv)q3g&ivKr82Xs>NaD2 z2O5MZWh(y+adtZHq;*rL@gi1T=($C^L}*c44w zA&;eGC2{&IPK>$2UyCQxxp#kg5I6wKe@|^~r|*X3MZbOL{9X+DoQ6i1SR~_KD=w6Z zS!ySVhC?pEVXA%n%)K4T1_VhzZ#;t!jz?b45c!CaZt z&7iu%FS=tA0NcOWMAT!|*_id^fw@zW5f6j`Fl_>X^{CvouKV-@Ql?koyXQIU&aw@B zwPC8NuvmR)v5SlO{g=bs&JGTrfJvHZgQwFbdsMz=%6BlW*!!L0;_Tyac8mpW_r*a$ zc676}8fUtP!W+W&ER8NVa(0BR7YZF3`fc}`H(M4_Bis(`dy;dXY4xDud$RpJpLC$* zHLWPPpQQXKNM%96GKA2~Vr15`YYmuJFIGDMs=vW|AHkQgrus{bI5y}DvLcgfQ;eN| z-gc#=m=E}yL-vId~d%NZWF6tKU98`TqU8`+-#1hO7ezUKCDh zO<%u;pyBBMMScFvlc^&P;g;!EML3|uIUvM3-e>CUrbIDuW4?%2;eHZ=UoGs^0_eFo zRIV~Ekx(-1g#rs!7iT9YLr4-3ZiRcveJlh;$Y!SZt{5ew%pno0GF`c{s%(>_e)ly6 zriAA#gBZ+tD_}UAme06G5nkZ+ z)j;v4r1kR8Bf!AmabI=#J10K~%8ruy(FAt(2>Tsw|G8vB+x6!_mW?<_3M;0VQ~oV? zv1@H&0(+HN&QB~zpLrKB-KorKP984#ScFh>N)smyptyLpDGJP_pO2O4M>7g3HlMy1 zXvDQ#p?B_kk7CwLtf{KP)2sSYXtLbZE?Qcp)`#$cZdv|-VBVL&8{tQ6qi z9Dgb}H$ClfH;o=;atj&Qtfv9BfzAlW^;+IrZKs;Ldyc>e#}p7M==$Y1D%4@X1kmh?>Sz|Lnj33DW#ikZ~ z9WV2Sqk$1+8{jcH@*z;p@c)mck_2LMa2Z${-4@87g^qw37P6E1a$Y*@de?mNy0jmI)EoNJd%N zj7+FKjABZqi1#cNZdhwlrD{2{4OQp-Y7l$C9Zs7TZXtY8($!a?{{`o%HX$ow!>8-e$7=Xt-HH59;{2{Db0xJ{b zelYLRAn(gz?E@)Be$=@04#92EvYr$O({-n#p%~Yc*sx=S0Gt9%`*55CPi3l34h(0p9V8HYmgTIB~173mM)> zY+e0w()fX-uPF@#z$o)ZrDkUVKye+&DrM1Hcm=BQ`+G3$(9|rN2prz~*z4>NX}2HC1A}FCvwae!%47wYGlj z)glnW6%mc0)(LG21UVu@YtlXg5%4Gg_C8?+5}g(QYmib07jqH6?U~$H{N{>?w+JQM zXLl(eu!St{Iqm(L-qQ5Occfg2a(_C)r3qM`=(iV@ zpNU1TtoS;Dm8yFY=BLlsKo-H^-I9EP;PpZXk4QJPwJCzM-U-CBXMp}*tVBxTLkAm0 zzQoeF_lZ%4(w?c~opS({3h@j#D|!4&UEneV_;k4fTYeX@uf{C_J%dD~a%wOxWenBU z@D=^)lkB+v!nmJ9#}!dEMevJj2J{3snY`s?PISUH+IR|id0yp2;z2#un}&s_$D_OF z+nWucA{A87g(j~!HHsiO$Gg?{v~8nIyOs|w?erME^7r3XAW((IzOtXW7@hpNO=n^3 zQ5tFXdFp_PC}@*-drG^%K|9SDBm;VL@E<)RlSxhmz%RyDKfC4`bvw9*Qhp9mdhcm@ z4$+CN)1FpfHNYeMdiPdlPe#qRe#-%~Uc~CO@F2bX{%PUo!qi@k`IcMWEjkEW(fjus zV@_i5p75KU*o+GS!gDF+uDW?^-}j7G8s>T+_bHOq0#_1w_^eS+l8{msH9vjq7LU#) zh$|B=F^C7P7_QP&$65-Uu5QhI^8d1NPssOwY{w#0=;<$D$rqvk{6VeP7{WpYvomvZ z@8DdUHcna?XB;Z13U32v6crfoV^@7Hr>ZOkab#N>rEZJh8LMkf3vCi&YLEDC&`05%Xu;oiFot6Iv$U@_=HE~t|WyGM`JTmL#j zq#_(|XI)%IUJL>+k;VV9vLiYCkVS6g^Ah-#1OR23tGD;jb%r(~py#MDqUo2UK;7nx zj4Y|?PwV~kwXwV=J;DIYwg6G@Vlwm_m_dpIGqtHre`Qejz^~Bre*oefe(1<_csBuh zAM=+7bo_L?7-*!?;c!T+ceY9f5TBkWBnkZ1XhoG|uYMvBX{bO%G^*(MC%^(G$Hwb( zh9ZA@t_u0ye-g#dbCdwVg+Jc*Bco>@g%Z^UPXCM9 z6p@kiud)2Mvuwf|Wuh9rK9-PdSjn>pRZy|3GIeFO)(pX%7ISkid@x5?*7Pg!=sP3d z*Ka6=TM>~rIkS?T%BAIa#S`rJ%%6G!+ zPmk+1R0Ie31z7174~+>QhN6DEyhSaWXD~Wa$Ihg}qi&HmY9TMqMl zn#l)(Q8)aqDJlT}IU1UO4DI#dGqJjrN077&Ox}U-n@OXh?A{+Nh9jk6ylgwRa)XlE z_r*(70F}$sW5n=eNNU<-Q9AzG^%A2I{V)KSwEX28ojA10U{g!;KlQZ0QOd4N98vQz zYgVIO0hLZe^XLm_%dLyt%*;%F#xYL=M47a6v4U+A!$zUP>vjt9THqUQ?-jU}7Xf;r zDNu{Hp>1qFwVp;$x9tS0j?ne>xma>dRBTfHw*x{}-^DH|Yh!w1qPSp`RY&1)-9I`3 zRgx7LoFvm8Nt5*J83O1h3Gow0n1?{zY4qy2dixn?>jjYNnh-~r5|Zi#jU&4g10*+x1(jt(oLZZ&3-! zL8=<46i9JscSI~)yLmrUzStWtgH3Du7HM%nNFT7_@Qwv+Qp5C|W6y8$)<|BQ?p~4l z*waHPK?CIjMe$Y{o}hI@aNmL6bfL!Y^pCjmLg{~gOVvR=8p~Gt3useV%!YsRoh<2| zx{G2!0=0p!G3~gaE`3I zA5spzH~{Mrf1H-v1+X!2VZWeQlbp(>=Y9L&438ke49Zje`)H|(Boj@vWJ zoTd7fLG{0MXu$9|Ey?q9R=3UX;C-f;IKw!8_L~mrJu`Er3w12YvcMO|SIbOy7XVvL zQfMYd=_@5PP!q*rclQ?+_i3=TLIUoTD%&F%5#|{9(MN94)*{2U4hMwa@-4PMTPQ@xS{` zcRWD9eiu#j_pL7-+9j5me|a#I5_s$W^(xVYPn*nB>!_ z5C&=e7klsn_mwvtExDD6B!g~AFL^88Zz}xu=SYiS$mccQh!SYVr_U@KXi)DcB^P3f zxPx1sF9~!)$=DaFtJc(Uw0-9*Oyu$|Ii)qHesw|Wbu>?Or!-(y-?{_w9-T21C*+Jx z_fHU_F?d8n2`~`CQDk|>jjvtHo0|Mb9_6RHJ(i01p-1BLw~irgtc3K!C_Uj))I)|9 z`1#)Ftu%zh1f|-%1SMoA4Q>Nz_D7XjvKAY+oUg2RmA_Z7D8*QS?})-;1n}|%F!E(3 zQh53l&7N~iM00c(%D|wwT*Q&&pJtoVal#$WhNSYy(ffo~;;TFvDgUQnt(g{=(vAfZ zRDbr_Nsn*c_f_soR0q|SA`mak%+fzmn{9i0gHM%)F%YV;jvZ+a~kP3g3DxIB~IZHeiSBq9(k8}pC%9`M0 z#{xVv`3RtY+TW4bqHYdQAQ@Ka3{gO7-2x3f9{U=6L~x668CdhiptmNY^3Zp+lkj@{ z@9WNqrk=^>;J8elJUIj&CdJ`QxyjD+Vqd;RJYWv;*{aj;h9RsnwYOF)hBVY*SF^m* zaOUoU~)+oUuVyiI>Y$yo`Y9z4T5Ba==bZw*PZqr^ivS63= zinMBelVrTRvzv@ig~_?>eH5)=UQi&o(kA1lf4$@}h%#oZQ{BBm>$$&2-nEH9AnlKxl)-=QZ|A%Ie38ZuxAI zPa9tYj-4iz^G?;$GSTVvF~WWgT#3LrFIY9B)VY zlKMS@rUY#gLKew4YL7>wp%6W&)CY#&j%0|$b0A%vFrC5Y;WRC{!t6jS3W=z*u`Hc7 z8|n<~rS14**_w*islba5>Bk1|lgOpuRfLD{rtS$*dU_|8Y5#RPaL`^9m@4$a8H!>VPAiupMf)c#vfRm&{K0%vHDKqG#GW-NOl z{5`7AAd;Dd!WXvhqMbm${xuCBUAE&k)BS4x6t|z#zT07LpJcNDs=h~b1A)3MAVhvz z@(eToMA6X9eBAN5^XEP-opo?hEX;*7=5uQr#EcFKlRL}kC!j=Z+ox@ zl9mdJG>!CP=aW5RekHktL2D_g=m>D1Q_4?YS@{D8UfqM}qNh*$>Dsmq4j=1dxza*u z8qWV58FRlD7U^0Zb)Nf~M)hd@r_ErOcPL~7M>tT~0Ni=kRFY0q0t}xNF4H7Lj3D#pM(t`^?N5 zy?E}YenN4HIWC2;zOz2-Ee#Fsf_t2Dov5T~+4Hc^-7y-?^3|b8Qri~Gr{>mf*9;1h z3v4NLjl4ud5KF1@Rr$~v*+G{5HrWL|zO~*aTgtHzDG%Ya&GF}aJ$Y2i`{@_GJ{`It zEl~w$sWrr}jd{mnW;PG5Gi)9nVX@cz+jEH<+alQTd8SJ%XrCNsZa(wmp$XSGqc=w! zZ7u8O)@IS36Fm>xd>e$cKhWt$Jg~0To2CM`BDdv2S|a^_T40|{J3c?{FQ%lpdueV-+C4ZJ>l_yDjVphY z4*s(~aB4L0tiVqm6;GI816d)r_LnGnYe`%L@*|OTLuq;>FH5euOn-D(WY`Tc1V2LW zv;HXweiF$4(g3j%Vj)~+^fmR&9(q(;si3+4Xx!4rStdlj1&Edg z)k~+Q+-DnDHxy!gzP2`>1gi2*vJyC%930odL>3#Q`h!V7Ld`D*xY7&`UEC&O!YGiVcVW9N=uda^nnt5Cs zdaP;F4^fGWJQpyN$zo)!1WGws-e8;8`=vRN?J}2}rj9LE`oP-cvy;7`7IAcO33I_0 z_J+MsuOW65nSYiXQk$Ouwfo&u?D7$~Jr7MtKj{f`#KeRYTf^|jvB8eh zD4|6zAFRCj;o+0gv$W=uSga@cA)!mQVnmM*OTt7o5KejsA&R3W@%ZDgN>pV+40q<( zg|y{vMqpFyr^v?Od09k4X7$R$sH*|i?KGCXN4qe(h*b`D&OxjWHYHgyUfXm*!Dhi98Y)1KKy^D^z>KIR z$kf+$>FM@&bbkuHte&~d#TD3gcnS7Q*_!wj(l%ZhUV`uGbUKKwv;ORmRzQn2mu(FW z2ykV^1L^z)dn78`hH89Nu|hdjpq{7J9x=+kuRBWR?l$0hh-+SEPV~9ioGIz?eV0M! zp852vyd}^+3l&TjW*gS4^7B<*PnK4H=<8YDA^ElrRLOmhpbtyPCLBqSTtyt~3DM^x zo@}UfGjRgOXtZhm`!^??8!`J89g}T(9}I}6F7PrbElKz}XuCyQ#H`)&<8ZhztlTqQ z!BRPX{OtB4UmxVv)EuE+WZ1DL=3Z@c8r`ZWAE6-hx$wrfa2_u!Cnx7E=*2<+LxhNe zO-;u99B1HeNzkOL6>jU0AM%Pj5Tz*~!x!bt#(wPa#pJ}e5kPe^^xzHi209@3J;O|T z>ASV{5|(f=YeWs%z1Mm^bc7Tn(j5JN>V=hNX5WS198}4Y`wBYxFfDsq$&J84=2WO5 z06p>7cLjA8Si>Ywj`871f)JSj-xteVe}!3z%pSSNdK0G;i$(O=uSNosAG4+dWq=~f z9QX^#uguTCPVUfRGv=*Ac5Z<;F-!-VFSvKFF4x^@BkJvXthY7*Du zvHR**KM(5ZKIOHwqDU)pyeR-zCja}9Hy2=fz5IF6c;e=(r1MCVTJXIDxk4p5krV@Eu`#aiV-(a)N)s-;VZs#inmen1MtIBZA3_mSrL z=cgNZ+0)lIQ}MhmI0}e#YYUzBh4gJHRKltSa;`Pwz`|6g{rXK`#`co-OLo7k`@@;0 z!q>M|xtV-~Tb6M#WHxy$F}S~77ero$hVQMpYcr`t<91$-ehY(0P0RY5s!Lq;c@s~{tF?=wsH|>SGIZJa#6b@%j4_FpKptq&Bd@8uZ zj`W(d#KPhiU|jpl!Wiv;Wng$q>6Uy^0pWuuKX(D@q~s{2ScqG~MWI z6DYy(nk7vO)LkrIFyzGzW4pD8Gt3EMB-EtCrn*iF68iF1e;I zZ+=)-JzQ5tUAscEwtaw9@$~7%_kct|ojQJwX?*tLX5L%<$;us1o|VELc<_G`*&0f* z9zFA?2&%wej2n4EaXU5U9{s)X+PiesGI>Ye#iO9rpm;i@hJYNu?c2V3I&8;OyZmM6 z&qFn5ce(a|>M;zT`{!Mo`D0A&?Uxqb@S5>Lk7{~kHMsn8Bb<#zmf6)aJxmIwHf}?K zHHV|^rq36tpcQb2K0_F!B`E6L2H(-2%V?~>|o z*!cXWXtYaAwKY|-BW>xn<};mwmt`fvyk^c{UEGp$=RL9S>dBVRFD=Q;&rcOa4iAcucJ!&1 zHvmSS#sCBpD%zP8;0>FO{D@D{Pw@ydj^2+an6bPAppu`6;qy3{3>9SjTl`My{?W5F zTtUUD*;|b^>YEa1U8tc7?IO&^bGc?mAndZw(RYxW7hqkvXS6ro8SR6@7PnkLpozB~ zuC(BO*?-!3Vo6t65%MY)RvaP1(O-M>LVAy$z^Nplg+Jyr6%;tMNc{x$Qy|UAhUXzG z#tIg)JD7(fg-0W1tZ;Oyn6$s}PE8?22;04k2;QhgF{A^%2XoaDep=uhnapt=^gE^< zE`vZ4KpJj|R(w41?|&JofM3tu%WIvM@adStUsVp)1i+D^|A?@3a?(H^!nXI^!hhpd z``?E6nii5p8{aQ!9s@QZv}9bI2-#ZY+E!DL+x{F>(}LsTD34s$%eHolY}#}4ua(hg0@MmRdhbuMrzVSb|^Ti1O diff --git a/docs/images/sparv_detailed.png b/docs/images/sparv_detailed.png new file mode 100644 index 0000000000000000000000000000000000000000..930a303aa8a8d6df45f5097ea9d75dd9ae142dd2 GIT binary patch literal 9243 zcmYLv1z1!6`!*rnAySh@N^)Z`x<@yT20@S-AOj2pkp}5*kdlxR=@tZ$(2)WYX-0^W z((ggPzyG`I?CkzL&;8s_oY<~&Ey_SogMyTq6bA=~LKCiPgoA@mje~<1N{kO!3d)3p zfe&0CBMoKTZ=ZyMfCY&c+}sBThm7v}jf<0&!wAr840KG?0H?IHG!zPz0)wR>5GhGX z07wD=0)hT-gZu>&5)yz#R#q0+Ut>vt^_K$%Uo(Lh1R@ED{0*0slmrNX+O_L{+qLm8 zzE1QP02Y{-7$5|YpPHp_@D7#dHDxc#_q5J<#**~@@K!n=Ph?%l^}aM)Y%^u+j=nGYC8gcZJr73wQI+*g{cjJv z50-?r65q)ZwU;;Ps#f6GOz}CdlNN>5aQ0yb%C}2Vh|!0K6xDYvvPEVd(KpL(;D_E4 z(+}?4dPNSB*&$K(pCTHm5m4H9#f7}%67s!KOx5!ZjLvN&XlU(Y8hBrx?`f~jci^Jt zedlOC!eAlnYu&Rn?#agt@rJ2GF{doCj0q}k0iT}R3i_5m zt!}vCgrSL*7l`X_6DPMW0Y2Es5Z~&}{G-YS9W5V@q2YRNiO)QOvu7%;$8BD_LirNT zqequQHR~dK-6CE;Pudrb_n!nc`fM{M2#kGRZE7S9+u6x|-qzMeboH*odS+-{7VC^J zUY}2J^0xOV5Rd-(`=y_O_Y-c_ru%bJysWn~wQyf8W@Ck?rHi&Z zTv{N%S6bXjwGrYB@&KE_ zMa2)C+*|AnvfDb?bb>8x2Tt9OstD~aVT9fl$+Wbyc9+P^NNvEc3k`Ue*vxP!K_tNg ztUuwi%sL#gn)*8zQ+9%M`#r~OLKm}k1HEgL%1?fr#is=N)+X??NSaBd zkJG@jdH~zHW|!QIdJ&mm(c_eZvk+_YF`}z;4ec(anZ5%c%j*NdP_^MPqJ=9GY&x;q zOafKs6(e?<*b4itBB|RiD;oq@zk^z?`S1tS3&ReLGNQiPyuD| zp>&^v@s-z)-)Iqf^>=sK&V58u0!u_Qe1ZJ8G`oy5KKBBtDh@2e3bzsYHfG6zb)rdc z0jDqa9vS|(Y^p$&Q%tWc!+OXrv}8e8$H7S9vKkpft@B-$#@d9wFX0rFsl#BBmu{(*Xs^jKo|Q(eW6_XUe9qGx`rjoM-VW5 zO%7cClIeExL&9Ad=iXZSTFT7F;K_MJ5MpZMpu|=V?09g@w!|+f%^6IPw(Du_Y_M#h zz`E)uzg>xnKzrakpYdfgSH&lF&oM}dQ`4odqHVtn_1#cjx_RT&Elc zXPq72C=@F5At%Na{;5PiA?~h(p~ejM4jV7XJY1meYEn7sCA%fY z8I4MgJ?lgnWX7Os$HvDJqD+`#JXH}JYODc~ku}8T8~q}Ql!7Y1bEO64 zPD{|i4@!+9OkG$ybYQi^;0#SDEBe>3bZw2a#0*6JguX2o$bzr}Twc6$AW;MKj1QTe zS8*!8_b9fzRKWV9*3{{@Xw>7(Qd0+;PZk&X$&?}=8el>&#I|Wo8l`9vc-Apw(uNCU zukPZ#n}M-8gJG#9v@}{f5~}mB2mq`Z_Ok~$eYtdk7IUIMC2F%CxWguiPC|+nZyvJJ zW5_|Flhq3MVBI<&8hNrWl7LhZvD!ziKPk{(ALNl{L#k4Piclz~$k5s;5$%DrOh)l< zSl#&KI3bm~h)-Icfwd^R%;=i_ZyR@^@PX7iI5C;rx)8Qj{1znC7b&X&dqZxPf|TRl z>D>n==175EQ|bNwGz+D}m5Iqv<*j^+6((-#;hEn0Ax$xK2 zNa8^zBQ?2$u_eW|O=E<+Yn9%b92`PkyyMzYi)f1dj)?4&Bo!+A*5k`o6uEgAU2%BZ z1tpfce(1!i)xI%eo*$&_*MewOqWOH5Cje4Pds2#u;f#|w*+li~r6Y~{3>=>FyX>YT zja3J9KYn6?Lj_*99^j*U9Ezy4pn|WThM~3*SXQi)8rc6L9JMjyqS0zk^U1o4A0(Z| z$?*y$agvZ^jV)EXk$#ninn%QB3sD?^_3%p~+TvpO3!SxzVn-gKM`9>2eqpF#gok$P z^SVqbZws*L9Gc=)>|-8mWF6~%wR+2iJN8ALbz;RVyK?{nX0x51j&uy2T5o=MN{W8_ zs#bV;e4NSd?JWItpeoF4zFRAlLtX&HA7?l07#^=l6fMXo!L&$#_Fkg=s6+V21Iy=4 z?7(~R+zJM_>dnx6mY)zu(^M;2)}kM%1yh~xB~<9JhNi|*PuJ#aIzu~cVe&a23W;;Q z7j@S7Dj`RX*bOflZJOEB29�R0*;51?8;(ZM)&(a#p%(aYXbNwC{W^M`#H#$a*xi zlK!I(b88CSqeqMq*mW&zffv)h&g_XbHp7eAfftifqSEVKHG*9|mgfS|#5pJ=n<1~N zP>RiQQTKAKW|QKG)_-H^bI&_bQZG1_xq{e1*5d`@K+ z`iZQ9$S*3UFlsM6K^lvKf++$)8rWl~;6!Ud#ng*qX{=~BzscmD6kZ`+g{yZE9|&LS zrHV2-j}wF^75(1Tz6m36ja+!m)6QKFlei7s)FI4YX^@1{d+nov124vx@T<+pc^+J>5cfw5@= zT2;eQ%`q=6jJZQ271EIYe6=Qb?RhcV2K;*j$J@_?wKERXlpU+D)UfIi%p{4@@^r-# ze#Z}X>d?Az?A)p!MANbhDspWS5~y9CO?Lc4{4&cvXx&%3zjWWGJU3|#tD~Xiw6+A!?2!cO z@~3}V{vnsvZCUJ5e>gl63@ORJmA%x~&cCGcXpMsdz3&_VAC=`+b)DJw;Ov~w%Th>3 zTB<547q9977Zp9Xs#d2uu6?lww!3aZCXXFc9~l}7Cp!Ml5*-Tf7aU7MB2-nBi!d0NfKYyXg3LPa&aUL++_j2SCX2zFmEE`!dm(+N>JIHgtVM4P2TWP zM{d<#;pc^@WKPP)pgY?v=)2JfDdnuH>3tW!cZt+dt>>C(is;A9Eh2{2*dtY)JQT6{ z171u{WH}#5*{tQ%9=#Kl;aB1?C=kR0SYvWoNKW0bQ>yM&pLy;K6teHv38!aV=AvVx z{$%*-4&N)8qb0GMg7D`jAhoz(XqFbYy9i*?;N5X3R`X>oY^f@3hNBiSKCevedl9Bv zfm*FRiNw5_wBFcTB`ma2c6W@$Z*yT1;Diug&Q0_Va2HSJC%IrLo}(eLUsk7;=fu2% zlR|v)Iyo`Xf5B;!7L+SVSvU8EOtB?RRAFh{ZZc-Sz&{qn#VI{8Q5Rb>NBxQa z5hG`H4seTBVhxYnnM8X;-Y_&CIJ}MJRZV(R?l5G{g9(MxLVR_Eln`OQn;DJ8mef)A zb78l4?_(KMU%x3|{+0&Z^<%PN41_w3K%NhRT1{+8{qU}k2_|oukFKTjdCcWLrHkG4S~lH z@-xwtoTR)~9%@811a#F?_YH;d_IfO=u`}W{WYkBk8n9+^Re8krbKtA-c06d|2&MsR z9afQ_+4d~{@L6glWJI1FWV6P>CggV+hF*NYy4T}0+=7e_sQpHvpByLh!9ZbV$Fl@~ zSac->)5!y}3kRtF`KZv$uz=e61x<|%^FgAy4mDWcE1;I29%v3H`n?{%h40C6V#x_q zQ*B1Dk2=coh+PS`Buep+1gfL6a8v?QXh7{k6EZDM+$VvG@=r7>B{L+zt9z@sXqa^+ zgseJ^QsUb2_|<=on^3KYOtI!)-wA?5Ku#ao*{ptnu%Z%#CqG^eDZmFZ`Twv-)BP#1 zg0F+kq$z-tBini2<=5MA7iT(=blxvCcw`1MT>vm;c>DIz38zv*xJ2z9 z^|{z418Ajqj)bYj1~xI=O`govDFAYDveO);v2E9eO}y$(%HgU$d5L80H3=F?W@f8k>lC2kOevb3UuorsSPS{4}^55km{cEZW3lYt@p zvZsgU_+_JtGlK)XKC~!Y55_>Dc~PJmLK$|#1GPFN2lBogI1S+%0xAiW-5QU3oOnHt z#U#N?bJ&EA;_;7Hf+Y?kIMNkF&hsNBsauZ;7}#>WBTx-k&lr@~fIl*Y5{mIKl$CPy zdWULviL;pH&rgqoj;yw5b@P|KTa+O-){bjYge^6El&6@UAJQzcuc9)@v`1M)~8-*;8zPa zdNC=P6zyHcAD_#tH=9T)tD8uL6fjb(>H{6e4QXrSl==g^5|{G`Ot9WA1~QbtkfXuSInCo7C95CRI#Mg$uagm}>kQj?;n z^=1!?@%8WNh_yuClukV>$tJ{$cE=k3K|euz;Nz}@3}r5)Az2*b0}M|6v5iVcgGBGw zwV;&Z8uD+%);C!TmXP!o;9Z2PqtyhrlnZgu6%k1Vi3qO_npe zpjWub#(tSLq#t!z^a^uqLhyR|Q>ffl_}Q(LPilb+gmx&xcaSLEK)y=l;s>qS^3>wQ z8*RR_>A||oAJmdf)$b1Zv69MuGxM$LanE9{m8CmAVpt3PNLEp`XJe^I4#%iRurJTY zQjqM9<%653gt<;WjN%~uDym>Wvx%x+O&YXfc<(>-<5`&|luBIvq~HkO)n&T2#dN2W ztk{yse6BRW;K}!I=-7}dvAGKH(-71$yN}Fpqrl^JcoT)V#&N;H>a!9SqRhF{X9mqH z_koG5Wmp;U+j3we2U&}FQon-G9wEaw$%fwxSoJ0nttU`wJ-=&qn;GNkYxM0Z=G}L` z3qDGA(1cA7|1t`dQcJ1dyVhxv*2+j&wiCu z!Pis|fS~eJ{dD(CBcJBI@}~iDJZdN^t&W(LtOqdU^v!rvkxz1Iha0z3j>Pi^GbJsO zy_?KEro**O4+?2c5^lE=mAJ2ak0R5LPYd(hxH~!Ti;T+hjntn*0>=H$W^1X|5~<1y zpBXScX!1t-KXXgwDUc_dZ~-h?4_HlyB^Bn!!8 zZD=Z&>@0UO&~Z@-&Imd4kMEy^hKjto`EI)JQ=;`wcJh7KrE~2Y2<<}LnEf;(*gH&Y zFCm^WVkSInMztU8_z_vATmFqp=fctXRQ;Hp-l^c;BjL8f^N`VC6)LylrnjeQy3UAO zeQ(+PNKcBe8>>yAtG;t&a8${mkwt#zqQdf}Ky*5l$W@K$bK}6d$-vjm-(26cp{1LH z#^RUc?^PW(Tjp202Y7yNLB4KQ5f7cpFa3gdtK82!0vcUsTXZ(^BgjASP79lQMx=1#mZYg(J19Er#%(u7-{OUeuyc_W zQpxxn$$9xGSC_;Uv4oxePKIZAM zdc6ARdsNO|`@EU`Kyw4UQpt*Dy?f)yzMG9^w789cnqCCX_cYCShK?#atBfB?VvGWg z4gJh`33wnfxIM2mzC%#zkV$1?BE zzwJQCp`0y$eB;wk4vnc;XvSS<9h!KgJ&xpcZIchjkd>nS2M0l9nFF)<%6TJ<&IWLi zeK4cs#e(U0db{aTM3do@9rB=ZOKm3!BNKb^#z`6$Sx+8nv{*pnWZ=|lp-7)v?blY4 zz+l6{r1!@v@6k0|A;hDS6tm9;GEB5aTU(}-n6rZ=8Lal8v-KC#r+0L8H^(t?nJUMjjY-<};sOL|9c~vo@iOY} zGQmBn)&t6cowu7qgpVE^xRM;wE`QrVblD)1>5QKTT7Tns+zdQLQCN_zY5jGGN#a#0 zd7_y!Se;juIIZKqad0+7*iq2GU@|({2X<|`Js1^CC}ArAJlxHti_nefU;4a@l=W{j z;B#j?@9;YL0^2@%>6AVG{qt5MMOZZsm~V@dw=I|Ff;04bi$RTW=0 z4JrtM4phQU%cNVnKr7f~XF^u?3KK8i_B!C0Xie+E1E>v+wSS$S^+jdbr)<4x>w7_` z(FLIyGDj8jeQL_w<P6vpMPakDQiH9Oj z)D0$|S{FJu>DS1aV-$4WjEj#`%@|1Moh1(`(LH~{8NTy-vym}@(wwx_#98wAj;ZCs z9dRmTYGBK(g`9=!aB=>C*$SC&tZSMlrz zC2P><+H%IG`d{q5WHXiE?6c$w!N<{iZQn;A0(?&+?x!+&FyZ=7>a7QE|K$Z77RB@SU!K>rPzSiHuwk!hkK@t3`@=#I^y z!{k5UWyj)=#uSt^Zjczd=$4fBg6N($^x{JAggYVrw`t;Ee$4uQ!Fb#jxX-qut#4_(!+6n6>|U--n3l`*6e5Xp!`4Qwme z;6!h4Y`X~DJvW@3-%?!Z8Ckh&gAwHbf$2maS}>CDlwYzARH{^t91j+UginN&F7*l| ziu*Hy!@vrQTCj)wc3&F)RNDGRzEm=ndN#@B>1E_#YN5aR*@=NlPtadMY@9xP`ib4o zR`IVpNiwWP`FWPxoGF$crR8t+x(C$1&Wpo>P^I6GT4&Vwwp`QJr&uJ325RD2z|O@X z-zS{I7}|&*SdY-u_&(jyEHYt!MV6B~Wz$U=*Ds-15s%C8ZJge{#tfwYagsSio9@ll zX8)X1itW%3i!8zpFdY2gsV9z;cSOEoTLg5jc)f>i-zDmd$aok z!3yU0iEwkVi|=%evVT&lDgE$%5A%_M=QHd1k&4c)@LpiCp!M$YMN;dJht=m$d|sn; znwSvY!wZPxjqO4qE(Yf;JiVJ@t}h`I-knn8^j;!Dw>o>7hdhPM7#ceS^ws|iLwY!e z?y|-h-u|U~T!PDBCveCR7Z%H8^JRSj%VkHln%Ug1?UGVdi9bP88@gvqtv#->nd*`M z?H)o!E~w$xUE)51wmCZF)x@oUO65mk?JQ88Zzo z-{qB3WWpsw;dS;kGcNa7-?+132dkMUABrgHReVnSY%-tl04m^LvHa;rvyqqNA8ofU z!EQ923fApwRfPqWh~H9$6rZ0(fZT8VmiPq*IY>XL%x z%>{@8x=}U`yr;`}!4oVNDivVo4ttp&*}I?ll|*no)H1E3JuC{+UZg3Vt#*)3`>N-D$ZSq%Y1t+V+$L8scy^S(bojnQ-O-jU& zjWW5){v$V#hVMIXfOEx&$Ru0w>;i*Hag8d^p-e9gUk#rEzoOOCi{CJND{!r5+KXkaPb zra;~vtuK>o{?U`tkNI#jy8PlPbOX=fB4tJ_^3N&PVR+=D_oi74G2=$%Um3wu3)1Xw z`;{e^m?w|%7=k)FPA{)^bKh8W*vOoZ&J5YfBKt9j&hjCl>~uoQ*zos$Z4z5y80xRP zp|5`b{(ZrCVZ*UxzVoJ;g5v9>+lZM1EH{n$t@nFL^3NkYz*3I!V%N!ngVRJuu zzeMc!drtSNMoGo85Y2Z6EW$PZPYREu-!la1J4<#^C%BS$_tkZ^ACb{*G%Q1e(kyOm8=ca4uEf)>-Ix(af z+2k588QJ8d>EJ^|WgH7j&F`BmZ#A3dKuu;BMg7?MiRlWf8)l?Xx2#B|ZqH4_4y&z( zV`Fz&#F=7m5B;V_2s~&z^14Nm{dIA9f2~F|$nK55(WsudGsKXAPF1d<&t$un~Iv zH-d|4EFY1DT2d?;+hU}#j>y~syvTe($WIbEHZpb72`;4O$ z8RBy1#ub$5y!K}uN3;5~b+2C>uzq)y#U7|XuW9;9apbK#u(S95Q`JWN)Az1|s!AUM zXwW{7`cMZiC%VV#%L)*#>T<>FeUg*E=S7S29^9e9b#sU7IveCPG7xcy z$v0(0eUPrW{O&cyPhUaA7PH~4!gQZya`ydAH?zV~D6#9S%E`56(=0Y8Q?2B+C3P!y zl12Xzw=yYgtx#;=T{d3GVCxt)F3mQD+m5Th_?N4_k_TIj5jbLxpLOhQ|C!o$@|Sga z&s_55UGe>_iJ3uzAQKV)inFjC`(s0XJ?)Sw$9ar?R(L=!dGj_S?dkW2o}QklP2n!% zpeCU}3ips#FZ9&WY?axft}o3td$j{{io=~;%@;gmox6M3AC`(pBWM4hKnbmK*1KN=vcU9fD>ABM+f^5aoH&h>Uf7v>S_c?+;yvRxG0&TEOEZvlUZz|mCGQ>|9E Hd-?wWScCEs literal 0 HcmV?d00001 diff --git a/docs/md2pdf/make_pdf.sh b/docs/md2pdf/make_pdf.sh index 67afe419..205f9e38 100755 --- a/docs/md2pdf/make_pdf.sh +++ b/docs/md2pdf/make_pdf.sh @@ -61,7 +61,7 @@ author: | | | | - | ![](../images/sparv.png){width=2.5cm} + | ![](../images/sparv_detailed.png){width=2.5cm} --- " From 21831b29d8cf0d6250196306bcd7a4eea124285c Mon Sep 17 00:00:00 2001 From: Anne Schumacher Date: Thu, 9 Nov 2023 13:31:03 +0100 Subject: [PATCH 162/175] change tag line on documentation cover --- docs/docsify/_coverpage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docsify/_coverpage.md b/docs/docsify/_coverpage.md index 2d235052..899e503a 100644 --- a/docs/docsify/_coverpage.md +++ b/docs/docsify/_coverpage.md @@ -2,7 +2,7 @@ # Sparv Pipeline Documentation -> Språkbanken's text analysis tool +> Språkbanken's analysis platform

version 5.1.1dev0

From b8d3fd57adbaad1e667537d2817e4d87cc702bff Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 17 Nov 2023 18:00:22 +0100 Subject: [PATCH 163/175] Switch to using importlib instead of pkg_resources in run_snake.py ... to prevent deprecation warning from messing up the progressbar --- sparv/core/run_snake.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sparv/core/run_snake.py b/sparv/core/run_snake.py index 5c8fa8df..038cc9e6 100644 --- a/sparv/core/run_snake.py +++ b/sparv/core/run_snake.py @@ -5,7 +5,7 @@ import sys import traceback -from pkg_resources import iter_entry_points +from importlib.metadata import entry_points from sparv.core import io, log_handler, paths from sparv.core import registry @@ -16,7 +16,7 @@ # The snakemake variable is provided automatically by Snakemake; the below is just to please the IDE try: - snakemake + snakemake # noqa except NameError: from snakemake.script import Snakemake snakemake: Snakemake @@ -96,7 +96,7 @@ def flush(): module = importlib.import_module(".".join((modules_path, module_name))) except ModuleNotFoundError: # Try to find plugin module - entry_points = dict((e.name, e) for e in iter_entry_points(f"sparv.{plugin_name}")) + entry_points = dict((e.name, e) for e in entry_points(group=f"sparv.{plugin_name}")) entry_point = entry_points.get(module_name) if entry_point: module = entry_point.load() From e0b780d87775a3562f967e0c96a765990c91a476 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Thu, 23 Nov 2023 13:48:08 +0100 Subject: [PATCH 164/175] Switch to using importlib instead of pkg_resources in registry.py --- pyproject.toml | 1 + sparv/core/registry.py | 30 +++++++++++++++++++++++++----- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e31e4aae..c55aa0bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ dependencies = [ "docx2python==1.27.1", "jsonschema==4.19.0", "nltk==3.8.1", + "packaging>=21.0", "pdfplumber==0.9.0", "protobuf~=3.19.0", # Used by Stanza; see https://github.com/spraakbanken/sparv-pipeline/issues/161 "pycountry==22.3.5", diff --git a/sparv/core/registry.py b/sparv/core/registry.py index deedcfa3..4ad3d970 100644 --- a/sparv/core/registry.py +++ b/sparv/core/registry.py @@ -107,7 +107,10 @@ def find_modules(no_import: bool = False, find_custom: bool = False) -> list: Returns: A list of available module names. """ - from pkg_resources import iter_entry_points, VersionConflict + from importlib.metadata import entry_points + from packaging.requirements import Requirement + + from sparv import __version__ as sparv_version modules_full_path = paths.sparv_path / paths.modules_dir core_modules_full_path = paths.sparv_path / paths.core_modules_dir @@ -143,12 +146,29 @@ def find_modules(no_import: bool = False, find_custom: bool = False) -> list: add_module_to_registry(m, module_name) # Search for installed plugins - for entry_point in iter_entry_points("sparv.plugin"): + for entry_point in entry_points(group="sparv.plugin"): + skip = False try: m = entry_point.load() - except VersionConflict as e: - console.print(f"[red]:warning-emoji: The plugin {entry_point.dist} could not be loaded. " - f"It requires {e.req}, but the current installed version is {e.dist}.\n") + # Check compatibility with Sparv version + for requirement in entry_point.dist.requires: + req = Requirement(requirement) + if req.name == "sparv-pipeline": + if not sparv_version in req.specifier: + console.print( + f"[red]:warning-emoji: The plugin {entry_point.name} ({entry_point.dist.name}) could not " + f"be loaded. It requires Sparv version {req.specifier}, but the currently running Sparv " + f"version is {sparv_version}.\n" + ) + skip = True + break + if skip: + continue + except Exception as e: + console.print( + f"[red]:warning-emoji: The plugin {entry_point.name} ({entry_point.dist.name}) could not be loaded:\n" + f"\n {e}" + ) continue add_module_to_registry(m, entry_point.name) module_names.append(entry_point.name) From 06193c56157c29d49698ccbbce21cc6472c45e1e Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Thu, 23 Nov 2023 13:48:54 +0100 Subject: [PATCH 165/175] Add TODO regarding pkg_resources in setup.py --- sparv/core/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sparv/core/setup.py b/sparv/core/setup.py index 594bebb8..bff6a1ff 100644 --- a/sparv/core/setup.py +++ b/sparv/core/setup.py @@ -34,6 +34,7 @@ def check_sparv_version() -> Optional[bool]: def copy_resource_files(data_dir: pathlib.Path): """Copy resource files to data dir.""" + # TODO: Use importlib.resources.files instead once we require Python 3.9 resources_dir = pathlib.Path(pkg_resources.resource_filename("sparv", "resources")) for f in resources_dir.rglob("*"): From 2046a35e5fcb848e49f6a3641085783da7a9bb41 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 24 Nov 2023 12:01:47 +0100 Subject: [PATCH 166/175] Print friendlier error message when user interrupts a module --- sparv/core/run_snake.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sparv/core/run_snake.py b/sparv/core/run_snake.py index 038cc9e6..dc1bf90f 100644 --- a/sparv/core/run_snake.py +++ b/sparv/core/run_snake.py @@ -133,6 +133,8 @@ def flush(): registry.modules[module_name].functions[f_name]["function"](**parameters) if snakemake.params.export_dirs: logger.export_dirs(snakemake.params.export_dirs) + except KeyboardInterrupt as e: + exit_with_error_message("Execution was terminated by an interrupt signal", "sparv.modules." + module_name) except SparvErrorMessage as e: # Any exception raised here would be printed directly to the terminal, due to how Snakemake runs the script. # Instead, we log the error message and exit with a non-zero status to signal to Snakemake that From d3394ef6a32195da50cf56829918559edb4a223e Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Fri, 24 Nov 2023 12:03:49 +0100 Subject: [PATCH 167/175] Print error message when Sparv subprocess is killed --- sparv/core/log_handler.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/sparv/core/log_handler.py b/sparv/core/log_handler.py index fe976b88..bd56c2c7 100644 --- a/sparv/core/log_handler.py +++ b/sparv/core/log_handler.py @@ -587,7 +587,16 @@ def missing_annotations_or_files(source, files): # an unexpected exception. Either way, the error message has already been logged, so it doesn't need to # be printed again. self.handled_error = True - + elif "died with Date: Fri, 24 Nov 2023 17:17:50 +0100 Subject: [PATCH 168/175] Add config validation and schema to documentation --- docs/developers-guide/sparv-classes.md | 11 ++++++-- docs/user-manual/corpus-configuration.md | 4 +++ docs/user-manual/running-sparv.md | 35 +++++++++++++----------- 3 files changed, 32 insertions(+), 18 deletions(-) diff --git a/docs/developers-guide/sparv-classes.md b/docs/developers-guide/sparv-classes.md index 66bb3c91..92518c3d 100644 --- a/docs/developers-guide/sparv-classes.md +++ b/docs/developers-guide/sparv-classes.md @@ -183,14 +183,21 @@ the `bin` path inside the Sparv data directory. ## Config -An instance of this class holds a configuration key name and its default value. +An instance of this class holds a configuration key name and its default value. The datatype and values allowed can be +specified, and will be used both for validating the config and when generating the Sparv config JSON schema. **Arguments:** - `name`: The name of the configuration key. - `default`: An optional default value of the configuration key. - `description`: An obligatory description. - +- `datatype`: Typehint specifying the allowed datatype(s). +- `choices`: Iterable with valid choices. +- `pattern`: Regular expression matching valid values (only for the datatype `str`). +- `min`: A `float` representing the minimum numeric value. +- `max`: A `float` representing the maximum numeric value. +- `const`: Restrict the value to a single value. +- `conditions`: List of `Config` objects with conditions that must also be met. ## Corpus An instance of this class holds the name (ID) of the corpus. diff --git a/docs/user-manual/corpus-configuration.md b/docs/user-manual/corpus-configuration.md index 4c94eac1..f2de7345 100644 --- a/docs/user-manual/corpus-configuration.md +++ b/docs/user-manual/corpus-configuration.md @@ -32,6 +32,10 @@ export: > has its own section in the file, like the `metadata` and `export` sections in the example above. > By using the `sparv modules` command you can get a list of the available configuration keys and their descriptions. +## Config Schema +Running `sparv schema` will output a JSON schema which can be used in many text editors to validate your config file as +you are creating it, and in some editors can be used to provide autocompletion. + ## Corpus Config Wizard The corpus config wizard is a tool designed to help you create a corpus config file by asking questions about your corpus and the annotations you would like Sparv to add to it. Run `sparv wizard` in order to start the tool. When diff --git a/docs/user-manual/running-sparv.md b/docs/user-manual/running-sparv.md index 4064f9d9..6cecb546 100644 --- a/docs/user-manual/running-sparv.md +++ b/docs/user-manual/running-sparv.md @@ -18,30 +18,33 @@ corpora](https://github.com/spraakbanken/sparv-pipeline/releases/latest/download When running `sparv` (or `sparv -h`) the available sparv commands will be listed: ``` Annotating a corpus: - run Annotate a corpus and generate export files - install Annotate and install a corpus on remote server - clean Remove output directories + run Annotate a corpus and generate export files + install Install a corpus + uninstall Uninstall a corpus + clean Remove output directories Inspecting corpus details: - config Display the corpus config - files List available corpus source files (input for Sparv) + config Display the corpus config + files List available corpus source files (input for Sparv) Show annotation info: - modules List available modules and annotations - presets List available annotation presets - classes List available annotation classes - languages List supported languages + modules List available modules and annotations + presets List available annotation presets + classes List available annotation classes + languages List supported languages Setting up the Sparv Pipeline: - setup Set up the Sparv data directory - wizard Run config wizard to create a corpus config - build-models Download and build the Sparv models (optional) + setup Set up the Sparv data directory + wizard Run config wizard to create a corpus config + build-models Download and build the Sparv models (optional) Advanced commands: - run-rule Run specified rule(s) for creating annotations - create-file Create specified file(s) - run-module Run annotator module independently - preload Preload annotators and models + run-rule Run specified rule(s) for creating annotations + create-file Create specified file(s) + run-module Run annotator module independently + preload Preload annotators and models + autocomplete Enable tab completion in bash + schema Print a JSON schema for the Sparv config format ``` Every command in the Sparv command line interface has a help text which can be accessed with the `-h` flag. Below we From 123c0d517abcb80e675c311ce1f189e2e5adf154 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Wed, 29 Nov 2023 16:17:36 +0100 Subject: [PATCH 169/175] Use importlib_metadata instead of importlib.metadata for compatibility with Python <3.10 --- pyproject.toml | 1 + sparv/core/registry.py | 2 +- sparv/core/run_snake.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c55aa0bc..fcc08aa0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ dependencies = [ "appdirs==1.4.4", "argcomplete==3.0.5", "docx2python==1.27.1", + "importlib-metadata==6.8.0", # For Python <3.10 compatibility "jsonschema==4.19.0", "nltk==3.8.1", "packaging>=21.0", diff --git a/sparv/core/registry.py b/sparv/core/registry.py index 4ad3d970..f4b24a55 100644 --- a/sparv/core/registry.py +++ b/sparv/core/registry.py @@ -107,7 +107,7 @@ def find_modules(no_import: bool = False, find_custom: bool = False) -> list: Returns: A list of available module names. """ - from importlib.metadata import entry_points + from importlib_metadata import entry_points from packaging.requirements import Requirement from sparv import __version__ as sparv_version diff --git a/sparv/core/run_snake.py b/sparv/core/run_snake.py index dc1bf90f..2af073f2 100644 --- a/sparv/core/run_snake.py +++ b/sparv/core/run_snake.py @@ -5,7 +5,7 @@ import sys import traceback -from importlib.metadata import entry_points +from importlib_metadata import entry_points from sparv.core import io, log_handler, paths from sparv.core import registry From 0bb39826a5a57da0b3e4966fabbc4a135c1180e2 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Wed, 29 Nov 2023 17:13:36 +0100 Subject: [PATCH 170/175] Accept pre-release versions of Sparv when checking plugin compatibility --- sparv/core/registry.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sparv/core/registry.py b/sparv/core/registry.py index f4b24a55..b8699475 100644 --- a/sparv/core/registry.py +++ b/sparv/core/registry.py @@ -154,6 +154,7 @@ def find_modules(no_import: bool = False, find_custom: bool = False) -> list: for requirement in entry_point.dist.requires: req = Requirement(requirement) if req.name == "sparv-pipeline": + req.specifier.prereleases = True # Accept pre-release versions of Sparv if not sparv_version in req.specifier: console.print( f"[red]:warning-emoji: The plugin {entry_point.name} ({entry_point.dist.name}) could not " From 723695a48b74ccd2c3efb66b2659abef3bd18bf7 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Mon, 4 Dec 2023 14:45:01 +0100 Subject: [PATCH 171/175] Bump protobuf dependency --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index fcc08aa0..f47be19f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ dependencies = [ "nltk==3.8.1", "packaging>=21.0", "pdfplumber==0.9.0", - "protobuf~=3.19.0", # Used by Stanza; see https://github.com/spraakbanken/sparv-pipeline/issues/161 + "protobuf>=3.19.0,<4.0.0", # Used by Stanza; see https://github.com/spraakbanken/sparv-pipeline/issues/161 "pycountry==22.3.5", "python-dateutil==2.8.2", "python-json-logger==2.0.7", From eee406585306a64e9ed219802d7dd12d351aadb4 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Mon, 4 Dec 2023 14:48:27 +0100 Subject: [PATCH 172/175] Format pyproject.toml --- pyproject.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f47be19f..90460f5a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,12 +16,12 @@ dependencies = [ "appdirs==1.4.4", "argcomplete==3.0.5", "docx2python==1.27.1", - "importlib-metadata==6.8.0", # For Python <3.10 compatibility + "importlib-metadata==6.8.0", # For Python <3.10 compatibility "jsonschema==4.19.0", "nltk==3.8.1", "packaging>=21.0", "pdfplumber==0.9.0", - "protobuf>=3.19.0,<4.0.0", # Used by Stanza; see https://github.com/spraakbanken/sparv-pipeline/issues/161 + "protobuf>=3.19.0,<4.0.0", # Used by Stanza; see https://github.com/spraakbanken/sparv-pipeline/issues/161 "pycountry==22.3.5", "python-dateutil==2.8.2", "python-json-logger==2.0.7", @@ -30,7 +30,7 @@ dependencies = [ "rich==13.3.3", "snakemake==7.25.0", "stanza==1.5.1", - "torch>=1.9.1", # Used by Stanza; see https://github.com/spraakbanken/sparv-pipeline/issues/82 + "torch>=1.9.1", # Used by Stanza; see https://github.com/spraakbanken/sparv-pipeline/issues/82 "typing-inspect==0.8.0", ] @@ -51,7 +51,7 @@ sparv = "sparv.__main__:main" [tool.hatch] version.path = "sparv/__init__.py" build.include = ["/sparv"] -publish.index.disable = true # Require confirmation to publish +publish.index.disable = true # Require confirmation to publish [tool.black] line-length = 120 From 24500aac00a723637fa03b126c06f83fe5e810ae Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Wed, 6 Dec 2023 14:04:51 +0100 Subject: [PATCH 173/175] Upgrade Snakemake --- pyproject.toml | 2 +- sparv/core/Snakefile | 6 +++--- sparv/core/log_handler.py | 2 +- sparv/core/snake_utils.py | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 90460f5a..e7353643 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ dependencies = [ "PyYAML==6.0", "questionary==1.10.0", "rich==13.3.3", - "snakemake==7.25.0", + "snakemake==7.32.3", "stanza==1.5.1", "torch>=1.9.1", # Used by Stanza; see https://github.com/spraakbanken/sparv-pipeline/issues/82 "typing-inspect==0.8.0", diff --git a/sparv/core/Snakefile b/sparv/core/Snakefile index e24565d3..8428c89a 100644 --- a/sparv/core/Snakefile +++ b/sparv/core/Snakefile @@ -119,7 +119,7 @@ def make_all_files_rule(rule_storage: snake_utils.RuleStorage) -> None: return # Get Snakemake rule object - sm_rule = getattr(rules, rule_storage.rule_name).rule + sm_rule = workflow.get_rule(rule_storage.rule_name) dependencies = rule_storage.outputs if not rule_storage.abstract else rule_storage.inputs @@ -141,7 +141,7 @@ def make_all_files_rule(rule_storage: snake_utils.RuleStorage) -> None: # Set rule dependencies for every file, so Snakemake knows which rule to use in case of ambiguity. # Converting the values of rule_outputs to snakemake.io.IOFile objects is not enough, since file paths must match # for that to work (which they don't do once we've expanded the {file} wildcard). - this_sm_rule = getattr(rules, rule_storage.target_name).rule + this_sm_rule = workflow.get_rule(rule_storage.target_name) for f in this_sm_rule.input: this_sm_rule.dependencies[f] = sm_rule @@ -441,7 +441,7 @@ rule list_uninstalls: # Rule for making exports defined in corpus config if "export_corpus" in selected_targets: - export_targets = snake_utils.get_export_targets(snake_storage, rules, + export_targets = snake_utils.get_export_targets(snake_storage, workflow, file=snake_utils.get_file_values(config, snake_storage), wildcards=snake_utils.get_wildcard_values(config)) rule export_corpus: diff --git a/sparv/core/log_handler.py b/sparv/core/log_handler.py index bd56c2c7..9a609100 100644 --- a/sparv/core/log_handler.py +++ b/sparv/core/log_handler.py @@ -501,7 +501,7 @@ def missing_annotations_or_files(source, files): lines = msg["msg"].splitlines()[3:] total_jobs = lines[-1].split()[1] for j in lines[:-1]: - job, count, _, _ = j.split() + job, count = j.split() if ":" in job and not "::" in job: job = job + "*" # Differentiate entrypoints from actual rules in the list self.jobs[job.replace("::", ":")] = int(count) diff --git a/sparv/core/snake_utils.py b/sparv/core/snake_utils.py index 6920229b..e2320c85 100644 --- a/sparv/core/snake_utils.py +++ b/sparv/core/snake_utils.py @@ -792,7 +792,7 @@ def get_install_outputs(snake_storage: SnakeStorage, install_types: Optional[Lis return install_outputs -def get_export_targets(snake_storage, rules, file, wildcards): +def get_export_targets(snake_storage, workflow: snakemake.Workflow, file, wildcards): """Get export targets from sparv_config.""" all_outputs = [] config_exports = set(sparv_config.get("export.default", [])) @@ -803,7 +803,7 @@ def get_export_targets(snake_storage, rules, file, wildcards): # Get all output files for all source files rule_outputs = expand(rule.outputs if not rule.abstract else rule.inputs, file=file, **wildcards) # Get Snakemake rule object - sm_rule = getattr(rules, rule.rule_name).rule + sm_rule = workflow.get_rule(rule.rule_name) all_outputs.append((sm_rule if not rule.abstract else None, rule_outputs)) if config_exports: From be05a88907afa8fc7ff468bb084730ede4ef3bfe Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Wed, 6 Dec 2023 14:15:54 +0100 Subject: [PATCH 174/175] Upgrade dependencies --- pyproject.toml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e7353643..80fa77a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,24 +14,24 @@ authors = [ ] dependencies = [ "appdirs==1.4.4", - "argcomplete==3.0.5", + "argcomplete==3.1.6", "docx2python==1.27.1", "importlib-metadata==6.8.0", # For Python <3.10 compatibility - "jsonschema==4.19.0", + "jsonschema==4.20.0", "nltk==3.8.1", "packaging>=21.0", - "pdfplumber==0.9.0", + "pdfplumber==0.10.3", "protobuf>=3.19.0,<4.0.0", # Used by Stanza; see https://github.com/spraakbanken/sparv-pipeline/issues/161 "pycountry==22.3.5", "python-dateutil==2.8.2", "python-json-logger==2.0.7", - "PyYAML==6.0", + "PyYAML==6.0.1", "questionary==1.10.0", - "rich==13.3.3", + "rich==13.7.0", "snakemake==7.32.3", "stanza==1.5.1", "torch>=1.9.1", # Used by Stanza; see https://github.com/spraakbanken/sparv-pipeline/issues/82 - "typing-inspect==0.8.0", + "typing-inspect==0.9.0", ] [project.optional-dependencies] From be1432251e4de1d23e20c9b2ba7dfa4e6d38fa97 Mon Sep 17 00:00:00 2001 From: Martin Hammarstedt Date: Thu, 7 Dec 2023 15:57:06 +0100 Subject: [PATCH 175/175] Update changelog and bump version number --- CHANGELOG.md | 6 +++++- docs/docsify/_coverpage.md | 2 +- sparv/__init__.py | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c646dd07..8a6d5bb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,11 @@ # Changelog -## [5.2.0.dev0] +## [5.2.0] - 2023-12-07 ### Added - Added support for tab autocompletion in bash. +- Added importer for PDF files. - Added new `misc:inherit` annotator for inheriting attributes. - Added `korp.wordpicture_no_sentences` setting to disable generation of Word Picture sentences table. - `util.mysql_wrapper` can now execute SQL queries remotely over SSH. @@ -35,6 +36,7 @@ - Running `sparv schema` will now generate a JSON schema which can be used to validate corpus config files. - More strict config validation, including validation of config values and data types. - Most Sparv decorators now have a `priority` parameter, to control the order in which functions are run. +- Added `util.misc.dump_yaml()` utility function for exporting YAML. ### Changed @@ -59,6 +61,7 @@ - Not specifying a corpus language now excludes all language specific annotators. - When an unhandled exception occurs, the relevant source document will be displayed in the log. - `localhost` as an installation target is no longer handled as if host was omitted. +- Removed `critical` log level. ### Fixed @@ -304,6 +307,7 @@ - Increased independence between modules and language models - This facilitates adding new annotation modules and import/export formats. +[5.2.0]: https://github.com/spraakbanken/sparv-pipeline/releases/tag/v5.2.0 [5.1.0]: https://github.com/spraakbanken/sparv-pipeline/releases/tag/v5.1.0 [5.0.0]: https://github.com/spraakbanken/sparv-pipeline/releases/tag/v5.0.0 [4.1.1]: https://github.com/spraakbanken/sparv-pipeline/releases/tag/v4.1.1 diff --git a/docs/docsify/_coverpage.md b/docs/docsify/_coverpage.md index 899e503a..17f94ee5 100644 --- a/docs/docsify/_coverpage.md +++ b/docs/docsify/_coverpage.md @@ -4,7 +4,7 @@ > Språkbanken's analysis platform -

version 5.1.1dev0

+

version 5.2.0

Sparv on GitHub diff --git a/sparv/__init__.py b/sparv/__init__.py index cb9e8c52..1cfd4801 100644 --- a/sparv/__init__.py +++ b/sparv/__init__.py @@ -1,3 +1,3 @@ """Main Sparv package.""" -__version__ = "5.1.1dev0" +__version__ = "5.2.0"