From c98c0fd55fa6a3ec507f6e42112fa48d0ecca243 Mon Sep 17 00:00:00 2001 From: "Federico M. Facca" Date: Wed, 11 Oct 2023 15:58:46 +0200 Subject: [PATCH 01/10] update poetry to support modernpython Signed-off-by: Federico M. Facca use enum for Signed-off-by: Federico M. Facca don't generate not used classes Signed-off-by: Federico M. Facca surface cimdatatype and primitive stereoteypes Signed-off-by: Federico M. Facca normalize attribute names to avoid issues when pointing to classes Signed-off-by: Federico M. Facca Delete PositionPoint.py Signed-off-by: Federico M. Facca generate references to classes * clean support for primitives and cim data types * class importer * cyclic reference validator Signed-off-by: Federico M. Facca copy util.py Signed-off-by: Federico M. Facca update dataclass config * support populate by name * support deferred evaluation * support build from attributes * change from forbid to ignore (to be tested also without change) Signed-off-by: Federico M. Facca spring cleans Signed-off-by: Federico M. Facca attempt to manage cyclic references across multiple files this is not working unfortunately (today); a similar approach may work in the future when superclass inherited type hints may work as well (probably python 3.13+) Signed-off-by: Federico M. Facca --- CIMgen.py | 59 +++- cpp/langPack.py | 6 + java/langPack.py | 6 + javascript/langPack.py | 6 + modernpython/langPack.py | 261 ++++++++++++++++-- .../templates/cimpy_class_template.mustache | 27 +- .../templates/pydantic_enum_template.mustache | 15 + modernpython/util.py | 37 +++ modernpython/utils/base.py | 4 +- python/langPack.py | 6 + 10 files changed, 386 insertions(+), 41 deletions(-) create mode 100644 modernpython/templates/pydantic_enum_template.mustache create mode 100644 modernpython/util.py diff --git a/CIMgen.py b/CIMgen.py index b759e0ea..5202346d 100644 --- a/CIMgen.py +++ b/CIMgen.py @@ -208,6 +208,7 @@ def __init__(self, rdfsEntry): self.origin_list = [] self.super = rdfsEntry.subClassOf() self.subclasses = [] + self.stereotype = rdfsEntry.stereotype() def attributes(self): return self.attribute_list @@ -275,6 +276,12 @@ def is_a_float(self): return False return True + def is_a_primitive(self): + return self.stereotype == 'Primitive' + + def is_a_cim_datatype(self): + return self.stereotype == 'CIMDatatype' + def get_profile_name(descriptions): for list_elem in descriptions: # only for CGMES-Standard @@ -446,6 +453,8 @@ def _write_python_files(elem_dict, langPack, outputPath, version): float_classes = {} enum_classes = {} + primitive_classes = {} + cim_data_type_classes = {} # Iterate over Classes for class_definition in elem_dict: @@ -453,9 +462,15 @@ def _write_python_files(elem_dict, langPack, outputPath, version): float_classes[class_definition] = True if elem_dict[class_definition].has_instances(): enum_classes[class_definition] = True + if elem_dict[class_definition].is_a_primitive(): + primitive_classes[class_definition] = True + if elem_dict[class_definition].is_a_cim_datatype(): + cim_data_type_classes[class_definition] = True langPack.set_float_classes(float_classes) langPack.set_enum_classes(enum_classes) + langPack.set_primitive_classes(primitive_classes) + langPack.set_cim_data_type_classes(cim_data_type_classes) for class_name in elem_dict.keys(): @@ -467,6 +482,8 @@ def _write_python_files(elem_dict, langPack, outputPath, version): "instances": elem_dict[class_name].instances(), "has_instances": elem_dict[class_name].has_instances(), "is_a_float": elem_dict[class_name].is_a_float(), + "is_a_primitive": elem_dict[class_name].is_a_primitive(), + "is_a_cim_data_type": elem_dict[class_name].is_a_cim_datatype(), "langPack": langPack, "sub_class_of": elem_dict[class_name].superClass(), "sub_classes": elem_dict[class_name].subClasses(), @@ -638,18 +655,42 @@ def _merge_classes(profiles_dict): class_dict[class_key].addAttribute(attr) return class_dict -def recursivelyAddSubClasses(class_dict, class_name): +def recursively_add_sub_classes(class_dict, class_name): newSubClasses = [] theClass = class_dict[class_name] for name in theClass.subClasses(): newSubClasses.append(name) - newNewSubClasses = recursivelyAddSubClasses(class_dict, name) + newNewSubClasses = recursively_add_sub_classes(class_dict, name) newSubClasses = newSubClasses + newNewSubClasses return newSubClasses -def addSubClassesOfSubClasses(class_dict): - for className in class_dict: - class_dict[className].setSubClasses(recursivelyAddSubClasses(class_dict, className)) +def add_sub_classes_of_sub_classes(class_dict): + for class_name in class_dict: + class_dict[class_name].setSubClasses(recursively_add_sub_classes(class_dict, class_name)) + +def add_sub_classes_of_sub_classes_clean(class_dict, source): + temp = {} + for class_name in class_dict: + for name in class_dict[class_name].subClasses(): + if name not in class_dict: + temp[name] = source[name] + add_sub_classes_of_sub_classes_clean(temp, source) + class_dict.update(temp) + +# Order classes based on dependency order + +def generate_clean_sub_classes(class_dict_with_origins, clean_class_dict): + for class_name in class_dict_with_origins: + super_class_name = class_dict_with_origins[class_name].superClass() + if super_class_name == None and class_dict_with_origins[class_name].has_instances(): + clean_class_dict[class_name] = class_dict_with_origins[class_name] + + for class_name in class_dict_with_origins: + super_class_name = class_dict_with_origins[class_name].superClass() + if super_class_name == None and not class_dict_with_origins[class_name].has_instances(): + clean_class_dict[class_name] = class_dict_with_origins[class_name] + + add_sub_classes_of_sub_classes_clean(clean_class_dict, class_dict_with_origins) def cim_generate(directory, outputPath, version, langPack): """Generates cgmes python classes from cgmes ontology @@ -691,6 +732,8 @@ def cim_generate(directory, outputPath, version, langPack): # merge classes from different profiles into one class and track origin of the classes and their attributes class_dict_with_origins = _merge_classes(profiles_dict) + clean_class_dict = {} + # work out the subclasses for each class by noting the reverse relationship for className in class_dict_with_origins: superClassName = class_dict_with_origins[className].superClass() @@ -702,10 +745,12 @@ def cim_generate(directory, outputPath, version, langPack): print("No match for superClass in dict: :", superClassName) # recursively add the subclasses of subclasses - addSubClassesOfSubClasses(class_dict_with_origins) + add_sub_classes_of_sub_classes(class_dict_with_origins) + + generate_clean_sub_classes(class_dict_with_origins, clean_class_dict) # get information for writing python files and write python files - _write_python_files(class_dict_with_origins, langPack, outputPath, version) + _write_python_files(clean_class_dict, langPack, outputPath, version) if "modernpython" in langPack.__name__: langPack.resolve_headers(outputPath, version) diff --git a/cpp/langPack.py b/cpp/langPack.py index c6e7c2c7..062075a2 100644 --- a/cpp/langPack.py +++ b/cpp/langPack.py @@ -118,6 +118,12 @@ def is_an_enum_class(name): if name in enum_classes: return enum_classes[name] +def set_primitive_classes(new_primitive_classes): + return + +def set_cim_data_type_classes(new_cim_data_type_classes): + return + # These insert_ functions are used to generate the entries in the dynamic_switch # maps, for use in assignments.cpp and Task.cpp # TODO: implement this as one function, determine in template if it should be called. diff --git a/java/langPack.py b/java/langPack.py index 10ba104b..6d404b55 100644 --- a/java/langPack.py +++ b/java/langPack.py @@ -117,6 +117,12 @@ def is_an_enum_class(name): if name in enum_classes: return enum_classes[name] +def set_primitive_classes(new_primitive_classes): + return + +def set_cim_data_type_classes(new_cim_data_type_classes): + return + # These insert_ functions are used to generate the entries in the dynamic_switch # maps, for use in assignments.cpp and Task.cpp # TODO: implement this as one function, determine in template if it should be called. diff --git a/javascript/langPack.py b/javascript/langPack.py index 8c6c49ca..bc1e5a1d 100644 --- a/javascript/langPack.py +++ b/javascript/langPack.py @@ -63,6 +63,12 @@ def is_an_enum_class(name): if name in enum_classes: return enum_classes[name] +def set_primitive_classes(new_primitive_classes): + return + +def set_cim_data_type_classes(new_cim_data_type_classes): + return + def get_class_location(class_name, class_map, version): pass diff --git a/modernpython/langPack.py b/modernpython/langPack.py index 5339a01a..62a3baee 100644 --- a/modernpython/langPack.py +++ b/modernpython/langPack.py @@ -2,6 +2,9 @@ import os import re from distutils.dir_util import copy_tree +import shutil +import sys +import textwrap from pathlib import Path import chevron @@ -32,14 +35,64 @@ def location(version): base = {"base_class": "Base", "class_location": location} template_files = [{"filename": "cimpy_class_template.mustache", "ext": ".py"}] - +enum_template_files = [{"filename": "pydantic_enum_template.mustache", "ext": ".py"}] def get_class_location(class_name, class_map, version): return f".{class_map[class_name].superClass()}" + # Check if the current class has a parent class +""" if class_map[class_name].superClass(): + if class_map[class_name].superClass() in class_map: + return "modernpython." + version + "." + class_map[class_name].superClass() + elif class_map[class_name].superClass() == "Base" or class_map[class_name].superClass() == None: + return location(version) + else: + return location(version) """ partials = {} +# computes the data type +def _compute_data_type(attribute): + if "label" in attribute and attribute["label"] == "mRID": + return "str" + + if "range" in attribute: + # return "'"+attribute["range"].split("#")[1]+"'" + return "'"+attribute["range"].split("#")[1]+"'" + if "dataType" in attribute and "class_name" in attribute: + # for whatever weird reason String is not created as class from CIMgen + if is_primitive_class(attribute["class_name"]) or attribute["class_name"] == "String": + datatype = attribute["dataType"].split("#")[1] + if datatype == "Integer" or datatype == "integer": + return "int" + if datatype == "Boolean": + return "bool" + if datatype == "String": + return "str" + if datatype == "DateTime": + return "datetime" + if datatype == "MonthDay": + return "str" # TO BE FIXED? + if datatype == "Date": + return "str" # TO BE FIXED? + if datatype == "Time": + return "time" + if datatype == "Float": + return "float" + if datatype == "String": + return "str" + else: + # this actually never happens + return "float" + # the assumption is that cim data type e.g. Voltage, ActivePower, always + # maps to a float + if is_cim_data_type_class(attribute["class_name"]): + return "float" + # this is for example the case for 'StreetAddress.streetDetail' + return "'"+attribute["dataType"].split("#")[1]+"'" + +def _ends_with_s(attribute_name): + return attribute_name.endswith("s") # called by chevron, text contains the label {{dataType}}, which is evaluated by the renderer (see class template) def _set_default(text, render): @@ -49,30 +102,126 @@ def _set_default(text, render): def _set_type(text, render): return _get_type_and_default(text, render)[0] +def _lower_case_first_char(str): + return str[:1].lower() + str[1:] if str else "" + +# attributes should never have the same name as a class the may map to +# Python won't be happy with that, so let's normalize attribute names +# and leverage aliasing in pydantic to use the cim attribute capitalization +# during value setting. +def _set_normalized_name(text, render): + return _lower_case_first_char(render(text)) + +# called by chevron, text contains the label {{dataType}}, which is evaluated by the renderer (see class template) +def _set_instances(text, render): + instance = None + try: + instance = eval(render(text)) + except SyntaxError as se: + rendered = render(text) + rendered = rendered.replace(""", '"') + instance = eval(rendered) + logger.warning("Exception in evaluating %s : %s . Handled replacing quotes", rendered, se.msg) + if "label" in instance: + value = instance["label"] + ' = "' + instance["label"] + '"' + if "comment" in instance: + value += " #" + instance["comment"] + return value + else: + return "" + +# called by chevron, text contains the label {{dataType}}, which is evaluated by the renderer (see class template) +def _set_imports(text, render): + rendered = render(text) + res = None + classes = set() + try: + res = eval(rendered) + except Exception as e: + pass + if res: + for val in res: + if "range" in val: + classes.add(val["range"].split("#")[1]) + elif not is_primitive_class(val["class_name"]) and val["class_name"] != "String" and not is_cim_data_type_class(val["class_name"]): + classes.add(val["dataType"].split("#")[1]) + result = "" + for val in classes: + result += "from ." + val + " import " + val + "\n" + return result + +# called by chevron, text contains the label {{dataType}}, which is evaluated by the renderer (see class template) +def _set_validator(text, render): + attribute = eval(render(text)) + + if not is_primitive_class(attribute["class_name"]) and attribute["class_name"] != "String" and not is_cim_data_type_class(attribute["class_name"]): + return ( + "val_" + + _lower_case_first_char(attribute["label"]) + + '_wrap = field_validator("' + + _lower_case_first_char(attribute["label"]) + + '", mode="wrap")(cyclic_references_validator)' + ) + elif attribute["label"] == "mRID": + return ( + '@field_validator("mRID", mode="before")\n' + + " def validate_mrid_format(cls, v):\n" + + " if isinstance(v, uuid.UUID):\n" + + " return str(v)\n" + + " elif isinstance(v, str):\n" + + " return v\n" + + " else:\n" + + ' raise ValueError("must be a UUID or str")\n' + ) + else: + return "" def _get_type_and_default(text, renderer) -> tuple[str, str]: - result = renderer(text) # the field {{dataType}} either contains the multiplicity of an attribute if it is a reference or otherwise the # datatype of the attribute. If no datatype is set and there is also no multiplicity entry for an attribute, the # default value is set to None. The multiplicity is set for all attributes, but the datatype is only set for basic # data types. If the data type entry for an attribute is missing, the attribute contains a reference and therefore # the default value is either None or [] depending on the multiplicity. See also write_python_files - # The default will be copied as-is, hence the possibility to have default or default_factory. - if result in ["M:1", "M:0..1", "M:1..1", ""]: - return ("Optional[str]", "default=None") - elif result in ["M:0..n", "M:1..n"] or "M:" in result: - return ("list", "default_factory=list") - - result = result.split("#")[1] - if result in ["integer", "Integer"]: - return ("int", "default=0") - elif result in ["String", "DateTime", "Date"]: - return ("str", 'default=""') - elif result == "Boolean": - return ("bool", "default=False") - else: - # everything else should be a float - return ("float", "default=0.0") + # The default will be copied as-is, hence the possibility to have default or + # default_factory. + attribute = eval(renderer(text)) + datatype = _compute_data_type(attribute) + type = datatype + default = 'default=None' + if "multiplicity" in attribute: + multiplicity = attribute["multiplicity"] + if multiplicity in ["M:0..1"]: + type = "Optional[" + datatype + "]" + elif multiplicity in ["M:0..n"]: + type = "Optional[List[" + datatype + "]]" + elif multiplicity in ["M:1..n", "M:2..n"]: + type = "List[" + datatype + "]" + elif multiplicity in ["M:1"] and attribute['label'] == 'PowerSystemResources': + # Most probably there is a bug in the RDF that states multiplicity + # M:1 but should be M:1..N + type = "List[" + datatype + "]" + else: + type = datatype + + if "label" in attribute and attribute["label"] == "mRID": + default = "default_factory=uuid.uuid4" + elif "multiplicity" in attribute: + multiplicity = attribute["multiplicity"] + if multiplicity in ["M:1"] and attribute['label'] == 'PowerSystemResources': + # Most probably there is a bug in the RDF that states multiplicity + # M:1 but should be M:1..N + default = 'default_factory=list' + elif multiplicity in ["M:0..n"] or multiplicity in ["M:1..n"]: + default = 'default_factory=list' + elif type == 'int': + default = 'default=0' + elif type == 'str': + default = 'default=""' + elif type == 'float': + default = 'default=0.0' + elif type == 'bool': + default = 'default=False' + return (type, default) def set_enum_classes(new_enum_classes): @@ -82,20 +231,74 @@ def set_enum_classes(new_enum_classes): def set_float_classes(new_float_classes): return +primitive_classes = {} -def run_template(version_path, class_details): - for template_info in template_files: +def set_primitive_classes(new_primitive_classes): + for new_class in new_primitive_classes: + primitive_classes[new_class] = new_primitive_classes[new_class] + +def is_primitive_class(name): + if name in primitive_classes: + return primitive_classes[name] + +cim_data_type_classes = {} + +def set_cim_data_type_classes(new_cim_data_type_classes): + for new_class in new_cim_data_type_classes: + cim_data_type_classes[new_class] = new_cim_data_type_classes[new_class] - resource_file = Path(os.path.join(version_path, "resources", class_details["class_name"] + template_info["ext"])) - if not resource_file.exists() : - if not (parent:=resource_file.parent).exists(): - parent.mkdir() +def is_cim_data_type_class(name): + if name in cim_data_type_classes: + return cim_data_type_classes[name] - with open(resource_file, "w", encoding="utf-8") as file: +def has_unit_attribute(attributes): + for attr in attributes: + if attr["label"] == "unit": + return True + return False +def run_template(version_path, class_details): + if ( + # Primitives are never used in the in memory representation but only for + # the schema + class_details["is_a_primitive"] == True + # Datatypes based on primitives are never used in the in memory + # representation but only for the schema + or class_details["is_a_cim_data_type"] == True + ): + return + elif class_details["has_instances"] == True: + run_template_enum(version_path, class_details, enum_template_files) + else: + run_template_schema(version_path, class_details, template_files) + +def run_template_enum(version_path, class_details, templates): + for template_info in templates: + class_file = os.path.join(version_path, class_details["class_name"] + template_info["ext"]) + if not os.path.exists(class_file): + with open(class_file, "w", encoding="utf-8") as file: + template_path = os.path.join(os.getcwd(), "modernpython/templates", template_info["filename"]) + class_details["setInstances"] = _set_instances + with open(template_path, encoding="utf-8") as f: + args = { + "data": class_details, + "template": f, + "partials_dict": partials, + } + output = chevron.render(**args) + file.write(output) + +def run_template_schema(version_path, class_details, templates): + for template_info in templates: + class_file = os.path.join(version_path, class_details["class_name"] + template_info["ext"]) + if not os.path.exists(class_file): + with open(class_file, "w", encoding="utf-8") as file: template_path = os.path.join(os.getcwd(), "modernpython/templates", template_info["filename"]) class_details["setDefault"] = _set_default class_details["setType"] = _set_type + class_details["setImports"] = _set_imports + class_details["setValidator"] = _set_validator + class_details["setNormalizedName"] = _set_normalized_name with open(template_path, encoding="utf-8") as f: args = { "data": class_details, @@ -107,6 +310,14 @@ def run_template(version_path, class_details): +def _copy_files(path): + base_path = path + "/util.py" + with open(Path(__file__).parent / "util.py", encoding="utf-8") as src, open( + base_path, "w", encoding="utf-8" + ) as dst: + dst.write(src.read()) + + def resolve_headers(dest: str, version: str): """Add all classes in __init__.py""" diff --git a/modernpython/templates/cimpy_class_template.mustache b/modernpython/templates/cimpy_class_template.mustache index cf9b32d5..439a46c6 100644 --- a/modernpython/templates/cimpy_class_template.mustache +++ b/modernpython/templates/cimpy_class_template.mustache @@ -2,14 +2,16 @@ Generated from the CGMES 3 files via cimgen: https://github.com/sogno-platform/cimgen """ +from __future__ import annotations from functools import cached_property -from typing import Optional -from pydantic import Field -from pydantic.dataclasses import dataclass -from ..utils.dataclassconfig import DataclassConfig -from ..utils.profile import BaseProfile, Profile - -from {{class_location}} import {{sub_class_of}} +from typing import Optional, List +from pydantic import Field, field_validator +import uuid +from datetime import date, datetime, time +from pydantic.dataclasses import dataclass, rebuild_dataclass +from .Base import DataclassConfig, Profile +from .util import cyclic_references_validator +from .{{sub_class_of}} import {{sub_class_of}} @dataclass(config=DataclassConfig) class {{class_name}}({{sub_class_of}}): @@ -24,8 +26,13 @@ class {{class_name}}({{sub_class_of}}): {{#attributes}} {{^isAssociationUsed}}# *Association not used* # Type {{dataType}} in CIM # pylint: disable-next=line-too-long - # {{/isAssociationUsed}}{{label}} : {{#setType}}{{dataType}}{{/setType}} = Field({{#setDefault}}{{dataType}}{{/setDefault}}, in_profiles = [{{#attr_origin}}Profile.{{origin}}, {{/attr_origin}}]) {{^isAssociationUsed}}# noqa: E501{{/isAssociationUsed}} + # {{/isAssociationUsed}}{{#setNormalizedName}}{{label}}{{/setNormalizedName}} : {{#setType}}{{.}}{{/setType}} = Field({{#setDefault}}{{.}}{{/setDefault}}, in_profiles = [{{#attr_origin}}Profile.{{origin}}, {{/attr_origin}}], alias = "{{label}}") {{^isAssociationUsed}}# noqa: E501{{/isAssociationUsed}} + {{/attributes}} + {{#attributes}} + {{^isAssociationUsed}}# *Association not used* + # Type {{dataType}} in CIM # pylint: disable-next=line-too-long + # {{/isAssociationUsed}}{{#setValidator}}{{.}}{{/setValidator}}{{^isAssociationUsed}}# noqa: E501{{/isAssociationUsed}} {{/attributes}} {{^attributes}} # No attributes defined for this class. @@ -39,3 +46,7 @@ class {{class_name}}({{sub_class_of}}): where this element can be found. """ return { {{#class_origin}}Profile.{{origin}}, {{/class_origin}} } + +{{#setImports}}{{attributes}}{{/setImports}} + +rebuild_dataclass({{class_name}}) diff --git a/modernpython/templates/pydantic_enum_template.mustache b/modernpython/templates/pydantic_enum_template.mustache new file mode 100644 index 00000000..52770cd6 --- /dev/null +++ b/modernpython/templates/pydantic_enum_template.mustache @@ -0,0 +1,15 @@ +""" +Generated from the CGMES 3 files via cimgen: https://github.com/sogno-platform/cimgen +""" + +from enum import Enum + +class {{class_name}}(str,Enum): + + ''' + {{{class_comment}}} + ''' + + {{#instances}} + {{#setInstances}}{{.}}{{/setInstances}} + {{/instances}} diff --git a/modernpython/util.py b/modernpython/util.py new file mode 100644 index 00000000..d08616f5 --- /dev/null +++ b/modernpython/util.py @@ -0,0 +1,37 @@ +from contextlib import contextmanager +from pydantic import ( + ValidationError, + ValidationInfo, + ValidatorFunctionWrapHandler, +) +from typing import Iterator, Any + + +def is_recursion_validation_error(exc: ValidationError) -> bool: + errors = exc.errors() + return len(errors) == 1 and errors[0]["type"] == "recursion_loop" + + +@contextmanager +def suppress_recursion_validation_error() -> Iterator[None]: + try: + yield + except ValidationError as exc: + if not is_recursion_validation_error(exc): + raise exc + + +def cyclic_references_validator( + v: Any, handler: ValidatorFunctionWrapHandler, info: ValidationInfo +): + try: + return handler(v) + except ValidationError as exc: + if not (is_recursion_validation_error(exc) and isinstance(v, list)): + raise exc + + value_without_cyclic_refs = [] + for child in v: + with suppress_recursion_validation_error(): + value_without_cyclic_refs.extend(handler([child])) + return handler(value_without_cyclic_refs) diff --git a/modernpython/utils/base.py b/modernpython/utils/base.py index c307b33b..ffe947d6 100644 --- a/modernpython/utils/base.py +++ b/modernpython/utils/base.py @@ -36,7 +36,7 @@ def parse_json_as(attrs: dict[str, Any]) -> "Base": # Works because the module and the class have the same name. return getattr(mod, subclass)(**data_attrs) - def to_dict(self) -> dict[str, "CgmesAttributeTypes"]: + def to_dict(self, with_class: bool = True) -> dict[str, "CgmesAttributeTypes"]: """ Returns the class as dict, with: - only public attributes @@ -45,6 +45,8 @@ def to_dict(self) -> dict[str, "CgmesAttributeTypes"]: """ attrs = {f.name: getattr(self, f.name) for f in fields(self)} attrs["__class__"] = self.apparent_name() + if with_class: + attrs["__class__"] = self.resource_name return attrs @cached_property diff --git a/python/langPack.py b/python/langPack.py index e448e0f4..94b8d4c9 100644 --- a/python/langPack.py +++ b/python/langPack.py @@ -69,6 +69,12 @@ def set_enum_classes(new_enum_classes): def set_float_classes(new_float_classes): return +def set_primitive_classes(new_primitive_classes): + return + +def set_cim_data_type_classes(new_cim_data_type_classes): + return + def run_template(version_path, class_details): for template_info in template_files: class_file = os.path.join(version_path, class_details['class_name'] + template_info["ext"]) From f0037740b3975ecb0b677699c8f090525ea1f533 Mon Sep 17 00:00:00 2001 From: "Federico M. Facca" Date: Mon, 16 Oct 2023 22:30:08 +0200 Subject: [PATCH 02/10] support single file generation for enum and schema Signed-off-by: Federico M. Facca --- modernpython/enum_header.py | 2 + modernpython/langPack.py | 93 ++++++++++++------ modernpython/schema_header.py | 67 +++++++++++++ .../templates/cimpy_class_template.mustache | 18 +--- .../templates/pydantic_enum_template.mustache | 3 +- modernpython/utils/base.py | 94 +++++++++++++++++++ 6 files changed, 230 insertions(+), 47 deletions(-) create mode 100644 modernpython/enum_header.py create mode 100644 modernpython/schema_header.py diff --git a/modernpython/enum_header.py b/modernpython/enum_header.py new file mode 100644 index 00000000..c860071f --- /dev/null +++ b/modernpython/enum_header.py @@ -0,0 +1,2 @@ +from enum import Enum, IntEnum + diff --git a/modernpython/langPack.py b/modernpython/langPack.py index 62a3baee..34363040 100644 --- a/modernpython/langPack.py +++ b/modernpython/langPack.py @@ -37,6 +37,8 @@ def location(version): template_files = [{"filename": "cimpy_class_template.mustache", "ext": ".py"}] enum_template_files = [{"filename": "pydantic_enum_template.mustache", "ext": ".py"}] +required_profiles = ["EQ", "GL"] #temporary + def get_class_location(class_name, class_map, version): return f".{class_map[class_name].superClass()}" # Check if the current class has a parent class @@ -58,7 +60,7 @@ def _compute_data_type(attribute): if "range" in attribute: # return "'"+attribute["range"].split("#")[1]+"'" - return "'"+attribute["range"].split("#")[1]+"'" + return attribute["range"].split("#")[1] if "dataType" in attribute and "class_name" in attribute: # for whatever weird reason String is not created as class from CIMgen if is_primitive_class(attribute["class_name"]) or attribute["class_name"] == "String": @@ -89,7 +91,7 @@ def _compute_data_type(attribute): if is_cim_data_type_class(attribute["class_name"]): return "float" # this is for example the case for 'StreetAddress.streetDetail' - return "'"+attribute["dataType"].split("#")[1]+"'" + return attribute["dataType"].split("#")[1] def _ends_with_s(attribute_name): return attribute_name.endswith("s") @@ -257,6 +259,12 @@ def has_unit_attribute(attributes): return True return False +def is_required_profile(class_origin): + for origin in class_origin: + if origin["origin"] in required_profiles: + return True + return False + def run_template(version_path, class_details): if ( # Primitives are never used in the in memory representation but only for @@ -265,6 +273,7 @@ def run_template(version_path, class_details): # Datatypes based on primitives are never used in the in memory # representation but only for the schema or class_details["is_a_cim_data_type"] == True + or class_details["class_name"] == 'PositionPoint' ): return elif class_details["has_instances"] == True: @@ -274,40 +283,68 @@ def run_template(version_path, class_details): def run_template_enum(version_path, class_details, templates): for template_info in templates: - class_file = os.path.join(version_path, class_details["class_name"] + template_info["ext"]) + class_file = os.path.join(version_path, "enum" + template_info["ext"]) if not os.path.exists(class_file): with open(class_file, "w", encoding="utf-8") as file: - template_path = os.path.join(os.getcwd(), "modernpython/templates", template_info["filename"]) - class_details["setInstances"] = _set_instances - with open(template_path, encoding="utf-8") as f: - args = { - "data": class_details, - "template": f, - "partials_dict": partials, - } - output = chevron.render(**args) - file.write(output) + header_file_path = os.path.join( + os.getcwd(), "modernpython", "enum_header.py" + ) + header_file = open(header_file_path, "r") + file.write(header_file.read()) + with open(class_file, "a", encoding="utf-8") as file: + template_path = os.path.join(os.getcwd(), "modernpython/templates", template_info["filename"]) + class_details["setInstances"] = _set_instances + with open(template_path, encoding="utf-8") as f: + args = { + "data": class_details, + "template": f, + "partials_dict": partials, + } + output = chevron.render(**args) + file.write(output) def run_template_schema(version_path, class_details, templates): for template_info in templates: - class_file = os.path.join(version_path, class_details["class_name"] + template_info["ext"]) + class_file = os.path.join(version_path, "schema" + template_info["ext"]) if not os.path.exists(class_file): with open(class_file, "w", encoding="utf-8") as file: - template_path = os.path.join(os.getcwd(), "modernpython/templates", template_info["filename"]) - class_details["setDefault"] = _set_default - class_details["setType"] = _set_type - class_details["setImports"] = _set_imports - class_details["setValidator"] = _set_validator - class_details["setNormalizedName"] = _set_normalized_name - with open(template_path, encoding="utf-8") as f: - args = { - "data": class_details, - "template": f, - "partials_dict": partials, - } - output = chevron.render(**args) - file.write(output) + schema_file_path = os.path.join( + os.getcwd(), "modernpython", "schema_header.py" + ) + schema_file = open(schema_file_path, "r") + file.write(schema_file.read()) + with open(class_file, "a", encoding="utf-8") as file: + template_path = os.path.join(os.getcwd(), "modernpython/templates", template_info["filename"]) + class_details["setDefault"] = _set_default + class_details["setType"] = _set_type + class_details["setImports"] = _set_imports + class_details["setValidator"] = _set_validator + class_details["setNormalizedName"] = _set_normalized_name + with open(template_path, encoding="utf-8") as f: + args = { + "data": class_details, + "template": f, + "partials_dict": partials, + } + output = chevron.render(**args) + file.write(output) + + +def _create_init(path): + init_file = path + "/__init__.py" + + with open(init_file, "w", encoding="utf-8") as init: + #init.write("# pylint: disable=too-many-lines,missing-module-docstring\n") + pass +# creates the Base class file, all classes inherit from this class +def _create_base(path): + # TODO: Check export priority of OP en SC, see Profile class + base_path = path + "/Base.py" + with open(Path(__file__).parent / "Base.py", encoding="utf-8") as src, open( + base_path, "w", encoding="utf-8" + ) as dst: + dst.write(src.read()) def _copy_files(path): diff --git a/modernpython/schema_header.py b/modernpython/schema_header.py new file mode 100644 index 00000000..5ba4122b --- /dev/null +++ b/modernpython/schema_header.py @@ -0,0 +1,67 @@ +from __future__ import annotations +import uuid +from functools import cached_property +from pydantic import ConfigDict, Field, field_validator, computed_field +from geoalchemy2.shape import to_shape +from geoalchemy2.elements import WKBElement +from shapely.geometry import Point +from datetime import date, datetime, time +from typing import Optional, Iterator, List +from pydantic.dataclasses import dataclass +from .Base import DataclassConfig, GeoDataclassConfig, Profile, Base +from .util import cyclic_references_validator +from .enum import * + +@dataclass(config=GeoDataclassConfig) +class PositionPoint(Base): + """ + Set of spatial coordinates that determine a point, defined in the coordinate system specified in 'Location.CoordinateSystem'. Use a single position point instance to desribe a point-oriented location. Use a sequence of position points to describe a line-oriented object (physical location of non-point oriented objects like cables or lines), or area of an object (like a substation or a geographical zone - in this case, have first and last position point with the same values). + + :Location: Location described by this position point. + :sequenceNumber: Zero-relative sequence number of this point within a series of points. + :xPosition: X axis position. + :yPosition: Y axis position. + :zPosition: (if applicable) Z axis position. + """ + + location: "Location" = Field(alias="Location", in_profiles = [Profile.GL, ]) + sequenceNumber: Optional[int] = Field(default=None, in_profiles = [Profile.GL, ]) + point: Point = Field( + repr=False, in_profiles = [Profile.GL, ] + ) # we introduce this field compared to CIM definition because we want to store a proper geometry "point" in the database + + @computed_field + @property + def xPosition(self) -> str: + return str(self.point.x) + + @computed_field + @property + def yPosition(self) -> str: + return str(self.point.y) + + @computed_field + @property + def zPosition(self) -> str: + return str(self.point.z) + + @cached_property + def possible_profiles(self)->set[Profile]: + """ + A resource can be used by multiple profiles. This is the set of profiles + where this element can be found. + """ + return { Profile.GL, } + + # Pydantic needs help to map GeoAlchemy classes to Shapely + @field_validator("point", mode="before") + def validate_point_format(cls, v): + if isinstance(v, Point): + return v + elif isinstance(v, WKBElement): + point = to_shape(v) + if point.geom_type != "Point": + raise ValueError("must be a Point") + return Point(point) + else: + raise ValueError("must be a Point or a WKBElement") diff --git a/modernpython/templates/cimpy_class_template.mustache b/modernpython/templates/cimpy_class_template.mustache index 439a46c6..f71c4ea9 100644 --- a/modernpython/templates/cimpy_class_template.mustache +++ b/modernpython/templates/cimpy_class_template.mustache @@ -1,18 +1,8 @@ + """ Generated from the CGMES 3 files via cimgen: https://github.com/sogno-platform/cimgen """ -from __future__ import annotations -from functools import cached_property -from typing import Optional, List -from pydantic import Field, field_validator -import uuid -from datetime import date, datetime, time -from pydantic.dataclasses import dataclass, rebuild_dataclass -from .Base import DataclassConfig, Profile -from .util import cyclic_references_validator -from .{{sub_class_of}} import {{sub_class_of}} - @dataclass(config=DataclassConfig) class {{class_name}}({{sub_class_of}}): """ @@ -37,8 +27,6 @@ class {{class_name}}({{sub_class_of}}): {{^attributes}} # No attributes defined for this class. {{/attributes}} - - @cached_property def possible_profiles(self)->set[BaseProfile]: """ @@ -46,7 +34,3 @@ class {{class_name}}({{sub_class_of}}): where this element can be found. """ return { {{#class_origin}}Profile.{{origin}}, {{/class_origin}} } - -{{#setImports}}{{attributes}}{{/setImports}} - -rebuild_dataclass({{class_name}}) diff --git a/modernpython/templates/pydantic_enum_template.mustache b/modernpython/templates/pydantic_enum_template.mustache index 52770cd6..1ad7d9f6 100644 --- a/modernpython/templates/pydantic_enum_template.mustache +++ b/modernpython/templates/pydantic_enum_template.mustache @@ -1,9 +1,8 @@ + """ Generated from the CGMES 3 files via cimgen: https://github.com/sogno-platform/cimgen """ -from enum import Enum - class {{class_name}}(str,Enum): ''' diff --git a/modernpython/utils/base.py b/modernpython/utils/base.py index ffe947d6..d3bee06c 100644 --- a/modernpython/utils/base.py +++ b/modernpython/utils/base.py @@ -10,6 +10,100 @@ from .dataclassconfig import DataclassConfig from .profile import BaseProfile +class Profile(Enum): + """ + Enum containing all CGMES profiles and their export priority. + todo: enums are ordered, so we can have a short->long enum without explicit prio + """ + + EQ = 0 + SSH = 1 + TP = 2 + SV = 3 + DY = 4 + OP = 5 + SC = 6 + GL = 7 + # DI = 8 # Initially mentioned but does not seem used? + DL = 9 + TPBD = 10 + EQBD = 11 + + @cached_property + def long_name(self): + """From the short name, return the long name of the profile.""" + return self._short_to_long()[self.name] + + @classmethod + def from_long_name(cls, long_name): + """From the long name, return the short name of the profile.""" + return cls[cls._long_to_short()[long_name]] + + @classmethod + @cache + def _short_to_long(cls) -> dict[str, str]: + """Returns the long name from a short name""" + return { + "DL": "DiagramLayout", + # "DI": "DiagramLayout", + "DY": "Dynamics", + "EQ": "Equipment", + "EQBD": "EquipmentBoundary", # Not too sure about that one + "GL": "GeographicalLocation", + "OP": "Operation", + "SC": "ShortCircuit", + "SV": "StateVariables", + "SSH": "SteadyStateHypothesis", + "TP": "Topology", + "TPBD": "TopologyBoundary", # Not too sure about that one + } + + @classmethod + @cache + def _long_to_short(cls) -> dict[str, str]: + """Returns the short name from a long name""" + return {_long: _short for _short, _long in cls._short_to_long().items()} + + +class DataclassConfig: # pylint: disable=too-few-public-methods + """ + Used to configure pydantic dataclasses. + + See doc at + https://docs.pydantic.dev/latest/usage/model_config/#options + """ + + # By default with pydantic extra arguments given to a dataclass are silently ignored. + # This matches the default behaviour by failing noisily. + extra = "forbid" + populate_by_name = True + defer_build = True + from_attributes = True + +class GeoDataclassConfig: # pylint: disable=too-few-public-methods + """ + Used to configure pydantic dataclasses. + + See doc at + https://docs.pydantic.dev/latest/usage/model_config/#options + """ + + # By default with pydantic extra arguments given to a dataclass are silently ignored. + # This matches the default behaviour by failing noisily. + extra = "ignore" + populate_by_name = True + defer_build = True + from_attributes = True + arbitrary_types_allowed=True + +# Default namespaces used by CGMES. +NAMESPACES = { + "cim": "http://iec.ch/TC57/2013/CIM-schema-cim16#", # NOSONAR + "entsoe": "http://entsoe.eu/CIM/SchemaExtension/3/1#", # NOSONAR + "md": "http://iec.ch/TC57/61970-552/ModelDescription/1#", # NOSONAR + "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", # NOSONAR +} + @dataclass(config=DataclassConfig) class Base: From 9cb0f1e61cdd65b37a725af1c5b2b3bc3996477e Mon Sep 17 00:00:00 2001 From: "Federico M. Facca" Date: Tue, 17 Oct 2023 09:21:34 +0200 Subject: [PATCH 03/10] remove header resolution, uncomment associations Signed-off-by: Federico M. Facca --- modernpython/templates/cimpy_class_template.mustache | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/modernpython/templates/cimpy_class_template.mustache b/modernpython/templates/cimpy_class_template.mustache index f71c4ea9..dff0904f 100644 --- a/modernpython/templates/cimpy_class_template.mustache +++ b/modernpython/templates/cimpy_class_template.mustache @@ -14,19 +14,15 @@ class {{class_name}}({{sub_class_of}}): """ {{#attributes}} - {{^isAssociationUsed}}# *Association not used* - # Type {{dataType}} in CIM # pylint: disable-next=line-too-long - # {{/isAssociationUsed}}{{#setNormalizedName}}{{label}}{{/setNormalizedName}} : {{#setType}}{{.}}{{/setType}} = Field({{#setDefault}}{{.}}{{/setDefault}}, in_profiles = [{{#attr_origin}}Profile.{{origin}}, {{/attr_origin}}], alias = "{{label}}") {{^isAssociationUsed}}# noqa: E501{{/isAssociationUsed}} - + {{#setNormalizedName}}{{label}}{{/setNormalizedName}} : {{#setType}}{{.}}{{/setType}} = Field({{#setDefault}}{{.}}{{/setDefault}}, in_profiles = [{{#attr_origin}}Profile.{{origin}}, {{/attr_origin}}], alias = "{{label}}") {{/attributes}} {{#attributes}} - {{^isAssociationUsed}}# *Association not used* - # Type {{dataType}} in CIM # pylint: disable-next=line-too-long - # {{/isAssociationUsed}}{{#setValidator}}{{.}}{{/setValidator}}{{^isAssociationUsed}}# noqa: E501{{/isAssociationUsed}} + {{#setValidator}}{{.}}{{/setValidator}} {{/attributes}} {{^attributes}} # No attributes defined for this class. {{/attributes}} + @cached_property def possible_profiles(self)->set[BaseProfile]: """ From 02cd04e4ec3cfca99103cdd7f00e7b2afc6ed0cb Mon Sep 17 00:00:00 2001 From: "Federico M. Facca" Date: Sun, 26 Nov 2023 17:26:26 +0100 Subject: [PATCH 04/10] Apply suggestions from code review Co-authored-by: Guillaume Roger <61284231+guillaume-alliander@users.noreply.github.com> Signed-off-by: Federico M. Facca --- CIMgen.py | 2 +- modernpython/langPack.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CIMgen.py b/CIMgen.py index 5202346d..1a304c2c 100644 --- a/CIMgen.py +++ b/CIMgen.py @@ -682,7 +682,7 @@ def add_sub_classes_of_sub_classes_clean(class_dict, source): def generate_clean_sub_classes(class_dict_with_origins, clean_class_dict): for class_name in class_dict_with_origins: super_class_name = class_dict_with_origins[class_name].superClass() - if super_class_name == None and class_dict_with_origins[class_name].has_instances(): + if super_class_name is None and class_dict_with_origins[class_name].has_instances(): clean_class_dict[class_name] = class_dict_with_origins[class_name] for class_name in class_dict_with_origins: diff --git a/modernpython/langPack.py b/modernpython/langPack.py index 34363040..024f059a 100644 --- a/modernpython/langPack.py +++ b/modernpython/langPack.py @@ -64,8 +64,8 @@ def _compute_data_type(attribute): if "dataType" in attribute and "class_name" in attribute: # for whatever weird reason String is not created as class from CIMgen if is_primitive_class(attribute["class_name"]) or attribute["class_name"] == "String": - datatype = attribute["dataType"].split("#")[1] - if datatype == "Integer" or datatype == "integer": + datatype = attribute["dataType"].split("#")[1].lower() + if datatype == "integer": return "int" if datatype == "Boolean": return "bool" @@ -118,7 +118,8 @@ def _set_normalized_name(text, render): def _set_instances(text, render): instance = None try: - instance = eval(render(text)) + # render(text) returns a python dict. Some fileds might be quoted by '"' instead of '"', making the first evel fail. + instance = ast.literal_eval(render(text)) except SyntaxError as se: rendered = render(text) rendered = rendered.replace(""", '"') @@ -269,7 +270,7 @@ def run_template(version_path, class_details): if ( # Primitives are never used in the in memory representation but only for # the schema - class_details["is_a_primitive"] == True + class_details["is_a_primitive"] is True # Datatypes based on primitives are never used in the in memory # representation but only for the schema or class_details["is_a_cim_data_type"] == True From 5f5f9c00d5bcf3df8b6a5456918b5acff3b6d25d Mon Sep 17 00:00:00 2001 From: "Federico M. Facca" Date: Sun, 26 Nov 2023 17:59:36 +0100 Subject: [PATCH 05/10] fix import Signed-off-by: Federico M. Facca --- modernpython/langPack.py | 1 + 1 file changed, 1 insertion(+) diff --git a/modernpython/langPack.py b/modernpython/langPack.py index 024f059a..691dd1f2 100644 --- a/modernpython/langPack.py +++ b/modernpython/langPack.py @@ -6,6 +6,7 @@ import sys import textwrap from pathlib import Path +import ast import chevron From 626863fab9770c92d25879be5cceedfec063a9bd Mon Sep 17 00:00:00 2001 From: "Federico M. Facca" Date: Sun, 26 Nov 2023 19:03:07 +0100 Subject: [PATCH 06/10] complete rebase Signed-off-by: Federico M. Facca --- modernpython/enum_header.py | 2 +- modernpython/langPack.py | 62 ++++-------- modernpython/schema_header.py | 6 +- modernpython/utils/base.py | 97 +------------------ modernpython/utils/dataclassconfig.py | 19 ++++ modernpython/{util.py => utils/validation.py} | 0 6 files changed, 46 insertions(+), 140 deletions(-) rename modernpython/{util.py => utils/validation.py} (100%) diff --git a/modernpython/enum_header.py b/modernpython/enum_header.py index c860071f..27067ef9 100644 --- a/modernpython/enum_header.py +++ b/modernpython/enum_header.py @@ -1,2 +1,2 @@ -from enum import Enum, IntEnum +from enum import Enum diff --git a/modernpython/langPack.py b/modernpython/langPack.py index 691dd1f2..60dd11f9 100644 --- a/modernpython/langPack.py +++ b/modernpython/langPack.py @@ -58,41 +58,43 @@ def get_class_location(class_name, class_map, version): def _compute_data_type(attribute): if "label" in attribute and attribute["label"] == "mRID": return "str" - - if "range" in attribute: + elif "range" in attribute: # return "'"+attribute["range"].split("#")[1]+"'" return attribute["range"].split("#")[1] - if "dataType" in attribute and "class_name" in attribute: + elif "dataType" in attribute and "class_name" in attribute: # for whatever weird reason String is not created as class from CIMgen if is_primitive_class(attribute["class_name"]) or attribute["class_name"] == "String": datatype = attribute["dataType"].split("#")[1].lower() if datatype == "integer": return "int" - if datatype == "Boolean": + if datatype == "boolean": return "bool" - if datatype == "String": + if datatype == "string": return "str" - if datatype == "DateTime": + if datatype == "datetime": return "datetime" - if datatype == "MonthDay": + if datatype == "monthday": return "str" # TO BE FIXED? - if datatype == "Date": + if datatype == "date": return "str" # TO BE FIXED? - if datatype == "Time": + if datatype == "time": return "time" - if datatype == "Float": + if datatype == "float": return "float" - if datatype == "String": + if datatype == "string": return "str" else: # this actually never happens return "float" # the assumption is that cim data type e.g. Voltage, ActivePower, always # maps to a float - if is_cim_data_type_class(attribute["class_name"]): + elif is_cim_data_type_class(attribute["class_name"]): return "float" + else: # this is for example the case for 'StreetAddress.streetDetail' - return attribute["dataType"].split("#")[1] + return attribute["dataType"].split("#")[1] + else: + raise ValueError(f"Cannot parse {attribute} to extract a data type.") def _ends_with_s(attribute_name): return attribute_name.endswith("s") @@ -285,8 +287,10 @@ def run_template(version_path, class_details): def run_template_enum(version_path, class_details, templates): for template_info in templates: - class_file = os.path.join(version_path, "enum" + template_info["ext"]) + class_file = Path(version_path, "resources", "enum" + template_info["ext"]) if not os.path.exists(class_file): + if not (parent:=class_file.parent).exists(): + parent.mkdir() with open(class_file, "w", encoding="utf-8") as file: header_file_path = os.path.join( os.getcwd(), "modernpython", "enum_header.py" @@ -307,8 +311,10 @@ def run_template_enum(version_path, class_details, templates): def run_template_schema(version_path, class_details, templates): for template_info in templates: - class_file = os.path.join(version_path, "schema" + template_info["ext"]) + class_file =Path(version_path, "resources", "schema" + template_info["ext"]) if not os.path.exists(class_file): + if not (parent:=class_file.parent).exists(): + parent.mkdir() with open(class_file, "w", encoding="utf-8") as file: schema_file_path = os.path.join( os.getcwd(), "modernpython", "schema_header.py" @@ -331,32 +337,6 @@ def run_template_schema(version_path, class_details, templates): output = chevron.render(**args) file.write(output) - -def _create_init(path): - init_file = path + "/__init__.py" - - with open(init_file, "w", encoding="utf-8") as init: - #init.write("# pylint: disable=too-many-lines,missing-module-docstring\n") - pass - -# creates the Base class file, all classes inherit from this class -def _create_base(path): - # TODO: Check export priority of OP en SC, see Profile class - base_path = path + "/Base.py" - with open(Path(__file__).parent / "Base.py", encoding="utf-8") as src, open( - base_path, "w", encoding="utf-8" - ) as dst: - dst.write(src.read()) - - -def _copy_files(path): - base_path = path + "/util.py" - with open(Path(__file__).parent / "util.py", encoding="utf-8") as src, open( - base_path, "w", encoding="utf-8" - ) as dst: - dst.write(src.read()) - - def resolve_headers(dest: str, version: str): """Add all classes in __init__.py""" diff --git a/modernpython/schema_header.py b/modernpython/schema_header.py index 5ba4122b..19a92033 100644 --- a/modernpython/schema_header.py +++ b/modernpython/schema_header.py @@ -8,8 +8,10 @@ from datetime import date, datetime, time from typing import Optional, Iterator, List from pydantic.dataclasses import dataclass -from .Base import DataclassConfig, GeoDataclassConfig, Profile, Base -from .util import cyclic_references_validator +from ..utils.base import Base +from ..utils.dataclassconfig import DataclassConfig, GeoDataclassConfig +from ..utils.profile import Profile, BaseProfile +from ..utils.validation import cyclic_references_validator from .enum import * @dataclass(config=GeoDataclassConfig) diff --git a/modernpython/utils/base.py b/modernpython/utils/base.py index d3bee06c..3fb558ee 100644 --- a/modernpython/utils/base.py +++ b/modernpython/utils/base.py @@ -4,107 +4,12 @@ from functools import cached_property from typing import Any, TypeAlias, TypedDict -from pycgmes.utils.constants import NAMESPACES +from .constants import NAMESPACES from pydantic.dataclasses import dataclass from .dataclassconfig import DataclassConfig from .profile import BaseProfile -class Profile(Enum): - """ - Enum containing all CGMES profiles and their export priority. - todo: enums are ordered, so we can have a short->long enum without explicit prio - """ - - EQ = 0 - SSH = 1 - TP = 2 - SV = 3 - DY = 4 - OP = 5 - SC = 6 - GL = 7 - # DI = 8 # Initially mentioned but does not seem used? - DL = 9 - TPBD = 10 - EQBD = 11 - - @cached_property - def long_name(self): - """From the short name, return the long name of the profile.""" - return self._short_to_long()[self.name] - - @classmethod - def from_long_name(cls, long_name): - """From the long name, return the short name of the profile.""" - return cls[cls._long_to_short()[long_name]] - - @classmethod - @cache - def _short_to_long(cls) -> dict[str, str]: - """Returns the long name from a short name""" - return { - "DL": "DiagramLayout", - # "DI": "DiagramLayout", - "DY": "Dynamics", - "EQ": "Equipment", - "EQBD": "EquipmentBoundary", # Not too sure about that one - "GL": "GeographicalLocation", - "OP": "Operation", - "SC": "ShortCircuit", - "SV": "StateVariables", - "SSH": "SteadyStateHypothesis", - "TP": "Topology", - "TPBD": "TopologyBoundary", # Not too sure about that one - } - - @classmethod - @cache - def _long_to_short(cls) -> dict[str, str]: - """Returns the short name from a long name""" - return {_long: _short for _short, _long in cls._short_to_long().items()} - - -class DataclassConfig: # pylint: disable=too-few-public-methods - """ - Used to configure pydantic dataclasses. - - See doc at - https://docs.pydantic.dev/latest/usage/model_config/#options - """ - - # By default with pydantic extra arguments given to a dataclass are silently ignored. - # This matches the default behaviour by failing noisily. - extra = "forbid" - populate_by_name = True - defer_build = True - from_attributes = True - -class GeoDataclassConfig: # pylint: disable=too-few-public-methods - """ - Used to configure pydantic dataclasses. - - See doc at - https://docs.pydantic.dev/latest/usage/model_config/#options - """ - - # By default with pydantic extra arguments given to a dataclass are silently ignored. - # This matches the default behaviour by failing noisily. - extra = "ignore" - populate_by_name = True - defer_build = True - from_attributes = True - arbitrary_types_allowed=True - -# Default namespaces used by CGMES. -NAMESPACES = { - "cim": "http://iec.ch/TC57/2013/CIM-schema-cim16#", # NOSONAR - "entsoe": "http://entsoe.eu/CIM/SchemaExtension/3/1#", # NOSONAR - "md": "http://iec.ch/TC57/61970-552/ModelDescription/1#", # NOSONAR - "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", # NOSONAR -} - - @dataclass(config=DataclassConfig) class Base: """ diff --git a/modernpython/utils/dataclassconfig.py b/modernpython/utils/dataclassconfig.py index 961f0d8a..5688c710 100644 --- a/modernpython/utils/dataclassconfig.py +++ b/modernpython/utils/dataclassconfig.py @@ -9,3 +9,22 @@ class DataclassConfig: # pylint: disable=too-few-public-methods # By default, with pydantic extra arguments given to a dataclass are silently ignored. # This matches the default behaviour by failing noisily. extra = "forbid" + populate_by_name = True + defer_build = True + from_attributes = True + +class GeoDataclassConfig: # pylint: disable=too-few-public-methods + """ + Used to configure pydantic dataclasses. + + See doc at + https://docs.pydantic.dev/latest/usage/model_config/#options + """ + + # By default, with pydantic extra arguments given to a dataclass are silently ignored. + # This matches the default behaviour by failing noisily. + extra = "ignore" + populate_by_name = True + defer_build = True + from_attributes = True + arbitrary_types_allowed=True diff --git a/modernpython/util.py b/modernpython/utils/validation.py similarity index 100% rename from modernpython/util.py rename to modernpython/utils/validation.py From 707c6966acaa3bdcff472e678dbe2f4f38f57a06 Mon Sep 17 00:00:00 2001 From: "Federico M. Facca" Date: Sun, 26 Nov 2023 22:52:34 +0100 Subject: [PATCH 07/10] Update dataclassconfig.py Signed-off-by: Federico M. Facca --- modernpython/utils/dataclassconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modernpython/utils/dataclassconfig.py b/modernpython/utils/dataclassconfig.py index 5688c710..fb6e723b 100644 --- a/modernpython/utils/dataclassconfig.py +++ b/modernpython/utils/dataclassconfig.py @@ -8,7 +8,7 @@ class DataclassConfig: # pylint: disable=too-few-public-methods # By default, with pydantic extra arguments given to a dataclass are silently ignored. # This matches the default behaviour by failing noisily. - extra = "forbid" + extra = "ignore" populate_by_name = True defer_build = True from_attributes = True From 691c5df09f00a3922c8670d63021b3c7e34706d1 Mon Sep 17 00:00:00 2001 From: "Federico M. Facca" Date: Mon, 27 Nov 2023 01:02:23 +0100 Subject: [PATCH 08/10] generate primitives and datatypes instances Signed-off-by: Federico M. Facca annotate datatype Signed-off-by: Federico M. Facca --- CIMgen.py | 7 +- modernpython/cimdatatype_header.py | 3 + modernpython/langPack.py | 205 +++++++++++------- modernpython/primitive_header.py | 6 + modernpython/schema_header.py | 2 + .../templates/cimdatatype_template.mustache | 11 + .../templates/cimpy_class_template.mustache | 2 +- ....mustache => enum_class_template.mustache} | 0 .../templates/primitive_template.mustache | 11 + modernpython/utils/base.py | 10 +- modernpython/utils/datatypes.py | 23 ++ 11 files changed, 189 insertions(+), 91 deletions(-) create mode 100644 modernpython/cimdatatype_header.py create mode 100644 modernpython/primitive_header.py create mode 100644 modernpython/templates/cimdatatype_template.mustache rename modernpython/templates/{pydantic_enum_template.mustache => enum_class_template.mustache} (100%) create mode 100644 modernpython/templates/primitive_template.mustache create mode 100644 modernpython/utils/datatypes.py diff --git a/CIMgen.py b/CIMgen.py index 1a304c2c..fa633071 100644 --- a/CIMgen.py +++ b/CIMgen.py @@ -4,7 +4,8 @@ import os import textwrap from time import time - +from distutils.dir_util import remove_tree +from pathlib import Path import xmltodict from bs4 import BeautifulSoup @@ -709,6 +710,10 @@ def cim_generate(directory, outputPath, version, langPack): :param outputPath: CGMES version, e.g. version = "cgmes_v2_4_15" :param langPack: python module containing language specific functions """ + + # clean directory + remove_tree(Path(outputPath)) + profiles_array = [] t0 = time() diff --git a/modernpython/cimdatatype_header.py b/modernpython/cimdatatype_header.py new file mode 100644 index 00000000..280725c1 --- /dev/null +++ b/modernpython/cimdatatype_header.py @@ -0,0 +1,3 @@ +from ..utils.datatypes import CIMDatatype +from ..utils.profile import Profile +from .enum import * diff --git a/modernpython/langPack.py b/modernpython/langPack.py index 60dd11f9..4fae4599 100644 --- a/modernpython/langPack.py +++ b/modernpython/langPack.py @@ -36,56 +36,75 @@ def location(version): base = {"base_class": "Base", "class_location": location} template_files = [{"filename": "cimpy_class_template.mustache", "ext": ".py"}] -enum_template_files = [{"filename": "pydantic_enum_template.mustache", "ext": ".py"}] - -required_profiles = ["EQ", "GL"] #temporary +enum_template_files = [{"filename": "enum_class_template.mustache", "ext": ".py"}] +primitive_template_files = [{"filename": "primitive_template.mustache", "ext": ".py"}] +cimdatatype_template_files = [{"filename": "cimdatatype_template.mustache", "ext": ".py"}] def get_class_location(class_name, class_map, version): return f".{class_map[class_name].superClass()}" # Check if the current class has a parent class -""" if class_map[class_name].superClass(): - if class_map[class_name].superClass() in class_map: - return "modernpython." + version + "." + class_map[class_name].superClass() - elif class_map[class_name].superClass() == "Base" or class_map[class_name].superClass() == None: - return location(version) - else: - return location(version) """ - partials = {} +def _primitive_to_data_type(datatype): + if datatype.lower() == "integer": + return "int" + if datatype.lower() == "boolean": + return "bool" + if datatype.lower() == "string": + return "str" + if datatype.lower() == "datetime": + return "datetime" + if datatype.lower() == "monthday": + return "str" # TO BE FIXED? I could not find a datatype in python that holds only month and day. + if datatype.lower() == "date": + return "date" + # as of today no CIM model is using only time. + if datatype.lower() == "time": + return "time" + if datatype.lower() == "float": + return "float" + if datatype.lower() == "string": + return "str" + else: + # this actually never happens + return "float" + +def _compute_cim_data_type(attributes) -> tuple[str, str, str]: + python_type = 'None' + unit = 'UnitSymbol.none' + multiplier = 'UnitMultiplier.none' + for attribute in attributes: + if 'about' in attribute and attribute['about'] and "value" in attribute['about'] and 'class_name' in attribute: + python_type = _primitive_to_data_type(attribute['class_name']) + if 'about' in attribute and attribute['about'] and "multiplier" in attribute['about'] and 'isFixed' in attribute: + multiplier = "UnitMultiplier."+attribute['isFixed'] + if 'about' in attribute and attribute['about'] and "unit" in attribute['about'] and 'isFixed' in attribute: + unit = "UnitSymbol."+attribute['isFixed'] + return (python_type, unit, multiplier) + + +# set the cim datatype for a field +def _set_cim_data_type(text, render) -> str: + attribute = eval(render(text)) + if is_primitive_class(attribute["class_name"]): + return "data_type = " + attribute["class_name"] + "," + elif is_primitive_class(attribute["class_name"]) or attribute["class_name"] == "String": + return "data_type = String," + elif is_cim_data_type_class(attribute["class_name"]): + return "data_type = " + attribute["class_name"] + "," + return "" + # computes the data type def _compute_data_type(attribute): if "label" in attribute and attribute["label"] == "mRID": return "str" elif "range" in attribute: - # return "'"+attribute["range"].split("#")[1]+"'" return attribute["range"].split("#")[1] elif "dataType" in attribute and "class_name" in attribute: # for whatever weird reason String is not created as class from CIMgen if is_primitive_class(attribute["class_name"]) or attribute["class_name"] == "String": - datatype = attribute["dataType"].split("#")[1].lower() - if datatype == "integer": - return "int" - if datatype == "boolean": - return "bool" - if datatype == "string": - return "str" - if datatype == "datetime": - return "datetime" - if datatype == "monthday": - return "str" # TO BE FIXED? - if datatype == "date": - return "str" # TO BE FIXED? - if datatype == "time": - return "time" - if datatype == "float": - return "float" - if datatype == "string": - return "str" - else: - # this actually never happens - return "float" + return _primitive_to_data_type(attribute["dataType"].split("#")[1]) # the assumption is that cim data type e.g. Voltage, ActivePower, always # maps to a float elif is_cim_data_type_class(attribute["class_name"]): @@ -143,7 +162,7 @@ def _set_imports(text, render): classes = set() try: res = eval(rendered) - except Exception as e: + except Exception: pass if res: for val in res: @@ -192,22 +211,22 @@ def _get_type_and_default(text, renderer) -> tuple[str, str]: # default_factory. attribute = eval(renderer(text)) datatype = _compute_data_type(attribute) - type = datatype + field_type = datatype default = 'default=None' if "multiplicity" in attribute: multiplicity = attribute["multiplicity"] if multiplicity in ["M:0..1"]: - type = "Optional[" + datatype + "]" + field_type = "Optional[" + datatype + "]" elif multiplicity in ["M:0..n"]: - type = "Optional[List[" + datatype + "]]" + field_type = "Optional[List[" + datatype + "]]" elif multiplicity in ["M:1..n", "M:2..n"]: - type = "List[" + datatype + "]" + field_type = "List[" + datatype + "]" elif multiplicity in ["M:1"] and attribute['label'] == 'PowerSystemResources': # Most probably there is a bug in the RDF that states multiplicity # M:1 but should be M:1..N - type = "List[" + datatype + "]" + field_type = "List[" + datatype + "]" else: - type = datatype + field_type = datatype if "label" in attribute and attribute["label"] == "mRID": default = "default_factory=uuid.uuid4" @@ -219,15 +238,15 @@ def _get_type_and_default(text, renderer) -> tuple[str, str]: default = 'default_factory=list' elif multiplicity in ["M:0..n"] or multiplicity in ["M:1..n"]: default = 'default_factory=list' - elif type == 'int': + elif field_type == 'int': default = 'default=0' - elif type == 'str': + elif field_type == 'str': default = 'default=""' - elif type == 'float': + elif field_type == 'float': default = 'default=0.0' - elif type == 'bool': + elif field_type == 'bool': default = 'default=False' - return (type, default) + return (field_type, default) def set_enum_classes(new_enum_classes): @@ -263,24 +282,19 @@ def has_unit_attribute(attributes): return True return False -def is_required_profile(class_origin): - for origin in class_origin: - if origin["origin"] in required_profiles: - return True - return False - def run_template(version_path, class_details): - if ( + if class_details["class_name"] == 'PositionPoint': + #this class is created manually to support types conversions + return + elif class_details["is_a_primitive"] is True: # Primitives are never used in the in memory representation but only for # the schema - class_details["is_a_primitive"] is True + run_template_primitive(version_path, class_details, primitive_template_files) + elif class_details["is_a_cim_data_type"] is True: # Datatypes based on primitives are never used in the in memory # representation but only for the schema - or class_details["is_a_cim_data_type"] == True - or class_details["class_name"] == 'PositionPoint' - ): - return - elif class_details["has_instances"] == True: + run_template_cimdatatype(version_path, class_details, cimdatatype_template_files) + elif class_details["has_instances"] is True: run_template_enum(version_path, class_details, enum_template_files) else: run_template_schema(version_path, class_details, template_files) @@ -328,6 +342,57 @@ def run_template_schema(version_path, class_details, templates): class_details["setImports"] = _set_imports class_details["setValidator"] = _set_validator class_details["setNormalizedName"] = _set_normalized_name + class_details["setCimDataType"] = _set_cim_data_type + with open(template_path, encoding="utf-8") as f: + args = { + "data": class_details, + "template": f, + "partials_dict": partials, + } + output = chevron.render(**args) + file.write(output) + +def run_template_primitive(version_path, class_details, templates): + for template_info in templates: + class_file =Path(version_path, "resources", "primitive" + template_info["ext"]) + if not os.path.exists(class_file): + if not (parent:=class_file.parent).exists(): + parent.mkdir() + with open(class_file, "w", encoding="utf-8") as file: + schema_file_path = os.path.join( + os.getcwd(), "modernpython", "primitive_header.py" + ) + schema_file = open(schema_file_path, "r") + file.write(schema_file.read()) + with open(class_file, "a", encoding="utf-8") as file: + template_path = os.path.join(os.getcwd(), "modernpython/templates", template_info["filename"]) + class_details["data_type"] = _primitive_to_data_type(class_details["class_name"]) + with open(template_path, encoding="utf-8") as f: + args = { + "data": class_details, + "template": f, + "partials_dict": partials, + } + output = chevron.render(**args) + file.write(output) + +def run_template_cimdatatype(version_path, class_details, templates): + for template_info in templates: + class_file =Path(version_path, "resources", "cimdatatype" + template_info["ext"]) + if not os.path.exists(class_file): + if not (parent:=class_file.parent).exists(): + parent.mkdir() + with open(class_file, "w", encoding="utf-8") as file: + schema_file_path = os.path.join( + os.getcwd(), "modernpython", "cimdatatype_header.py" + ) + schema_file = open(schema_file_path, "r") + file.write(schema_file.read()) + with open(class_file, "a", encoding="utf-8") as file: + template_path = os.path.join(os.getcwd(), "modernpython/templates", template_info["filename"]) + class_details["data_type"] = _compute_cim_data_type(class_details["attributes"])[0] + class_details["unit"] = _compute_cim_data_type(class_details["attributes"])[1] + class_details["multiplier"] = _compute_cim_data_type(class_details["attributes"])[2] with open(template_path, encoding="utf-8") as f: args = { "data": class_details, @@ -349,25 +414,3 @@ def resolve_headers(dest: str, version: str): with open(dest / "__init__.py", "a", encoding="utf-8") as header_file: header_file.write("# pylint: disable=too-many-lines,missing-module-docstring\n") header_file.write(f"CGMES_VERSION='{version_number}'\n") - - # # Under this, add all imports in init. Disabled becasue loading 600 unneeded classes is slow. - # _all = ["CGMES_VERSION"] - - # for include_name in sorted(dest.glob("*.py")): - # stem = include_name.stem - # if stem in[ "__init__", "Base"]: - # continue - # _all.append(stem) - # header_file.write(f"from .{stem} import {stem}\n") - - # header_file.write( - # "\n".join( - # [ - # "# This is not needed per se, but by referencing all imports", - # "# this prevents a potential autoflake from cleaning up the whole file.", - # "# FYA, if __all__ is present, only what's in there will be import with a import *", - # "", - # ] - # ) - # ) - # header_file.write(f"__all__={_all}") diff --git a/modernpython/primitive_header.py b/modernpython/primitive_header.py new file mode 100644 index 00000000..c8e79e7d --- /dev/null +++ b/modernpython/primitive_header.py @@ -0,0 +1,6 @@ +from datetime import date, datetime, time +from ..utils.datatypes import Primitive +from ..utils.profile import Profile +from .enum import * + +String = Primitive(name="String",type=str, profiles=[Profile.EQBD, Profile.OP, Profile.SSH, Profile.EQ, Profile.DY, Profile.DL, Profile.SV, Profile.SC, ]) \ No newline at end of file diff --git a/modernpython/schema_header.py b/modernpython/schema_header.py index 19a92033..5b1718e5 100644 --- a/modernpython/schema_header.py +++ b/modernpython/schema_header.py @@ -13,6 +13,8 @@ from ..utils.profile import Profile, BaseProfile from ..utils.validation import cyclic_references_validator from .enum import * +from .primitive import * +from .cimdatatype import * @dataclass(config=GeoDataclassConfig) class PositionPoint(Base): diff --git a/modernpython/templates/cimdatatype_template.mustache b/modernpython/templates/cimdatatype_template.mustache new file mode 100644 index 00000000..d2d553fe --- /dev/null +++ b/modernpython/templates/cimdatatype_template.mustache @@ -0,0 +1,11 @@ + +""" +Generated from the CGMES 3 files via cimgen: https://github.com/sogno-platform/cimgen +""" + +{{class_name}} = CIMDatatype(name="{{class_name}}", type={{data_type}}, symbol={{unit}}, multiplier={{multiplier}}, profiles=[{{#class_origin}}Profile.{{origin}},{{/class_origin}}]) + +""" +{{{wrapped_class_comment}}} +""" + diff --git a/modernpython/templates/cimpy_class_template.mustache b/modernpython/templates/cimpy_class_template.mustache index dff0904f..d866896d 100644 --- a/modernpython/templates/cimpy_class_template.mustache +++ b/modernpython/templates/cimpy_class_template.mustache @@ -14,7 +14,7 @@ class {{class_name}}({{sub_class_of}}): """ {{#attributes}} - {{#setNormalizedName}}{{label}}{{/setNormalizedName}} : {{#setType}}{{.}}{{/setType}} = Field({{#setDefault}}{{.}}{{/setDefault}}, in_profiles = [{{#attr_origin}}Profile.{{origin}}, {{/attr_origin}}], alias = "{{label}}") + {{#setNormalizedName}}{{label}}{{/setNormalizedName}} : {{#setType}}{{.}}{{/setType}} = Field({{#setDefault}}{{.}}{{/setDefault}}, in_profiles = [{{#attr_origin}}Profile.{{origin}}, {{/attr_origin}}], {{#setCimDataType}}{{.}}{{/setCimDataType}}alias = "{{label}}") {{/attributes}} {{#attributes}} {{#setValidator}}{{.}}{{/setValidator}} diff --git a/modernpython/templates/pydantic_enum_template.mustache b/modernpython/templates/enum_class_template.mustache similarity index 100% rename from modernpython/templates/pydantic_enum_template.mustache rename to modernpython/templates/enum_class_template.mustache diff --git a/modernpython/templates/primitive_template.mustache b/modernpython/templates/primitive_template.mustache new file mode 100644 index 00000000..91282052 --- /dev/null +++ b/modernpython/templates/primitive_template.mustache @@ -0,0 +1,11 @@ + +""" +Generated from the CGMES 3 files via cimgen: https://github.com/sogno-platform/cimgen +""" + +{{class_name}} = Primitive(name="{{class_name}}", type={{data_type}}, profiles=[{{#class_origin}}Profile.{{origin}}, {{/class_origin}}]) + +""" +{{{wrapped_class_comment}}} +""" + diff --git a/modernpython/utils/base.py b/modernpython/utils/base.py index 3fb558ee..fd9dfcf1 100644 --- a/modernpython/utils/base.py +++ b/modernpython/utils/base.py @@ -43,16 +43,10 @@ def to_dict(self, with_class: bool = True) -> dict[str, "CgmesAttributeTypes"]: """ attrs = {f.name: getattr(self, f.name) for f in fields(self)} - attrs["__class__"] = self.apparent_name() if with_class: - attrs["__class__"] = self.resource_name + attrs["__class__"] = self.apparent_name() return attrs - @cached_property - def resource_name(self) -> str: - """Returns the resource type.""" - return self.__class__.__name__ - @cached_property def namespace(self) -> str: """Returns the namespace. By default, the namespace is the cim namespace for all resources. @@ -71,7 +65,7 @@ def apparent_name(cls) -> str: def cgmes_attribute_names_in_profile(self, profile: BaseProfile | None) -> set[Field]: """ - Returns all fields accross the parent tree which are in the profile in parameter. + Returns all fields across the parent tree which are in the profile in parameter. Mostly useful during export to find all the attributes relevant to one profile only. diff --git a/modernpython/utils/datatypes.py b/modernpython/utils/datatypes.py new file mode 100644 index 00000000..b14b2ee2 --- /dev/null +++ b/modernpython/utils/datatypes.py @@ -0,0 +1,23 @@ +from pydantic import Field +from typing import List + +from .constants import NAMESPACES +from pydantic.dataclasses import dataclass + +from .dataclassconfig import DataclassConfig +from .profile import BaseProfile +from ..resources.enum import UnitMultiplier, UnitSymbol + +@dataclass(config=DataclassConfig) +class Primitive: + + name: str = Field(frozen=True) + type: object = Field(frozen=True) + namespace: str = Field(frozen=True, default=NAMESPACES["cim"]) + profiles: List[BaseProfile] = Field(frozen=True) + +@dataclass(config=DataclassConfig) +class CIMDatatype(Primitive): + + multiplier: UnitMultiplier = Field(frozen=True) + symbol: UnitSymbol = Field(frozen=True) From 5e0c5a8ddf2b02c4e8a41f5e81ab4c93bdb2d7d4 Mon Sep 17 00:00:00 2001 From: "Federico M. Facca" Date: Tue, 28 Nov 2023 13:15:15 +0100 Subject: [PATCH 09/10] clean imports Signed-off-by: Federico M. Facca --- modernpython/cimdatatype_header.py | 2 +- modernpython/primitive_header.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/modernpython/cimdatatype_header.py b/modernpython/cimdatatype_header.py index 280725c1..ff44a22c 100644 --- a/modernpython/cimdatatype_header.py +++ b/modernpython/cimdatatype_header.py @@ -1,3 +1,3 @@ from ..utils.datatypes import CIMDatatype from ..utils.profile import Profile -from .enum import * +from .enum import UnitMultiplier, UnitSymbol diff --git a/modernpython/primitive_header.py b/modernpython/primitive_header.py index c8e79e7d..d8981a5f 100644 --- a/modernpython/primitive_header.py +++ b/modernpython/primitive_header.py @@ -1,6 +1,5 @@ from datetime import date, datetime, time from ..utils.datatypes import Primitive from ..utils.profile import Profile -from .enum import * String = Primitive(name="String",type=str, profiles=[Profile.EQBD, Profile.OP, Profile.SSH, Profile.EQ, Profile.DY, Profile.DL, Profile.SV, Profile.SC, ]) \ No newline at end of file From 84f8d1095c26949c49dc4088603f73149c688bc6 Mon Sep 17 00:00:00 2001 From: "Federico M. Facca" Date: Thu, 21 Dec 2023 13:35:26 +0100 Subject: [PATCH 10/10] clean directory only if flag Signed-off-by: Federico M. Facca --- CIMgen.py | 5 +++-- build.py | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CIMgen.py b/CIMgen.py index fa633071..6cbf16c5 100644 --- a/CIMgen.py +++ b/CIMgen.py @@ -693,7 +693,7 @@ def generate_clean_sub_classes(class_dict_with_origins, clean_class_dict): add_sub_classes_of_sub_classes_clean(clean_class_dict, class_dict_with_origins) -def cim_generate(directory, outputPath, version, langPack): +def cim_generate(directory, outputPath, version, langPack, clean_outdir): """Generates cgmes python classes from cgmes ontology This function uses package xmltodict to parse the RDF files. The parse_rdf function sorts the classes to @@ -712,7 +712,8 @@ def cim_generate(directory, outputPath, version, langPack): """ # clean directory - remove_tree(Path(outputPath)) + if clean_outdir: + remove_tree(Path(outputPath)) profiles_array = [] diff --git a/build.py b/build.py index 9140803f..6bd9dd3b 100644 --- a/build.py +++ b/build.py @@ -9,8 +9,9 @@ parser.add_argument('--schemadir', type=str, help='The schema directory', required=True) parser.add_argument('--langdir', type=str, help='The langpack directory', required=True) parser.add_argument('--cgmes_version', type=str, choices=['cgmes_v2_4_13', 'cgmes_v2_4_15', 'cgmes_v3_0_0'], default='cgmes_v2_4_15', help='CGMES Version') +parser.add_argument('--clean_outdir', type=bool, help='Clean the output directory', required=False, default=False) args = parser.parse_args() langPack = importlib.import_module(args.langdir + ".langPack") schema_path = os.path.join(os.getcwd(), args.schemadir) -CIMgen.cim_generate(schema_path, args.outdir, args.cgmes_version, langPack) +CIMgen.cim_generate(schema_path, args.outdir, args.cgmes_version, langPack, args.clean_outdir)