diff --git a/CIMgen.py b/CIMgen.py index b759e0ea..6cbf16c5 100644 --- a/CIMgen.py +++ b/CIMgen.py @@ -4,7 +4,8 @@ import os import textwrap from time import time - +from distutils.dir_util import remove_tree +from pathlib import Path import xmltodict from bs4 import BeautifulSoup @@ -208,6 +209,7 @@ def __init__(self, rdfsEntry): self.origin_list = [] self.super = rdfsEntry.subClassOf() self.subclasses = [] + self.stereotype = rdfsEntry.stereotype() def attributes(self): return self.attribute_list @@ -275,6 +277,12 @@ def is_a_float(self): return False return True + def is_a_primitive(self): + return self.stereotype == 'Primitive' + + def is_a_cim_datatype(self): + return self.stereotype == 'CIMDatatype' + def get_profile_name(descriptions): for list_elem in descriptions: # only for CGMES-Standard @@ -446,6 +454,8 @@ def _write_python_files(elem_dict, langPack, outputPath, version): float_classes = {} enum_classes = {} + primitive_classes = {} + cim_data_type_classes = {} # Iterate over Classes for class_definition in elem_dict: @@ -453,9 +463,15 @@ def _write_python_files(elem_dict, langPack, outputPath, version): float_classes[class_definition] = True if elem_dict[class_definition].has_instances(): enum_classes[class_definition] = True + if elem_dict[class_definition].is_a_primitive(): + primitive_classes[class_definition] = True + if elem_dict[class_definition].is_a_cim_datatype(): + cim_data_type_classes[class_definition] = True langPack.set_float_classes(float_classes) langPack.set_enum_classes(enum_classes) + langPack.set_primitive_classes(primitive_classes) + langPack.set_cim_data_type_classes(cim_data_type_classes) for class_name in elem_dict.keys(): @@ -467,6 +483,8 @@ def _write_python_files(elem_dict, langPack, outputPath, version): "instances": elem_dict[class_name].instances(), "has_instances": elem_dict[class_name].has_instances(), "is_a_float": elem_dict[class_name].is_a_float(), + "is_a_primitive": elem_dict[class_name].is_a_primitive(), + "is_a_cim_data_type": elem_dict[class_name].is_a_cim_datatype(), "langPack": langPack, "sub_class_of": elem_dict[class_name].superClass(), "sub_classes": elem_dict[class_name].subClasses(), @@ -638,20 +656,44 @@ def _merge_classes(profiles_dict): class_dict[class_key].addAttribute(attr) return class_dict -def recursivelyAddSubClasses(class_dict, class_name): +def recursively_add_sub_classes(class_dict, class_name): newSubClasses = [] theClass = class_dict[class_name] for name in theClass.subClasses(): newSubClasses.append(name) - newNewSubClasses = recursivelyAddSubClasses(class_dict, name) + newNewSubClasses = recursively_add_sub_classes(class_dict, name) newSubClasses = newSubClasses + newNewSubClasses return newSubClasses -def addSubClassesOfSubClasses(class_dict): - for className in class_dict: - class_dict[className].setSubClasses(recursivelyAddSubClasses(class_dict, className)) +def add_sub_classes_of_sub_classes(class_dict): + for class_name in class_dict: + class_dict[class_name].setSubClasses(recursively_add_sub_classes(class_dict, class_name)) + +def add_sub_classes_of_sub_classes_clean(class_dict, source): + temp = {} + for class_name in class_dict: + for name in class_dict[class_name].subClasses(): + if name not in class_dict: + temp[name] = source[name] + add_sub_classes_of_sub_classes_clean(temp, source) + class_dict.update(temp) + +# Order classes based on dependency order + +def generate_clean_sub_classes(class_dict_with_origins, clean_class_dict): + for class_name in class_dict_with_origins: + super_class_name = class_dict_with_origins[class_name].superClass() + if super_class_name is None and class_dict_with_origins[class_name].has_instances(): + clean_class_dict[class_name] = class_dict_with_origins[class_name] -def cim_generate(directory, outputPath, version, langPack): + for class_name in class_dict_with_origins: + super_class_name = class_dict_with_origins[class_name].superClass() + if super_class_name == None and not class_dict_with_origins[class_name].has_instances(): + clean_class_dict[class_name] = class_dict_with_origins[class_name] + + add_sub_classes_of_sub_classes_clean(clean_class_dict, class_dict_with_origins) + +def cim_generate(directory, outputPath, version, langPack, clean_outdir): """Generates cgmes python classes from cgmes ontology This function uses package xmltodict to parse the RDF files. The parse_rdf function sorts the classes to @@ -668,6 +710,11 @@ def cim_generate(directory, outputPath, version, langPack): :param outputPath: CGMES version, e.g. version = "cgmes_v2_4_15" :param langPack: python module containing language specific functions """ + + # clean directory + if clean_outdir: + remove_tree(Path(outputPath)) + profiles_array = [] t0 = time() @@ -691,6 +738,8 @@ def cim_generate(directory, outputPath, version, langPack): # merge classes from different profiles into one class and track origin of the classes and their attributes class_dict_with_origins = _merge_classes(profiles_dict) + clean_class_dict = {} + # work out the subclasses for each class by noting the reverse relationship for className in class_dict_with_origins: superClassName = class_dict_with_origins[className].superClass() @@ -702,10 +751,12 @@ def cim_generate(directory, outputPath, version, langPack): print("No match for superClass in dict: :", superClassName) # recursively add the subclasses of subclasses - addSubClassesOfSubClasses(class_dict_with_origins) + add_sub_classes_of_sub_classes(class_dict_with_origins) + + generate_clean_sub_classes(class_dict_with_origins, clean_class_dict) # get information for writing python files and write python files - _write_python_files(class_dict_with_origins, langPack, outputPath, version) + _write_python_files(clean_class_dict, langPack, outputPath, version) if "modernpython" in langPack.__name__: langPack.resolve_headers(outputPath, version) diff --git a/build.py b/build.py index 9140803f..6bd9dd3b 100644 --- a/build.py +++ b/build.py @@ -9,8 +9,9 @@ parser.add_argument('--schemadir', type=str, help='The schema directory', required=True) parser.add_argument('--langdir', type=str, help='The langpack directory', required=True) parser.add_argument('--cgmes_version', type=str, choices=['cgmes_v2_4_13', 'cgmes_v2_4_15', 'cgmes_v3_0_0'], default='cgmes_v2_4_15', help='CGMES Version') +parser.add_argument('--clean_outdir', type=bool, help='Clean the output directory', required=False, default=False) args = parser.parse_args() langPack = importlib.import_module(args.langdir + ".langPack") schema_path = os.path.join(os.getcwd(), args.schemadir) -CIMgen.cim_generate(schema_path, args.outdir, args.cgmes_version, langPack) +CIMgen.cim_generate(schema_path, args.outdir, args.cgmes_version, langPack, args.clean_outdir) diff --git a/cpp/langPack.py b/cpp/langPack.py index c6e7c2c7..062075a2 100644 --- a/cpp/langPack.py +++ b/cpp/langPack.py @@ -118,6 +118,12 @@ def is_an_enum_class(name): if name in enum_classes: return enum_classes[name] +def set_primitive_classes(new_primitive_classes): + return + +def set_cim_data_type_classes(new_cim_data_type_classes): + return + # These insert_ functions are used to generate the entries in the dynamic_switch # maps, for use in assignments.cpp and Task.cpp # TODO: implement this as one function, determine in template if it should be called. diff --git a/java/langPack.py b/java/langPack.py index 10ba104b..6d404b55 100644 --- a/java/langPack.py +++ b/java/langPack.py @@ -117,6 +117,12 @@ def is_an_enum_class(name): if name in enum_classes: return enum_classes[name] +def set_primitive_classes(new_primitive_classes): + return + +def set_cim_data_type_classes(new_cim_data_type_classes): + return + # These insert_ functions are used to generate the entries in the dynamic_switch # maps, for use in assignments.cpp and Task.cpp # TODO: implement this as one function, determine in template if it should be called. diff --git a/javascript/langPack.py b/javascript/langPack.py index 8c6c49ca..bc1e5a1d 100644 --- a/javascript/langPack.py +++ b/javascript/langPack.py @@ -63,6 +63,12 @@ def is_an_enum_class(name): if name in enum_classes: return enum_classes[name] +def set_primitive_classes(new_primitive_classes): + return + +def set_cim_data_type_classes(new_cim_data_type_classes): + return + def get_class_location(class_name, class_map, version): pass diff --git a/modernpython/cimdatatype_header.py b/modernpython/cimdatatype_header.py new file mode 100644 index 00000000..ff44a22c --- /dev/null +++ b/modernpython/cimdatatype_header.py @@ -0,0 +1,3 @@ +from ..utils.datatypes import CIMDatatype +from ..utils.profile import Profile +from .enum import UnitMultiplier, UnitSymbol diff --git a/modernpython/enum_header.py b/modernpython/enum_header.py new file mode 100644 index 00000000..27067ef9 --- /dev/null +++ b/modernpython/enum_header.py @@ -0,0 +1,2 @@ +from enum import Enum + diff --git a/modernpython/langPack.py b/modernpython/langPack.py index 5339a01a..4fae4599 100644 --- a/modernpython/langPack.py +++ b/modernpython/langPack.py @@ -2,7 +2,11 @@ import os import re from distutils.dir_util import copy_tree +import shutil +import sys +import textwrap from pathlib import Path +import ast import chevron @@ -32,14 +36,87 @@ def location(version): base = {"base_class": "Base", "class_location": location} template_files = [{"filename": "cimpy_class_template.mustache", "ext": ".py"}] - +enum_template_files = [{"filename": "enum_class_template.mustache", "ext": ".py"}] +primitive_template_files = [{"filename": "primitive_template.mustache", "ext": ".py"}] +cimdatatype_template_files = [{"filename": "cimdatatype_template.mustache", "ext": ".py"}] def get_class_location(class_name, class_map, version): return f".{class_map[class_name].superClass()}" - + # Check if the current class has a parent class partials = {} +def _primitive_to_data_type(datatype): + if datatype.lower() == "integer": + return "int" + if datatype.lower() == "boolean": + return "bool" + if datatype.lower() == "string": + return "str" + if datatype.lower() == "datetime": + return "datetime" + if datatype.lower() == "monthday": + return "str" # TO BE FIXED? I could not find a datatype in python that holds only month and day. + if datatype.lower() == "date": + return "date" + # as of today no CIM model is using only time. + if datatype.lower() == "time": + return "time" + if datatype.lower() == "float": + return "float" + if datatype.lower() == "string": + return "str" + else: + # this actually never happens + return "float" + +def _compute_cim_data_type(attributes) -> tuple[str, str, str]: + python_type = 'None' + unit = 'UnitSymbol.none' + multiplier = 'UnitMultiplier.none' + for attribute in attributes: + if 'about' in attribute and attribute['about'] and "value" in attribute['about'] and 'class_name' in attribute: + python_type = _primitive_to_data_type(attribute['class_name']) + if 'about' in attribute and attribute['about'] and "multiplier" in attribute['about'] and 'isFixed' in attribute: + multiplier = "UnitMultiplier."+attribute['isFixed'] + if 'about' in attribute and attribute['about'] and "unit" in attribute['about'] and 'isFixed' in attribute: + unit = "UnitSymbol."+attribute['isFixed'] + return (python_type, unit, multiplier) + + +# set the cim datatype for a field +def _set_cim_data_type(text, render) -> str: + attribute = eval(render(text)) + if is_primitive_class(attribute["class_name"]): + return "data_type = " + attribute["class_name"] + "," + elif is_primitive_class(attribute["class_name"]) or attribute["class_name"] == "String": + return "data_type = String," + elif is_cim_data_type_class(attribute["class_name"]): + return "data_type = " + attribute["class_name"] + "," + return "" + +# computes the data type +def _compute_data_type(attribute): + if "label" in attribute and attribute["label"] == "mRID": + return "str" + elif "range" in attribute: + return attribute["range"].split("#")[1] + elif "dataType" in attribute and "class_name" in attribute: + # for whatever weird reason String is not created as class from CIMgen + if is_primitive_class(attribute["class_name"]) or attribute["class_name"] == "String": + return _primitive_to_data_type(attribute["dataType"].split("#")[1]) + # the assumption is that cim data type e.g. Voltage, ActivePower, always + # maps to a float + elif is_cim_data_type_class(attribute["class_name"]): + return "float" + else: + # this is for example the case for 'StreetAddress.streetDetail' + return attribute["dataType"].split("#")[1] + else: + raise ValueError(f"Cannot parse {attribute} to extract a data type.") + +def _ends_with_s(attribute_name): + return attribute_name.endswith("s") # called by chevron, text contains the label {{dataType}}, which is evaluated by the renderer (see class template) def _set_default(text, render): @@ -49,30 +126,127 @@ def _set_default(text, render): def _set_type(text, render): return _get_type_and_default(text, render)[0] +def _lower_case_first_char(str): + return str[:1].lower() + str[1:] if str else "" + +# attributes should never have the same name as a class the may map to +# Python won't be happy with that, so let's normalize attribute names +# and leverage aliasing in pydantic to use the cim attribute capitalization +# during value setting. +def _set_normalized_name(text, render): + return _lower_case_first_char(render(text)) + +# called by chevron, text contains the label {{dataType}}, which is evaluated by the renderer (see class template) +def _set_instances(text, render): + instance = None + try: + # render(text) returns a python dict. Some fileds might be quoted by '"' instead of '"', making the first evel fail. + instance = ast.literal_eval(render(text)) + except SyntaxError as se: + rendered = render(text) + rendered = rendered.replace(""", '"') + instance = eval(rendered) + logger.warning("Exception in evaluating %s : %s . Handled replacing quotes", rendered, se.msg) + if "label" in instance: + value = instance["label"] + ' = "' + instance["label"] + '"' + if "comment" in instance: + value += " #" + instance["comment"] + return value + else: + return "" + +# called by chevron, text contains the label {{dataType}}, which is evaluated by the renderer (see class template) +def _set_imports(text, render): + rendered = render(text) + res = None + classes = set() + try: + res = eval(rendered) + except Exception: + pass + if res: + for val in res: + if "range" in val: + classes.add(val["range"].split("#")[1]) + elif not is_primitive_class(val["class_name"]) and val["class_name"] != "String" and not is_cim_data_type_class(val["class_name"]): + classes.add(val["dataType"].split("#")[1]) + result = "" + for val in classes: + result += "from ." + val + " import " + val + "\n" + return result + +# called by chevron, text contains the label {{dataType}}, which is evaluated by the renderer (see class template) +def _set_validator(text, render): + attribute = eval(render(text)) + + if not is_primitive_class(attribute["class_name"]) and attribute["class_name"] != "String" and not is_cim_data_type_class(attribute["class_name"]): + return ( + "val_" + + _lower_case_first_char(attribute["label"]) + + '_wrap = field_validator("' + + _lower_case_first_char(attribute["label"]) + + '", mode="wrap")(cyclic_references_validator)' + ) + elif attribute["label"] == "mRID": + return ( + '@field_validator("mRID", mode="before")\n' + + " def validate_mrid_format(cls, v):\n" + + " if isinstance(v, uuid.UUID):\n" + + " return str(v)\n" + + " elif isinstance(v, str):\n" + + " return v\n" + + " else:\n" + + ' raise ValueError("must be a UUID or str")\n' + ) + else: + return "" def _get_type_and_default(text, renderer) -> tuple[str, str]: - result = renderer(text) # the field {{dataType}} either contains the multiplicity of an attribute if it is a reference or otherwise the # datatype of the attribute. If no datatype is set and there is also no multiplicity entry for an attribute, the # default value is set to None. The multiplicity is set for all attributes, but the datatype is only set for basic # data types. If the data type entry for an attribute is missing, the attribute contains a reference and therefore # the default value is either None or [] depending on the multiplicity. See also write_python_files - # The default will be copied as-is, hence the possibility to have default or default_factory. - if result in ["M:1", "M:0..1", "M:1..1", ""]: - return ("Optional[str]", "default=None") - elif result in ["M:0..n", "M:1..n"] or "M:" in result: - return ("list", "default_factory=list") - - result = result.split("#")[1] - if result in ["integer", "Integer"]: - return ("int", "default=0") - elif result in ["String", "DateTime", "Date"]: - return ("str", 'default=""') - elif result == "Boolean": - return ("bool", "default=False") - else: - # everything else should be a float - return ("float", "default=0.0") + # The default will be copied as-is, hence the possibility to have default or + # default_factory. + attribute = eval(renderer(text)) + datatype = _compute_data_type(attribute) + field_type = datatype + default = 'default=None' + if "multiplicity" in attribute: + multiplicity = attribute["multiplicity"] + if multiplicity in ["M:0..1"]: + field_type = "Optional[" + datatype + "]" + elif multiplicity in ["M:0..n"]: + field_type = "Optional[List[" + datatype + "]]" + elif multiplicity in ["M:1..n", "M:2..n"]: + field_type = "List[" + datatype + "]" + elif multiplicity in ["M:1"] and attribute['label'] == 'PowerSystemResources': + # Most probably there is a bug in the RDF that states multiplicity + # M:1 but should be M:1..N + field_type = "List[" + datatype + "]" + else: + field_type = datatype + + if "label" in attribute and attribute["label"] == "mRID": + default = "default_factory=uuid.uuid4" + elif "multiplicity" in attribute: + multiplicity = attribute["multiplicity"] + if multiplicity in ["M:1"] and attribute['label'] == 'PowerSystemResources': + # Most probably there is a bug in the RDF that states multiplicity + # M:1 but should be M:1..N + default = 'default_factory=list' + elif multiplicity in ["M:0..n"] or multiplicity in ["M:1..n"]: + default = 'default_factory=list' + elif field_type == 'int': + default = 'default=0' + elif field_type == 'str': + default = 'default=""' + elif field_type == 'float': + default = 'default=0.0' + elif field_type == 'bool': + default = 'default=False' + return (field_type, default) def set_enum_classes(new_enum_classes): @@ -82,30 +256,151 @@ def set_enum_classes(new_enum_classes): def set_float_classes(new_float_classes): return +primitive_classes = {} -def run_template(version_path, class_details): - for template_info in template_files: +def set_primitive_classes(new_primitive_classes): + for new_class in new_primitive_classes: + primitive_classes[new_class] = new_primitive_classes[new_class] - resource_file = Path(os.path.join(version_path, "resources", class_details["class_name"] + template_info["ext"])) - if not resource_file.exists() : - if not (parent:=resource_file.parent).exists(): - parent.mkdir() +def is_primitive_class(name): + if name in primitive_classes: + return primitive_classes[name] - with open(resource_file, "w", encoding="utf-8") as file: +cim_data_type_classes = {} - template_path = os.path.join(os.getcwd(), "modernpython/templates", template_info["filename"]) - class_details["setDefault"] = _set_default - class_details["setType"] = _set_type - with open(template_path, encoding="utf-8") as f: - args = { - "data": class_details, - "template": f, - "partials_dict": partials, - } - output = chevron.render(**args) - file.write(output) +def set_cim_data_type_classes(new_cim_data_type_classes): + for new_class in new_cim_data_type_classes: + cim_data_type_classes[new_class] = new_cim_data_type_classes[new_class] +def is_cim_data_type_class(name): + if name in cim_data_type_classes: + return cim_data_type_classes[name] + +def has_unit_attribute(attributes): + for attr in attributes: + if attr["label"] == "unit": + return True + return False + +def run_template(version_path, class_details): + if class_details["class_name"] == 'PositionPoint': + #this class is created manually to support types conversions + return + elif class_details["is_a_primitive"] is True: + # Primitives are never used in the in memory representation but only for + # the schema + run_template_primitive(version_path, class_details, primitive_template_files) + elif class_details["is_a_cim_data_type"] is True: + # Datatypes based on primitives are never used in the in memory + # representation but only for the schema + run_template_cimdatatype(version_path, class_details, cimdatatype_template_files) + elif class_details["has_instances"] is True: + run_template_enum(version_path, class_details, enum_template_files) + else: + run_template_schema(version_path, class_details, template_files) +def run_template_enum(version_path, class_details, templates): + for template_info in templates: + class_file = Path(version_path, "resources", "enum" + template_info["ext"]) + if not os.path.exists(class_file): + if not (parent:=class_file.parent).exists(): + parent.mkdir() + with open(class_file, "w", encoding="utf-8") as file: + header_file_path = os.path.join( + os.getcwd(), "modernpython", "enum_header.py" + ) + header_file = open(header_file_path, "r") + file.write(header_file.read()) + with open(class_file, "a", encoding="utf-8") as file: + template_path = os.path.join(os.getcwd(), "modernpython/templates", template_info["filename"]) + class_details["setInstances"] = _set_instances + with open(template_path, encoding="utf-8") as f: + args = { + "data": class_details, + "template": f, + "partials_dict": partials, + } + output = chevron.render(**args) + file.write(output) + +def run_template_schema(version_path, class_details, templates): + for template_info in templates: + class_file =Path(version_path, "resources", "schema" + template_info["ext"]) + if not os.path.exists(class_file): + if not (parent:=class_file.parent).exists(): + parent.mkdir() + with open(class_file, "w", encoding="utf-8") as file: + schema_file_path = os.path.join( + os.getcwd(), "modernpython", "schema_header.py" + ) + schema_file = open(schema_file_path, "r") + file.write(schema_file.read()) + with open(class_file, "a", encoding="utf-8") as file: + template_path = os.path.join(os.getcwd(), "modernpython/templates", template_info["filename"]) + class_details["setDefault"] = _set_default + class_details["setType"] = _set_type + class_details["setImports"] = _set_imports + class_details["setValidator"] = _set_validator + class_details["setNormalizedName"] = _set_normalized_name + class_details["setCimDataType"] = _set_cim_data_type + with open(template_path, encoding="utf-8") as f: + args = { + "data": class_details, + "template": f, + "partials_dict": partials, + } + output = chevron.render(**args) + file.write(output) + +def run_template_primitive(version_path, class_details, templates): + for template_info in templates: + class_file =Path(version_path, "resources", "primitive" + template_info["ext"]) + if not os.path.exists(class_file): + if not (parent:=class_file.parent).exists(): + parent.mkdir() + with open(class_file, "w", encoding="utf-8") as file: + schema_file_path = os.path.join( + os.getcwd(), "modernpython", "primitive_header.py" + ) + schema_file = open(schema_file_path, "r") + file.write(schema_file.read()) + with open(class_file, "a", encoding="utf-8") as file: + template_path = os.path.join(os.getcwd(), "modernpython/templates", template_info["filename"]) + class_details["data_type"] = _primitive_to_data_type(class_details["class_name"]) + with open(template_path, encoding="utf-8") as f: + args = { + "data": class_details, + "template": f, + "partials_dict": partials, + } + output = chevron.render(**args) + file.write(output) + +def run_template_cimdatatype(version_path, class_details, templates): + for template_info in templates: + class_file =Path(version_path, "resources", "cimdatatype" + template_info["ext"]) + if not os.path.exists(class_file): + if not (parent:=class_file.parent).exists(): + parent.mkdir() + with open(class_file, "w", encoding="utf-8") as file: + schema_file_path = os.path.join( + os.getcwd(), "modernpython", "cimdatatype_header.py" + ) + schema_file = open(schema_file_path, "r") + file.write(schema_file.read()) + with open(class_file, "a", encoding="utf-8") as file: + template_path = os.path.join(os.getcwd(), "modernpython/templates", template_info["filename"]) + class_details["data_type"] = _compute_cim_data_type(class_details["attributes"])[0] + class_details["unit"] = _compute_cim_data_type(class_details["attributes"])[1] + class_details["multiplier"] = _compute_cim_data_type(class_details["attributes"])[2] + with open(template_path, encoding="utf-8") as f: + args = { + "data": class_details, + "template": f, + "partials_dict": partials, + } + output = chevron.render(**args) + file.write(output) def resolve_headers(dest: str, version: str): """Add all classes in __init__.py""" @@ -119,25 +414,3 @@ def resolve_headers(dest: str, version: str): with open(dest / "__init__.py", "a", encoding="utf-8") as header_file: header_file.write("# pylint: disable=too-many-lines,missing-module-docstring\n") header_file.write(f"CGMES_VERSION='{version_number}'\n") - - # # Under this, add all imports in init. Disabled becasue loading 600 unneeded classes is slow. - # _all = ["CGMES_VERSION"] - - # for include_name in sorted(dest.glob("*.py")): - # stem = include_name.stem - # if stem in[ "__init__", "Base"]: - # continue - # _all.append(stem) - # header_file.write(f"from .{stem} import {stem}\n") - - # header_file.write( - # "\n".join( - # [ - # "# This is not needed per se, but by referencing all imports", - # "# this prevents a potential autoflake from cleaning up the whole file.", - # "# FYA, if __all__ is present, only what's in there will be import with a import *", - # "", - # ] - # ) - # ) - # header_file.write(f"__all__={_all}") diff --git a/modernpython/primitive_header.py b/modernpython/primitive_header.py new file mode 100644 index 00000000..d8981a5f --- /dev/null +++ b/modernpython/primitive_header.py @@ -0,0 +1,5 @@ +from datetime import date, datetime, time +from ..utils.datatypes import Primitive +from ..utils.profile import Profile + +String = Primitive(name="String",type=str, profiles=[Profile.EQBD, Profile.OP, Profile.SSH, Profile.EQ, Profile.DY, Profile.DL, Profile.SV, Profile.SC, ]) \ No newline at end of file diff --git a/modernpython/schema_header.py b/modernpython/schema_header.py new file mode 100644 index 00000000..5b1718e5 --- /dev/null +++ b/modernpython/schema_header.py @@ -0,0 +1,71 @@ +from __future__ import annotations +import uuid +from functools import cached_property +from pydantic import ConfigDict, Field, field_validator, computed_field +from geoalchemy2.shape import to_shape +from geoalchemy2.elements import WKBElement +from shapely.geometry import Point +from datetime import date, datetime, time +from typing import Optional, Iterator, List +from pydantic.dataclasses import dataclass +from ..utils.base import Base +from ..utils.dataclassconfig import DataclassConfig, GeoDataclassConfig +from ..utils.profile import Profile, BaseProfile +from ..utils.validation import cyclic_references_validator +from .enum import * +from .primitive import * +from .cimdatatype import * + +@dataclass(config=GeoDataclassConfig) +class PositionPoint(Base): + """ + Set of spatial coordinates that determine a point, defined in the coordinate system specified in 'Location.CoordinateSystem'. Use a single position point instance to desribe a point-oriented location. Use a sequence of position points to describe a line-oriented object (physical location of non-point oriented objects like cables or lines), or area of an object (like a substation or a geographical zone - in this case, have first and last position point with the same values). + + :Location: Location described by this position point. + :sequenceNumber: Zero-relative sequence number of this point within a series of points. + :xPosition: X axis position. + :yPosition: Y axis position. + :zPosition: (if applicable) Z axis position. + """ + + location: "Location" = Field(alias="Location", in_profiles = [Profile.GL, ]) + sequenceNumber: Optional[int] = Field(default=None, in_profiles = [Profile.GL, ]) + point: Point = Field( + repr=False, in_profiles = [Profile.GL, ] + ) # we introduce this field compared to CIM definition because we want to store a proper geometry "point" in the database + + @computed_field + @property + def xPosition(self) -> str: + return str(self.point.x) + + @computed_field + @property + def yPosition(self) -> str: + return str(self.point.y) + + @computed_field + @property + def zPosition(self) -> str: + return str(self.point.z) + + @cached_property + def possible_profiles(self)->set[Profile]: + """ + A resource can be used by multiple profiles. This is the set of profiles + where this element can be found. + """ + return { Profile.GL, } + + # Pydantic needs help to map GeoAlchemy classes to Shapely + @field_validator("point", mode="before") + def validate_point_format(cls, v): + if isinstance(v, Point): + return v + elif isinstance(v, WKBElement): + point = to_shape(v) + if point.geom_type != "Point": + raise ValueError("must be a Point") + return Point(point) + else: + raise ValueError("must be a Point or a WKBElement") diff --git a/modernpython/templates/cimdatatype_template.mustache b/modernpython/templates/cimdatatype_template.mustache new file mode 100644 index 00000000..d2d553fe --- /dev/null +++ b/modernpython/templates/cimdatatype_template.mustache @@ -0,0 +1,11 @@ + +""" +Generated from the CGMES 3 files via cimgen: https://github.com/sogno-platform/cimgen +""" + +{{class_name}} = CIMDatatype(name="{{class_name}}", type={{data_type}}, symbol={{unit}}, multiplier={{multiplier}}, profiles=[{{#class_origin}}Profile.{{origin}},{{/class_origin}}]) + +""" +{{{wrapped_class_comment}}} +""" + diff --git a/modernpython/templates/cimpy_class_template.mustache b/modernpython/templates/cimpy_class_template.mustache index cf9b32d5..d866896d 100644 --- a/modernpython/templates/cimpy_class_template.mustache +++ b/modernpython/templates/cimpy_class_template.mustache @@ -1,16 +1,8 @@ + """ Generated from the CGMES 3 files via cimgen: https://github.com/sogno-platform/cimgen """ -from functools import cached_property -from typing import Optional -from pydantic import Field -from pydantic.dataclasses import dataclass -from ..utils.dataclassconfig import DataclassConfig -from ..utils.profile import BaseProfile, Profile - -from {{class_location}} import {{sub_class_of}} - @dataclass(config=DataclassConfig) class {{class_name}}({{sub_class_of}}): """ @@ -22,16 +14,15 @@ class {{class_name}}({{sub_class_of}}): """ {{#attributes}} - {{^isAssociationUsed}}# *Association not used* - # Type {{dataType}} in CIM # pylint: disable-next=line-too-long - # {{/isAssociationUsed}}{{label}} : {{#setType}}{{dataType}}{{/setType}} = Field({{#setDefault}}{{dataType}}{{/setDefault}}, in_profiles = [{{#attr_origin}}Profile.{{origin}}, {{/attr_origin}}]) {{^isAssociationUsed}}# noqa: E501{{/isAssociationUsed}} - + {{#setNormalizedName}}{{label}}{{/setNormalizedName}} : {{#setType}}{{.}}{{/setType}} = Field({{#setDefault}}{{.}}{{/setDefault}}, in_profiles = [{{#attr_origin}}Profile.{{origin}}, {{/attr_origin}}], {{#setCimDataType}}{{.}}{{/setCimDataType}}alias = "{{label}}") + {{/attributes}} + {{#attributes}} + {{#setValidator}}{{.}}{{/setValidator}} {{/attributes}} {{^attributes}} # No attributes defined for this class. {{/attributes}} - - + @cached_property def possible_profiles(self)->set[BaseProfile]: """ diff --git a/modernpython/templates/enum_class_template.mustache b/modernpython/templates/enum_class_template.mustache new file mode 100644 index 00000000..1ad7d9f6 --- /dev/null +++ b/modernpython/templates/enum_class_template.mustache @@ -0,0 +1,14 @@ + +""" +Generated from the CGMES 3 files via cimgen: https://github.com/sogno-platform/cimgen +""" + +class {{class_name}}(str,Enum): + + ''' + {{{class_comment}}} + ''' + + {{#instances}} + {{#setInstances}}{{.}}{{/setInstances}} + {{/instances}} diff --git a/modernpython/templates/primitive_template.mustache b/modernpython/templates/primitive_template.mustache new file mode 100644 index 00000000..91282052 --- /dev/null +++ b/modernpython/templates/primitive_template.mustache @@ -0,0 +1,11 @@ + +""" +Generated from the CGMES 3 files via cimgen: https://github.com/sogno-platform/cimgen +""" + +{{class_name}} = Primitive(name="{{class_name}}", type={{data_type}}, profiles=[{{#class_origin}}Profile.{{origin}}, {{/class_origin}}]) + +""" +{{{wrapped_class_comment}}} +""" + diff --git a/modernpython/utils/base.py b/modernpython/utils/base.py index c307b33b..fd9dfcf1 100644 --- a/modernpython/utils/base.py +++ b/modernpython/utils/base.py @@ -4,13 +4,12 @@ from functools import cached_property from typing import Any, TypeAlias, TypedDict -from pycgmes.utils.constants import NAMESPACES +from .constants import NAMESPACES from pydantic.dataclasses import dataclass from .dataclassconfig import DataclassConfig from .profile import BaseProfile - @dataclass(config=DataclassConfig) class Base: """ @@ -36,7 +35,7 @@ def parse_json_as(attrs: dict[str, Any]) -> "Base": # Works because the module and the class have the same name. return getattr(mod, subclass)(**data_attrs) - def to_dict(self) -> dict[str, "CgmesAttributeTypes"]: + def to_dict(self, with_class: bool = True) -> dict[str, "CgmesAttributeTypes"]: """ Returns the class as dict, with: - only public attributes @@ -44,14 +43,10 @@ def to_dict(self) -> dict[str, "CgmesAttributeTypes"]: """ attrs = {f.name: getattr(self, f.name) for f in fields(self)} - attrs["__class__"] = self.apparent_name() + if with_class: + attrs["__class__"] = self.apparent_name() return attrs - @cached_property - def resource_name(self) -> str: - """Returns the resource type.""" - return self.__class__.__name__ - @cached_property def namespace(self) -> str: """Returns the namespace. By default, the namespace is the cim namespace for all resources. @@ -70,7 +65,7 @@ def apparent_name(cls) -> str: def cgmes_attribute_names_in_profile(self, profile: BaseProfile | None) -> set[Field]: """ - Returns all fields accross the parent tree which are in the profile in parameter. + Returns all fields across the parent tree which are in the profile in parameter. Mostly useful during export to find all the attributes relevant to one profile only. diff --git a/modernpython/utils/dataclassconfig.py b/modernpython/utils/dataclassconfig.py index 961f0d8a..fb6e723b 100644 --- a/modernpython/utils/dataclassconfig.py +++ b/modernpython/utils/dataclassconfig.py @@ -8,4 +8,23 @@ class DataclassConfig: # pylint: disable=too-few-public-methods # By default, with pydantic extra arguments given to a dataclass are silently ignored. # This matches the default behaviour by failing noisily. - extra = "forbid" + extra = "ignore" + populate_by_name = True + defer_build = True + from_attributes = True + +class GeoDataclassConfig: # pylint: disable=too-few-public-methods + """ + Used to configure pydantic dataclasses. + + See doc at + https://docs.pydantic.dev/latest/usage/model_config/#options + """ + + # By default, with pydantic extra arguments given to a dataclass are silently ignored. + # This matches the default behaviour by failing noisily. + extra = "ignore" + populate_by_name = True + defer_build = True + from_attributes = True + arbitrary_types_allowed=True diff --git a/modernpython/utils/datatypes.py b/modernpython/utils/datatypes.py new file mode 100644 index 00000000..b14b2ee2 --- /dev/null +++ b/modernpython/utils/datatypes.py @@ -0,0 +1,23 @@ +from pydantic import Field +from typing import List + +from .constants import NAMESPACES +from pydantic.dataclasses import dataclass + +from .dataclassconfig import DataclassConfig +from .profile import BaseProfile +from ..resources.enum import UnitMultiplier, UnitSymbol + +@dataclass(config=DataclassConfig) +class Primitive: + + name: str = Field(frozen=True) + type: object = Field(frozen=True) + namespace: str = Field(frozen=True, default=NAMESPACES["cim"]) + profiles: List[BaseProfile] = Field(frozen=True) + +@dataclass(config=DataclassConfig) +class CIMDatatype(Primitive): + + multiplier: UnitMultiplier = Field(frozen=True) + symbol: UnitSymbol = Field(frozen=True) diff --git a/modernpython/utils/validation.py b/modernpython/utils/validation.py new file mode 100644 index 00000000..d08616f5 --- /dev/null +++ b/modernpython/utils/validation.py @@ -0,0 +1,37 @@ +from contextlib import contextmanager +from pydantic import ( + ValidationError, + ValidationInfo, + ValidatorFunctionWrapHandler, +) +from typing import Iterator, Any + + +def is_recursion_validation_error(exc: ValidationError) -> bool: + errors = exc.errors() + return len(errors) == 1 and errors[0]["type"] == "recursion_loop" + + +@contextmanager +def suppress_recursion_validation_error() -> Iterator[None]: + try: + yield + except ValidationError as exc: + if not is_recursion_validation_error(exc): + raise exc + + +def cyclic_references_validator( + v: Any, handler: ValidatorFunctionWrapHandler, info: ValidationInfo +): + try: + return handler(v) + except ValidationError as exc: + if not (is_recursion_validation_error(exc) and isinstance(v, list)): + raise exc + + value_without_cyclic_refs = [] + for child in v: + with suppress_recursion_validation_error(): + value_without_cyclic_refs.extend(handler([child])) + return handler(value_without_cyclic_refs) diff --git a/python/langPack.py b/python/langPack.py index e448e0f4..94b8d4c9 100644 --- a/python/langPack.py +++ b/python/langPack.py @@ -69,6 +69,12 @@ def set_enum_classes(new_enum_classes): def set_float_classes(new_float_classes): return +def set_primitive_classes(new_primitive_classes): + return + +def set_cim_data_type_classes(new_cim_data_type_classes): + return + def run_template(version_path, class_details): for template_info in template_files: class_file = os.path.join(version_path, class_details['class_name'] + template_info["ext"])