From c17edebec0751b32ee094bfe863b22b768a1c606 Mon Sep 17 00:00:00 2001 From: James Bruten Date: Mon, 13 Jan 2025 10:54:01 +0000 Subject: [PATCH 1/7] initial changes --- lfric_macros/apply_macros.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/lfric_macros/apply_macros.py b/lfric_macros/apply_macros.py index 4cf1544..03d211d 100755 --- a/lfric_macros/apply_macros.py +++ b/lfric_macros/apply_macros.py @@ -245,7 +245,13 @@ def set_rose_meta_path(self): When Jules Shared from Jules is enabled, self.jules_source will need adding here """ - rose_meta_path = f"{self.root_path}:{self.core_source}" + if os.path.isdir(os.path.join(self.root_path, "rose-meta")): + rose_meta_path = ( + f"{os.path.join(self.root_path, 'rose-meta')}:" + f"{os.path.join(self.core_source, 'rose-meta')}" + ) + else: + rose_meta_path = f"{self.root_path}:{self.core_source}" os.environ["ROSE_META_PATH"] = rose_meta_path def parse_application_section(self, meta_dir): @@ -560,20 +566,20 @@ def get_full_import_path(self, imp): not found """ - core_imp = os.path.join(self.core_source, imp) + core_imp = os.path.join(self.core_source, "rose-meta", imp) if os.path.exists(core_imp) or os.path.exists( os.path.dirname(core_imp) ): return core_imp # Reinstate when using Jules Shared from Jules - # jules_imp = os.path.join(self.jules_source, imp) + # jules_imp = os.path.join(self.jules_source, "rose-meta", imp) # if os.path.exists(jules_imp) or os.path.exists( # os.path.dirname(jules_imp) # ): # return jules_imp - apps_imp = os.path.join(self.root_path, imp) + apps_imp = os.path.join(self.root_path, "rose-meta", imp) if os.path.exists(apps_imp) or os.path.exists( os.path.dirname(apps_imp) ): From 22624e09b068847fbe0ead7880436cccdb2c3923 Mon Sep 17 00:00:00 2001 From: James Bruten Date: Mon, 13 Jan 2025 14:48:31 +0000 Subject: [PATCH 2/7] not working for central meta --- lfric_macros/apply_macros.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lfric_macros/apply_macros.py b/lfric_macros/apply_macros.py index 03d211d..84ecb7b 100755 --- a/lfric_macros/apply_macros.py +++ b/lfric_macros/apply_macros.py @@ -680,7 +680,9 @@ def determine_import_order(self, app): - A list of meta imports in the correct order """ - import_list = [app] + app_name = os.path.basename(app) + import_list = [app_name] + try: imports = self.parsed_macros[app]["imports"] except KeyError: @@ -764,8 +766,8 @@ def preprocess_macros(self): """ # Get list of versions files to check - in both core and apps - self.find_meta_dirs(self.root_path) - self.find_meta_dirs(self.core_source) + self.find_meta_dirs(os.path.join(self.root_path, "rose-meta")) + self.find_meta_dirs(os.path.join(self.core_source, "rose-meta")) for meta_dir in self.meta_dirs: print( @@ -809,6 +811,7 @@ def preprocess_macros(self): # added macro or import metadata with the new macro for meta_dir in self.meta_dirs: import_order = self.determine_import_order(meta_dir) + print(import_order) full_command = self.combine_macros(import_order) # If there are commands to write out, do so and record this # application as having the macro From 6b5ae2d6a7806fc2a2098810add03ac5ec0dcb2c Mon Sep 17 00:00:00 2001 From: James Bruten Date: Mon, 13 Jan 2025 16:43:23 +0000 Subject: [PATCH 3/7] apply_macros working with central meta --- lfric_macros/apply_macros.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/lfric_macros/apply_macros.py b/lfric_macros/apply_macros.py index 84ecb7b..fec38b2 100755 --- a/lfric_macros/apply_macros.py +++ b/lfric_macros/apply_macros.py @@ -392,7 +392,7 @@ def find_meta_dirs(self, path): - str, stdout of find command looking for versions.py files """ - for dirpath, dirnames, filenames in os.walk(path): + for dirpath, dirnames, filenames in os.walk(path, followlinks=True): dirnames[:] = [d for d in dirnames if d not in [".svn"]] if "versions.py" in filenames: self.meta_dirs.add(dirpath) @@ -565,13 +565,6 @@ def get_full_import_path(self, imp): - the import statement containing the full path - raises an error if not found """ - - core_imp = os.path.join(self.core_source, "rose-meta", imp) - if os.path.exists(core_imp) or os.path.exists( - os.path.dirname(core_imp) - ): - return core_imp - # Reinstate when using Jules Shared from Jules # jules_imp = os.path.join(self.jules_source, "rose-meta", imp) # if os.path.exists(jules_imp) or os.path.exists( @@ -579,10 +572,16 @@ def get_full_import_path(self, imp): # ): # return jules_imp + core_imp = os.path.join(self.core_source, "rose-meta", imp) apps_imp = os.path.join(self.root_path, "rose-meta", imp) - if os.path.exists(apps_imp) or os.path.exists( - os.path.dirname(apps_imp) - ): + + if os.path.exists(core_imp): + return core_imp + if os.path.exists(apps_imp): + return apps_imp + if os.path.exists(os.path.dirname(core_imp)): + return core_imp + if os.path.exists(os.path.dirname(apps_imp)): return apps_imp raise Exception( @@ -707,6 +706,7 @@ def combine_macros(self, import_order): full_command = "" for meta_import in import_order: + meta_import = self.get_full_import_path(meta_import) if ( meta_import in self.parsed_macros and self.parsed_macros[meta_import]["commands"] @@ -811,7 +811,6 @@ def preprocess_macros(self): # added macro or import metadata with the new macro for meta_dir in self.meta_dirs: import_order = self.determine_import_order(meta_dir) - print(import_order) full_command = self.combine_macros(import_order) # If there are commands to write out, do so and record this # application as having the macro From 8c18905bd9d525a4b5edc7f402c65b465f1772b1 Mon Sep 17 00:00:00 2001 From: James Bruten Date: Tue, 14 Jan 2025 09:06:11 +0000 Subject: [PATCH 4/7] changes for backwards compatibility --- lfric_macros/apply_macros.py | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/lfric_macros/apply_macros.py b/lfric_macros/apply_macros.py index fec38b2..58e2781 100755 --- a/lfric_macros/apply_macros.py +++ b/lfric_macros/apply_macros.py @@ -236,6 +236,7 @@ def __init__(self, tag, cname, apps, core, jules): self.sections_with_macro = [] self.python_imports = set() self.upgraded_core = False + self.central_rose_meta = False def set_rose_meta_path(self): """ @@ -246,10 +247,12 @@ def set_rose_meta_path(self): adding here """ if os.path.isdir(os.path.join(self.root_path, "rose-meta")): + # For backwards compatibility with central rose-meta imports rose_meta_path = ( f"{os.path.join(self.root_path, 'rose-meta')}:" f"{os.path.join(self.core_source, 'rose-meta')}" ) + self.central_rose_meta = True else: rose_meta_path = f"{self.root_path}:{self.core_source}" os.environ["ROSE_META_PATH"] = rose_meta_path @@ -565,15 +568,16 @@ def get_full_import_path(self, imp): - the import statement containing the full path - raises an error if not found """ - # Reinstate when using Jules Shared from Jules - # jules_imp = os.path.join(self.jules_source, "rose-meta", imp) - # if os.path.exists(jules_imp) or os.path.exists( - # os.path.dirname(jules_imp) - # ): - # return jules_imp - core_imp = os.path.join(self.core_source, "rose-meta", imp) - apps_imp = os.path.join(self.root_path, "rose-meta", imp) + # TODO: Reinstate Jules checks when using Jules Metadata from Jules + + # For backwards compatibility with central rose-meta imports + if self.central_rose_meta: + core_imp = os.path.join(self.core_source, "rose-meta", imp) + apps_imp = os.path.join(self.root_path, "rose-meta", imp) + else: + core_imp = os.path.join(self.core_source, imp) + apps_imp = os.path.join(self.root_path, imp) if os.path.exists(core_imp): return core_imp @@ -766,8 +770,13 @@ def preprocess_macros(self): """ # Get list of versions files to check - in both core and apps - self.find_meta_dirs(os.path.join(self.root_path, "rose-meta")) - self.find_meta_dirs(os.path.join(self.core_source, "rose-meta")) + # Duplicated for backwards compatibility with central rose-meta imports + if self.central_rose_meta: + self.find_meta_dirs(os.path.join(self.root_path, "rose-meta")) + self.find_meta_dirs(os.path.join(self.core_source, "rose-meta")) + else: + self.find_meta_dirs(self.root_path) + self.find_meta_dirs(self.core_source) for meta_dir in self.meta_dirs: print( From 937d6bb612252d9e10fe743280c8960bf7e59316 Mon Sep 17 00:00:00 2001 From: James Bruten Date: Tue, 28 Jan 2025 16:58:08 +0000 Subject: [PATCH 5/7] updates for central metadata --- lfric_macros/apply_macros.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lfric_macros/apply_macros.py b/lfric_macros/apply_macros.py index ef4a14a..5ac7013 100755 --- a/lfric_macros/apply_macros.py +++ b/lfric_macros/apply_macros.py @@ -234,6 +234,7 @@ def __init__(self, tag, cname, version, apps, core, jules): # copy in LFRic, rather than using the Jules version. The LFRic build # system needs modifying to enable this # self.jules_source = self.get_dependency_paths(jules, "jules") + self.central_rose_meta = False self.set_rose_meta_path() if version is None: self.version = re.search(r".*vn(\d+\.\d+)(_.*)?", tag).group(1) @@ -246,7 +247,6 @@ def __init__(self, tag, cname, version, apps, core, jules): self.sections_with_macro = [] self.python_imports = set() self.upgraded_core = False - self.central_rose_meta = False def set_rose_meta_path(self): """ @@ -693,7 +693,12 @@ def determine_import_order(self, app): - A list of meta imports in the correct order """ - app_name = os.path.basename(app) + # If using central metadata, use the basename, otherwise the full path + if self.central_rose_meta: + app_name = os.path.basename(app) + else: + app_name = app + import_list = [app_name] try: From e9b05bc77e252906151e646630257543db37a313 Mon Sep 17 00:00:00 2001 From: James Bruten Date: Wed, 5 Feb 2025 15:27:05 +0000 Subject: [PATCH 6/7] remove unused variable --- lfric_macros/apply_macros.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lfric_macros/apply_macros.py b/lfric_macros/apply_macros.py index 5ac7013..a927bc1 100755 --- a/lfric_macros/apply_macros.py +++ b/lfric_macros/apply_macros.py @@ -130,7 +130,6 @@ def split_macros(parsed_versions): macros = [] macro = "" in_macro = False - in_comment = False for line in parsed_versions: if line.startswith("class vn") and not line.startswith( "class vnXX_txxx" From c90e1df684c9bf8b6b72241d042ff5c8d8b3e647 Mon Sep 17 00:00:00 2001 From: James Bruten Date: Wed, 5 Feb 2025 16:17:33 +0000 Subject: [PATCH 7/7] additional typos --- lfric_macros/apply_macros.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lfric_macros/apply_macros.py b/lfric_macros/apply_macros.py index a927bc1..84b8f90 100755 --- a/lfric_macros/apply_macros.py +++ b/lfric_macros/apply_macros.py @@ -774,7 +774,7 @@ def write_new_macro(self, meta_dir, full_command): def preprocess_macros(self): """ - Overraching function to pre-process added macros + Overarching function to pre-process added macros Run before running any rose macro upgrade commands" Search through versions.py files for macros with the correct after-tag Save info and then delete the macro when found