Merge remote-tracking branch 'origin/feature/mtg2_tablesVersion_switch' into feature/mtg2_tablesVersion_switch_chem

Author: mjg41

.github/workflows/build-wheel-macos.yml

@@ -0,0 +1,64 @@
+# (C) Copyright 2024- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+
+name: Build Python Wheel for MacOS
+
+on:
+  # Trigger the workflow manually
+  workflow_dispatch: ~
+
+  # Allow to be called from another workflow
+  workflow_call: ~
+
+  # TODO automation trigger
+
+jobs:
+  build:
+    name: Build macos wheel
+    # TODO enable after all tested
+    # strategy:
+    #   matrix:
+    #     arch_type: [ARM64, X64]
+    # runs-on: [self-hosted, macOS, "${{ matrix.arch_type }}"]
+    runs-on: [self-hosted, macOS, ARM64 ]
+    steps:
+        # TODO convert this to be matrix-friendly for python versions. Note it's a bit tricky since
+        # we'd ideally not reexecute the compile step multiple times, but it
+        # (non-essentially) depends on a matrix-based step
+        # NOTE we dont use action checkout because it doesnt cleanup after itself correctly
+      - run: |
+          if [ -z "$(which uv)" ] ; then curl -LsSf https://astral.sh/uv/install.sh | sh ; fi
+          rm -rf ecbuild wheelmaker
+          git clone --depth=1 https://github.com/ecmwf/ecbuild ecbuild
+          # git clone --depth=1 --branch="wheelmaker" https://github.com/ecmwf/ci-utils wheelmaker # TODO use token here to get rid of the checkout action below
+      - uses: actions/checkout@v4
+        with:
+          repository: ecmwf/ci-utils
+          ref: develop
+          path: ci-utils
+          token: ${{ secrets.GH_REPO_READ_TOKEN }}
+      - run: rm -rf proj && git clone --depth=1 --branch="${GITHUB_REF#refs/heads/}" https://github.com/$GITHUB_REPOSITORY proj
+      - run: |
+          cd proj && $GITHUB_WORKSPACE/ci-utils/wheelmaker/buildscripts/prepare_deps.sh ./python_wrapper/buildconfig 3.11
+      - run: |
+          cd proj
+          if [[ -f ./python_wrapper/pre-compile.sh ]] ; then ./python_wrapper/pre-compile.sh ; fi
+          PATH="$PATH:$GITHUB_WORKSPACE/ecbuild/bin/" $GITHUB_WORKSPACE/ci-utils/wheelmaker/buildscripts/compile.sh ./python_wrapper/buildconfig
+      - run: |
+          cd proj
+          rm -rf /tmp/buildvenv && uv python install python3.11 && uv venv --python python3.11 /tmp/buildvenv && source /tmp/buildvenv/bin/activate && uv pip install build twine
+          PYTHONPATH=$GITHUB_WORKSPACE/ci-utils/wheelmaker/buildscripts $GITHUB_WORKSPACE/ci-utils/wheelmaker/buildscripts/wheel-linux.sh ./python_wrapper/buildconfig 3.11
+          if [[ -f ./python_wrapper/post-build.sh ]] ; then ./python_wrapper/post-build.sh ; fi
+          $GITHUB_WORKSPACE/ci-utils/wheelmaker/buildscripts/test-wheel.sh ./python_wrapper/buildconfig 3.11
+          $GITHUB_WORKSPACE/ci-utils/wheelmaker/buildscripts/upload-pypi.sh ./python_wrapper/buildconfig
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+        # NOTE temporary thing until all the mess gets cleared
+      - run: rm -rf ./* ./.git ./.github

definitions/grib2/boot.def

@@ -31,9 +31,6 @@ transient missingValue = 9999;
 constant ieeeFloats = 1 : edition_specific;
 constant isHindcast = 0;
 
-# Set MTG2 defaults
-constant tablesVersionMTG2Switch=33: hidden;
-constant MTG2SwitchDefault = 1;
 
 include "grib2/section.0.def"
 

definitions/grib2/local/ecmf/section2_extras.def

@@ -1,36 +1,15 @@
 # (C) Copyright 2005- ECMWF.
 
 transient dummyc=0: hidden;
-# Evaluate MTG2Switch concept.
-# Default is 1 unless changed by centre or pseudocentre, set in boot.def.
-# To be discussed. This means that all centres are default post-MTG2 unless entry is added in concept.
 
-concept MTG2Switch(MTG2SwitchDefault, "MTG2SwitchConcept.def", conceptsDir2,conceptsDir1): no_copy, read_only, long_type;
+# With value of the switch defined in section.4.def, we read the corresponding def files. This works as follows:
+#      Pre-MTG2 ( MTG2Switch=0 ): static def files [concept].[tablesVersionMTG2Switch].def
+#      Else ( MTG2Switch!=0 ): default def files [concept].def. Ensures continuity of "status quo".
 
-# If MTG2Switch is 0 (pre-MTG2) we need to look at pre-MTG2 definitions, tableVersion defined in boot.def.
-# Otherwise look at new definitions.
-
-if ( MTG2Switch == 0 ) {
-
-    constant paramIdFilename = "paramId.[tablesVersionMTG2Switch].def" : hidden ;
-    constant shortNameFilename = "shortName.[tablesVersionMTG2Switch].def" : hidden ;
-    constant unitsFilename = "units.[tablesVersionMTG2Switch].def" : hidden ;
-    constant nameFilename = "name.[tablesVersionMTG2Switch].def" : hidden ;
-
-# All other cases other than pre-MTG2 fall into default parameter files
-} else {
-
-    constant paramIdFilename = "paramId.def" : hidden ;
-    constant shortNameFilename = "shortName.def" : hidden ;
-    constant unitsFilename = "units.def" : hidden ;
-    constant nameFilename = "name.def" : hidden ;
-
-}
-
-# meta paramIdFilename   g2_param_concept_filename("paramId",   MTG2Switch, tablesVersionMTG2Switch, datasetForLocal): hidden;
-# meta shortNameFilename g2_param_concept_filename("shortName", MTG2Switch, tablesVersionMTG2Switch, datasetForLocal): hidden;
-# meta unitsFilename     g2_param_concept_filename("units",     MTG2Switch, tablesVersionMTG2Switch, datasetForLocal): hidden;
-# meta nameFilename      g2_param_concept_filename("name",      MTG2Switch, tablesVersionMTG2Switch, datasetForLocal): hidden;
+meta paramIdFilename   g2_param_concept_filename("paramId",   MTG2Switch, tablesVersionMTG2Switch, datasetForLocal): hidden;
+meta shortNameFilename g2_param_concept_filename("shortName", MTG2Switch, tablesVersionMTG2Switch, datasetForLocal): hidden;
+meta unitsFilename     g2_param_concept_filename("units",     MTG2Switch, tablesVersionMTG2Switch, datasetForLocal): hidden;
+meta nameFilename      g2_param_concept_filename("name",      MTG2Switch, tablesVersionMTG2Switch, datasetForLocal): hidden;
 
 # Now deal with chemical parameter split, pre and post MTG2
 if (defined(MTG2Switch) and MTG2Switch == 2) {
@@ -65,7 +44,7 @@ concept nameLegacyECMF(defaultName,"name.legacy.def",conceptsMasterDir,conceptsL
 concept nameECMF(nameLegacyECMF,nameFilename,conceptsMasterDir,conceptsLocalDirECMF): no_copy;
 concept name(nameECMF,nameFilename,conceptsDir2,conceptsDir1): no_copy,dump;
 
-# cfName does not have pre and post MTG2 switch
+# cfName does not have pre and post MTG2 change in definition files
 
 concept cfNameLegacyECMF(defaultShortName,"cfName.legacy.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,hidden;
 concept cfNameECMF(cfNameLegacyECMF,"cfName.def",conceptsMasterDir,conceptsLocalDirECMF) : no_copy;

definitions/grib2/localConcepts/s2s/MTG2SwitchConcept.def

@@ -141,12 +141,3 @@ if (marsStream is "dame") {
   unalias mars.time;
   unalias mars.step;
 }
-# MTG2 support given lack of local section
-# This comes after the paramId is already decided, so doesn't work.
-# if (tablesVersion <= tablesVersionMTG2Switch) {
-#     # Pre-MTG2
-#     constant MTG2SwitchDefault = 0;
-# } else {
-#     # All other cases we are post-MTG2
-#     constant MTG2SwitchDefault = 1;
-# }

definitions/grib2/localConcepts/tigge/MTG2SwitchConcept.def

@@ -109,13 +109,3 @@ if (is_ocean3d_param) {
    alias mars.levtype = oceanLevName;
    unalias mars.levelist;
 }
-
-# MTG2 support given lack of local section
-# This comes after the paramId is already decided, so doesn't work.
-# if (tablesVersion <= tablesVersionMTG2Switch) {
-#     # Pre-MTG2
-#     constant MTG2SwitchDefault = 0;
-# } else {
-#     # All other cases we are post-MTG2
-#     constant MTG2SwitchDefault = 1;
-# }

definitions/grib2/localConcepts/uerra/MTG2SwitchConcept.def

@@ -81,13 +81,3 @@ concept marsStream(unknown) {
 
 alias mars.stream = marsStream;
 alias mars.type   = marsType;
-
-# MTG2 support given lack of local section
-# This comes after the paramId is already decided, so doesn't work.
-# if (tablesVersion <= tablesVersionMTG2Switch) {
-#     # Pre-MTG2
-#     constant MTG2SwitchDefault = 0;
-# } else {
-#     # All other cases we are post-MTG2
-#     constant MTG2SwitchDefault = 1;
-# }