diff --git a/src/multio/action/interpolate/CMakeLists.txt b/src/multio/action/interpolate/CMakeLists.txt index f483e003c..6d57531a8 100644 --- a/src/multio/action/interpolate/CMakeLists.txt +++ b/src/multio/action/interpolate/CMakeLists.txt @@ -27,6 +27,8 @@ ecbuild_add_executable( SOURCES FesomCacheGenerator.cc + HEALPix.cc + HEALPix.h ../../tools/MultioTool.cc CONDITION diff --git a/src/multio/action/interpolate/FesomCacheGenerator.cc b/src/multio/action/interpolate/FesomCacheGenerator.cc index abc931af3..d01984104 100644 --- a/src/multio/action/interpolate/FesomCacheGenerator.cc +++ b/src/multio/action/interpolate/FesomCacheGenerator.cc @@ -22,6 +22,8 @@ #include "mir/method/WeightMatrix.h" #include "multio/tools/MultioTool.h" +#include "HEALPix.h" + namespace multio::action::interpolateFESOM2MIR { namespace { @@ -96,7 +98,7 @@ class Fesom2mirCacheGenerator final : public multio::MultioTool { void usage(const std::string& tool) const override { eckit::Log::info() << std::endl << "Usage: " << tool << " [options]" << std::endl; eckit::Log::info() << "EXAMPLE: " << std::endl - << "fesom-cache-generator --inputPath=. --inputFile=CORE2_ngrid_NSIDE32_0_ring.csv " + << tool << " --inputPath=. --inputFile=CORE2_ngrid_NSIDE32_0_ring.csv --outputPath=. --nCols=126858 --inputOrdering=ring --outputOrdering=nested" << std::endl << std::endl; } @@ -122,30 +124,30 @@ class Fesom2mirCacheGenerator final : public multio::MultioTool { size_t level_; size_t Nrow_; size_t Ncol_; - orderingConvention_e orderingConvention_; + orderingConvention_e inputOrdering_; + orderingConvention_e outputOrdering_; + void loadTriplets(std::vector& triplets) const; }; Fesom2mirCacheGenerator::Fesom2mirCacheGenerator(int argc, char** argv) : - multio::MultioTool{argc, argv}, - inputPath_{"."}, - outputPath_{"."}, - inputFile_{"CORE2_ngrid_NSIDE32_0_ring.csv"}, - fesomName_{"CORE2"}, - domain_{"ngrid"}, - NSide_{0}, - level_{0}, - orderingConvention_{orderingConvention_e::RING} { + multio::MultioTool{argc, argv} + { options_.push_back(new eckit::option::SimpleOption( - "inputPath", "Path of the input files with the triplets. Default( \".\" )")); + "inputPath", "Path of the input files with the triplets. Default( \".\" )", ".")); + options_.push_back(new eckit::option::SimpleOption( + "outputPath", "Path of the output files with the triplets. Default( \".\" )", ".")); + options_.push_back(new eckit::option::SimpleOption( + "inputFile", "Name of the input file. Default( \"CORE2_ngrid_NSIDE32_0_ring.csv\" )", "CORE2_ngrid_NSIDE32_0_ring.csv")); + options_.push_back(new eckit::option::SimpleOption( + "nCols", "Size of the fesom grid.", 126858)); options_.push_back(new eckit::option::SimpleOption( - "outputPath", "Path of the output files with the triplets. Default( \".\" )")); + "inputOrdering", "Ordering of the input files. Options( \"ring\", \"nested\") Default( \"ring\" )", "ring")); options_.push_back(new eckit::option::SimpleOption( - "inputFile", "Name of the input file. Default( \"CORE2_ngrid_NSIDE32_0_ring.csv\" )")); - options_.push_back(new eckit::option::SimpleOption("nCols", "Size of the fesom grid.")); + "outputOrdering", "Ordering of the output files. Options( \"ring\", \"nested\", \"input\") Default( \"input\" )", "input")); return; } @@ -167,7 +169,20 @@ void Fesom2mirCacheGenerator::loadTriplets(std::vector& "([0-9][0-9]*)\\s+([0-9][0-9]*)\\s*([+]?([0-9]*[.])?[0-9]+([eE][-+][0-9]+)?)"); std::smatch matchLine; if (std::regex_match(line, matchLine, lineGrammar)) { - iHEALPix = std::stoi(matchLine[1].str()); + if ( inputOrdering_ == outputOrdering_ ){ + iHEALPix = std::stoi(matchLine[1].str()); + } + else if (inputOrdering_ == orderingConvention_e::RING && outputOrdering_ == orderingConvention_e::NESTED) { + HEALPix Idx(static_cast(NSide_)); + iHEALPix = Idx.ring_to_nest(static_cast(std::stoi(matchLine[1].str()))); + } + else if (inputOrdering_ == orderingConvention_e::NESTED && outputOrdering_ == orderingConvention_e::RING){ + HEALPix Idx(static_cast(NSide_)); + iHEALPix = Idx.nest_to_ring(static_cast(std::stoi(matchLine[1].str()))); + } + else { + throw eckit::SeriousBug("Ordering not supported: " + line, Here()); + } iFESOM = std::stoi(matchLine[2].str()); weight = std::stod(matchLine[3].str()); } @@ -185,13 +200,27 @@ void Fesom2mirCacheGenerator::init(const eckit::option::CmdArgs& args) { args.get("outputPath", outputPath_); args.get("inputFile", inputFile_); + std::string inputOrderingType; + std::string outputOrderingType; + args.get("inputOrdering", inputOrderingType); + args.get("outputOrdering", outputOrderingType); + eckit::PathName inputPath_tmp{inputPath_}; ASSERT(inputPath_tmp.exists()); eckit::PathName inputFile_tmp{inputPath_ + "/" + inputFile_}; ASSERT(inputFile_tmp.exists()); eckit::PathName outputPath_tmp{outputPath_}; outputPath_tmp.mkdir(); - parseInputFileName(inputFile_, fesomName_, domain_, NSide_, level_, orderingConvention_); + parseInputFileName(inputFile_, fesomName_, domain_, NSide_, level_, inputOrdering_); + + if (inputOrderingType == "ring") { inputOrdering_ = orderingConvention_e::RING; } + else if (inputOrderingType == "nested") { inputOrdering_ = orderingConvention_e::NESTED; } + else { throw eckit::SeriousBug("Unsupported input ordering convention", Here()); } + + if (outputOrderingType == "input") { outputOrdering_ = inputOrdering_; } + else if (outputOrderingType == "ring") { outputOrdering_ = orderingConvention_e::RING; } + else if (outputOrderingType == "nested") { outputOrdering_ = orderingConvention_e::NESTED; } + else { throw eckit::SeriousBug("Unsupported output ordering convention", Here()); } args.get("nCols", Ncol_); Nrow_ = NSide_ * NSide_ * 12; @@ -203,7 +232,7 @@ void Fesom2mirCacheGenerator::execute(const eckit::option::CmdArgs& args) { std::sort(begin(triplets), end(triplets), [](const auto& a, const auto& b) { return a.row() < b.row(); }); - const auto orderingConvention = orderingConvention_enum2string(orderingConvention_); + const auto orderingConvention = orderingConvention_enum2string(outputOrdering_); const auto cacheFileName = fesomCacheName(fesomName_, domain_, "double", NSide_, orderingConvention, level_); mir::method::WeightMatrix W(Nrow_, Ncol_); diff --git a/src/multio/action/interpolate/HEALPix.cc b/src/multio/action/interpolate/HEALPix.cc new file mode 100644 index 000000000..ebac11aa5 --- /dev/null +++ b/src/multio/action/interpolate/HEALPix.cc @@ -0,0 +1,198 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Pedro Maciel + +/// @date Sep 2023 + +#include "HEALPix.h" + +#include +#include +#include +#include +#include +#include + +namespace { + + +struct CodecFijNest { + static constexpr uint64_t __masks[] = {0x00000000ffffffff, 0x0000ffff0000ffff, 0x00ff00ff00ff00ff, + 0x0f0f0f0f0f0f0f0f, 0x3333333333333333, 0x5555555555555555}; + + inline static int nest_encode_bits(int n) { + auto b = static_cast(n) & __masks[0]; + b = (b ^ (b << 16)) & __masks[1]; + b = (b ^ (b << 8)) & __masks[2]; + b = (b ^ (b << 4)) & __masks[3]; + b = (b ^ (b << 2)) & __masks[4]; + b = (b ^ (b << 1)) & __masks[5]; + return static_cast(b); + } + + inline static int nest_decode_bits(int n) { + auto b = static_cast(n) & __masks[5]; + b = (b ^ (b >> 1)) & __masks[4]; + b = (b ^ (b >> 2)) & __masks[3]; + b = (b ^ (b >> 4)) & __masks[2]; + b = (b ^ (b >> 8)) & __masks[1]; + b = (b ^ (b >> 16)) & __masks[0]; + return static_cast(b); + } + + static std::tuple nest_to_fij(int n, int k) { + assert(0 <= n); + auto f = n >> (2 * k); // f = n / (Nside * Nside) + n &= (1 << (2 * k)) - 1; // n = n % (Nside * Nside) + auto i = nest_decode_bits(n); + auto j = nest_decode_bits(n >> 1); + return {f, i, j}; + } + + static int fij_to_nest(int f, int i, int j, int k) { + return (f << (2 * k)) + nest_encode_bits(i) + (nest_encode_bits(j) << 1); + } +}; + + +inline int sqrt(int n) { + return static_cast(std::sqrt(static_cast(n) + 0.5)); +} + + +// for division result within [0; 3] +inline int div_03(int a, int b) { + int t = (a >= (b << 1)) ? 1 : 0; + a -= t * (b << 1); + return (t << 1) + (a >= b ? 1 : 0); +} + + +inline bool is_power_of_2(int n) { + return std::bitset(n).count() == 1; +} + + +inline int pll(int f) { + constexpr int __pll[] = {1, 3, 5, 7, 0, 2, 4, 6, 1, 3, 5, 7}; + return __pll[f]; +} + + +} // unnamed namespace + + +HEALPix::HEALPix(int Nside) : + Nside_(Nside), + Npix_(size()), + Ncap_((Nside * (Nside - 1)) << 1), + k_(is_power_of_2(Nside_) ? static_cast(std::log2(Nside)) : -1) { + assert(0 <= k_); // (specific to nested ordering) + assert(0 < Nside_); +} + + +int HEALPix::ring_to_nest(int r) const { + auto to_nest = [&](int f, //!< base pixel index + int ring, //!< 1-based ring number + int Nring, //!< number of pixels in ring + int phi, //!< index in longitude + int shift //!< if ring's first pixel is not at phi=0 + ) -> int { + int r = ((2 + (f >> 2)) << k_) - ring - 1; + int p = 2 * phi - pll(f) * Nring - shift - 1; + if (p >= 2 * Nside_) { + p -= 8 * Nside_; + } + + int i = (r + p) >> 1; + int j = (r - p) >> 1; + + assert(f < 12 && i < Nside_ && j < Nside_); + return CodecFijNest::fij_to_nest(f, i, j, k_); + }; + + if (r < Ncap_) { + // North polar cap + int Nring = (1 + sqrt(2 * r + 1)) >> 1; + int phi = 1 + r - 2 * Nring * (Nring - 1); + int r = Nring; + int f = div_03(phi - 1, Nring); + + return to_nest(f, r, Nring, phi, 0); + } + + if (Npix_ - Ncap_ <= r) { + // South polar cap + int Nring = (1 + sqrt(2 * Npix_ - 2 * r - 1)) >> 1; + int phi = 1 + r + 2 * Nring * (Nring - 1) + 4 * Nring - Npix_; + int ring = 4 * Nside_ - Nring; // (from South pole) + int f = div_03(phi - 1, Nring) + 8; + + return to_nest(f, ring, Nring, phi, 0); + } + + // Equatorial belt + int ip = r - Ncap_; + int tmp = ip >> (k_ + 2); + + int Nring = Nside_; + int phi = ip - tmp * 4 * Nside_ + 1; + int ring = tmp + Nside_; + + int ifm = 1 + ((phi - 1 - ((1 + tmp) >> 1)) >> k_); + int ifp = 1 + ((phi - 1 - ((1 - tmp + 2 * Nside_) >> 1)) >> k_); + int f = (ifp == ifm) ? (ifp | 4) : ((ifp < ifm) ? ifp : (ifm + 8)); + + return to_nest(f, ring, Nring, phi, ring & 1); +} + + +int HEALPix::nest_to_ring(int n) const { + auto [f, i, j] = CodecFijNest::nest_to_fij(n, k_); + assert(f < 12 && i < Nside_ && j < Nside_); + + auto to_ring_local = [&](int f, int i, int j, + int Nring, //!< number of pixels in ring + int shift //!< if ring's first pixel is/is not at phi=0 + ) -> int { + Nring >>= 2; + int r = (pll(f) * Nring + i - j + 1 + shift) / 2 - 1; + assert(r < 4 * Nring); + + return r < 0 ? r + 4 * Nside_ : r; + }; + + const int ring = ((f >> 2) + 2) * Nside_ - i - j - 1; // 1-based ring number + if (ring < Nside_) { + // North polar cap + int Nring = 4 * ring; + int r0 = 2 * ring * (ring - 1); // index of first ring pixel (ring numbering) + + return r0 + to_ring_local(f, i, j, Nring, 0); + } + + if (ring < 3 * Nside_) { + // South polar cap + int Nring = 4 * Nside_; + int r0 = Ncap_ + (ring - Nside_) * Nring; // index of first ring pixel (ring numbering) + int shift = (ring - Nside_) & 1; + + return r0 + to_ring_local(f, i, j, Nring, shift); + } + + // Equatorial belt + int N = 4 * Nside_ - ring; + int Nring = 4 * N; + int r0 = Npix_ - 2 * N * (N + 1); // index of first ring pixel (ring numbering) + + return r0 + to_ring_local(f, i, j, Nring, 0); +} diff --git a/src/multio/action/interpolate/HEALPix.h b/src/multio/action/interpolate/HEALPix.h new file mode 100644 index 000000000..92f8ca38a --- /dev/null +++ b/src/multio/action/interpolate/HEALPix.h @@ -0,0 +1,32 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Pedro Maciel + +/// @date Sep 2023 + +#pragma once + +class HEALPix { +public: + explicit HEALPix(int Nside); + + int size() const { return 12 * Nside_ * Nside_; } + int nside() const { return Nside_; } + + int nest_to_ring(int) const; + int ring_to_nest(int) const; + +private: + const int Nside_; // up to 2^13 + const int Npix_; + const int Ncap_; + const int k_; +}; diff --git a/src/multio/action/interpolate/Interpolate.cc b/src/multio/action/interpolate/Interpolate.cc index 634879898..c64067e56 100644 --- a/src/multio/action/interpolate/Interpolate.cc +++ b/src/multio/action/interpolate/Interpolate.cc @@ -357,7 +357,7 @@ void fill_job(const eckit::LocalConfiguration& cfg, mir::param::SimpleParametris }); #define fp "([+]?([0-9]*[.])?[0-9]+([eE][-+][0-9]+)?)" static const std::regex ll(fp "/" fp); - static const std::regex H("([h|H])([1-9][0-9]*)"); + static const std::regex H("([h|H])([1-9][0-9]*)(_nested)?"); #undef fp std::smatch matchll; std::smatch matchH; @@ -368,7 +368,11 @@ void fill_job(const eckit::LocalConfiguration& cfg, mir::param::SimpleParametris grid[1] = std::stod(matchll[4].str()); } if (std::regex_match(input, matchH, H)) { - gridKind = "HEALPix"; + if (matchH[3].str() == "_nested") { + gridKind = "HEALPix_nested"; + } else { + gridKind = "HEALPix"; + } grid[0] = std::stod(matchH[2].str()); } } @@ -401,12 +405,11 @@ void fill_job(const eckit::LocalConfiguration& cfg, mir::param::SimpleParametris throw eckit::SeriousBug(os.str(), Here()); } } - else if (gridKind == "HEALPix") { - // + else if (gridKind == "HEALPix" || gridKind == "HEALPix_nested") { md.set("gridded", true); md.set("gridType", "HEALPix"); md.set("Nside", (std::int64_t)grid[0]); - md.set("orderingConvention", "ring"); + md.set("orderingConvention", gridKind == "HEALPix_nested" ? "nested" : "ring"); // If no interpolation matrix name is provided, generate one if (interpolationMatrix) { @@ -427,7 +430,7 @@ void fill_job(const eckit::LocalConfiguration& cfg, mir::param::SimpleParametris char* env = ::getenv(lookUpKey.c_str()); return env ? std::optional{env} : std::optional{}; }); - const auto weights_file = generateKey(msg, expanded_cache_path, grid[0], "ring"); + const auto weights_file = generateKey(msg, expanded_cache_path, grid[0], gridKind == "HEALPix_nested" ? "nested" : "ring"); destination.set("interpolation-matrix", weights_file); } } diff --git a/src/multio/action/interpolate/generate_fesom_caches.sh b/src/multio/action/interpolate/generate_fesom_caches.sh index 9db603a3c..4c5d2f4c0 100755 --- a/src/multio/action/interpolate/generate_fesom_caches.sh +++ b/src/multio/action/interpolate/generate_fesom_caches.sh @@ -12,12 +12,27 @@ if [[ -n $SLURM_JOB_ID ]]; then echo "Searching in $1 for files matching $2..." echo "Using generator tool: $BDIR/bin/cache-generator-fesom-2-mir" + echo "Using nCols=${3}" files=$(find ${1} -iname "${2}") - for f in $files - do - fname=$(basename ${f}) - ${BDIR}/bin/cache-generator-fesom-2-mir --inputPath=${1} --inputFile=${fname} --nCols=${3} - done + MODE="${4:-ring}" # Default mode is ring to ring + + if [[ "$MODE" == "ring" ]]; then + echo "Using mode ring to ring" + for f in $files + do + fname=$(basename ${f}) + ${BDIR}/bin/cache-generator-fesom-2-mir --inputPath=${1} --inputFile=${fname} --nCols=${3} --inputOrdering=ring --outputOrdering=ring + done + + elif [[ "$MODE" == "nested" ]]; then + echo "Using mode ring to nested" + for f in $files + do + fname=$(basename ${f}) + ${BDIR}/bin/cache-generator-fesom-2-mir --inputPath=${1} --inputFile=${fname} --nCols=${3} --inputOrdering=ring --outputOrdering=nested + done + fi + else - sbatch --export=BDIR $0 $1 $2 $3 + sbatch --export=BDIR $0 $1 $2 $3 $4 fi diff --git a/src/multio/action/renumber-healpix/HEALPix_ring2nest.cc b/src/multio/action/renumber-healpix/HEALPix_ring2nest.cc index 97168b6dd..c15db09eb 100644 --- a/src/multio/action/renumber-healpix/HEALPix_ring2nest.cc +++ b/src/multio/action/renumber-healpix/HEALPix_ring2nest.cc @@ -115,6 +115,8 @@ HEALPixRingToNest::HEALPixRingToNest(const ComponentConfiguration& compConf) : void HEALPixRingToNest::executeImpl(message::Message msg) { + eckit::Log::warning() << "Action 'renumber-healpix' is no longer needed to interpolate to HEALPix nested. Interpolating to HEALPix nested can now be done in the 'interpolate' action directly!" << std::endl; + // Bypass if it is not a field if (msg.tag() != message::Message::Tag::Field) { executeNext(msg); diff --git a/tests/multio/action/CMakeLists.txt b/tests/multio/action/CMakeLists.txt index ad0c0d362..8b764a21e 100644 --- a/tests/multio/action/CMakeLists.txt +++ b/tests/multio/action/CMakeLists.txt @@ -4,3 +4,4 @@ add_subdirectory(interpolate) add_subdirectory(interpolate-fesom) add_subdirectory(statistics) add_subdirectory(renumber-healpix) +add_subdirectory(interpolate-healpix-nested) diff --git a/tests/multio/action/interpolate-healpix-nested/CMakeLists.txt b/tests/multio/action/interpolate-healpix-nested/CMakeLists.txt new file mode 100644 index 000000000..c4ef9d87a --- /dev/null +++ b/tests/multio/action/interpolate-healpix-nested/CMakeLists.txt @@ -0,0 +1,168 @@ +set(DATA_PREFIX multio_tests_action_interpolate_reduced_gg_to_HEALPix_nested) + +ecbuild_get_test_multidata( + TARGET ${DATA_PREFIX}_get_data_interpolate + DIRNAME multio/tests/actions/interpolate + DIRLOCAL ${CMAKE_CURRENT_BINARY_DIR} + NAMES "MARS_reduced_gg.grib" +) +if ( TEST ${DATA_PREFIX}_get_data_interpolate ) + set_tests_properties(${DATA_PREFIX}_get_data_interpolate PROPERTIES + FIXTURES_SETUP ${DATA_PREFIX}_get_data_interpolate + ) +endif() + +ecbuild_get_test_multidata( + TARGET ${DATA_PREFIX}_get_data_healpix_ring2nest + DIRNAME multio/tests/actions/healpix_ring2nest + DIRLOCAL ${CMAKE_CURRENT_BINARY_DIR} + NAMES "Reference_HEALPix_32_nested.grib" + "Reference_HEALPix_1024_nested.grib" +) +if ( TEST ${DATA_PREFIX}_get_data_healpix_ring2nest ) + set_tests_properties(${DATA_PREFIX}_get_data_healpix_ring2nest PROPERTIES + FIXTURES_SETUP ${DATA_PREFIX}_get_data_healpix_ring2nest + ) +endif() + +ecbuild_get_test_multidata( + TARGET ${DATA_PREFIX}_get_data_interpolate_healpix + DIRNAME multio/tests/actions/interpolate-healpix-nested + DIRLOCAL ${CMAKE_CURRENT_BINARY_DIR} + NAMES "Reference_reduced_gg_to_HEALPix_32_nested.grib" + "Reference_reduced_gg_to_HEALPix_1024_nested.grib" + "HEALPix_ring2nest.atlas" + "CORE2_ngrid_NSIDE128_0_ring.csv" +) +if ( TEST ${DATA_PREFIX}_get_data_interpolate_healpix ) + set_tests_properties(${DATA_PREFIX}_get_data_interpolate_healpix PROPERTIES + FIXTURES_SETUP ${DATA_PREFIX}_get_data_interpolate_healpix + ) +endif() + +foreach (_n + 32 + 1024 +) + set(PREFIX multio_tests_action_interpolate_reduced_gg_to_HEALPix_${_n}_nested) + + ecbuild_add_test( + TARGET ${PREFIX}_original + COMMAND multio-feed + ARGS ${CMAKE_CURRENT_BINARY_DIR}/MARS_reduced_gg.grib --decode --plans=${CMAKE_CURRENT_SOURCE_DIR}/reduced_gg_to_HEALPix_${_n}_nested_original.yaml + ) + if ( TEST ${PREFIX}_original ) + set_tests_properties(${PREFIX}_original PROPERTIES + FIXTURES_SETUP ${PREFIX}_original + FIXTURES_REQUIRED "${DATA_PREFIX}_get_data_interpolate;${DATA_PREFIX}_get_data_healpix_ring2nest" + ) + endif() + + ecbuild_add_test( + TARGET ${PREFIX}_direct + COMMAND multio-feed + ARGS ${CMAKE_CURRENT_BINARY_DIR}/MARS_reduced_gg.grib --decode --plans=${CMAKE_CURRENT_SOURCE_DIR}/reduced_gg_to_HEALPix_${_n}_nested_direct.yaml + ) + if ( TEST ${PREFIX}_direct ) + set_tests_properties(${PREFIX}_direct PROPERTIES + FIXTURES_SETUP ${PREFIX}_direct + FIXTURES_REQUIRED "${DATA_PREFIX}_get_data_interpolate;${DATA_PREFIX}_get_data_healpix_ring2nest" + ) + endif() + + ecbuild_add_test( + TARGET ${PREFIX}_compare_original + COMMAND grib_compare + ARGS MultIO_reduced_gg_to_HEALPix_${_n}_nested_original.grib MultIO_reduced_gg_to_HEALPix_${_n}_nested_direct.grib + ) + if ( TEST ${PREFIX}_compare_original ) + set_tests_properties(${PREFIX}_compare_original PROPERTIES + FIXTURES_REQUIRED "${PREFIX}_original;${PREFIX}_direct" + ) + endif() + + ecbuild_add_test( + TARGET ${PREFIX}_compare_reference + COMMAND grib_compare + ARGS Reference_reduced_gg_to_HEALPix_${_n}_nested.grib MultIO_reduced_gg_to_HEALPix_${_n}_nested_direct.grib + ) + if ( TEST ${PREFIX}_compare_reference ) + set_tests_properties(${PREFIX}_compare_reference PROPERTIES + FIXTURES_REQUIRED "${DATA_PREFIX}_get_data_interpolate_healpix;${PREFIX}_direct" + ) + endif() + +endforeach() + + +### FESOM to MIR Cache Generator + +set(PREFIX multio_tests_fesom_cache_generator) + +ecbuild_add_test( + TARGET ${PREFIX}_fesom_csv_ring2mat_ring + COMMAND cache-generator-fesom-2-mir + ARGS --inputPath=${CMAKE_CURRENT_BINARY_DIR} --inputFile=CORE2_ngrid_NSIDE128_0_ring.csv --nCols=126858 --inputOrdering=ring --outputOrdering=ring +) +if ( TEST ${PREFIX}_fesom_csv_ring2mat_ring ) + set_tests_properties(${PREFIX}_fesom_csv_ring2mat_ring PROPERTIES + FIXTURES_SETUP ${PREFIX}_fesom_csv_ring2mat_ring + FIXTURES_REQUIRED ${DATA_PREFIX}_get_data_interpolate_healpix + ) +endif() + +ecbuild_add_test( + TARGET ${PREFIX}_fesom_csv_ring2mat_nested + COMMAND cache-generator-fesom-2-mir + ARGS --inputPath=${CMAKE_CURRENT_BINARY_DIR} --inputFile=CORE2_ngrid_NSIDE128_0_ring.csv --nCols=126858 --inputOrdering=ring --outputOrdering=nested +) +if ( TEST ${PREFIX}_fesom_csv_ring2mat_nested ) + set_tests_properties(${PREFIX}_fesom_csv_ring2mat_nested PROPERTIES + FIXTURES_SETUP ${PREFIX}_fesom_csv_ring2mat_nested + FIXTURES_REQUIRED ${DATA_PREFIX}_get_data_interpolate_healpix + ) +endif() + +ecbuild_get_test_multidata( + TARGET ${DATA_PREFIX}_get_data_fesom + DIRNAME multio/tests/actions/interpolate-fesom + DIRLOCAL ${CMAKE_CURRENT_BINARY_DIR} + NAMES + "fesom_CORE2_ngrid_feed_o2d.grib" + "fesom_to_HEALPix32.tmpl" +) + +ecbuild_add_test( + TARGET ${PREFIX}_original + COMMAND multio-feed + ARGS ${CMAKE_CURRENT_BINARY_DIR}/fesom_CORE2_ngrid_feed_o2d.grib --decode --plans=${CMAKE_CURRENT_SOURCE_DIR}/fesom_CORE2_ngrid_to_HEALPix_128_nested_original.yaml +) +if ( TEST ${PREFIX}_original ) + set_tests_properties(${PREFIX}_original PROPERTIES + FIXTURES_SETUP ${PREFIX}_original + FIXTURES_REQUIRED "${DATA_PREFIX}_get_data_fesom;${PREFIX}_fesom_csv_ring2mat_ring" + ) +endif() + +ecbuild_add_test( + TARGET ${PREFIX}_direct + COMMAND multio-feed + ARGS ${CMAKE_CURRENT_BINARY_DIR}/fesom_CORE2_ngrid_feed_o2d.grib --decode --plans=${CMAKE_CURRENT_SOURCE_DIR}/fesom_CORE2_ngrid_to_HEALPix_128_nested_direct.yaml +) +if ( TEST ${PREFIX}_direct ) + set_tests_properties(${PREFIX}_direct PROPERTIES + FIXTURES_SETUP ${PREFIX}_direct + FIXTURES_REQUIRED "${DATA_PREFIX}_get_data_fesom;${PREFIX}_fesom_csv_ring2mat_nested" + ) +endif() + +ecbuild_add_test( + TARGET ${PREFIX}_compare + COMMAND grib_compare + ARGS MultIO_fesom_CORE2_ngrid_to_HEALPix_128_nested_original.grib MultIO_fesom_CORE2_ngrid_to_HEALPix_128_nested_direct.grib +) +if ( TEST ${PREFIX}_compare ) + set_tests_properties(${PREFIX}_compare PROPERTIES + FIXTURES_REQUIRED "${PREFIX}_original;${PREFIX}_direct" + ) +endif() diff --git a/tests/multio/action/interpolate-healpix-nested/fesom_CORE2_ngrid_mappings.yaml b/tests/multio/action/interpolate-healpix-nested/fesom_CORE2_ngrid_mappings.yaml new file mode 120000 index 000000000..10319b573 --- /dev/null +++ b/tests/multio/action/interpolate-healpix-nested/fesom_CORE2_ngrid_mappings.yaml @@ -0,0 +1 @@ +../interpolate-fesom/fesom_CORE2_ngrid_mappings.yaml \ No newline at end of file diff --git a/tests/multio/action/interpolate-healpix-nested/fesom_CORE2_ngrid_to_HEALPix_128_nested_direct.yaml b/tests/multio/action/interpolate-healpix-nested/fesom_CORE2_ngrid_to_HEALPix_128_nested_direct.yaml new file mode 100644 index 000000000..815baa557 --- /dev/null +++ b/tests/multio/action/interpolate-healpix-nested/fesom_CORE2_ngrid_to_HEALPix_128_nested_direct.yaml @@ -0,0 +1,21 @@ +plans: + - name: fesom_CORE2_ngrid_to_HEALPix_128_nested_direct + actions: + - type: metadata-mapping + mapping: "{~}/fesom_CORE2_ngrid_mappings.yaml" + overwrite-existing: true + - type: interpolate + input: fesom + grid: H128_nested + cache-path: . + options: + interpolation: matrix + - type: encode + format: grib + template: Reference_HEALPix_32_nested.grib # This seems to work for H128_nested + - type: sink + sinks: + - type: file + append: false + per-server: false + path: MultIO_fesom_CORE2_ngrid_to_HEALPix_128_nested_direct.grib diff --git a/tests/multio/action/interpolate-healpix-nested/fesom_CORE2_ngrid_to_HEALPix_128_nested_original.yaml b/tests/multio/action/interpolate-healpix-nested/fesom_CORE2_ngrid_to_HEALPix_128_nested_original.yaml new file mode 100644 index 000000000..de63ae5a2 --- /dev/null +++ b/tests/multio/action/interpolate-healpix-nested/fesom_CORE2_ngrid_to_HEALPix_128_nested_original.yaml @@ -0,0 +1,25 @@ +plans: + - name: fesom_CORE2_ngrid_to_HEALPix_128_nested_original + actions: + - type: metadata-mapping + mapping: "{~}/fesom_CORE2_ngrid_mappings.yaml" + overwrite-existing: true + - type: interpolate + input: fesom + grid: H128 + cache-path: . + options: + interpolation: matrix + - type: metadata-mapping + mapping: '{~}/mapping_128.yaml' + - type: renumber-healpix + cache-file-name: HEALPix_ring2nest.atlas + - type: encode + format: grib + template: Reference_HEALPix_32_nested.grib # This seems to work for H128_nested + - type: sink + sinks: + - type: file + append: false + per-server: false + path: MultIO_fesom_CORE2_ngrid_to_HEALPix_128_nested_original.grib diff --git a/tests/multio/action/interpolate-healpix-nested/mapping_1024.yaml b/tests/multio/action/interpolate-healpix-nested/mapping_1024.yaml new file mode 120000 index 000000000..0981b04d7 --- /dev/null +++ b/tests/multio/action/interpolate-healpix-nested/mapping_1024.yaml @@ -0,0 +1 @@ +../renumber-healpix/mapping_1024.yaml \ No newline at end of file diff --git a/tests/multio/action/interpolate-healpix-nested/mapping_128.yaml b/tests/multio/action/interpolate-healpix-nested/mapping_128.yaml new file mode 100644 index 000000000..a42e95f0c --- /dev/null +++ b/tests/multio/action/interpolate-healpix-nested/mapping_128.yaml @@ -0,0 +1,10 @@ +mappings: + - match: + paramId: paramId + map: + Nside: n-side + orderingConvention: ordering-convention +data: + - paramId: 263101 + n-side: 128 + ordering-convention: ring diff --git a/tests/multio/action/interpolate-healpix-nested/mapping_32.yaml b/tests/multio/action/interpolate-healpix-nested/mapping_32.yaml new file mode 120000 index 000000000..7615d2971 --- /dev/null +++ b/tests/multio/action/interpolate-healpix-nested/mapping_32.yaml @@ -0,0 +1 @@ +../renumber-healpix/mapping_32.yaml \ No newline at end of file diff --git a/tests/multio/action/interpolate-healpix-nested/reduced_gg_to_HEALPix_1024_nested_direct.yaml b/tests/multio/action/interpolate-healpix-nested/reduced_gg_to_HEALPix_1024_nested_direct.yaml new file mode 100644 index 000000000..74bc162b6 --- /dev/null +++ b/tests/multio/action/interpolate-healpix-nested/reduced_gg_to_HEALPix_1024_nested_direct.yaml @@ -0,0 +1,20 @@ +plans: + - name: reduced_gg_to_HEALPix_1024_nested_direct + actions: + - type: select + match: + - paramId: 133 + - type: interpolate + input: O1280 + grid: H1024_nested + options: + caching: false + - type: encode + format: grib + template: Reference_HEALPix_1024_nested.grib + - type: sink + sinks: + - type: file + append: false + per-server: false + path: MultIO_reduced_gg_to_HEALPix_1024_nested_direct.grib diff --git a/tests/multio/action/interpolate-healpix-nested/reduced_gg_to_HEALPix_1024_nested_original.yaml b/tests/multio/action/interpolate-healpix-nested/reduced_gg_to_HEALPix_1024_nested_original.yaml new file mode 100644 index 000000000..0f9e07d8c --- /dev/null +++ b/tests/multio/action/interpolate-healpix-nested/reduced_gg_to_HEALPix_1024_nested_original.yaml @@ -0,0 +1,24 @@ +plans: + - name: reduced_gg_to_HEALPix_1024_nested_original + actions: + - type: select + match: + - paramId: 133 + - type: interpolate + input: O1280 + grid: H1024 + options: + caching: false + - type: metadata-mapping + mapping: '{~}/mapping_1024.yaml' + - type: renumber-healpix + cache-file-name: HEALPix_ring2nest.atlas + - type: encode + format: grib + template: Reference_HEALPix_1024_nested.grib + - type: sink + sinks: + - type: file + append: false + per-server: false + path: MultIO_reduced_gg_to_HEALPix_1024_nested_original.grib diff --git a/tests/multio/action/interpolate-healpix-nested/reduced_gg_to_HEALPix_32_nested_direct.yaml b/tests/multio/action/interpolate-healpix-nested/reduced_gg_to_HEALPix_32_nested_direct.yaml new file mode 100644 index 000000000..18bae2430 --- /dev/null +++ b/tests/multio/action/interpolate-healpix-nested/reduced_gg_to_HEALPix_32_nested_direct.yaml @@ -0,0 +1,20 @@ +plans: + - name: reduced_gg_to_HEALPix_32_nested_direct + actions: + - type: select + match: + - paramId: 133 + - type: interpolate + input: O1280 + grid: H32_nested + options: + caching: false + - type: encode + format: grib + template: Reference_HEALPix_32_nested.grib + - type: sink + sinks: + - type: file + append: false + per-server: false + path: MultIO_reduced_gg_to_HEALPix_32_nested_direct.grib diff --git a/tests/multio/action/interpolate-healpix-nested/reduced_gg_to_HEALPix_32_nested_original.yaml b/tests/multio/action/interpolate-healpix-nested/reduced_gg_to_HEALPix_32_nested_original.yaml new file mode 100644 index 000000000..5df6cf982 --- /dev/null +++ b/tests/multio/action/interpolate-healpix-nested/reduced_gg_to_HEALPix_32_nested_original.yaml @@ -0,0 +1,24 @@ +plans: + - name: reduced_gg_to_HEALPix_32_nested_original + actions: + - type: select + match: + - paramId: 133 + - type: interpolate + input: O1280 + grid: H32 + options: + caching: false + - type: metadata-mapping + mapping: '{~}/mapping_32.yaml' + - type: renumber-healpix + cache-file-name: HEALPix_ring2nest.atlas + - type: encode + format: grib + template: Reference_HEALPix_32_nested.grib + - type: sink + sinks: + - type: file + append: false + per-server: false + path: MultIO_reduced_gg_to_HEALPix_32_nested_original.grib