Merge branch 'release/0.31.0'

* release/0.31.0: (31 commits)
  Version 0.31.0
  Remove redundant code after previous commit
  Add FunctionSpace::gather and FunctionSpace::scatter
  Make Cells::global_index() using StructuredMeshGenerator for RegularLonLatGrid independent of partitioning
  Use OpenMP in MatchingMeshPartitionerLonLatPolygon
  Improve performance of BuildHalo, using unordered_map and OpenMP
  Output max threads in Library print
  Add OpenMP support to computing bounds in StructuredColumns::generate_region
  Add progress timer to StructuredColumns::generate_region
  Fix building with ATLAS_BITS_LOCAL=64
  Reduce memory peak in GatherScatter::setup
  Fix compilation warning
  Add missing this%return() in atlas_Trace_module, but likely no impact as mostly unused subroutines
  Improve performance of ATLAS_TRACE by improving hashing function
  Add interface to adjoint of direct spectral transform implemented via ectrans (#113)
  Fix compilation with ATLAS_BITS_LOCAL=64
  Fortran interfaces for PointCloud functionspace halo exchange (#112)
  Extend PointCloud functionspace to do halo exchanges (#111)
  Implement IndexViev::assign( initializer_list ) ; for now only with NativeIndexView
  Fix Github Actions missing brew
  ...

Author: wdeconinck

.github/workflows/build.yml

@@ -98,6 +98,10 @@ jobs:
     - name: Checkout Repository
       uses: actions/checkout@v2
 
+    - name: Set up Homebrew
+      id: set-up-homebrew
+      uses: Homebrew/actions/setup-homebrew@master
+
     - name: Environment
       run:  |
         echo "DEPS_DIR=${{ runner.temp }}/deps" >> $GITHUB_ENV

CHANGELOG.md

@@ -7,6 +7,25 @@ This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html
 
 ## [Unreleased]
 
+## [0.31.0] - 2022-11-10
+### Added
+- Extend PointCloud functionspace to do halo exchanges, including Fortran API
+- Add FunctionSpace::gather and FunctionSpace::scatter abstraction
+
+### Changed
+- Improve performance of MatchingMeshPartitionerLonLatPolygon using OpenMP
+- Improve performance of BuildHalo using OpenMP and unordered_map
+- Improve performance of StructuredMeshGenerator using OpenMP
+- Improve performance of ATLAS_TRACE
+- Reduce memory peak in GatherScatter setup
+- Global element numbering of RegularLonLat grids is now following rows independent of partitioning
+
+### Fixed
+- Running CI with Github Actions
+- Fix output of atlas-grids y-range for shifted grids
+- Fix building with ATLAS_BITS_LOCAL=64
+
+
 ## [0.30.0] - 2022-08-22
 ### Added
 - Fortran API for Interpolation::execute_adjoint()
@@ -386,6 +405,7 @@ This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html
 ## 0.13.0 - 2018-02-16
 
 [Unreleased]: https://github.com/ecmwf/atlas/compare/master...develop
+[0.31.0]: https://github.com/ecmwf/atlas/compare/0.30.0...0.31.0
 [0.30.0]: https://github.com/ecmwf/atlas/compare/0.29.0...0.30.0
 [0.29.0]: https://github.com/ecmwf/atlas/compare/0.28.1...0.29.0
 [0.28.1]: https://github.com/ecmwf/atlas/compare/0.28.0...0.28.1

VERSION

@@ -1 +1 @@
-0.30.0
+0.31.0

atlas_io/src/atlas_io/Data.h

@@ -23,7 +23,7 @@ class Data {
 public:
     Data() = default;
     Data(void*, size_t);
-    Data(Data&&)  = default;
+    Data(Data&&)            = default;
     Data& operator=(Data&&) = default;
 
     operator const void*() const { return data(); }

atlas_io/src/atlas_io/Metadata.h

@@ -83,6 +83,14 @@ class Metadata : public eckit::LocalConfiguration {
         root.remove(name);
         return *this;
     }
+
+
+    std::vector<std::string> keys() const {
+        // Preserves order of keys
+        std::vector<std::string> result;
+        eckit::fromValue(result, get().keys());
+        return result;
+    }
 };
 
 //---------------------------------------------------------------------------------------------------------------------

atlas_io/src/atlas_io/Record.cc

@@ -132,6 +132,20 @@ Record::operator const ParsedRecord&() const {
 
 //---------------------------------------------------------------------------------------------------------------------
 
+static void parse_record(ParsedRecord& record, const std::string& key, const Metadata& metadata) {
+    if (metadata.type() || metadata.link()) {
+        record.items.emplace(key, metadata);
+        record.keys.emplace_back(key);
+    }
+    else {
+        for (auto& next_key : metadata.keys()) {
+            parse_record(record, key + "." + next_key, metadata.getSubConfiguration(next_key));
+        }
+    }
+}
+
+//---------------------------------------------------------------------------------------------------------------------
+
 Record& Record::read(Stream& in, bool read_to_end) {
     if (not empty()) {
         return *this;
@@ -206,9 +220,9 @@ Record& Record::read(Stream& in, bool read_to_end) {
 
     ATLAS_IO_ASSERT(r.metadata_format == "yaml");
     Metadata metadata = eckit::YAMLConfiguration(metadata_str);
-    record_->keys     = metadata.keys();
-    for (const auto& key : record_->keys) {
-        record_->items.emplace(key, metadata.getSubConfiguration(key));
+
+    for (auto& key : metadata.keys()) {
+        parse_record(*record_, key, metadata.getSubConfiguration(key));
     }
 
 

atlas_io/src/atlas_io/RecordWriter.h

@@ -24,6 +24,9 @@
 #include "atlas_io/detail/TypeTraits.h"
 
 #include "atlas_io/types/array/ArrayReference.h"
+#include "atlas_io/types/scalar.h"
+#include "atlas_io/types/string.h"
+
 
 #include "atlas_io/detail/Defaults.h"
 
@@ -88,6 +91,14 @@ class RecordWriter {
         set(key, Encoder{value}, config);
     }
 
+    void set(const Key& key, const char* value, const eckit::Configuration& config = NoConfig()) {
+        set(key, Encoder{std::string(value)}, config);
+    }
+
+    void set(const Key& key, const std::string& value, const eckit::Configuration& config = NoConfig()) {
+        set(key, Encoder{std::string(value)}, config);
+    }
+
     /// @brief Write new record to path
     size_t write(const eckit::PathName&, Mode = Mode::write) const;
 

atlas_io/src/atlas_io/Trace.cc

@@ -16,20 +16,26 @@ namespace atlas {
 namespace io {
 
 atlas::io::Trace::Trace(const eckit::CodeLocation& loc) {
-    for (auto& hook : TraceHookRegistry::instance().hooks) {
-        hooks_.emplace_back(hook(loc, loc.func()));
+    for (size_t id = 0; id < TraceHookRegistry::size(); ++id) {
+        if (TraceHookRegistry::enabled(id)) {
+            hooks_.emplace_back(TraceHookRegistry::hook(id)(loc, loc.func()));
+        }
     }
 }
 
 Trace::Trace(const eckit::CodeLocation& loc, const std::string& title) {
-    for (auto& hook : TraceHookRegistry::instance().hooks) {
-        hooks_.emplace_back(hook(loc, title));
+    for (size_t id = 0; id < TraceHookRegistry::size(); ++id) {
+        if (TraceHookRegistry::enabled(id)) {
+            hooks_.emplace_back(TraceHookRegistry::hook(id)(loc, title));
+        }
     }
 }
 
 Trace::Trace(const eckit::CodeLocation& loc, const std::string& title, const Labels& labels) {
-    for (auto& hook : TraceHookRegistry::instance().hooks) {
-        hooks_.emplace_back(hook(loc, title));
+    for (size_t id = 0; id < TraceHookRegistry::size(); ++id) {
+        if (TraceHookRegistry::enabled(id)) {
+            hooks_.emplace_back(TraceHookRegistry::hook(id)(loc, title));
+        }
     }
 }
 

atlas_io/src/atlas_io/Trace.h

@@ -31,12 +31,26 @@ struct TraceHook {
 struct TraceHookRegistry {
     using TraceHookBuilder = std::function<std::unique_ptr<TraceHook>(const eckit::CodeLocation&, const std::string&)>;
     std::vector<TraceHookBuilder> hooks;
+    std::vector<int> enabled_;
     static TraceHookRegistry& instance() {
         static TraceHookRegistry instance;
         return instance;
     }
-    static void add(TraceHookBuilder&& hook) { instance().hooks.emplace_back(hook); }
-    static void add(const TraceHookBuilder& hook) { instance().hooks.emplace_back(hook); }
+    static size_t add(TraceHookBuilder&& hook) {
+        instance().hooks.emplace_back(hook);
+        instance().enabled_.emplace_back(true);
+        return instance().hooks.size() - 1;
+    }
+    static size_t add(const TraceHookBuilder& hook) {
+        instance().hooks.emplace_back(hook);
+        instance().enabled_.emplace_back(true);
+        return instance().hooks.size() - 1;
+    }
+    static void enable(size_t id) { instance().enabled_[id] = true; }
+    static void disable(size_t id) { instance().enabled_[id] = false; }
+    static bool enabled(size_t id) { return instance().enabled_[id]; }
+    static size_t size() { return instance().hooks.size(); }
+    static TraceHookBuilder& hook(size_t id) { return instance().hooks[id]; }
 
 private:
     TraceHookRegistry() = default;

atlas_io/src/atlas_io/detail/Encoder.h

@@ -38,6 +38,10 @@ class Encoder {
 
     Encoder(Encoder&& other): self_(std::move(other.self_)) {}
 
+    template <typename T, enable_if_scalar_t<T> = 0>
+    explicit Encoder(const T& x): self_(new EncodableValue<T>(x)) {}
+
+
     Encoder& operator=(Encoder&& rhs) {
         self_ = std::move(rhs.self_);
         return *this;
@@ -61,6 +65,11 @@ class Encoder {
     struct EncodableValue : Encodable {
         EncodableValue(Value&& v): value_{std::move(v)} { sfinae::encode_metadata(value_, metadata_, data_size_); }
 
+        template <bool EnableBool = true, enable_if_scalar_t<Value, EnableBool> = 0>
+        EncodableValue(const Value& v): value_{v} {
+            sfinae::encode_metadata(value_, metadata_, data_size_);
+        }
+
         size_t encode_metadata_(atlas::io::Metadata& metadata) const override {
             metadata.set(metadata_);
             return data_size_;