Skip to content

Commit 93f41b8

Browse files
wdeconinckwdeconinck
committed
Merge branch 'release/0.31.0'
* release/0.31.0: (31 commits) Version 0.31.0 Remove redundant code after previous commit Add FunctionSpace::gather and FunctionSpace::scatter Make Cells::global_index() using StructuredMeshGenerator for RegularLonLatGrid independent of partitioning Use OpenMP in MatchingMeshPartitionerLonLatPolygon Improve performance of BuildHalo, using unordered_map and OpenMP Output max threads in Library print Add OpenMP support to computing bounds in StructuredColumns::generate_region Add progress timer to StructuredColumns::generate_region Fix building with ATLAS_BITS_LOCAL=64 Reduce memory peak in GatherScatter::setup Fix compilation warning Add missing this%return() in atlas_Trace_module, but likely no impact as mostly unused subroutines Improve performance of ATLAS_TRACE by improving hashing function Add interface to adjoint of direct spectral transform implemented via ectrans (#113) Fix compilation with ATLAS_BITS_LOCAL=64 Fortran interfaces for PointCloud functionspace halo exchange (#112) Extend PointCloud functionspace to do halo exchanges (#111) Implement IndexViev::assign( initializer_list ) ; for now only with NativeIndexView Fix Github Actions missing brew ...
2 parents d818872 + 2d5a53b commit 93f41b8

File tree

74 files changed

+2476
-887
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

74 files changed

+2476
-887
lines changed

‎.github/workflows/build.yml

+4
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,10 @@ jobs:
9898
- name: Checkout Repository
9999
uses: actions/checkout@v2
100100

101+
- name: Set up Homebrew
102+
id: set-up-homebrew
103+
uses: Homebrew/actions/setup-homebrew@master
104+
101105
- name: Environment
102106
run: |
103107
echo "DEPS_DIR=${{ runner.temp }}/deps" >> $GITHUB_ENV

‎CHANGELOG.md

+20
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,25 @@ This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html
77

88
## [Unreleased]
99

10+
## [0.31.0] - 2022-11-10
11+
### Added
12+
- Extend PointCloud functionspace to do halo exchanges, including Fortran API
13+
- Add FunctionSpace::gather and FunctionSpace::scatter abstraction
14+
15+
### Changed
16+
- Improve performance of MatchingMeshPartitionerLonLatPolygon using OpenMP
17+
- Improve performance of BuildHalo using OpenMP and unordered_map
18+
- Improve performance of StructuredMeshGenerator using OpenMP
19+
- Improve performance of ATLAS_TRACE
20+
- Reduce memory peak in GatherScatter setup
21+
- Global element numbering of RegularLonLat grids is now following rows independent of partitioning
22+
23+
### Fixed
24+
- Running CI with Github Actions
25+
- Fix output of atlas-grids y-range for shifted grids
26+
- Fix building with ATLAS_BITS_LOCAL=64
27+
28+
1029
## [0.30.0] - 2022-08-22
1130
### Added
1231
- Fortran API for Interpolation::execute_adjoint()
@@ -386,6 +405,7 @@ This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html
386405
## 0.13.0 - 2018-02-16
387406

388407
[Unreleased]: https://github.com/ecmwf/atlas/compare/master...develop
408+
[0.31.0]: https://github.com/ecmwf/atlas/compare/0.30.0...0.31.0
389409
[0.30.0]: https://github.com/ecmwf/atlas/compare/0.29.0...0.30.0
390410
[0.29.0]: https://github.com/ecmwf/atlas/compare/0.28.1...0.29.0
391411
[0.28.1]: https://github.com/ecmwf/atlas/compare/0.28.0...0.28.1

‎VERSION

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
0.30.0
1+
0.31.0

‎atlas_io/src/atlas_io/Data.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ class Data {
2323
public:
2424
Data() = default;
2525
Data(void*, size_t);
26-
Data(Data&&) = default;
26+
Data(Data&&) = default;
2727
Data& operator=(Data&&) = default;
2828

2929
operator const void*() const { return data(); }

‎atlas_io/src/atlas_io/Metadata.h

+8
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,14 @@ class Metadata : public eckit::LocalConfiguration {
8383
root.remove(name);
8484
return *this;
8585
}
86+
87+
88+
std::vector<std::string> keys() const {
89+
// Preserves order of keys
90+
std::vector<std::string> result;
91+
eckit::fromValue(result, get().keys());
92+
return result;
93+
}
8694
};
8795

8896
//---------------------------------------------------------------------------------------------------------------------

‎atlas_io/src/atlas_io/Record.cc

+17-3
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,20 @@ Record::operator const ParsedRecord&() const {
132132

133133
//---------------------------------------------------------------------------------------------------------------------
134134

135+
static void parse_record(ParsedRecord& record, const std::string& key, const Metadata& metadata) {
136+
if (metadata.type() || metadata.link()) {
137+
record.items.emplace(key, metadata);
138+
record.keys.emplace_back(key);
139+
}
140+
else {
141+
for (auto& next_key : metadata.keys()) {
142+
parse_record(record, key + "." + next_key, metadata.getSubConfiguration(next_key));
143+
}
144+
}
145+
}
146+
147+
//---------------------------------------------------------------------------------------------------------------------
148+
135149
Record& Record::read(Stream& in, bool read_to_end) {
136150
if (not empty()) {
137151
return *this;
@@ -206,9 +220,9 @@ Record& Record::read(Stream& in, bool read_to_end) {
206220

207221
ATLAS_IO_ASSERT(r.metadata_format == "yaml");
208222
Metadata metadata = eckit::YAMLConfiguration(metadata_str);
209-
record_->keys = metadata.keys();
210-
for (const auto& key : record_->keys) {
211-
record_->items.emplace(key, metadata.getSubConfiguration(key));
223+
224+
for (auto& key : metadata.keys()) {
225+
parse_record(*record_, key, metadata.getSubConfiguration(key));
212226
}
213227

214228

‎atlas_io/src/atlas_io/RecordWriter.h

+11
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,9 @@
2424
#include "atlas_io/detail/TypeTraits.h"
2525

2626
#include "atlas_io/types/array/ArrayReference.h"
27+
#include "atlas_io/types/scalar.h"
28+
#include "atlas_io/types/string.h"
29+
2730

2831
#include "atlas_io/detail/Defaults.h"
2932

@@ -88,6 +91,14 @@ class RecordWriter {
8891
set(key, Encoder{value}, config);
8992
}
9093

94+
void set(const Key& key, const char* value, const eckit::Configuration& config = NoConfig()) {
95+
set(key, Encoder{std::string(value)}, config);
96+
}
97+
98+
void set(const Key& key, const std::string& value, const eckit::Configuration& config = NoConfig()) {
99+
set(key, Encoder{std::string(value)}, config);
100+
}
101+
91102
/// @brief Write new record to path
92103
size_t write(const eckit::PathName&, Mode = Mode::write) const;
93104

‎atlas_io/src/atlas_io/Trace.cc

+12-6
Original file line numberDiff line numberDiff line change
@@ -16,20 +16,26 @@ namespace atlas {
1616
namespace io {
1717

1818
atlas::io::Trace::Trace(const eckit::CodeLocation& loc) {
19-
for (auto& hook : TraceHookRegistry::instance().hooks) {
20-
hooks_.emplace_back(hook(loc, loc.func()));
19+
for (size_t id = 0; id < TraceHookRegistry::size(); ++id) {
20+
if (TraceHookRegistry::enabled(id)) {
21+
hooks_.emplace_back(TraceHookRegistry::hook(id)(loc, loc.func()));
22+
}
2123
}
2224
}
2325

2426
Trace::Trace(const eckit::CodeLocation& loc, const std::string& title) {
25-
for (auto& hook : TraceHookRegistry::instance().hooks) {
26-
hooks_.emplace_back(hook(loc, title));
27+
for (size_t id = 0; id < TraceHookRegistry::size(); ++id) {
28+
if (TraceHookRegistry::enabled(id)) {
29+
hooks_.emplace_back(TraceHookRegistry::hook(id)(loc, title));
30+
}
2731
}
2832
}
2933

3034
Trace::Trace(const eckit::CodeLocation& loc, const std::string& title, const Labels& labels) {
31-
for (auto& hook : TraceHookRegistry::instance().hooks) {
32-
hooks_.emplace_back(hook(loc, title));
35+
for (size_t id = 0; id < TraceHookRegistry::size(); ++id) {
36+
if (TraceHookRegistry::enabled(id)) {
37+
hooks_.emplace_back(TraceHookRegistry::hook(id)(loc, title));
38+
}
3339
}
3440
}
3541

‎atlas_io/src/atlas_io/Trace.h

+16-2
Original file line numberDiff line numberDiff line change
@@ -31,12 +31,26 @@ struct TraceHook {
3131
struct TraceHookRegistry {
3232
using TraceHookBuilder = std::function<std::unique_ptr<TraceHook>(const eckit::CodeLocation&, const std::string&)>;
3333
std::vector<TraceHookBuilder> hooks;
34+
std::vector<int> enabled_;
3435
static TraceHookRegistry& instance() {
3536
static TraceHookRegistry instance;
3637
return instance;
3738
}
38-
static void add(TraceHookBuilder&& hook) { instance().hooks.emplace_back(hook); }
39-
static void add(const TraceHookBuilder& hook) { instance().hooks.emplace_back(hook); }
39+
static size_t add(TraceHookBuilder&& hook) {
40+
instance().hooks.emplace_back(hook);
41+
instance().enabled_.emplace_back(true);
42+
return instance().hooks.size() - 1;
43+
}
44+
static size_t add(const TraceHookBuilder& hook) {
45+
instance().hooks.emplace_back(hook);
46+
instance().enabled_.emplace_back(true);
47+
return instance().hooks.size() - 1;
48+
}
49+
static void enable(size_t id) { instance().enabled_[id] = true; }
50+
static void disable(size_t id) { instance().enabled_[id] = false; }
51+
static bool enabled(size_t id) { return instance().enabled_[id]; }
52+
static size_t size() { return instance().hooks.size(); }
53+
static TraceHookBuilder& hook(size_t id) { return instance().hooks[id]; }
4054

4155
private:
4256
TraceHookRegistry() = default;

‎atlas_io/src/atlas_io/detail/Encoder.h

+9
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,10 @@ class Encoder {
3838

3939
Encoder(Encoder&& other): self_(std::move(other.self_)) {}
4040

41+
template <typename T, enable_if_scalar_t<T> = 0>
42+
explicit Encoder(const T& x): self_(new EncodableValue<T>(x)) {}
43+
44+
4145
Encoder& operator=(Encoder&& rhs) {
4246
self_ = std::move(rhs.self_);
4347
return *this;
@@ -61,6 +65,11 @@ class Encoder {
6165
struct EncodableValue : Encodable {
6266
EncodableValue(Value&& v): value_{std::move(v)} { sfinae::encode_metadata(value_, metadata_, data_size_); }
6367

68+
template <bool EnableBool = true, enable_if_scalar_t<Value, EnableBool> = 0>
69+
EncodableValue(const Value& v): value_{v} {
70+
sfinae::encode_metadata(value_, metadata_, data_size_);
71+
}
72+
6473
size_t encode_metadata_(atlas::io::Metadata& metadata) const override {
6574
metadata.set(metadata_);
6675
return data_size_;

‎atlas_io/src/atlas_io/detail/TypeTraits.h

+15
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212

1313
#include <type_traits>
1414

15+
#include "atlas_io/detail/DataType.h"
1516

1617
namespace atlas {
1718
namespace io {
@@ -134,6 +135,20 @@ template <typename T>
134135
using enable_if_move_constructible_decodable_rvalue_t =
135136
enable_if_t<is_decodable<T>() && std::is_rvalue_reference<T&&>() && std::is_move_constructible<T>()>;
136137

138+
template <typename T, bool EnableBool = true>
139+
using enable_if_scalar_t = enable_if_t<std::is_scalar<T>::value && EnableBool>;
140+
141+
142+
template <typename T>
143+
constexpr bool is_array_datatype() {
144+
return std::is_same<T, double>::value || std::is_same<T, float>::value ||
145+
std::is_same<T, int>::value || std::is_same<T, long>::value ||
146+
std::is_same<T, size_t>::value || std::is_same<T, std::byte>::value;
147+
}
148+
149+
template <typename T>
150+
using enable_if_array_datatype = typename std::enable_if<is_array_datatype<T>(), int>::type;
151+
137152

138153
} // namespace io
139154
} // namespace atlas

‎atlas_io/src/atlas_io/print/JSONFormat.cc

+7-5
Original file line numberDiff line numberDiff line change
@@ -52,11 +52,13 @@ void JSONFormat::print(std::ostream& out) const {
5252
}
5353

5454
if (print_details_) {
55-
m.set("data.compression.type", item.data.compression());
56-
m.set("data.compression.size", item.data.compressed_size());
57-
m.set("data.size", item.data.size());
58-
m.set("data.byte_order", (item.data.endian() == Endian::little) ? "little endian" : "big endian");
59-
m.set("data.checksum", item.data.checksum().str());
55+
if (item.data.size()) {
56+
m.set("data.compression.type", item.data.compression());
57+
m.set("data.compression.size", item.data.compressed_size());
58+
m.set("data.size", item.data.size());
59+
m.set("data.byte_order", (item.data.endian() == Endian::little) ? "little endian" : "big endian");
60+
m.set("data.checksum", item.data.checksum().str());
61+
}
6062
m.set("version", item.record.version().str());
6163
m.set("created", item.record.created().str());
6264
}

‎atlas_io/src/atlas_io/print/TableFormat.cc

-1
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,6 @@ class ScalarMetadataPrettyPrint : public MetadataPrettyPrintBase {
135135
std::string type = metadata_.getString("type");
136136
ATLAS_IO_ASSERT(type == "scalar");
137137
std::string datatype = metadata_.getString("datatype");
138-
std::string base64 = metadata_.getString("base64");
139138
out << std::setw(7) << std::left << datatype << ": ";
140139
if (datatype == DataType::str<double>()) {
141140
out << decode<double>();

‎atlas_io/src/atlas_io/types/array/ArrayReference.cc

+2-2
Original file line numberDiff line numberDiff line change
@@ -75,8 +75,8 @@ ArrayReference::ArrayReference(ArrayReference&& other): ArrayMetadata(std::move(
7575

7676
ArrayReference& ArrayReference::operator=(ArrayReference&& rhs) {
7777
ArrayMetadata::operator=(std::move(rhs));
78-
data_ = rhs.data_;
79-
rhs.data_ = nullptr;
78+
data_ = rhs.data_;
79+
rhs.data_ = nullptr;
8080
return *this;
8181
}
8282

‎atlas_io/src/atlas_io/types/array/adaptors/StdVectorAdaptor.h

+3-2
Original file line numberDiff line numberDiff line change
@@ -15,22 +15,23 @@
1515
#include "atlas_io/Data.h"
1616
#include "atlas_io/Exceptions.h"
1717
#include "atlas_io/Metadata.h"
18+
#include "atlas_io/detail/TypeTraits.h"
1819
#include "atlas_io/types/array/ArrayMetadata.h"
1920
#include "atlas_io/types/array/ArrayReference.h"
2021

2122
namespace std {
2223

2324
//---------------------------------------------------------------------------------------------------------------------
2425

25-
template <typename T>
26+
template <typename T, atlas::io::enable_if_array_datatype<T> = 0>
2627
void interprete(const std::vector<T>& vector, atlas::io::ArrayReference& out) {
2728
using atlas::io::ArrayReference;
2829
out = ArrayReference{vector.data(), {int(vector.size())}};
2930
}
3031

3132
//---------------------------------------------------------------------------------------------------------------------
3233

33-
template <typename T>
34+
template <typename T, atlas::io::enable_if_array_datatype<T> = 0>
3435
void decode(const atlas::io::Metadata& m, const atlas::io::Data& encoded, std::vector<T>& out) {
3536
atlas::io::ArrayMetadata array(m);
3637
if (array.datatype().kind() != atlas::io::ArrayMetadata::DataType::kind<T>()) {

‎atlas_io/src/atlas_io/types/scalar.cc

+4-4
Original file line numberDiff line numberDiff line change
@@ -135,16 +135,16 @@ void encode_data(const double&, atlas::io::Data&) {}
135135
//---------------------------------------------------------------------------------------------------------------------
136136

137137
void decode(const atlas::io::Metadata& metadata, const atlas::io::Data&, int& value) {
138-
decode_scalar_b64(metadata, value);
138+
decode_scalar(metadata, value);
139139
}
140140
void decode(const atlas::io::Metadata& metadata, const atlas::io::Data&, long& value) {
141-
decode_scalar_b64(metadata, value);
141+
decode_scalar(metadata, value);
142142
}
143143
void decode(const atlas::io::Metadata& metadata, const atlas::io::Data&, long long& value) {
144-
decode_scalar_b64(metadata, value);
144+
decode_scalar(metadata, value);
145145
}
146146
void decode(const atlas::io::Metadata& metadata, const atlas::io::Data&, unsigned long& value) {
147-
decode_scalar_b64(metadata, value);
147+
decode_scalar(metadata, value);
148148
}
149149
void decode(const atlas::io::Metadata& metadata, const atlas::io::Data&, unsigned long long& value) {
150150
decode_scalar_b64(metadata, value);

‎atlas_io/tests/test_io_encoding.cc

+15-5
Original file line numberDiff line numberDiff line change
@@ -635,11 +635,21 @@ void test_encode_decode_scalar() {
635635

636636
CASE("Encode/Decode scalar") {
637637
// bit identical encoding via Base64 string within the metadata!
638-
SECTION("int32") { test_encode_decode_scalar<std::int32_t>(); }
639-
SECTION("int64") { test_encode_decode_scalar<std::int64_t>(); }
640-
SECTION("real32") { test_encode_decode_scalar<float>(); }
641-
SECTION("real64") { test_encode_decode_scalar<double>(); }
642-
SECTION("uint64") { test_encode_decode_scalar<std::uint64_t>(); }
638+
SECTION("int32") {
639+
test_encode_decode_scalar<std::int32_t>();
640+
}
641+
SECTION("int64") {
642+
test_encode_decode_scalar<std::int64_t>();
643+
}
644+
SECTION("real32") {
645+
test_encode_decode_scalar<float>();
646+
}
647+
SECTION("real64") {
648+
test_encode_decode_scalar<double>();
649+
}
650+
SECTION("uint64") {
651+
test_encode_decode_scalar<std::uint64_t>();
652+
}
643653
}
644654

645655
// -------------------------------------------------------------------------------------------------------

‎atlas_io/tests/test_io_record.cc

+9-3
Original file line numberDiff line numberDiff line change
@@ -137,9 +137,15 @@ CASE("Write records, each in separate file (offset=0)") {
137137
write_length(length, path + ".length");
138138
};
139139

140-
SECTION("record1.atlas" + suffix()) { write_record(globals::record1.data, "record1.atlas" + suffix()); }
141-
SECTION("record2.atlas" + suffix()) { write_record(globals::record2.data, "record2.atlas" + suffix()); }
142-
SECTION("record3.atlas" + suffix()) { write_record(globals::record3.data, "record3.atlas" + suffix()); }
140+
SECTION("record1.atlas" + suffix()) {
141+
write_record(globals::record1.data, "record1.atlas" + suffix());
142+
}
143+
SECTION("record2.atlas" + suffix()) {
144+
write_record(globals::record2.data, "record2.atlas" + suffix());
145+
}
146+
SECTION("record3.atlas" + suffix()) {
147+
write_record(globals::record3.data, "record3.atlas" + suffix());
148+
}
143149
}
144150

145151
//-----------------------------------------------------------------------------

‎cmake/features/TRANS.cmake

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
set( atlas_HAVE_ECTRANS 0 )
44
if( ENABLE_TRANS OR NOT DEFINED ENABLE_TRANS )
5-
find_package( ectrans 1.0 COMPONENTS transi double QUIET )
5+
find_package( ectrans 1.1 COMPONENTS transi double QUIET )
66
if( TARGET transi_dp )
77
set( transi_FOUND TRUE )
88
if( NOT TARGET transi )

‎src/apps/atlas-grids.cc

+4-3
Original file line numberDiff line numberDiff line change
@@ -282,9 +282,10 @@ int AtlasGrids::execute(const Args& args) {
282282
Log::info() << " x : [ " << std::setw(10) << std::fixed << structuredgrid.xspace().min() << " , "
283283
<< std::setw(10) << std::fixed << structuredgrid.xspace().max() << " ] deg"
284284
<< std::endl;
285-
Log::info() << " y : [ " << std::setw(10) << std::fixed << structuredgrid.yspace().min() << " , "
286-
<< std::setw(10) << std::fixed << structuredgrid.yspace().max() << " ] deg"
287-
<< std::endl;
285+
double ymin = std::min(structuredgrid.yspace().front(), structuredgrid.yspace().back());
286+
double ymax = std::max(structuredgrid.yspace().front(), structuredgrid.yspace().back());
287+
Log::info() << " y : [ " << std::setw(10) << std::fixed << ymin << " , " << std::setw(10)
288+
<< std::fixed << ymax << " ] deg" << std::endl;
288289
}
289290
auto it = grid.lonlat().begin();
290291
Log::info() << " lonlat(first) : " << *it << std::endl;

0 commit comments

Comments
 (0)