Feature/parametrization

VERSION

@@ -1 +1 @@
-2.2.1
+2.2.2

src/multio/CMakeLists.txt

@@ -91,6 +91,8 @@ list( APPEND multio_domain_srcs
 )
 
 list( APPEND multio_message_srcs
+    message/BaseMetadata.h
+    message/BaseMetadata.cc
     message/Message.cc
     message/MessageHeader.cc
     message/Message.h
@@ -112,6 +114,8 @@ list( APPEND multio_message_srcs
     message/SharedPayload.h
     message/SharedPayloadException.cc
     message/SharedPayloadException.h
+    message/Parametrization.cc
+    message/Parametrization.h
     message/Peer.cc
     message/Peer.h
     message/MetadataMapping.cc

src/multio/action/aggregate/Aggregate.cc

@@ -47,7 +47,6 @@ bool Aggregate::handleField(const Message& msg) {
 }
 
 auto Aggregate::flushCount(const Message& msg) {
-
     auto res = flushes_[msg.fieldId()].emplace(msg.source());
 
     if (not res.second) {

src/multio/action/aggregate/AggregationCatalogue.cc

@@ -3,6 +3,8 @@
 
 #include "multio/domain/Mappings.h"
 
+#include "multio/message/Parametrization.h"
+
 namespace multio::action {
 
 message::Message& AggregationCatalogue::getMessage(const std::string& key) {

src/multio/action/encode/GribEncoder.cc

@@ -32,9 +32,9 @@
 
 #include "multio/LibMultio.h"
 #include "multio/util/DateTime.h"
+#include "multio/util/Environment.h"
 #include "multio/util/Metadata.h"
 #include "multio/util/Substitution.h"
-#include "multio/util/Environment.h"
 
 #include "multio/util/PrecisionTag.h"
 
@@ -547,7 +547,7 @@ QueriedMarsKeys setMarsKeys(GribEncoder& g, const Dict& md) {
 void applyOverwrites(GribEncoder& g, const message::Metadata& md) {
     if (auto searchOverwrites = md.find("encoder-overwrites"); searchOverwrites != md.end()) {
         // TODO Refactor with visitor
-        for (const auto& kv : searchOverwrites->second.get<message::Metadata>()) {
+        for (const auto& kv : searchOverwrites->second.get<message::BaseMetadata>()) {
             if (g.hasKey(kv.first.value().c_str())) {
                 kv.second.visit(eckit::Overloaded{
                     [](const auto& v) -> util::IfTypeOf<decltype(v), MetadataTypes::AllNested> {},
@@ -629,7 +629,7 @@ void setDateAndStatisticalFields(GribEncoder& g, const message::Metadata& in,
     auto significanceOfReferenceTime = lookUp<std::int64_t>(md, "significanceOfReferenceTime")();
     if (!significanceOfReferenceTime) {
         if (auto searchEncoderOverwrites = md.find("encoder-overwrites"); searchEncoderOverwrites != md.end()) {
-            const auto& overwrites = md.get<message::Metadata>("encoder-overwrites");
+            const auto& overwrites = md.get<message::BaseMetadata>("encoder-overwrites");
             significanceOfReferenceTime = lookUp<std::int64_t>(overwrites, "significanceOfReferenceTime")();
         }
     }

src/multio/action/encode/GridDownloader.cc

@@ -25,8 +25,8 @@
 #include "eckit/mpi/Comm.h"
 
 #include "multio/message/Glossary.h"
-#include "multio/util/Substitution.h"
 #include "multio/util/Environment.h"
+#include "multio/util/Substitution.h"
 
 namespace {
 const std::unordered_map<std::string, int> latParamIds{

src/multio/action/scale/Mapping.cc

@@ -3,10 +3,10 @@
 #include <cstdlib>
 #include <string>
 
+#include "MetadataUtils.h"
 #include "multio/LibMultio.h"
 #include "multio/message/Glossary.h"
 #include "multio/util/Substitution.h"
-#include "MetadataUtils.h"
 
 
 namespace multio::action {
@@ -35,7 +35,7 @@ void ScaleMapping::applyMapping(message::Metadata& md) const {
 
         auto it = scaleMap_.find(cparam);
         if (it != scaleMap_.end()) {
-            md.set(glossary().paramId, std::stoll(it->second)); 
+            md.set(glossary().paramId, std::stoll(it->second));
             md.set(glossary().param, it->second.c_str());
         }
     }

src/multio/action/scale/MetadataUtils.cc

@@ -14,7 +14,8 @@ std::string extractParam(const multio::message::Metadata& md) {
     std::string cparam{};
     if (auto param = md.getOpt<std::string>(glossary().param); param) {
         cparam = *param;
-    } else if (auto paramId = md.getOpt<std::int64_t>(glossary().paramId); paramId) {
+    }
+    else if (auto paramId = md.getOpt<std::int64_t>(glossary().paramId); paramId) {
         cparam = std::to_string(*paramId);
     }
     else {
@@ -23,4 +24,4 @@ std::string extractParam(const multio::message::Metadata& md) {
 
     return cparam;
 }
-}
+}  // namespace multio::action

src/multio/action/scale/MetadataUtils.h

@@ -6,6 +6,5 @@
 #include "multio/message/Metadata.h"
 
 namespace multio::action {
-    std::string extractParam(const multio::message::Metadata& md);
-
+std::string extractParam(const multio::message::Metadata& md);
 }

src/multio/action/scale/Scale.cc

@@ -21,8 +21,8 @@
 #include "multio/util/PrecisionTag.h"
 
 #include "multio/action/scale/Mapping.h"
-#include "multio/action/scale/Scaling.h"
 #include "multio/action/scale/MetadataUtils.h"
+#include "multio/action/scale/Scaling.h"
 
 
 namespace multio::action {
@@ -57,12 +57,12 @@ void Scale::executeImpl(message::Message msg) {
         executeNext(std::move(msg));
         return;
     }
-    //Scale the message
+    // Scale the message
     util::dispatchPrecisionTag(msg.precision(), [&](auto pt) {
         using Precision = typename decltype(pt)::type;
-        ScaleMessage<Precision>(msg); // Modify msg in place
+        ScaleMessage<Precision>(msg);  // Modify msg in place
     });
-    //pass on the modified message
+    // pass on the modified message
     executeNext(std::move(msg));
     return;
 }

src/multio/action/scale/Scale.h

@@ -1,7 +1,7 @@
 #pragma once
 
 #include "multio/action/ChainedAction.h"
-#include "multio/config/ComponentConfiguration.h" 
+#include "multio/config/ComponentConfiguration.h"
 
 #include "multio/action/scale/Mapping.h"
 #include "multio/action/scale/Scaling.h"

src/multio/action/scale/Scaling.h

@@ -4,10 +4,10 @@
 #include <string>
 
 #include "eckit/config/LocalConfiguration.h"
+#include "multio/action/scale/MetadataUtils.h"
 #include "multio/config/ComponentConfiguration.h"
 #include "multio/message/Glossary.h"
 #include "multio/message/Message.h"
-#include "multio/action/scale/MetadataUtils.h"
 
 namespace multio::action {
 

src/multio/action/select/Select.cc

@@ -23,7 +23,8 @@ namespace multio::action {
 
 //--------------------------------------------------------------------------------------------------
 
-Select::Select(const ComponentConfiguration& compConf) : ChainedAction{compConf}, selectors_{MatchReduce::construct( compConf.parsedConfig())} {}
+Select::Select(const ComponentConfiguration& compConf) :
+    ChainedAction{compConf}, selectors_{MatchReduce::construct(compConf.parsedConfig())} {}
 
 void Select::executeImpl(Message msg) {
     if (matches(msg)) {

src/multio/action/statistics/OperationWindow.cc

@@ -57,16 +57,16 @@ eckit::DateTime yyyymmdd_hhmmss2DateTime(uint64_t yyyymmdd, uint64_t hhmmss) {
 }  // namespace
 
 
-OperationWindow make_window( const std::unique_ptr<PeriodUpdater>& periodUpdater, const StatisticsConfiguration& cfg) {
+OperationWindow make_window(const std::unique_ptr<PeriodUpdater>& periodUpdater, const StatisticsConfiguration& cfg) {
     eckit::DateTime epochPoint{cfg.epoch()};
     eckit::DateTime startPoint{periodUpdater->computeWinStartTime(cfg.winStart())};
     eckit::DateTime creationPoint{periodUpdater->computeWinCreationTime(cfg.winStart())};
     eckit::DateTime endPoint{periodUpdater->computeWinEndTime(startPoint)};
     long windowType = 0;
-    if ( cfg.options().windowType() == "forward-offset" ){
+    if (cfg.options().windowType() == "forward-offset") {
         windowType = 0;
     }
-    else if ( cfg.options().windowType() == "backward-offset" ) {
+    else if (cfg.options().windowType() == "backward-offset") {
         windowType = 1;
     }
     else {
@@ -77,8 +77,8 @@ OperationWindow make_window( const std::unique_ptr<PeriodUpdater>& periodUpdater
     return OperationWindow{epochPoint, startPoint, creationPoint, endPoint, cfg.timeStep(), windowType};
 };
 
-OperationWindow load_window( std::shared_ptr<StatisticsIO>& IOmanager, const StatisticsOptions& opt ) {
-    IOmanager->pushDir( "operationWindow" );
+OperationWindow load_window(std::shared_ptr<StatisticsIO>& IOmanager, const StatisticsOptions& opt) {
+    IOmanager->pushDir("operationWindow");
     // std::ostringstream logos;
     // logos << "     - Loading operationWindow from: " << IOmanager->getCurrentDir()  << std::endl;
     // LOG_DEBUG_LIB(LibMultio) << logos.str() << std::endl;
@@ -125,15 +125,15 @@ long OperationWindow::count() const {
 void OperationWindow::dump(std::shared_ptr<StatisticsIO>& IOmanager, const StatisticsOptions& opt) const {
     IOBuffer restartState{IOmanager->getBuffer(restartSize())};
     restartState.zero();
-    serialize(restartState, IOmanager->getCurrentDir() + "/operationWindow_dump.txt", opt );
-    IOmanager->write("operationWindow", static_cast<size_t>(16), restartSize() );
+    serialize(restartState, IOmanager->getCurrentDir() + "/operationWindow_dump.txt", opt);
+    IOmanager->write("operationWindow", static_cast<size_t>(16), restartSize());
     IOmanager->flush();
     return;
 }
 
 void OperationWindow::load(std::shared_ptr<StatisticsIO>& IOmanager, const StatisticsOptions& opt) {
     IOBuffer restartState{IOmanager->getBuffer(restartSize())};
-    IOmanager->read( "operationWindow", restartSize() );
+    IOmanager->read("operationWindow", restartSize());
     deserialize(restartState, IOmanager->getCurrentDir() + "/operationWindow_load.txt", opt);
     restartState.zero();
     return;
@@ -163,9 +163,11 @@ void OperationWindow::updateWindow(const eckit::DateTime& startPoint, const ecki
 std::string OperationWindow::windowType() const {
     if (type_ == 0) {
         return std::string{"forward-offset"};
-    } else if (type_ == 1) {
+    }
+    else if (type_ == 1) {
         return std::string{"backward-offset"};
-    } else {
+    }
+    else {
         std::ostringstream os;
         os << *this << " Unknown window type " << std::endl;
         throw eckit::SeriousBug(os.str(), Here());
@@ -175,10 +177,10 @@ std::string OperationWindow::windowType() const {
 
 bool OperationWindow::isWithin(const eckit::DateTime& dt) const {
     bool ret;
-    if ( type_ == 0 ) {
+    if (type_ == 0) {
         ret = gtLowerBound(dt, false) && leUpperBound(dt, false);
     }
-    else if ( type_ == 1 ) {
+    else if (type_ == 1) {
         ret = geLowerBound(dt, false) && ltUpperBound(dt, false);
     }
     else {
@@ -434,7 +436,7 @@ long OperationWindow::lastFlushInSteps() const {
 
 void OperationWindow::serialize(IOBuffer& currState, const std::string& fname, const StatisticsOptions& opt) const {
 
-    if ( opt.debugRestart() ) {
+    if (opt.debugRestart()) {
         std::ofstream outFile(fname);
         outFile << "epochPoint_ :: " << epochPoint_ << std::endl;
         outFile << "startPoint_ :: " << startPoint_ << std::endl;
@@ -493,7 +495,7 @@ void OperationWindow::deserialize(const IOBuffer& currState, const std::string&
     count_ = static_cast<long>(currState[15]);
     type_ = static_cast<long>(currState[16]);
 
-    if ( opt.debugRestart() ) {
+    if (opt.debugRestart()) {
         std::ofstream outFile(fname);
         outFile << "epochPoint_ :: " << epochPoint_ << std::endl;
         outFile << "startPoint_ :: " << startPoint_ << std::endl;