diff --git a/src/multio/action/encode/GribEncoder.cc b/src/multio/action/encode/GribEncoder.cc index 3d6de2b6..51f49a89 100644 --- a/src/multio/action/encode/GribEncoder.cc +++ b/src/multio/action/encode/GribEncoder.cc @@ -32,9 +32,9 @@ #include "multio/LibMultio.h" #include "multio/util/DateTime.h" +#include "multio/util/Environment.h" #include "multio/util/Metadata.h" #include "multio/util/Substitution.h" -#include "multio/util/Environment.h" #include "multio/util/PrecisionTag.h" diff --git a/src/multio/action/encode/GridDownloader.cc b/src/multio/action/encode/GridDownloader.cc index 2bd2eac6..2eb07cd8 100644 --- a/src/multio/action/encode/GridDownloader.cc +++ b/src/multio/action/encode/GridDownloader.cc @@ -25,8 +25,8 @@ #include "eckit/mpi/Comm.h" #include "multio/message/Glossary.h" -#include "multio/util/Substitution.h" #include "multio/util/Environment.h" +#include "multio/util/Substitution.h" namespace { const std::unordered_map latParamIds{ diff --git a/src/multio/action/scale/Mapping.cc b/src/multio/action/scale/Mapping.cc index 3d6a405a..d34122a6 100644 --- a/src/multio/action/scale/Mapping.cc +++ b/src/multio/action/scale/Mapping.cc @@ -3,10 +3,10 @@ #include #include +#include "MetadataUtils.h" #include "multio/LibMultio.h" #include "multio/message/Glossary.h" #include "multio/util/Substitution.h" -#include "MetadataUtils.h" namespace multio::action { @@ -35,7 +35,7 @@ void ScaleMapping::applyMapping(message::Metadata& md) const { auto it = scaleMap_.find(cparam); if (it != scaleMap_.end()) { - md.set(glossary().paramId, std::stoll(it->second)); + md.set(glossary().paramId, std::stoll(it->second)); md.set(glossary().param, it->second.c_str()); } } diff --git a/src/multio/action/scale/MetadataUtils.cc b/src/multio/action/scale/MetadataUtils.cc index f586e994..32a00c4d 100644 --- a/src/multio/action/scale/MetadataUtils.cc +++ b/src/multio/action/scale/MetadataUtils.cc @@ -14,7 +14,8 @@ std::string extractParam(const multio::message::Metadata& md) { std::string cparam{}; if (auto param = md.getOpt(glossary().param); param) { cparam = *param; - } else if (auto paramId = md.getOpt(glossary().paramId); paramId) { + } + else if (auto paramId = md.getOpt(glossary().paramId); paramId) { cparam = std::to_string(*paramId); } else { @@ -23,4 +24,4 @@ std::string extractParam(const multio::message::Metadata& md) { return cparam; } -} \ No newline at end of file +} // namespace multio::action \ No newline at end of file diff --git a/src/multio/action/scale/MetadataUtils.h b/src/multio/action/scale/MetadataUtils.h index 6b42440c..a8742d70 100644 --- a/src/multio/action/scale/MetadataUtils.h +++ b/src/multio/action/scale/MetadataUtils.h @@ -6,6 +6,5 @@ #include "multio/message/Metadata.h" namespace multio::action { - std::string extractParam(const multio::message::Metadata& md); - +std::string extractParam(const multio::message::Metadata& md); } \ No newline at end of file diff --git a/src/multio/action/scale/Scale.cc b/src/multio/action/scale/Scale.cc index 96ad4984..9b1cceb9 100644 --- a/src/multio/action/scale/Scale.cc +++ b/src/multio/action/scale/Scale.cc @@ -21,8 +21,8 @@ #include "multio/util/PrecisionTag.h" #include "multio/action/scale/Mapping.h" -#include "multio/action/scale/Scaling.h" #include "multio/action/scale/MetadataUtils.h" +#include "multio/action/scale/Scaling.h" namespace multio::action { @@ -57,12 +57,12 @@ void Scale::executeImpl(message::Message msg) { executeNext(std::move(msg)); return; } - //Scale the message + // Scale the message util::dispatchPrecisionTag(msg.precision(), [&](auto pt) { using Precision = typename decltype(pt)::type; - ScaleMessage(msg); // Modify msg in place + ScaleMessage(msg); // Modify msg in place }); - //pass on the modified message + // pass on the modified message executeNext(std::move(msg)); return; } diff --git a/src/multio/action/scale/Scale.h b/src/multio/action/scale/Scale.h index 26605c6a..ebbf4242 100644 --- a/src/multio/action/scale/Scale.h +++ b/src/multio/action/scale/Scale.h @@ -1,7 +1,7 @@ #pragma once #include "multio/action/ChainedAction.h" -#include "multio/config/ComponentConfiguration.h" +#include "multio/config/ComponentConfiguration.h" #include "multio/action/scale/Mapping.h" #include "multio/action/scale/Scaling.h" diff --git a/src/multio/action/scale/Scaling.h b/src/multio/action/scale/Scaling.h index 52921fb8..0ee4f224 100644 --- a/src/multio/action/scale/Scaling.h +++ b/src/multio/action/scale/Scaling.h @@ -4,10 +4,10 @@ #include #include "eckit/config/LocalConfiguration.h" +#include "multio/action/scale/MetadataUtils.h" #include "multio/config/ComponentConfiguration.h" #include "multio/message/Glossary.h" #include "multio/message/Message.h" -#include "multio/action/scale/MetadataUtils.h" namespace multio::action { diff --git a/src/multio/action/select/Select.cc b/src/multio/action/select/Select.cc index 3a397ebe..2f236e05 100644 --- a/src/multio/action/select/Select.cc +++ b/src/multio/action/select/Select.cc @@ -23,7 +23,8 @@ namespace multio::action { //-------------------------------------------------------------------------------------------------- -Select::Select(const ComponentConfiguration& compConf) : ChainedAction{compConf}, selectors_{MatchReduce::construct( compConf.parsedConfig())} {} +Select::Select(const ComponentConfiguration& compConf) : + ChainedAction{compConf}, selectors_{MatchReduce::construct(compConf.parsedConfig())} {} void Select::executeImpl(Message msg) { if (matches(msg)) { diff --git a/src/multio/action/statistics/OperationWindow.cc b/src/multio/action/statistics/OperationWindow.cc index a55bd5f9..0eb5c3e7 100644 --- a/src/multio/action/statistics/OperationWindow.cc +++ b/src/multio/action/statistics/OperationWindow.cc @@ -57,16 +57,16 @@ eckit::DateTime yyyymmdd_hhmmss2DateTime(uint64_t yyyymmdd, uint64_t hhmmss) { } // namespace -OperationWindow make_window( const std::unique_ptr& periodUpdater, const StatisticsConfiguration& cfg) { +OperationWindow make_window(const std::unique_ptr& periodUpdater, const StatisticsConfiguration& cfg) { eckit::DateTime epochPoint{cfg.epoch()}; eckit::DateTime startPoint{periodUpdater->computeWinStartTime(cfg.winStart())}; eckit::DateTime creationPoint{periodUpdater->computeWinCreationTime(cfg.winStart())}; eckit::DateTime endPoint{periodUpdater->computeWinEndTime(startPoint)}; long windowType = 0; - if ( cfg.options().windowType() == "forward-offset" ){ + if (cfg.options().windowType() == "forward-offset") { windowType = 0; } - else if ( cfg.options().windowType() == "backward-offset" ) { + else if (cfg.options().windowType() == "backward-offset") { windowType = 1; } else { @@ -77,8 +77,8 @@ OperationWindow make_window( const std::unique_ptr& periodUpdater return OperationWindow{epochPoint, startPoint, creationPoint, endPoint, cfg.timeStep(), windowType}; }; -OperationWindow load_window( std::shared_ptr& IOmanager, const StatisticsOptions& opt ) { - IOmanager->pushDir( "operationWindow" ); +OperationWindow load_window(std::shared_ptr& IOmanager, const StatisticsOptions& opt) { + IOmanager->pushDir("operationWindow"); // std::ostringstream logos; // logos << " - Loading operationWindow from: " << IOmanager->getCurrentDir() << std::endl; // LOG_DEBUG_LIB(LibMultio) << logos.str() << std::endl; @@ -125,15 +125,15 @@ long OperationWindow::count() const { void OperationWindow::dump(std::shared_ptr& IOmanager, const StatisticsOptions& opt) const { IOBuffer restartState{IOmanager->getBuffer(restartSize())}; restartState.zero(); - serialize(restartState, IOmanager->getCurrentDir() + "/operationWindow_dump.txt", opt ); - IOmanager->write("operationWindow", static_cast(16), restartSize() ); + serialize(restartState, IOmanager->getCurrentDir() + "/operationWindow_dump.txt", opt); + IOmanager->write("operationWindow", static_cast(16), restartSize()); IOmanager->flush(); return; } void OperationWindow::load(std::shared_ptr& IOmanager, const StatisticsOptions& opt) { IOBuffer restartState{IOmanager->getBuffer(restartSize())}; - IOmanager->read( "operationWindow", restartSize() ); + IOmanager->read("operationWindow", restartSize()); deserialize(restartState, IOmanager->getCurrentDir() + "/operationWindow_load.txt", opt); restartState.zero(); return; @@ -163,9 +163,11 @@ void OperationWindow::updateWindow(const eckit::DateTime& startPoint, const ecki std::string OperationWindow::windowType() const { if (type_ == 0) { return std::string{"forward-offset"}; - } else if (type_ == 1) { + } + else if (type_ == 1) { return std::string{"backward-offset"}; - } else { + } + else { std::ostringstream os; os << *this << " Unknown window type " << std::endl; throw eckit::SeriousBug(os.str(), Here()); @@ -175,10 +177,10 @@ std::string OperationWindow::windowType() const { bool OperationWindow::isWithin(const eckit::DateTime& dt) const { bool ret; - if ( type_ == 0 ) { + if (type_ == 0) { ret = gtLowerBound(dt, false) && leUpperBound(dt, false); } - else if ( type_ == 1 ) { + else if (type_ == 1) { ret = geLowerBound(dt, false) && ltUpperBound(dt, false); } else { @@ -434,7 +436,7 @@ long OperationWindow::lastFlushInSteps() const { void OperationWindow::serialize(IOBuffer& currState, const std::string& fname, const StatisticsOptions& opt) const { - if ( opt.debugRestart() ) { + if (opt.debugRestart()) { std::ofstream outFile(fname); outFile << "epochPoint_ :: " << epochPoint_ << std::endl; outFile << "startPoint_ :: " << startPoint_ << std::endl; @@ -493,7 +495,7 @@ void OperationWindow::deserialize(const IOBuffer& currState, const std::string& count_ = static_cast(currState[15]); type_ = static_cast(currState[16]); - if ( opt.debugRestart() ) { + if (opt.debugRestart()) { std::ofstream outFile(fname); outFile << "epochPoint_ :: " << epochPoint_ << std::endl; outFile << "startPoint_ :: " << startPoint_ << std::endl; diff --git a/src/multio/action/statistics/OperationWindow.h b/src/multio/action/statistics/OperationWindow.h index 4204a32c..b42e750f 100644 --- a/src/multio/action/statistics/OperationWindow.h +++ b/src/multio/action/statistics/OperationWindow.h @@ -6,10 +6,10 @@ #include "eckit/types/DateTime.h" -#include "multio/action/statistics/cfg/StatisticsOptions.h" +#include "StatisticsIO.h" #include "multio/action/statistics/cfg/StatisticsConfiguration.h" +#include "multio/action/statistics/cfg/StatisticsOptions.h" #include "multio/action/statistics/period-updaters/PeriodUpdater.h" -#include "StatisticsIO.h" #include "multio/message/Message.h" #include "multio/util/DateTime.h" @@ -20,8 +20,8 @@ class OperationWindow { OperationWindow(std::shared_ptr& IOmanager, const StatisticsOptions& opt); OperationWindow(const eckit::DateTime& epochPoint, const eckit::DateTime& startPoint, - const eckit::DateTime& creationPoint, const eckit::DateTime& endPoint, - long timeStepInSeconds, long windowType); + const eckit::DateTime& creationPoint, const eckit::DateTime& endPoint, long timeStepInSeconds, + long windowType); long count() const; @@ -29,7 +29,7 @@ class OperationWindow { void updateWindow(const eckit::DateTime& startPoint, const eckit::DateTime& endPoint); void dump(std::shared_ptr& IOmanager, const StatisticsOptions& opt) const; - void load(std::shared_ptr& IOmanager, const StatisticsOptions& opt ); + void load(std::shared_ptr& IOmanager, const StatisticsOptions& opt); std::string windowType() const; bool isWithin(const eckit::DateTime& dt) const; @@ -123,8 +123,7 @@ class OperationWindow { friend std::ostream& operator<<(std::ostream& os, const OperationWindow& a); }; -OperationWindow make_window( const std::unique_ptr& periodUpdater, const StatisticsConfiguration& cfg); -OperationWindow load_window( std::shared_ptr& IOmanager, const StatisticsOptions& opt ); +OperationWindow make_window(const std::unique_ptr& periodUpdater, const StatisticsConfiguration& cfg); +OperationWindow load_window(std::shared_ptr& IOmanager, const StatisticsOptions& opt); } // namespace multio::action - diff --git a/src/multio/action/statistics/Operations.cc b/src/multio/action/statistics/Operations.cc index a561ec81..03f0f950 100644 --- a/src/multio/action/statistics/Operations.cc +++ b/src/multio/action/statistics/Operations.cc @@ -5,12 +5,12 @@ namespace multio::action { -void parse_file_name( const std::string& file, std::string& opname, std::string& precision ){ +void parse_file_name(const std::string& file, std::string& opname, std::string& precision) { static const std::regex operation_grammar("([a-z]+)_([a-z]+)"); std::smatch match; if (std::regex_match(file, match, operation_grammar)) { precision = match[2].str(); - opname = match[1].str(); + opname = match[1].str(); } else { throw eckit::SeriousBug("Wrong grammar in operation definition : " + file, Here()); @@ -18,11 +18,10 @@ void parse_file_name( const std::string& file, std::string& opname, std::string& }; -std::vector> make_operations(const std::vector& opNames, - message::Message msg, +std::vector> make_operations(const std::vector& opNames, message::Message msg, std::shared_ptr& IOmanager, const OperationWindow& win, - const StatisticsConfiguration& cfg ) { + const StatisticsConfiguration& cfg) { return multio::util::dispatchPrecisionTag(msg.precision(), [&](auto pt) { using Precision = typename decltype(pt)::type; @@ -48,12 +47,10 @@ std::vector> make_operations(const std::vector> load_operations(std::shared_ptr& IOmanager, - const OperationWindow& win, - const StatisticsOptions& opt) { + const OperationWindow& win, const StatisticsOptions& opt) { std::vector> stats; - IOmanager->pushDir( "operations" ); + IOmanager->pushDir("operations"); // std::ostringstream logos; // logos << " - Loading operations from: " << IOmanager->getCurrentDir() << std::endl; // std::cout << logos.str() << std::endl; @@ -61,15 +58,14 @@ std::vector> load_operations(std::shared_ptr(opname, IOmanager, win,opt ); - }) ); + // logos << " - Loading " << opname << " operation from: " << file.baseName(false).asString() << + // std::endl; std::cout << logos.str() << std::endl; + stats.push_back(multio::util::dispatchPrecisionTag(precision, [&](auto pt) { + using Precision = typename decltype(pt)::type; + return load_operation(opname, IOmanager, win, opt); + })); } IOmanager->popDir(); return stats; diff --git a/src/multio/action/statistics/Operations.h b/src/multio/action/statistics/Operations.h index 55b393ef..01249b8e 100644 --- a/src/multio/action/statistics/Operations.h +++ b/src/multio/action/statistics/Operations.h @@ -66,8 +66,8 @@ std::unique_ptr make_operation(const std::string& opname, long sz, st template -std::unique_ptr load_operation( const std::string& opname, std::shared_ptr& IOmanager, - const OperationWindow& win, const StatisticsOptions& opt) { +std::unique_ptr load_operation(const std::string& opname, std::shared_ptr& IOmanager, + const OperationWindow& win, const StatisticsOptions& opt) { std::unique_ptr ret; bool found = false; @@ -105,14 +105,13 @@ std::unique_ptr load_operation( const std::string& opname, std::share ret = std::make_unique>(opname, win, IOmanager, opt); } - if ( !found ){ - std::ostringstream os; - os << "Invalid opname in statistics operation :: " << opname << std::endl; - throw eckit::UserError(os.str(), Here()); + if (!found) { + std::ostringstream os; + os << "Invalid opname in statistics operation :: " << opname << std::endl; + throw eckit::UserError(os.str(), Here()); } return ret; - } std::vector> make_operations(const std::vector& opNames, message::Message msg, diff --git a/src/multio/action/statistics/PeriodUpdaters.cc b/src/multio/action/statistics/PeriodUpdaters.cc index ca05b1a5..c7dae5c5 100644 --- a/src/multio/action/statistics/PeriodUpdaters.cc +++ b/src/multio/action/statistics/PeriodUpdaters.cc @@ -38,7 +38,7 @@ std::unique_ptr make_period_updater(std::string const& output_fre long span; std::string periodKind; - parsePeriodGrammar( output_freq, span, periodKind ); + parsePeriodGrammar(output_freq, span, periodKind); if (periodKind == "hour" || periodKind == "h") { return std::make_unique(span); @@ -50,23 +50,23 @@ std::unique_ptr make_period_updater(std::string const& output_fre return std::make_unique(span); } throw eckit::SeriousBug("Unknown period kind : " + periodKind, Here()); - }; -std::unique_ptr load_period_updater(std::shared_ptr& IOmanager, const StatisticsOptions& opt) { +std::unique_ptr load_period_updater(std::shared_ptr& IOmanager, + const StatisticsOptions& opt) { std::unique_ptr ret; - IOmanager->pushDir( "periodUpdater" ); + IOmanager->pushDir("periodUpdater"); std::vector files = IOmanager->getFiles(); if (files.empty()) { throw eckit::SeriousBug("No files found in: " + IOmanager->getCurrentDir(), Here()); } - if ( files.size() > 1 ) { + if (files.size() > 1) { throw eckit::SeriousBug("More than one file found in: " + IOmanager->getCurrentDir(), Here()); } - std::string periodKind=files[0].baseName(false).asString(); + std::string periodKind = files[0].baseName(false).asString(); // parsePeriodFileName( files[0].baseName(false).asString(), periodKind ); // std::ostringstream logos; @@ -84,16 +84,15 @@ std::unique_ptr load_period_updater(std::shared_ptr } if (periodKind == "month") { found = true; - ret = std::make_unique(IOmanager, opt); + ret = std::make_unique(IOmanager, opt); } - IOmanager->popDir( ); + IOmanager->popDir(); - if ( !found ){ + if (!found) { throw eckit::SeriousBug("Unknown period kind : " + periodKind, Here()); } return ret; - }; diff --git a/src/multio/action/statistics/PeriodUpdaters.h b/src/multio/action/statistics/PeriodUpdaters.h index 6788fa7e..b8ae17c6 100644 --- a/src/multio/action/statistics/PeriodUpdaters.h +++ b/src/multio/action/statistics/PeriodUpdaters.h @@ -3,20 +3,21 @@ #include "multio/action/statistics/cfg/StatisticsConfiguration.h" -#include "multio/action/statistics/period-updaters/PeriodUpdater.h" -#include "multio/action/statistics/period-updaters/HourPeriodUpdater.h" #include "multio/action/statistics/period-updaters/DayPeriodUpdater.h" +#include "multio/action/statistics/period-updaters/HourPeriodUpdater.h" #include "multio/action/statistics/period-updaters/MonthPeriodUpdater.h" +#include "multio/action/statistics/period-updaters/PeriodUpdater.h" -#include "multio/action/statistics/TimeUtils.h" #include "eckit/types/DateTime.h" +#include "multio/action/statistics/TimeUtils.h" #include "multio/message/Message.h" #include "OperationWindow.h" namespace multio::action { -std::unique_ptr make_period_updater(std::string const& output_freq, const StatisticsConfiguration& cfg ); -std::unique_ptr load_period_updater(std::shared_ptr& IOmanager, const StatisticsOptions& opt ); +std::unique_ptr make_period_updater(std::string const& output_freq, const StatisticsConfiguration& cfg); +std::unique_ptr load_period_updater(std::shared_ptr& IOmanager, + const StatisticsOptions& opt); } // namespace multio::action diff --git a/src/multio/action/statistics/RemapParamID.cc b/src/multio/action/statistics/RemapParamID.cc index eaf221fe..25446816 100644 --- a/src/multio/action/statistics/RemapParamID.cc +++ b/src/multio/action/statistics/RemapParamID.cc @@ -1,7 +1,7 @@ #include "RemapParamID.h" -#include #include +#include #include "multio/LibMultio.h" #include "multio/message/Glossary.h" @@ -12,48 +12,47 @@ namespace multio::action { using message::glossary; -RemapParamID::RemapParamID( const config::ComponentConfiguration& compConf ): -hasMapping_{false},paramIDMap_{} { - const auto mappings - = compConf.parsedConfig().has("mapping-param") ? compConf.parsedConfig().getSubConfigurations("mapping-param") : std::vector{}; +RemapParamID::RemapParamID(const config::ComponentConfiguration& compConf) : hasMapping_{false}, paramIDMap_{} { + const auto mappings = compConf.parsedConfig().has("mapping-param") + ? compConf.parsedConfig().getSubConfigurations("mapping-param") + : std::vector{}; if (!mappings.empty()) { hasMapping_ = true; for (const auto& mapping : mappings) { auto matcher = mapping.getSubConfiguration("where"); - std::string param = matcher.getString("param-is"); - std::string operation = matcher.getString("operation-is"); + std::string param = matcher.getString("param-is"); + std::string operation = matcher.getString("operation-is"); std::string outputFrequency = matcher.getString("output-frequency-is"); - std::string newParam = mapping.getString("map-to-param"); + std::string newParam = mapping.getString("map-to-param"); std::ostringstream key; // TODO: outptuFrequqncy should always be computed in seconds key << param << "_" << operation << "_" << outputFrequency << std::endl; paramIDMap_[key.str()] = newParam; } } - } -void RemapParamID::ApplyRemap( message::Metadata& md, const std::string& opname, const std::string& outputFrequency ) { - if ( hasMapping_ ) { +void RemapParamID::ApplyRemap(message::Metadata& md, const std::string& opname, const std::string& outputFrequency) { + if (hasMapping_) { std::ostringstream key; std::string cparam{"xxx"}; if (auto param = md.getOpt(glossary().param); param) { cparam = *param; } if (auto paramId = md.getOpt(glossary().paramId); paramId) { - cparam = std::to_string( *paramId ); + cparam = std::to_string(*paramId); } else { throw eckit::SeriousBug{"param/paramId metadata not present", Here()}; } key << cparam << "_" << opname << "_" << outputFrequency << std::endl; auto it = paramIDMap_.find(key.str()); - if ( it != paramIDMap_.end() ) { + if (it != paramIDMap_.end()) { md.set(glossary().paramId, std::int64_t(::atol(it->second.c_str()))); - md.set(glossary().param, it->second.c_str() ); + md.set(glossary().param, it->second.c_str()); } } } -} +} // namespace multio::action diff --git a/src/multio/action/statistics/RemapParamID.h b/src/multio/action/statistics/RemapParamID.h index 36da4646..e8896f3b 100644 --- a/src/multio/action/statistics/RemapParamID.h +++ b/src/multio/action/statistics/RemapParamID.h @@ -11,11 +11,12 @@ namespace multio::action { class RemapParamID { private: - bool hasMapping_; - std::map paramIDMap_; + bool hasMapping_; + std::map paramIDMap_; + public: - RemapParamID( const config::ComponentConfiguration& compConf ); - void ApplyRemap( message::Metadata& md, const std::string& opname, const std::string& outputFrequency ); + RemapParamID(const config::ComponentConfiguration& compConf); + void ApplyRemap(message::Metadata& md, const std::string& opname, const std::string& outputFrequency); }; -} +} // namespace multio::action diff --git a/src/multio/action/statistics/Statistics.cc b/src/multio/action/statistics/Statistics.cc index 12f50afa..ca0b52a8 100644 --- a/src/multio/action/statistics/Statistics.cc +++ b/src/multio/action/statistics/Statistics.cc @@ -10,7 +10,7 @@ #include "Statistics.h" -#include //currently needed because ECKIT PathName only has hardlinks for for the directories we need a symlink +#include //currently needed because ECKIT PathName only has hardlinks for for the directories we need a symlink #include #include @@ -19,9 +19,9 @@ #include "eckit/exception/Exceptions.h" #include "eckit/types/DateTime.h" #include "multio/LibMultio.h" +#include "multio/message/Glossary.h" #include "multio/message/Message.h" #include "multio/util/Timing.h" -#include "multio/message/Glossary.h" #include "multio/action/statistics/cfg/StatisticsConfiguration.h" @@ -45,23 +45,23 @@ std::string Statistics::generateRestartNameFromFlush(const message::Message& msg std::string folderName; // Restart flush directly provides the folderName - auto restartDateTime = msg.metadata().getOpt( "restartDateTime" ); - auto step = msg.metadata().getOpt(glossary().step); - auto timeStep = msg.metadata().getOpt(glossary().timeStep); - auto date = msg.metadata().getOpt(glossary().date); - auto time = msg.metadata().getOpt(glossary().time); + auto restartDateTime = msg.metadata().getOpt("restartDateTime"); + auto step = msg.metadata().getOpt(glossary().step); + auto timeStep = msg.metadata().getOpt(glossary().timeStep); + auto date = msg.metadata().getOpt(glossary().date); + auto time = msg.metadata().getOpt(glossary().time); - if ( restartDateTime ) { + if (restartDateTime) { folderName = *restartDateTime; } // Restart flush provides the step, timeStep, date and time - else if ( step && timeStep && date && time ) { + else if (step && timeStep && date && time) { - std::int64_t flushStep = *step; + std::int64_t flushStep = *step; std::int64_t flushTimeStep = *timeStep; - std::int64_t flushDate = *date; - std::int64_t flushTime = *time; + std::int64_t flushDate = *date; + std::int64_t flushTime = *time; // Compute the date and time from the step // TODO: Remove this multiple times repeated code @@ -86,8 +86,8 @@ std::string Statistics::generateRestartNameFromFlush(const message::Message& msg eckit::DateTime dt = epoch + static_cast(flushStep * flushTimeStep); std::ostringstream tmp; - tmp << std::setw(8) << std::setfill('0') << dt.date().yyyymmdd() << "-" - << std::setw(6) << std::setfill('0') << dt.time().hhmmss(); + tmp << std::setw(8) << std::setfill('0') << dt.date().yyyymmdd() << "-" << std::setw(6) << std::setfill('0') + << dt.time().hhmmss(); folderName = tmp.str(); } @@ -99,28 +99,29 @@ std::string Statistics::generateRestartNameFromFlush(const message::Message& msg return folderName; } -void Statistics::CreateMainRestartDirectory( const std::string& restartFolderName, bool is_master ) { +void Statistics::CreateMainRestartDirectory(const std::string& restartFolderName, bool is_master) { // Create the main restart directory // TODO: if statistics are client side opt_.clientSideStatistics() then // the restart directory should be created with appended the mpi-rank of // processor that is creating the directory and all following login should // be skipped since every processor will create its own directory. - IOmanager_->setDateTime( restartFolderName ); + IOmanager_->setDateTime(restartFolderName); // Only master create the directory - if ( !IOmanager_->currentDirExists() ) { - if ( is_master ) { + if (!IOmanager_->currentDirExists()) { + if (is_master) { IOmanager_->createCurrentDir(); } else { long cnt = 0; - while ( !IOmanager_->currentDirExists() && cnt < 100 ) { + while (!IOmanager_->currentDirExists() && cnt < 100) { cnt++; usleep(1000); - LOG_DEBUG_LIB(LibMultio) << "Waiting for Dump directory to be created by master: " << restartFolderName << std::endl; + LOG_DEBUG_LIB(LibMultio) << "Waiting for Dump directory to be created by master: " << restartFolderName + << std::endl; } - if ( cnt>= 100 ){ + if (cnt >= 100) { std::ostringstream os; os << "Unable to create the restart directory: " << restartFolderName << std::endl; throw eckit::SeriousBug(os.str(), Here()); @@ -131,14 +132,15 @@ void Statistics::CreateMainRestartDirectory( const std::string& restartFolderNam return; } -void Statistics::DumpTemporalStatistics(){ +void Statistics::DumpTemporalStatistics() { for (auto it = fieldStats_.begin(); it != fieldStats_.end(); it++) { LOG_DEBUG_LIB(LibMultio) << " - Restart for field with key :: " << it->first << ", " << it->second->cwin().currPointInSteps() << std::endl; IOmanager_->pushDir(it->first); - if ( IOmanager_->currentDirExists() ) { + if (IOmanager_->currentDirExists()) { std::ostringstream os; - os << "Current restart already exists (this means that two mpi tasks has the same field): " << IOmanager_->getCurrentDir() << std::endl; + os << "Current restart already exists (this means that two mpi tasks has the same field): " + << IOmanager_->getCurrentDir() << std::endl; throw eckit::SeriousBug(os.str(), Here()); } IOmanager_->createCurrentDir(); @@ -148,40 +150,38 @@ void Statistics::DumpTemporalStatistics(){ return; } -void Statistics::TryDumpRestart( const message::Message& msg ) { +void Statistics::TryDumpRestart(const message::Message& msg) { auto flushKind = msg.metadata().getOpt("flushKind"); - if ( flushKind && opt_.writeRestart() && needRestart_ ) { + if (flushKind && opt_.writeRestart() && needRestart_) { // Check the kind of flush // NOTE: This is a bit of a hack, in case no serverRank is provided, we // assume that all processors are "master" and rely on atomicity of // filesystem operation to avoid race conditions auto is_master = msg.metadata().getOpt("serverRank").value_or(true); - if ( *flushKind == "step-and-restart" || - *flushKind == "last-step" || - *flushKind == "end-of-simulation" || - *flushKind == "close-connection" ) { + if (*flushKind == "step-and-restart" || *flushKind == "last-step" || *flushKind == "end-of-simulation" + || *flushKind == "close-connection") { // Generate name of the main restart directory - std::string restartFolderName = generateRestartNameFromFlush( msg ); + std::string restartFolderName = generateRestartNameFromFlush(msg); // Log for Dump operation - LOG_DEBUG_LIB(LibMultio) << "Performing a Dump :: Flush kind :: " << - *flushKind << "Last DateTime :: " << restartFolderName << std::endl; + LOG_DEBUG_LIB(LibMultio) << "Performing a Dump :: Flush kind :: " << *flushKind + << "Last DateTime :: " << restartFolderName << std::endl; // Delete the latest symlink as soon as possible - if ( is_master ) { + if (is_master) { DeleteLatestSymLink(); } // Create the main restart directory: // - CreateMainRestartDirectory( restartFolderName, is_master ); + CreateMainRestartDirectory(restartFolderName, is_master); // Dump the temporal statistics restart directories DumpTemporalStatistics(); // Create the latest symlink to the latest restart directory - if ( is_master ) { + if (is_master) { CreateLatestSymLink(); } } @@ -193,13 +193,12 @@ void Statistics::DeleteLatestSymLink() { std::string latestPath = IOmanager_->getRestartSymLink(); std::string currentDateTime = IOmanager_->getDateTime(); - //Even though eckit::PathName::link is a hardlink and does not work blow exists() follows the link and isLink also works for symlinks and hence the eckit calls can be used here - if ( eckit::PathName{latestPath}.exists() && eckit::PathName{latestPath}.isLink()){ + // Even though eckit::PathName::link is a hardlink and does not work blow exists() follows the link and isLink also + // works for symlinks and hence the eckit calls can be used here + if (eckit::PathName{latestPath}.exists() && eckit::PathName{latestPath}.isLink()) { eckit::PathName{latestPath}.unlink(); - LOG_DEBUG_LIB(LibMultio) << "Deleted old Symlink from " << currentDateTime << " to " - << latestPath << std::endl; + LOG_DEBUG_LIB(LibMultio) << "Deleted old Symlink from " << currentDateTime << " to " << latestPath << std::endl; } - } void Statistics::CreateLatestSymLink() { @@ -208,32 +207,30 @@ void Statistics::CreateLatestSymLink() { // create latest symlink // TODO If eckit allows symlinks for directories instead of hard links it would be good to use eckit // //eckit::PathName::link(eckit::PathName{latestPath},eckit::PathName{IOmanager_->getCurrentDir()}); - symlink(currentDateTime.c_str(),latestPath.c_str()); - LOG_DEBUG_LIB(LibMultio) << "Created Symlink from " << currentDateTime << " to " - << latestPath << std::endl; - + symlink(currentDateTime.c_str(), latestPath.c_str()); + LOG_DEBUG_LIB(LibMultio) << "Created Symlink from " << currentDateTime << " to " << latestPath << std::endl; } -std::unique_ptr Statistics::LoadTemporalStatisticsFromKey( const std::string& key ) { +std::unique_ptr Statistics::LoadTemporalStatisticsFromKey(const std::string& key) { IOmanager_->setDateTime(opt_.restartTime()); IOmanager_->pushDir(key); - if ( !IOmanager_->currentDirExists() ) { + if (!IOmanager_->currentDirExists()) { std::ostringstream os; os << "Unable to find the restart field" << std::endl; throw eckit::SeriousBug(os.str(), Here()); } - std::unique_ptr tmp = std::make_unique( IOmanager_, opt_ ); + std::unique_ptr tmp = std::make_unique(IOmanager_, opt_); IOmanager_->popDir(); return tmp; } -bool Statistics::HasMainRestartDir( ) { +bool Statistics::HasMainRestartDir() { IOmanager_->setDateTime(opt_.restartTime()); bool has_restart = IOmanager_->currentDirExists() ? true : false; return has_restart; } -bool Statistics::HasRestartKey( const std::string& key ) { +bool Statistics::HasRestartKey(const std::string& key) { IOmanager_->setDateTime(opt_.restartTime()); IOmanager_->pushDir(key); bool has_restart = IOmanager_->currentDirExists() ? true : false; @@ -276,11 +273,11 @@ message::Metadata Statistics::outputMetadata(const message::Metadata& inputMetad } -void Statistics::updateLatestDateTime( const StatisticsConfiguration& cfg ){ +void Statistics::updateLatestDateTime(const StatisticsConfiguration& cfg) { std::ostringstream tmp; - tmp << std::setw(8) << std::setfill('0') << cfg.curr().date().yyyymmdd() << "-" - << std::setw(6) << std::setfill('0') << cfg.curr().time().hhmmss(); + tmp << std::setw(8) << std::setfill('0') << cfg.curr().date().yyyymmdd() << "-" << std::setw(6) << std::setfill('0') + << cfg.curr().time().hhmmss(); lastDateTime_ = tmp.str(); needRestart_ = true; @@ -291,7 +288,7 @@ void Statistics::updateLatestDateTime( const StatisticsConfiguration& cfg ){ void Statistics::executeImpl(message::Message msg) { // Handle flush - if (msg.tag() == message::Message::Tag::Flush ) { + if (msg.tag() == message::Message::Tag::Flush) { TryDumpRestart(msg); executeNext(msg); return; @@ -309,10 +306,10 @@ void Statistics::executeImpl(message::Message msg) { // Initialize local variables StatisticsConfiguration cfg{msg, opt_}; std::string key = cfg.key(); - updateLatestDateTime( cfg ); + updateLatestDateTime(cfg); // Check if the main restart directory exists - if ( opt_.readRestart() && !HasMainRestartDir() ) { + if (opt_.readRestart() && !HasMainRestartDir()) { std::ostringstream os; os << "Main restart directory does not exist :: " << opt_.restartPath() << std::endl; throw eckit::SeriousBug(os.str(), Here()); @@ -322,12 +319,13 @@ void Statistics::executeImpl(message::Message msg) { // Access or create the temporal statistics object auto stat = fieldStats_.find(key); - if ( stat == fieldStats_.end() ) { - if ( opt_.readRestart() && HasRestartKey(key) ) { + if (stat == fieldStats_.end()) { + if (opt_.readRestart() && HasRestartKey(key)) { fieldStats_[key] = LoadTemporalStatisticsFromKey(key); } else { - fieldStats_[key] = std::make_unique(outputFrequency_, operations_, msg, IOmanager_, cfg); + fieldStats_[key] + = std::make_unique(outputFrequency_, operations_, msg, IOmanager_, cfg); } // TODO: Reorganize the code to avoid this second search // which is not efficient @@ -339,7 +337,7 @@ void Statistics::executeImpl(message::Message msg) { // This can happen when the solver is sending the initial condition // and and the same point is already present in the restart auto& ts = *(stat->second); - if ( cfg.curr() == ts.cwin().currPoint() && opt_.solver_send_initial_condition() ) { + if (cfg.curr() == ts.cwin().currPoint() && opt_.solver_send_initial_condition()) { return; } @@ -350,11 +348,11 @@ void Statistics::executeImpl(message::Message msg) { // std::cout << os.str() << std::endl; // In any case if the current time is greater than the current point in the window, we have a problem - if ( cfg.curr() <= ts.cwin().currPoint() ) { + if (cfg.curr() <= ts.cwin().currPoint()) { std::ostringstream os; os << "Current time is greater than the current point in the window :: " << cfg.curr().date().yyyymmdd() << " " - << cfg.curr().time().hhmmss() << " " << ts.cwin().currPoint().date().yyyymmdd() << " " << ts.cwin().currPoint().time().hhmmss() - << std::endl; + << cfg.curr().time().hhmmss() << " " << ts.cwin().currPoint().date().yyyymmdd() << " " + << ts.cwin().currPoint().time().hhmmss() << std::endl; throw eckit::SeriousBug(os.str(), Here()); } @@ -372,11 +370,11 @@ void Statistics::executeImpl(message::Message msg) { std::string outputFrequency = compConf_.parsedConfig().getString("output-frequency"); md.set("operation", opname); md.set("operation-frequency", outputFrequency); - remapParamID_.ApplyRemap( md, opname, outputFrequency ); + remapParamID_.ApplyRemap(md, opname, outputFrequency); (*it)->compute(payload, cfg); executeNext(message::Message{message::Message::Header{message::Message::Tag::Field, msg.source(), msg.destination(), message::Metadata{md}}, - std::move(payload)}); + std::move(payload)}); } ts.updateWindow(msg, cfg); @@ -387,8 +385,7 @@ void Statistics::executeImpl(message::Message msg) { void Statistics::print(std::ostream& os) const { - os << "Statistics( output-frequency:" << outputFrequency_ - << ", operations:{"; + os << "Statistics( output-frequency:" << outputFrequency_ << ", operations:{"; bool first = true; for (const auto& ops : operations_) { os << (first ? "" : ", "); diff --git a/src/multio/action/statistics/Statistics.h b/src/multio/action/statistics/Statistics.h index 4d290a4f..be752278 100644 --- a/src/multio/action/statistics/Statistics.h +++ b/src/multio/action/statistics/Statistics.h @@ -17,8 +17,8 @@ #include "PeriodUpdaters.h" -#include "StatisticsIO.h" #include "RemapParamID.h" +#include "StatisticsIO.h" #include "multio/action/ChainedAction.h" #include "multio/action/statistics/cfg/StatisticsOptions.h" @@ -44,12 +44,12 @@ class Statistics : public ChainedAction { std::string generateRestartNameFromFlush(const message::Message& msg) const; void DeleteLatestSymLink(); void CreateLatestSymLink(); - void CreateMainRestartDirectory( const std::string& restartFolderName, bool is_master ); + void CreateMainRestartDirectory(const std::string& restartFolderName, bool is_master); void DumpTemporalStatistics(); std::unique_ptr LoadTemporalStatisticsFromKey(const std::string& key); bool HasRestartKey(const std::string& key); bool HasMainRestartDir(); - void updateLatestDateTime( const StatisticsConfiguration& cfg ); + void updateLatestDateTime(const StatisticsConfiguration& cfg); void print(std::ostream& os) const override; const StatisticsOptions opt_; const std::vector operations_; diff --git a/src/multio/action/statistics/StatisticsIO.cc b/src/multio/action/statistics/StatisticsIO.cc index 2f0576fe..589f3bfa 100644 --- a/src/multio/action/statistics/StatisticsIO.cc +++ b/src/multio/action/statistics/StatisticsIO.cc @@ -98,7 +98,7 @@ void IOBuffer::checkChecksum() const { StatisticsIO::StatisticsIO(const std::string& basePath, const std::string& uniqueID, const std::string& ext) : hasValidDateTime_{false}, basePath_{basePath}, uniqueID_{uniqueID}, ext_{ext}, dateTime_{""}, buffer_{8192, 0} { - if ( ! eckit::PathName{basePath_}.exists() ) { + if (!eckit::PathName{basePath_}.exists()) { std::ostringstream os; os << "ERROR : base path does not exist: " << basePath_; throw eckit::SeriousBug{os.str(), Here()}; @@ -108,7 +108,7 @@ StatisticsIO::StatisticsIO(const std::string& basePath, const std::string& uniqu return; }; -StatisticsIO::~StatisticsIO(){ +StatisticsIO::~StatisticsIO() { buffer_.clear(); return; }; @@ -124,7 +124,7 @@ std::string StatisticsIO::getDateTime() { } std::string StatisticsIO::pushDir(const std::string& directory) { - if ( !hasValidDateTime_ ) { + if (!hasValidDateTime_) { std::ostringstream os; os << "ERROR : no valid datetime found"; throw eckit::SeriousBug{os.str(), Here()}; @@ -149,8 +149,8 @@ IOBuffer StatisticsIO::getBuffer(std::size_t size) { }; -std::vector StatisticsIO::getFiles( ) { - if ( !currentDirExists() ) { +std::vector StatisticsIO::getFiles() { + if (!currentDirExists()) { std::ostringstream os; os << "ERROR : Curret director does not exists: " << getCurrentDir(); throw eckit::SeriousBug{os.str(), Here()}; @@ -159,9 +159,9 @@ std::vector StatisticsIO::getFiles( ) { std::vector files_tmp; std::vector files; std::vector dirs; - eckit::PathName{path}.children( files_tmp, dirs ); - for ( const auto& file : files_tmp ) { - if ( file.extension() != ".txt" ) { + eckit::PathName{path}.children(files_tmp, dirs); + for (const auto& file : files_tmp) { + if (file.extension() != ".txt") { files.push_back(file); LOG_DEBUG_LIB(LibMultio) << "File found :: " << file << ", " << file.extension() << std::endl; } @@ -170,8 +170,8 @@ std::vector StatisticsIO::getFiles( ) { }; -std::vector StatisticsIO::getDirs( ) { - if ( !currentDirExists() ) { +std::vector StatisticsIO::getDirs() { + if (!currentDirExists()) { std::ostringstream os; os << "ERROR : Curret directory does not exists: " << getCurrentDir(); throw eckit::SeriousBug{os.str(), Here()}; @@ -180,7 +180,7 @@ std::vector StatisticsIO::getDirs( ) { std::vector files; std::vector dirs; eckit::PathName path2{path}; - path2.children( files, dirs ); + path2.children(files, dirs); return dirs; }; @@ -192,14 +192,14 @@ std::string StatisticsIO::getUniqueRestartDir() const { }; std::string StatisticsIO::getCurrentDir() const { - if ( !hasValidDateTime_ ) { + if (!hasValidDateTime_) { std::ostringstream os; os << "ERROR : no valid datetime found"; throw eckit::SeriousBug{os.str(), Here()}; } std::ostringstream os; os << basePath_ << "/" << uniqueID_ << "/" << dateTime_; - for ( const auto& dir : path_ ) { + for (const auto& dir : path_) { os << "/" << dir; } return os.str(); @@ -207,7 +207,7 @@ std::string StatisticsIO::getCurrentDir() const { std::string StatisticsIO::getRestartSymLink() const { std::ostringstream os; - os << basePath_ << "/" << uniqueID_<< "/" << "latest"; + os << basePath_ << "/" << uniqueID_ << "/" << "latest"; return os.str(); } @@ -221,7 +221,7 @@ void StatisticsIO::createCurrentDir() const { }; void StatisticsIO::createDateTimeDir() const { - if ( !hasValidDateTime_ ) { + if (!hasValidDateTime_) { std::ostringstream os; os << "ERROR : no valid datetime found"; throw eckit::SeriousBug{os.str(), Here()}; @@ -269,7 +269,7 @@ void StatisticsIOFactory::list(std::ostream& out) { } std::shared_ptr StatisticsIOFactory::build(const std::string& name, const std::string& basePath, - const std::string& uniqueID ) { + const std::string& uniqueID) { std::lock_guard lock{mutex_}; LOG_DEBUG_LIB(LibMultio) << "Looking for StatisticsIOFactory [" << name << "]" << std::endl; diff --git a/src/multio/action/statistics/StatisticsIO.h b/src/multio/action/statistics/StatisticsIO.h index 810a2287..d9f00f2f 100644 --- a/src/multio/action/statistics/StatisticsIO.h +++ b/src/multio/action/statistics/StatisticsIO.h @@ -54,10 +54,10 @@ class StatisticsIO { std::string getDateTime(); - std::string pushDir( const std::string& directory ); - std::string popDir( ); - std::string getRestartSymLink( ) const; //This returns the restart dir for the current plan - std::string getCurrentDir( ) const; + std::string pushDir(const std::string& directory); + std::string popDir(); + std::string getRestartSymLink() const; // This returns the restart dir for the current plan + std::string getCurrentDir() const; std::string getUniqueRestartDir() const; bool currentDirExists() const; void createCurrentDir() const; @@ -68,9 +68,9 @@ class StatisticsIO { std::vector getDirs(); virtual void write(const std::string& name, std::size_t fieldSize, size_t writeSize) = 0; - virtual void readSize(const std::string& name, size_t& readSize ) = 0; - virtual void read(const std::string& name, size_t readSize ) = 0; - virtual void flush ( ) = 0; + virtual void readSize(const std::string& name, size_t& readSize) = 0; + virtual void read(const std::string& name, size_t readSize) = 0; + virtual void flush() = 0; protected: std::string generatePathName() const; @@ -87,12 +87,6 @@ class StatisticsIO { }; - - - - - - class StatisticsIOBuilderBase; class StatisticsIOFactory : private eckit::NonCopyable { @@ -107,7 +101,8 @@ class StatisticsIOFactory : private eckit::NonCopyable { void list(std::ostream&); - std::shared_ptr build(const std::string& name, const std::string& basePath, const std::string& uniqueID); + std::shared_ptr build(const std::string& name, const std::string& basePath, + const std::string& uniqueID); private: // members std::map factories_; diff --git a/src/multio/action/statistics/SynopticCollection.cc b/src/multio/action/statistics/SynopticCollection.cc index b67aa2f9..fdce3462 100644 --- a/src/multio/action/statistics/SynopticCollection.cc +++ b/src/multio/action/statistics/SynopticCollection.cc @@ -114,7 +114,7 @@ SynopticCollection::SynopticCollection(const std::string& operation, const messa const StatisticsConfiguration& cfg) : win_{win}, op_{parseOperationName(operation, filterConf)}, - filter_{op_[0] == op_[1] ? make_filter(op_[1], cfg) : make_filter(op_[1], filterConf.at(op_[0]), cfg)}//, + filter_{op_[0] == op_[1] ? make_filter(op_[1], cfg) : make_filter(op_[1], filterConf.at(op_[0]), cfg)} //, // statistics_{make_operations(op_[2], msg, filter_->size(), IOmanager, win, cfg)} { LOG_DEBUG_LIB(::multio::LibMultio) << cfg.logPrefix() << " *** SynopticCollection standard constructor " @@ -129,7 +129,7 @@ SynopticCollection::SynopticCollection(const std::string& operation, const std:: const StatisticsConfiguration& cfg) : win_{win}, op_{parseOperationName(operation, filterConf)}, - filter_{op_[0] == op_[1] ? make_filter(op_[1], cfg) : make_filter(op_[1], filterConf.at(op_[0]), cfg)}//, + filter_{op_[0] == op_[1] ? make_filter(op_[1], cfg) : make_filter(op_[1], filterConf.at(op_[0]), cfg)} //, // statistics_{load_operations(op_[2], precision, filter_->size(), IOmanager, win, cfg)} { LOG_DEBUG_LIB(::multio::LibMultio) << cfg.logPrefix() << " *** SynopticCollection load constructor " << std::endl; @@ -154,7 +154,7 @@ void SynopticCollection::resetWindow(const message::Message& msg, const Statisti // TODO: add profiling code // TODO: handling the time for a better metadata control // TODO: update data must have control over - statistics_[key]->updateWindow(msg,cfg); + statistics_[key]->updateWindow(msg, cfg); } return; }; @@ -166,7 +166,7 @@ void SynopticCollection::updateData(const message::Message& msg, const Statistic if (filter_->match(msg, cfg, key)) { // TODO: add profiling code // TODO: handling the time for a better metadata control - statistics_[key]->updateData(msg.Payload(),msg.size(),cfg); + statistics_[key]->updateData(msg.Payload(), msg.size(), cfg); } return; }; diff --git a/src/multio/action/statistics/SynopticCollection.h b/src/multio/action/statistics/SynopticCollection.h index f978e602..53a80b42 100644 --- a/src/multio/action/statistics/SynopticCollection.h +++ b/src/multio/action/statistics/SynopticCollection.h @@ -11,9 +11,9 @@ #include "multio/action/statistics/OperationWindow.h" -#include "multio/action/statistics/cfg/StatisticsConfiguration.h" #include "multio/action/statistics/StatisticsIO.h" #include "multio/action/statistics/SynopticFilters.h" +#include "multio/action/statistics/cfg/StatisticsConfiguration.h" #include "multio/action/statistics/operations/Operation.h" #include "multio/config/ComponentConfiguration.h" diff --git a/src/multio/action/statistics/TemporalStatistics.cc b/src/multio/action/statistics/TemporalStatistics.cc index 456f4b2b..8d106dbc 100644 --- a/src/multio/action/statistics/TemporalStatistics.cc +++ b/src/multio/action/statistics/TemporalStatistics.cc @@ -11,14 +11,14 @@ namespace multio::action { -TemporalStatistics::TemporalStatistics(const std::string& output_freq, - const std::vector& operations, const message::Message& msg, - std::shared_ptr& IOmanager, const StatisticsConfiguration& cfg) : +TemporalStatistics::TemporalStatistics(const std::string& output_freq, const std::vector& operations, + const message::Message& msg, std::shared_ptr& IOmanager, + const StatisticsConfiguration& cfg) : periodUpdater_{make_period_updater(output_freq, cfg)}, window_{make_window(periodUpdater_, cfg)}, statistics_{make_operations(operations, msg, IOmanager, window_, cfg)} {} -TemporalStatistics::TemporalStatistics( std::shared_ptr& IOmanager, const StatisticsOptions& opt ) : +TemporalStatistics::TemporalStatistics(std::shared_ptr& IOmanager, const StatisticsOptions& opt) : periodUpdater_{load_period_updater(IOmanager, opt)}, window_{load_window(IOmanager, opt)}, statistics_{load_operations(IOmanager, window_, opt)} { @@ -26,17 +26,17 @@ TemporalStatistics::TemporalStatistics( std::shared_ptr& IOmanager } -void TemporalStatistics::dump(std::shared_ptr& IOmanager, const StatisticsOptions& opt) const { +void TemporalStatistics::dump(std::shared_ptr& IOmanager, const StatisticsOptions& opt) const { LOG_DEBUG_LIB(LibMultio) << opt.logPrefix() << " *** Dump restart files" << std::endl; - IOmanager->pushDir( "periodUpdater" ); + IOmanager->pushDir("periodUpdater"); IOmanager->createCurrentDir(); periodUpdater_->dump(IOmanager, opt); IOmanager->popDir(); - IOmanager->pushDir( "operationWindow" ); + IOmanager->pushDir("operationWindow"); IOmanager->createCurrentDir(); window_.dump(IOmanager, opt); IOmanager->popDir(); - IOmanager->pushDir( "operations" ); + IOmanager->pushDir("operations"); IOmanager->createCurrentDir(); for (auto& stat : statistics_) { stat->dump(IOmanager, opt); diff --git a/src/multio/action/statistics/TemporalStatistics.h b/src/multio/action/statistics/TemporalStatistics.h index bde71c82..d18dd6e0 100644 --- a/src/multio/action/statistics/TemporalStatistics.h +++ b/src/multio/action/statistics/TemporalStatistics.h @@ -20,9 +20,9 @@ class TemporalStatistics { op::iterator begin() { return statistics_.begin(); }; op::iterator end() { return statistics_.end(); }; - TemporalStatistics(const std::string& output_freq, - const std::vector& operations, const message::Message& msg, - std::shared_ptr& IOmanager, const StatisticsConfiguration& cfg); + TemporalStatistics(const std::string& output_freq, const std::vector& operations, + const message::Message& msg, std::shared_ptr& IOmanager, + const StatisticsConfiguration& cfg); TemporalStatistics(std::shared_ptr& IOmanager, const StatisticsOptions& opt); @@ -31,7 +31,7 @@ class TemporalStatistics { void updateData(message::Message& msg, const StatisticsConfiguration& cfg); void updateWindow(const message::Message& msg, const StatisticsConfiguration& cfg); - void dump(std::shared_ptr& IOmanager, const StatisticsOptions& opt) const; + void dump(std::shared_ptr& IOmanager, const StatisticsOptions& opt) const; const OperationWindow& cwin() const; OperationWindow& win(); diff --git a/src/multio/action/statistics/TimeUtils.h b/src/multio/action/statistics/TimeUtils.h index 94abcaee..1e7813cd 100644 --- a/src/multio/action/statistics/TimeUtils.h +++ b/src/multio/action/statistics/TimeUtils.h @@ -1,7 +1,7 @@ #pragma once -#include "multio/action/statistics/cfg/StatisticsConfiguration.h" #include "eckit/types/DateTime.h" +#include "multio/action/statistics/cfg/StatisticsConfiguration.h" #include "multio/message/Message.h" diff --git a/src/multio/action/statistics/cfg/StatisticsConfiguration.cc b/src/multio/action/statistics/cfg/StatisticsConfiguration.cc index f20253e0..46949d63 100644 --- a/src/multio/action/statistics/cfg/StatisticsConfiguration.cc +++ b/src/multio/action/statistics/cfg/StatisticsConfiguration.cc @@ -34,40 +34,40 @@ StatisticsConfiguration::StatisticsConfiguration(const message::Message& msg, co logPrefix_{opt_.logPrefix()} { // Associate local procedure pointers - computeEpoch_ = std::bind(&StatisticsConfiguration::computeEpoch, this); - computePrev_ = std::bind(&StatisticsConfiguration::computePrev, this); - computeCurr_ = std::bind(&StatisticsConfiguration::computeCurr, this); - computeNext_ = std::bind(&StatisticsConfiguration::computeNext, this); + computeEpoch_ = std::bind(&StatisticsConfiguration::computeEpoch, this); + computePrev_ = std::bind(&StatisticsConfiguration::computePrev, this); + computeCurr_ = std::bind(&StatisticsConfiguration::computeCurr, this); + computeNext_ = std::bind(&StatisticsConfiguration::computeNext, this); computeWinStart_ = std::bind(&StatisticsConfiguration::computeWinStart, this); - computeBeginningOfHour_ = std::bind(&StatisticsConfiguration::computeBeginningOfHour, this); - computeBeginningOfDay_ = std::bind(&StatisticsConfiguration::computeBeginningOfDay, this); + computeBeginningOfHour_ = std::bind(&StatisticsConfiguration::computeBeginningOfHour, this); + computeBeginningOfDay_ = std::bind(&StatisticsConfiguration::computeBeginningOfDay, this); computeBeginningOfMonth_ = std::bind(&StatisticsConfiguration::computeBeginningOfMonth, this); - computeBeginningOfYear_ = std::bind(&StatisticsConfiguration::computeBeginningOfYear, this); + computeBeginningOfYear_ = std::bind(&StatisticsConfiguration::computeBeginningOfYear, this); // Extraact message metadata const auto& md = msg.metadata(); // Read the metadata - readStartDate( md, opt ); - readStartTime( md, opt ); - readStep( md, opt ); - readTimeStep( md, opt ); - readStepFrequency( md, opt ); - readLevel( md, opt ); - readParam( md, opt ); - readLevType( md, opt ); - readGridType( md, opt ); - readPrecision( md, opt ); - readMissingValue( md, opt ); + readStartDate(md, opt); + readStartTime(md, opt); + readStep(md, opt); + readTimeStep(md, opt); + readStepFrequency(md, opt); + readLevel(md, opt); + readParam(md, opt); + readLevType(md, opt); + readGridType(md, opt); + readPrecision(md, opt); + readMissingValue(md, opt); // Generate Key - generateKey( md, std::to_string(std::hash{}(msg.source())) ); + generateKey(md, std::to_string(std::hash{}(msg.source()))); return; }; -const std::string& StatisticsConfiguration::key() const{ +const std::string& StatisticsConfiguration::key() const { return key_; }; @@ -94,8 +94,9 @@ void StatisticsConfiguration::readGridType(const message::Metadata& md, const St void StatisticsConfiguration::readLevType(const message::Metadata& md, const StatisticsOptions& opt) { if (auto levType = md.getOpt(glossary().levtype); levType) { levType_ = *levType; - } else if ( auto category = md.getOpt(glossary().category); category) { - levType_ = *category;//TODO this needs proper handling once category is changed to levtype + } + else if (auto category = md.getOpt(glossary().category); category) { + levType_ = *category; // TODO this needs proper handling once category is changed to levtype } else { throw eckit::SeriousBug{"LevType metadata not present", Here()}; @@ -192,15 +193,10 @@ void StatisticsConfiguration::readMissingValue(const message::Metadata& md, cons }; -void StatisticsConfiguration::generateKey( const message::Metadata& md, const std::string& src ){ +void StatisticsConfiguration::generateKey(const message::Metadata& md, const std::string& src) { std::ostringstream os; - os << param_ << "-" - << level_ << "-" - << levType_ << "-" - << gridType_ << "-" - << precision_ << "-" - << src; + os << param_ << "-" << level_ << "-" << levType_ << "-" << gridType_ << "-" << precision_ << "-" << src; key_ = os.str(); // Exit point @@ -209,53 +205,51 @@ void StatisticsConfiguration::generateKey( const message::Metadata& md, const st eckit::DateTime StatisticsConfiguration::epoch() const { - epoch_ = computeEpoch_( ); + epoch_ = computeEpoch_(); return epoch_; }; -eckit::DateTime StatisticsConfiguration::prev()const { - prev_ = computePrev_( ); +eckit::DateTime StatisticsConfiguration::prev() const { + prev_ = computePrev_(); return prev_; }; -eckit::DateTime StatisticsConfiguration::curr()const { - curr_ = computeCurr_( ); +eckit::DateTime StatisticsConfiguration::curr() const { + curr_ = computeCurr_(); return curr_; }; -eckit::DateTime StatisticsConfiguration::next()const { - next_ = computeNext_( ); +eckit::DateTime StatisticsConfiguration::next() const { + next_ = computeNext_(); return next_; }; -eckit::DateTime StatisticsConfiguration::winStart()const { - winStart_ = computeWinStart_( ); +eckit::DateTime StatisticsConfiguration::winStart() const { + winStart_ = computeWinStart_(); return winStart_; }; -bool StatisticsConfiguration::beginningOfHour()const { - auto ret = computeBeginningOfHour_( ); +bool StatisticsConfiguration::beginningOfHour() const { + auto ret = computeBeginningOfHour_(); return ret; }; -bool StatisticsConfiguration::beginningOfDay()const { - auto ret = computeBeginningOfDay_( ); +bool StatisticsConfiguration::beginningOfDay() const { + auto ret = computeBeginningOfDay_(); return ret; }; -bool StatisticsConfiguration::beginningOfMonth()const { - auto ret = computeBeginningOfMonth_( ); +bool StatisticsConfiguration::beginningOfMonth() const { + auto ret = computeBeginningOfMonth_(); return ret; }; -bool StatisticsConfiguration::beginningOfYear()const { - auto ret = computeBeginningOfYear_( ); +bool StatisticsConfiguration::beginningOfYear() const { + auto ret = computeBeginningOfYear_(); return ret; }; - - eckit::DateTime StatisticsConfiguration::computeEpoch() const { eckit::Date startDate{date_}; auto hour = time_ / 10000; @@ -265,59 +259,57 @@ eckit::DateTime StatisticsConfiguration::computeEpoch() const { return epoch_; }; -eckit::DateTime StatisticsConfiguration::getEpoch() const{ +eckit::DateTime StatisticsConfiguration::getEpoch() const { return epoch_; }; -eckit::DateTime StatisticsConfiguration::computePrev() const{ +eckit::DateTime StatisticsConfiguration::computePrev() const { prev_ = epoch() + static_cast(std::max((step_ - stepFreq_), 0L) * timeStep_); computePrev_ = std::bind(&StatisticsConfiguration::getPrev, this); return prev_; }; -eckit::DateTime StatisticsConfiguration::getPrev() const{ +eckit::DateTime StatisticsConfiguration::getPrev() const { return prev_; }; -eckit::DateTime StatisticsConfiguration::computeCurr() const{ +eckit::DateTime StatisticsConfiguration::computeCurr() const { curr_ = epoch() + static_cast(std::max((step_), 0L) * timeStep_); computeCurr_ = std::bind(&StatisticsConfiguration::getCurr, this); return curr_; }; -eckit::DateTime StatisticsConfiguration::getCurr() const{ +eckit::DateTime StatisticsConfiguration::getCurr() const { return curr_; }; -eckit::DateTime StatisticsConfiguration::computeNext() const{ +eckit::DateTime StatisticsConfiguration::computeNext() const { next_ = epoch() + static_cast((step_ + stepFreq_) * timeStep_); computeNext_ = std::bind(&StatisticsConfiguration::getNext, this); return next_; }; -eckit::DateTime StatisticsConfiguration::getNext() const{ +eckit::DateTime StatisticsConfiguration::getNext() const { return next_; }; -eckit::DateTime StatisticsConfiguration::computeWinStart() const{ +eckit::DateTime StatisticsConfiguration::computeWinStart() const { winStart_ = opt_.solver_send_initial_condition() ? curr() : prev(); computeWinStart_ = std::bind(&StatisticsConfiguration::getWinStart, this); return winStart_; }; -eckit::DateTime StatisticsConfiguration::getWinStart() const{ +eckit::DateTime StatisticsConfiguration::getWinStart() const { return winStart_; }; - - bool StatisticsConfiguration::bitmapPresent() const { - return bitmapPresent_; + return bitmapPresent_; }; @@ -331,12 +323,8 @@ std::string StatisticsConfiguration::logPrefix() const { }; - - - - -bool StatisticsConfiguration::computeBeginningOfHour() const{ - eckit::DateTime now = curr(); +bool StatisticsConfiguration::computeBeginningOfHour() const { + eckit::DateTime now = curr(); long min = now.time().minutes(); long sec = now.time().seconds(); beginningOfHour_ = (min == 0 && sec == 0); @@ -344,26 +332,26 @@ bool StatisticsConfiguration::computeBeginningOfHour() const{ return beginningOfHour_; }; -bool StatisticsConfiguration::isBeginningOfHour () const{ +bool StatisticsConfiguration::isBeginningOfHour() const { return beginningOfHour_; }; -bool StatisticsConfiguration::computeBeginningOfDay() const{ - eckit::DateTime now = curr(); +bool StatisticsConfiguration::computeBeginningOfDay() const { + eckit::DateTime now = curr(); long hour = now.time().hours(); long min = now.time().minutes(); long sec = now.time().seconds(); - beginningOfDay_= (hour == 0 && min == 0 && sec == 0); + beginningOfDay_ = (hour == 0 && min == 0 && sec == 0); computeBeginningOfDay_ = std::bind(&StatisticsConfiguration::isBeginningOfDay, this); return beginningOfDay_; }; -bool StatisticsConfiguration::isBeginningOfDay () const{ +bool StatisticsConfiguration::isBeginningOfDay() const { return beginningOfDay_; }; -bool StatisticsConfiguration::computeBeginningOfMonth() const{ - eckit::DateTime now = curr(); +bool StatisticsConfiguration::computeBeginningOfMonth() const { + eckit::DateTime now = curr(); long day = now.date().day(); long hour = now.time().hours(); long min = now.time().minutes(); @@ -373,12 +361,12 @@ bool StatisticsConfiguration::computeBeginningOfMonth() const{ return beginningOfMonth_; }; -bool StatisticsConfiguration::isBeginningOfMonth () const{ +bool StatisticsConfiguration::isBeginningOfMonth() const { return beginningOfMonth_; }; -bool StatisticsConfiguration::computeBeginningOfYear() const{ - eckit::DateTime now = curr(); +bool StatisticsConfiguration::computeBeginningOfYear() const { + eckit::DateTime now = curr(); long month = now.date().month(); long day = now.date().day(); long hour = now.time().hours(); @@ -389,21 +377,43 @@ bool StatisticsConfiguration::computeBeginningOfYear() const{ return beginningOfYear_; }; -bool StatisticsConfiguration::isBeginningOfYear () const{ +bool StatisticsConfiguration::isBeginningOfYear() const { return beginningOfYear_; }; -long StatisticsConfiguration::date() const { return date_; }; -long StatisticsConfiguration::time() const {return time_; }; -long StatisticsConfiguration::level() const { return level_; }; -long StatisticsConfiguration::timeStep() const { return timeStep_; }; -long StatisticsConfiguration::stepFreq() const { return stepFreq_; }; -long StatisticsConfiguration::step() const { return step_; }; +long StatisticsConfiguration::date() const { + return date_; +}; +long StatisticsConfiguration::time() const { + return time_; +}; +long StatisticsConfiguration::level() const { + return level_; +}; +long StatisticsConfiguration::timeStep() const { + return timeStep_; +}; +long StatisticsConfiguration::stepFreq() const { + return stepFreq_; +}; +long StatisticsConfiguration::step() const { + return step_; +}; -std::string StatisticsConfiguration::param() const { return param_; }; -std::string StatisticsConfiguration::levType() const { return levType_; }; -std::string StatisticsConfiguration::gridType() const { return gridType_; }; -std::string StatisticsConfiguration::precision() const { return precision_; }; -const StatisticsOptions& StatisticsConfiguration::options() const { return opt_; }; -} \ No newline at end of file +std::string StatisticsConfiguration::param() const { + return param_; +}; +std::string StatisticsConfiguration::levType() const { + return levType_; +}; +std::string StatisticsConfiguration::gridType() const { + return gridType_; +}; +std::string StatisticsConfiguration::precision() const { + return precision_; +}; +const StatisticsOptions& StatisticsConfiguration::options() const { + return opt_; +}; +} // namespace multio::action \ No newline at end of file diff --git a/src/multio/action/statistics/cfg/StatisticsConfiguration.h b/src/multio/action/statistics/cfg/StatisticsConfiguration.h index 4e8f8548..2140babd 100644 --- a/src/multio/action/statistics/cfg/StatisticsConfiguration.h +++ b/src/multio/action/statistics/cfg/StatisticsConfiguration.h @@ -1,18 +1,17 @@ #pragma once -#include "multio/action/statistics/cfg/StatisticsOptions.h" #include "eckit/config/LocalConfiguration.h" #include "eckit/types/DateTime.h" +#include "multio/action/Action.h" +#include "multio/action/statistics/cfg/StatisticsOptions.h" #include "multio/config/ComponentConfiguration.h" #include "multio/message/Message.h" -#include "multio/action/Action.h" namespace multio::action { class StatisticsConfiguration { private: - // Options const StatisticsOptions& opt_; @@ -31,7 +30,7 @@ class StatisticsConfiguration { std::string logPrefix_; // Handle missing values - bool bitmapPresent_; + bool bitmapPresent_; double missingValue_; // Unique key used for statistics map @@ -76,15 +75,15 @@ class StatisticsConfiguration { eckit::DateTime getWinStart() const; bool computeBeginningOfHour() const; - bool isBeginningOfHour () const; + bool isBeginningOfHour() const; bool computeBeginningOfDay() const; - bool isBeginningOfDay () const; + bool isBeginningOfDay() const; bool computeBeginningOfMonth() const; - bool isBeginningOfMonth () const; + bool isBeginningOfMonth() const; bool computeBeginningOfYear() const; - bool isBeginningOfYear () const; + bool isBeginningOfYear() const; - void generateKey( const message::Metadata& md, const std::string& src ); + void generateKey(const message::Metadata& md, const std::string& src); void readPrecision(const message::Metadata& md, const StatisticsOptions& opt); void readGridType(const message::Metadata& md, const StatisticsOptions& opt); @@ -99,10 +98,8 @@ class StatisticsConfiguration { void readMissingValue(const message::Metadata& md, const StatisticsOptions& opt); - public: - - StatisticsConfiguration( const message::Message& msg, const StatisticsOptions& opt ); + StatisticsConfiguration(const message::Message& msg, const StatisticsOptions& opt); const StatisticsOptions& options() const; const std::string& key() const; @@ -135,7 +132,6 @@ class StatisticsConfiguration { bool beginningOfDay() const; bool beginningOfMonth() const; bool beginningOfYear() const; - }; -} \ No newline at end of file +} // namespace multio::action \ No newline at end of file diff --git a/src/multio/action/statistics/cfg/StatisticsOptions.cc b/src/multio/action/statistics/cfg/StatisticsOptions.cc index 07c7460c..51ed9b55 100644 --- a/src/multio/action/statistics/cfg/StatisticsOptions.cc +++ b/src/multio/action/statistics/cfg/StatisticsOptions.cc @@ -4,7 +4,7 @@ namespace multio::action { -StatisticsOptions::StatisticsOptions( const config::ComponentConfiguration& compConf ): +StatisticsOptions::StatisticsOptions(const config::ComponentConfiguration& compConf) : stepFreq_{1}, timeStep_{3600}, solverSendInitStep_{false}, @@ -13,14 +13,13 @@ StatisticsOptions::StatisticsOptions( const config::ComponentConfiguration& comp debugRestart_{false}, useDateTime_{false}, clientSideStatistics_{false}, - restartTime_{"latest"},//00000000-000000 + restartTime_{"latest"}, // 00000000-000000 restartPath_{"."}, restartPrefix_{"StatisticsRestartFile"}, restartLib_{"fstream_io"}, logPrefix_{"Plan"}, windowType_{"forward-offset"}, - accumulatedFieldsResetFreqency_{"month"} -{ + accumulatedFieldsResetFreqency_{"month"} { // Dump usage if (compConf.parsedConfig().has("help")) { usage(); @@ -29,7 +28,7 @@ StatisticsOptions::StatisticsOptions( const config::ComponentConfiguration& comp // Read the options - if ( compConf.parsedConfig().has("options") ) { + if (compConf.parsedConfig().has("options")) { const auto& options = compConf.parsedConfig().getSubConfiguration("options"); parseUseDateTime(options); parseStepFrequency(options); @@ -58,7 +57,7 @@ StatisticsOptions::StatisticsOptions( const config::ComponentConfiguration& comp void StatisticsOptions::parseUseDateTime(const eckit::LocalConfiguration& cfg) { // Distance in steps between two messages - useDateTime_ = cfg.getLong("use-current-time", false ); + useDateTime_ = cfg.getLong("use-current-time", false); return; }; @@ -155,7 +154,7 @@ void StatisticsOptions::parseRestartPath(const config::ComponentConfiguration& c if (!path.exists() || !path.isDir()) { std::ostringstream os; os << "Restart path does not exist :: " << restartPath_ << std::endl; - throw eckit::UserError{ os.str(), Here()}; + throw eckit::UserError{os.str(), Here()}; } } return; @@ -174,7 +173,7 @@ void StatisticsOptions::parseRestartTime(const config::ComponentConfiguration& c void StatisticsOptions::parseRestartPrefix(const config::ComponentConfiguration& compConf, - const eckit::LocalConfiguration& cfg) { + const eckit::LocalConfiguration& cfg) { // Prefix used for the restart file names in order // to make the file name unique across different plans restartPrefix_ = compConf.multioConfig().replaceCurly(cfg.getString("restart-prefix", "StatisticsDump")); @@ -187,14 +186,16 @@ void StatisticsOptions::parseRestartLib(const eckit::LocalConfiguration& cfg) { }; -void StatisticsOptions::parseLogPrefix(const config::ComponentConfiguration& compConf, const eckit::LocalConfiguration& cfg) { +void StatisticsOptions::parseLogPrefix(const config::ComponentConfiguration& compConf, + const eckit::LocalConfiguration& cfg) { logPrefix_ = cfg.getString("log-prefix", "Plan"); return; }; -void StatisticsOptions::parseWindowType(const config::ComponentConfiguration& compConf, const eckit::LocalConfiguration& cfg) { +void StatisticsOptions::parseWindowType(const config::ComponentConfiguration& compConf, + const eckit::LocalConfiguration& cfg) { windowType_ = cfg.getString("window-type", "forward-offset"); - if ( windowType_ != "forward-offset" && windowType_ != "backward-offset" ) { + if (windowType_ != "forward-offset" && windowType_ != "backward-offset") { std::ostringstream os; os << "Invalid window type :: " << windowType_ << std::endl; throw eckit::UserError(os.str(), Here()); @@ -203,7 +204,7 @@ void StatisticsOptions::parseWindowType(const config::ComponentConfiguration& co }; void StatisticsOptions::parseSolverResetAccumulatedFields(const config::ComponentConfiguration& compConf, - const eckit::LocalConfiguration& cfg) { + const eckit::LocalConfiguration& cfg) { // Used in the deaccumulate action to not deaccumulate twice accumulatedFieldsResetFreqency_ = cfg.getString("solver-reset-accumulate-fields-every", "month"); @@ -218,8 +219,7 @@ void StatisticsOptions::parseSolverResetAccumulatedFields(const config::Componen }; -const std::string& StatisticsOptions::logPrefix() const -{ +const std::string& StatisticsOptions::logPrefix() const { return logPrefix_; }; @@ -287,8 +287,6 @@ const std::string& StatisticsOptions::solverResetAccumulatedFields() const { }; - - void StatisticsOptions::dumpOptions() { // TODO: Implement this function return; @@ -300,4 +298,4 @@ void StatisticsOptions::usage() { return; } -} +} // namespace multio::action diff --git a/src/multio/action/statistics/cfg/StatisticsOptions.h b/src/multio/action/statistics/cfg/StatisticsOptions.h index 5d26f7b1..599ef1e4 100644 --- a/src/multio/action/statistics/cfg/StatisticsOptions.h +++ b/src/multio/action/statistics/cfg/StatisticsOptions.h @@ -3,9 +3,9 @@ #include #include "eckit/config/LocalConfiguration.h" +#include "multio/action/Action.h" #include "multio/config/ComponentConfiguration.h" #include "multio/message/Message.h" -#include "multio/action/Action.h" namespace multio::action { @@ -13,9 +13,8 @@ namespace multio::action { /* * This class handle all the statistics configurations under the option keyword in the yaml */ -class StatisticsOptions{ +class StatisticsOptions { private: - // Default values for configurations long stepFreq_; long timeStep_; @@ -35,7 +34,6 @@ class StatisticsOptions{ std::string accumulatedFieldsResetFreqency_; private: - void parseUseDateTime(const eckit::LocalConfiguration& cfg); void parseCheckMissingValues(const eckit::LocalConfiguration& cfg); void parseStepFrequency(const eckit::LocalConfiguration& cfg); @@ -58,8 +56,7 @@ class StatisticsOptions{ void usage(); public: - - StatisticsOptions( const config::ComponentConfiguration& compConf ); + StatisticsOptions(const config::ComponentConfiguration& compConf); const std::string& logPrefix() const; const std::string& windowType() const; @@ -83,7 +80,6 @@ class StatisticsOptions{ const std::string& restartPrefix() const; const std::string& restartLib() const; const std::string& solverResetAccumulatedFields() const; - }; -} +} // namespace multio::action diff --git a/src/multio/action/statistics/io/AtlasIO.cc b/src/multio/action/statistics/io/AtlasIO.cc index 4dab8afa..c7ece265 100644 --- a/src/multio/action/statistics/io/AtlasIO.cc +++ b/src/multio/action/statistics/io/AtlasIO.cc @@ -72,7 +72,7 @@ void AtlasIO::readSize(const std::string& name, std::size_t& readSize) { checkFileExist(fname); std::uint64_t sz; atlas::io::RecordReader record(fname); - record.read("size", sz ).wait(); + record.read("size", sz).wait(); readSize = static_cast(sz); return; }; diff --git a/src/multio/action/statistics/operations/Accumulate.h b/src/multio/action/statistics/operations/Accumulate.h index f735b05a..42841ee9 100644 --- a/src/multio/action/statistics/operations/Accumulate.h +++ b/src/multio/action/statistics/operations/Accumulate.h @@ -21,7 +21,7 @@ class Accumulate final : public OperationWithData { OperationWithData{name, "accumulate", sz, true, win, cfg} {} Accumulate(const std::string& name, const OperationWindow& win, std::shared_ptr& IOmanager, - const StatisticsOptions& opt) : + const StatisticsOptions& opt) : OperationWithData{name, "accumulate", true, win, IOmanager, opt} {}; void compute(eckit::Buffer& buf, const StatisticsConfiguration& cfg) override { @@ -32,10 +32,10 @@ class Accumulate final : public OperationWithData { } void updateData(const void* data, long sz, const StatisticsConfiguration& cfg) override { - checkSize(sz,cfg); + checkSize(sz, cfg); LOG_DEBUG_LIB(LibMultio) << logHeader_ << ".update().count=" << win_.count() << std::endl; const T* val = static_cast(data); - cfg.bitmapPresent() ? updateWithMissing(val,cfg) : updateWithoutMissing(val,cfg); + cfg.bitmapPresent() ? updateWithMissing(val, cfg) : updateWithoutMissing(val, cfg); return; } diff --git a/src/multio/action/statistics/operations/Average.h b/src/multio/action/statistics/operations/Average.h index 0a290aae..5b9bfff0 100644 --- a/src/multio/action/statistics/operations/Average.h +++ b/src/multio/action/statistics/operations/Average.h @@ -20,7 +20,7 @@ class Average final : public OperationWithData { OperationWithData{name, "average", sz, true, win, cfg} {} Average(const std::string& name, const OperationWindow& win, std::shared_ptr& IOmanager, - const StatisticsOptions& opt) : + const StatisticsOptions& opt) : OperationWithData{name, "average", true, win, IOmanager, opt} {}; void compute(eckit::Buffer& buf, const StatisticsConfiguration& cfg) override { @@ -31,10 +31,10 @@ class Average final : public OperationWithData { } void updateData(const void* data, long sz, const StatisticsConfiguration& cfg) override { - checkSize(sz,cfg); + checkSize(sz, cfg); LOG_DEBUG_LIB(LibMultio) << logHeader_ << ".update().count=" << win_.count() << std::endl; const T* val = static_cast(data); - cfg.bitmapPresent() ? updateWithMissing(val,cfg) : updateWithoutMissing(val,cfg); + cfg.bitmapPresent() ? updateWithMissing(val, cfg) : updateWithoutMissing(val, cfg); return; } diff --git a/src/multio/action/statistics/operations/DeAccumulate.h b/src/multio/action/statistics/operations/DeAccumulate.h index 3aad7f45..46c5306c 100644 --- a/src/multio/action/statistics/operations/DeAccumulate.h +++ b/src/multio/action/statistics/operations/DeAccumulate.h @@ -22,22 +22,22 @@ class DeAccumulate final : public OperationWithDeaccumulatedData { OperationWithDeaccumulatedData{name, "accumulate", sz, true, win, cfg} {} DeAccumulate(const std::string& name, const OperationWindow& win, std::shared_ptr& IOmanager, - const StatisticsOptions& opt) : + const StatisticsOptions& opt) : OperationWithDeaccumulatedData{name, "accumulate", true, win, IOmanager, opt} {}; void compute(eckit::Buffer& buf, const StatisticsConfiguration& cfg) override { checkTimeInterval(cfg); LOG_DEBUG_LIB(LibMultio) << logHeader_ << ".compute().count=" << win_.count() << std::endl; auto val = static_cast(buf.data()); - cfg.bitmapPresent() ? computeWithMissing(val,cfg) : computeWithoutMissing(val,cfg); + cfg.bitmapPresent() ? computeWithMissing(val, cfg) : computeWithoutMissing(val, cfg); return; } void updateData(const void* data, long sz, const StatisticsConfiguration& cfg) override { - checkSize(sz,cfg); + checkSize(sz, cfg); LOG_DEBUG_LIB(LibMultio) << logHeader_ << ".update().count=" << win_.count() << std::endl; const T* val = static_cast(data); - cfg.bitmapPresent() ? updateWithMissing(val,cfg) : updateWithoutMissing(val,cfg); + cfg.bitmapPresent() ? updateWithMissing(val, cfg) : updateWithoutMissing(val, cfg); return; } diff --git a/src/multio/action/statistics/operations/FixedWindowFluxAverage.h b/src/multio/action/statistics/operations/FixedWindowFluxAverage.h index 668424c7..ce171eb8 100644 --- a/src/multio/action/statistics/operations/FixedWindowFluxAverage.h +++ b/src/multio/action/statistics/operations/FixedWindowFluxAverage.h @@ -21,20 +21,20 @@ class FixedWindowFluxAverage final : public OperationWithDeaccumulatedData { const StatisticsConfiguration& cfg) : OperationWithDeaccumulatedData{name, "average", sz, true, win, cfg} {} - FixedWindowFluxAverage(const std::string& name, const OperationWindow& win, std::shared_ptr& IOmanager, - const StatisticsOptions& opt) : + FixedWindowFluxAverage(const std::string& name, const OperationWindow& win, + std::shared_ptr& IOmanager, const StatisticsOptions& opt) : OperationWithDeaccumulatedData{name, "average", true, win, IOmanager, opt} {}; void compute(eckit::Buffer& buf, const StatisticsConfiguration& cfg) override { checkTimeInterval(cfg); LOG_DEBUG_LIB(LibMultio) << logHeader_ << ".compute().count=" << win_.count() << std::endl; auto val = static_cast(buf.data()); - cfg.bitmapPresent() ? computeWithMissing(val,cfg) : computeWithoutMissing(val,cfg); + cfg.bitmapPresent() ? computeWithMissing(val, cfg) : computeWithoutMissing(val, cfg); return; } void updateData(const void* data, long sz, const StatisticsConfiguration& cfg) override { - checkSize(sz,cfg); + checkSize(sz, cfg); LOG_DEBUG_LIB(LibMultio) << logHeader_ << ".update().count=" << win_.count() << std::endl; const T* val = static_cast(data); std::copy(val, val + (sz / sizeof(T)), values_.begin()); @@ -44,16 +44,14 @@ class FixedWindowFluxAverage final : public OperationWithDeaccumulatedData { private: void computeWithMissing(T* buf, const StatisticsConfiguration& cfg) { const double m = cfg.missingValue(); - const double c - = static_cast(1.0) / static_cast(win_.count() * cfg.stepFreq() * cfg.timeStep()); + const double c = static_cast(1.0) / static_cast(win_.count() * cfg.stepFreq() * cfg.timeStep()); std::transform(values_.begin(), values_.end(), initValues_.begin(), buf, [c, m](T v1, T v2) { return static_cast(m == v1 ? m : (v1 - v2) * c); }); return; } void computeWithoutMissing(T* buf, const StatisticsConfiguration& cfg) { - const double c - = static_cast(1.0) / static_cast(win_.count() * cfg.stepFreq() * cfg.timeStep()); + const double c = static_cast(1.0) / static_cast(win_.count() * cfg.stepFreq() * cfg.timeStep()); std::transform(values_.begin(), values_.end(), initValues_.begin(), buf, [c](T v1, T v2) { return static_cast((v1 - v2) * c); }); return; diff --git a/src/multio/action/statistics/operations/FluxAverage.h b/src/multio/action/statistics/operations/FluxAverage.h index 68a476b2..833a3000 100644 --- a/src/multio/action/statistics/operations/FluxAverage.h +++ b/src/multio/action/statistics/operations/FluxAverage.h @@ -27,12 +27,12 @@ class FluxAverage final : public OperationWithData { checkTimeInterval(cfg); LOG_DEBUG_LIB(LibMultio) << logHeader_ << ".compute().count=" << win_.count() << std::endl; auto val = static_cast(buf.data()); - cfg.bitmapPresent() ? computeWithMissing(val,cfg) : computeWithoutMissing(val,cfg); + cfg.bitmapPresent() ? computeWithMissing(val, cfg) : computeWithoutMissing(val, cfg); return; } void updateData(const void* data, long sz, const StatisticsConfiguration& cfg) override { - checkSize(sz,cfg); + checkSize(sz, cfg); LOG_DEBUG_LIB(LibMultio) << logHeader_ << ".update().count=" << win_.count() << std::endl; const T* val = static_cast(data); std::copy(val, val + (sz / sizeof(T)), values_.begin()); @@ -42,15 +42,13 @@ class FluxAverage final : public OperationWithData { private: void computeWithMissing(T* buf, const StatisticsConfiguration& cfg) { const double m = cfg.missingValue(); - const double c - = static_cast(1.0) / static_cast(win_.count() * cfg.stepFreq() * cfg.timeStep()); + const double c = static_cast(1.0) / static_cast(win_.count() * cfg.stepFreq() * cfg.timeStep()); std::transform(values_.begin(), values_.end(), buf, [c, m](T v) { return static_cast(m == v ? m : v * c); }); return; } void computeWithoutMissing(T* buf, const StatisticsConfiguration& cfg) { - const double c - = static_cast(1.0) / static_cast(win_.count() * cfg.stepFreq() * cfg.timeStep()); + const double c = static_cast(1.0) / static_cast(win_.count() * cfg.stepFreq() * cfg.timeStep()); std::transform(values_.begin(), values_.end(), buf, [c](T v) { return static_cast(v * c); }); return; } diff --git a/src/multio/action/statistics/operations/Instant.h b/src/multio/action/statistics/operations/Instant.h index 30bebc3e..09b54233 100644 --- a/src/multio/action/statistics/operations/Instant.h +++ b/src/multio/action/statistics/operations/Instant.h @@ -32,7 +32,7 @@ class Instant final : public OperationWithData { } void updateData(const void* data, long sz, const StatisticsConfiguration& cfg) override { - checkSize(sz,cfg); + checkSize(sz, cfg); LOG_DEBUG_LIB(LibMultio) << logHeader_ << ".update().count=" << win_.count() << std::endl; const T* val = static_cast(data); std::copy(val, val + (sz / sizeof(T)), values_.begin()); diff --git a/src/multio/action/statistics/operations/Maximum.h b/src/multio/action/statistics/operations/Maximum.h index 9c8cbeed..7d1feb14 100644 --- a/src/multio/action/statistics/operations/Maximum.h +++ b/src/multio/action/statistics/operations/Maximum.h @@ -32,10 +32,10 @@ class Maximum final : public OperationWithData { }; void updateData(const void* data, long sz, const StatisticsConfiguration& cfg) override { - checkSize(sz,cfg); + checkSize(sz, cfg); LOG_DEBUG_LIB(LibMultio) << logHeader_ << ".update().count=" << win_.count() << std::endl; const T* val = static_cast(data); - cfg.bitmapPresent() ? updateWithMissing(val,cfg) : updateWithoutMissing(val,cfg); + cfg.bitmapPresent() ? updateWithMissing(val, cfg) : updateWithoutMissing(val, cfg); return; }; diff --git a/src/multio/action/statistics/operations/Minimum.h b/src/multio/action/statistics/operations/Minimum.h index a526c157..3fa224ec 100644 --- a/src/multio/action/statistics/operations/Minimum.h +++ b/src/multio/action/statistics/operations/Minimum.h @@ -32,10 +32,10 @@ class Minimum final : public OperationWithData { } void updateData(const void* data, long sz, const StatisticsConfiguration& cfg) override { - checkSize(sz,cfg); + checkSize(sz, cfg); LOG_DEBUG_LIB(LibMultio) << logHeader_ << ".update().count=" << win_.count() << std::endl; const T* val = static_cast(data); - cfg.bitmapPresent() ? updateWithMissing(val,cfg) : updateWithoutMissing(val,cfg); + cfg.bitmapPresent() ? updateWithMissing(val, cfg) : updateWithoutMissing(val, cfg); return; } diff --git a/src/multio/action/statistics/operations/Operation.h b/src/multio/action/statistics/operations/Operation.h index bcd0cbcb..9dadcc3e 100644 --- a/src/multio/action/statistics/operations/Operation.h +++ b/src/multio/action/statistics/operations/Operation.h @@ -4,9 +4,9 @@ #include #include "multio/action/statistics/OperationWindow.h" +#include "multio/action/statistics/StatisticsIO.h" #include "multio/action/statistics/cfg/StatisticsConfiguration.h" #include "multio/action/statistics/cfg/StatisticsOptions.h" -#include "multio/action/statistics/StatisticsIO.h" namespace multio::action { @@ -23,7 +23,8 @@ class Operation { virtual void updateData(const void* data, long sz, const StatisticsConfiguration& cfg) = 0; - virtual void updateWindow(const void* data, long sz, const message::Message& msg, const StatisticsConfiguration& cfg) + virtual void updateWindow(const void* data, long sz, const message::Message& msg, + const StatisticsConfiguration& cfg) = 0; virtual void updateWindow(const message::Message& msg, const StatisticsConfiguration& cfg) = 0; diff --git a/src/multio/action/statistics/operations/OperationWithData.h b/src/multio/action/statistics/operations/OperationWithData.h index e097b37d..eff014aa 100644 --- a/src/multio/action/statistics/operations/OperationWithData.h +++ b/src/multio/action/statistics/operations/OperationWithData.h @@ -21,14 +21,12 @@ class OperationWithData : public Operation { OperationWithData(const std::string& name, const std::string& operation, bool needRestart, const OperationWindow& win, std::shared_ptr& IOmanager, const StatisticsOptions& opt, T initial_value = 0.0) : - Operation{name, operation, win, opt}, - values_{}, - needRestart_{needRestart}, - initialValue_{initial_value} { + Operation{name, operation, win, opt}, values_{}, needRestart_{needRestart}, initialValue_{initial_value} { load(IOmanager, opt); } - void updateWindow(const void* data, long sz, const message::Message& msg, const StatisticsConfiguration& cfg) override { + void updateWindow(const void* data, long sz, const message::Message& msg, + const StatisticsConfiguration& cfg) override { std::fill(values_.begin(), values_.end(), initialValue_); }; @@ -80,7 +78,7 @@ class OperationWithData : public Operation { void serialize(IOBuffer& restartState, const std::string& fname, const StatisticsOptions& opt) const { size_t sz = values_.size(); - size_t cnt=0; + size_t cnt = 0; // restartState[cnt] = static_cast(sz); for (size_t i = 0; i < sz; ++i) { T lv = values_[i]; @@ -103,7 +101,7 @@ class OperationWithData : public Operation { void deserialize(const IOBuffer& restartState, const std::string& fname, const StatisticsOptions& opt) { restartState.checkChecksum(); - size_t cnt=0; + size_t cnt = 0; size_t sz = values_.size(); for (size_t i = 0; i < sz; ++i) { std::uint64_t lv = restartState[cnt]; @@ -129,7 +127,7 @@ class OperationWithData : public Operation { } }; - void checkTimeInterval( const StatisticsConfiguration& cfg ) { + void checkTimeInterval(const StatisticsConfiguration& cfg) { long sec = win_.count() * cfg.stepFreq() * cfg.timeStep(); if (sec == 0) { throw eckit::SeriousBug{logHeader_ + " :: Divide by zero", Here()}; @@ -141,7 +139,6 @@ class OperationWithData : public Operation { std::vector values_; private: - const std::string restartFileName() const { return name_ + "_" + (sizeof(T) == 4 ? "single" : "double"); }; bool needRestart_; diff --git a/src/multio/action/statistics/operations/OperationWithDeaccumulatedData.h b/src/multio/action/statistics/operations/OperationWithDeaccumulatedData.h index 88fe9cd2..42786820 100644 --- a/src/multio/action/statistics/operations/OperationWithDeaccumulatedData.h +++ b/src/multio/action/statistics/operations/OperationWithDeaccumulatedData.h @@ -21,17 +21,15 @@ class OperationWithDeaccumulatedData : public Operation { OperationWithDeaccumulatedData(const std::string& name, const std::string& operation, bool needRestart, const OperationWindow& win, std::shared_ptr& IOmanager, - const StatisticsOptions& opt) : - Operation{name, operation, win, opt}, - values_{}, - initValues_{}, - needRestart_{needRestart} { + const StatisticsOptions& opt) : + Operation{name, operation, win, opt}, values_{}, initValues_{}, needRestart_{needRestart} { load(IOmanager, opt); return; } - void updateWindow(const void* data, long sz, const message::Message& msg, const StatisticsConfiguration& cfg) override { - checkSize(sz,cfg); + void updateWindow(const void* data, long sz, const message::Message& msg, + const StatisticsConfiguration& cfg) override { + checkSize(sz, cfg); if (solverResetAccumulatedFields(msg, cfg)) { std::transform(initValues_.begin(), initValues_.end(), initValues_.begin(), [](const T& v1) { return static_cast(0.0); }); @@ -54,7 +52,7 @@ class OperationWithDeaccumulatedData : public Operation { }; void init(const void* data, long sz, const message::Message& msg, const StatisticsConfiguration& cfg) override { - checkSize(sz,cfg); + checkSize(sz, cfg); if (solverResetAccumulatedFields(msg, cfg)) { std::transform(initValues_.begin(), initValues_.end(), initValues_.begin(), [](const T& v1) { return static_cast(0.0); }); @@ -110,7 +108,7 @@ class OperationWithDeaccumulatedData : public Operation { protected: void serialize(IOBuffer& restartState, const std::string& fname, const StatisticsOptions& opt) const { size_t sz = values_.size(); - size_t cnt=0; + size_t cnt = 0; // restartState[cnt] = static_cast(sz); for (size_t i = 0; i < sz; ++i) { T lv = initValues_[i]; @@ -130,11 +128,11 @@ class OperationWithDeaccumulatedData : public Operation { std::ofstream outFile(fname); outFile << "initValues(" << sz << ")" << std::endl; for (size_t i = 0; i < sz; ++i) { - outFile << i << ", " << initValues_[i] << std::endl; + outFile << i << ", " << initValues_[i] << std::endl; } outFile << "values(" << sz << ")" << std::endl; for (size_t i = 0; i < sz; ++i) { - outFile << i << ", " << values_[i] << std::endl; + outFile << i << ", " << values_[i] << std::endl; } outFile.close(); } @@ -143,7 +141,7 @@ class OperationWithDeaccumulatedData : public Operation { void deserialize(const IOBuffer& restartState, const std::string& fname, const StatisticsOptions& opt) { restartState.checkChecksum(); - size_t cnt=0; + size_t cnt = 0; size_t sz = values_.size(); // size_t sz = static_cast(restartState[cnt]); for (size_t i = 0; i < sz; ++i) { @@ -163,11 +161,11 @@ class OperationWithDeaccumulatedData : public Operation { std::ofstream outFile(fname); outFile << "initValues(" << sz << ")" << std::endl; for (size_t i = 0; i < sz; ++i) { - outFile << i << ", " << initValues_[i] << std::endl; + outFile << i << ", " << initValues_[i] << std::endl; } outFile << "values(" << sz << ")" << std::endl; for (size_t i = 0; i < sz; ++i) { - outFile << i << ", " << values_[i] << std::endl; + outFile << i << ", " << values_[i] << std::endl; } outFile.close(); } @@ -181,7 +179,7 @@ class OperationWithDeaccumulatedData : public Operation { } }; - void checkTimeInterval( const StatisticsConfiguration& cfg ) { + void checkTimeInterval(const StatisticsConfiguration& cfg) { long sec = win_.count() * cfg.stepFreq() * cfg.timeStep(); if (sec == 0) { throw eckit::SeriousBug{logHeader_ + " :: Divide by zero", Here()}; @@ -217,7 +215,8 @@ class OperationWithDeaccumulatedData : public Operation { } std::ostringstream os; - os << "Invalid reset period of accumulated fields :: " << cfg.options().solverResetAccumulatedFields() << std::endl; + os << "Invalid reset period of accumulated fields :: " << cfg.options().solverResetAccumulatedFields() + << std::endl; throw eckit::UserError(os.str(), Here()); } }; diff --git a/src/multio/action/statistics/period-updaters/DayPeriodUpdater.h b/src/multio/action/statistics/period-updaters/DayPeriodUpdater.h index 8f817328..9eda79f1 100644 --- a/src/multio/action/statistics/period-updaters/DayPeriodUpdater.h +++ b/src/multio/action/statistics/period-updaters/DayPeriodUpdater.h @@ -1,13 +1,13 @@ #pragma once +#include // For setw #include -#include // For setw +#include "eckit/types/DateTime.h" +#include "multio/action/statistics/TimeUtils.h" #include "multio/action/statistics/cfg/StatisticsConfiguration.h" #include "multio/action/statistics/period-updaters/PeriodUpdater.h" -#include "multio/action/statistics/TimeUtils.h" -#include "eckit/types/DateTime.h" #include "multio/message/Message.h" @@ -16,23 +16,21 @@ namespace multio::action { class DayPeriodUpdater final : public PeriodUpdater { public: - DayPeriodUpdater(long span) : PeriodUpdater{span} {}; - DayPeriodUpdater(std::shared_ptr& IOmanager, const StatisticsOptions& opt): - PeriodUpdater{ timeUnit(), IOmanager, opt}{ }; + DayPeriodUpdater(std::shared_ptr& IOmanager, const StatisticsOptions& opt) : + PeriodUpdater{timeUnit(), IOmanager, opt} {}; - void dump( std::shared_ptr& IOmanager, const StatisticsOptions& opt ) const { - std::ostringstream os; - os << timeUnit() << "_" << std::setw(4) << std::setfill('0') << span_; - PeriodUpdater::baseDump( timeUnit(), IOmanager, opt); + void dump(std::shared_ptr& IOmanager, const StatisticsOptions& opt) const { + std::ostringstream os; + os << timeUnit() << "_" << std::setw(4) << std::setfill('0') << span_; + PeriodUpdater::baseDump(timeUnit(), IOmanager, opt); }; const std::string name() const { std::ostringstream os; - os << std::setw(4) << std::setfill('0') << span_ << "-" - << timeUnit(); + os << std::setw(4) << std::setfill('0') << span_ << "-" << timeUnit(); return os.str(); }; @@ -52,8 +50,7 @@ class DayPeriodUpdater final : public PeriodUpdater { eckit::DateTime tmp = startPoint + static_cast(86400 * span_); return eckit::DateTime{tmp.date(), eckit::Time{0}}; }; - }; -} +} // namespace multio::action diff --git a/src/multio/action/statistics/period-updaters/HourPeriodUpdater.h b/src/multio/action/statistics/period-updaters/HourPeriodUpdater.h index eb69226e..000e812d 100644 --- a/src/multio/action/statistics/period-updaters/HourPeriodUpdater.h +++ b/src/multio/action/statistics/period-updaters/HourPeriodUpdater.h @@ -1,13 +1,13 @@ #pragma once +#include // For setw #include -#include // For setw +#include "eckit/types/DateTime.h" +#include "multio/action/statistics/TimeUtils.h" #include "multio/action/statistics/cfg/StatisticsConfiguration.h" #include "multio/action/statistics/period-updaters/PeriodUpdater.h" -#include "multio/action/statistics/TimeUtils.h" -#include "eckit/types/DateTime.h" #include "multio/message/Message.h" @@ -16,20 +16,19 @@ namespace multio::action { class HourPeriodUpdater final : public PeriodUpdater { public: HourPeriodUpdater(long span) : PeriodUpdater{span} {}; - HourPeriodUpdater( std::shared_ptr& IOmanager, const StatisticsOptions& opt): - PeriodUpdater{ timeUnit(), IOmanager, opt}{ }; + HourPeriodUpdater(std::shared_ptr& IOmanager, const StatisticsOptions& opt) : + PeriodUpdater{timeUnit(), IOmanager, opt} {}; - void dump( std::shared_ptr& IOmanager, const StatisticsOptions& opt ) const { - std::ostringstream os; - os << timeUnit() << "_" << std::setw(4) << std::setfill('0') << span_; - PeriodUpdater::baseDump(timeUnit(), IOmanager, opt); + void dump(std::shared_ptr& IOmanager, const StatisticsOptions& opt) const { + std::ostringstream os; + os << timeUnit() << "_" << std::setw(4) << std::setfill('0') << span_; + PeriodUpdater::baseDump(timeUnit(), IOmanager, opt); }; - const std::string name() const{ + const std::string name() const { std::ostringstream os; - os << std::setw(4) << std::setfill('0') << span_ << "-" - << timeUnit(); + os << std::setw(4) << std::setfill('0') << span_ << "-" << timeUnit(); return os.str(); }; @@ -49,7 +48,6 @@ class HourPeriodUpdater final : public PeriodUpdater { eckit::DateTime tmp = startPoint + static_cast(3600 * span_); return eckit::DateTime{tmp.date(), eckit::Time{tmp.time().hours(), 0, 0}}; }; - }; -} +} // namespace multio::action diff --git a/src/multio/action/statistics/period-updaters/MonthPeriodUpdater.h b/src/multio/action/statistics/period-updaters/MonthPeriodUpdater.h index 27ca5140..9f868841 100644 --- a/src/multio/action/statistics/period-updaters/MonthPeriodUpdater.h +++ b/src/multio/action/statistics/period-updaters/MonthPeriodUpdater.h @@ -1,10 +1,10 @@ #pragma once +#include "eckit/types/DateTime.h" +#include "multio/action/statistics/TimeUtils.h" #include "multio/action/statistics/cfg/StatisticsConfiguration.h" #include "multio/action/statistics/period-updaters/PeriodUpdater.h" -#include "multio/action/statistics/TimeUtils.h" -#include "eckit/types/DateTime.h" #include "multio/message/Message.h" @@ -13,19 +13,18 @@ namespace multio::action { class MonthPeriodUpdater final : public PeriodUpdater { public: MonthPeriodUpdater(long span) : PeriodUpdater{span} {}; - MonthPeriodUpdater( std::shared_ptr& IOmanager, const StatisticsOptions& opt): - PeriodUpdater{ timeUnit(), IOmanager, opt}{ }; + MonthPeriodUpdater(std::shared_ptr& IOmanager, const StatisticsOptions& opt) : + PeriodUpdater{timeUnit(), IOmanager, opt} {}; - void dump( std::shared_ptr& IOmanager, const StatisticsOptions& opt ) const { - std::ostringstream os; - os << timeUnit() << "_" << std::setw(4) << std::setfill('0') << span_; - PeriodUpdater::baseDump(timeUnit(), IOmanager, opt); + void dump(std::shared_ptr& IOmanager, const StatisticsOptions& opt) const { + std::ostringstream os; + os << timeUnit() << "_" << std::setw(4) << std::setfill('0') << span_; + PeriodUpdater::baseDump(timeUnit(), IOmanager, opt); }; const std::string name() const { std::ostringstream os; - os << std::setw(4) << std::setfill('0') << span_ << "-" - << timeUnit(); + os << std::setw(4) << std::setfill('0') << span_ << "-" << timeUnit(); return os.str(); }; @@ -50,7 +49,6 @@ class MonthPeriodUpdater final : public PeriodUpdater { auto endMonth = totalSpan % 12 + 1; return eckit::DateTime{eckit::Date{endYear, endMonth, 1}, eckit::Time{0}}; }; - }; -} +} // namespace multio::action diff --git a/src/multio/action/statistics/period-updaters/PeriodUpdater.h b/src/multio/action/statistics/period-updaters/PeriodUpdater.h index 8f42f341..ef8030e9 100644 --- a/src/multio/action/statistics/period-updaters/PeriodUpdater.h +++ b/src/multio/action/statistics/period-updaters/PeriodUpdater.h @@ -2,20 +2,19 @@ #pragma once -#include "multio/action/statistics/cfg/StatisticsConfiguration.h" -#include "multio/action/statistics/TimeUtils.h" #include "eckit/types/DateTime.h" -#include "multio/message/Message.h" #include "multio/action/statistics/StatisticsIO.h" +#include "multio/action/statistics/TimeUtils.h" +#include "multio/action/statistics/cfg/StatisticsConfiguration.h" +#include "multio/message/Message.h" namespace multio::action { class PeriodUpdater { public: - PeriodUpdater(long span) : span_{span} {}; - PeriodUpdater( const std::string& name, std::shared_ptr& IOmanager, const StatisticsOptions& opt) { + PeriodUpdater(const std::string& name, std::shared_ptr& IOmanager, const StatisticsOptions& opt) { load(name, IOmanager, opt); }; @@ -23,15 +22,11 @@ class PeriodUpdater { virtual const std::string timeUnit() const = 0; - virtual void dump( std::shared_ptr& IOmanager, const StatisticsOptions& opt ) const = 0; + virtual void dump(std::shared_ptr& IOmanager, const StatisticsOptions& opt) const = 0; - long timeSpan() const { - return span_; - }; + long timeSpan() const { return span_; }; - eckit::DateTime computeWinCreationTime(const eckit::DateTime& currentTime) const { - return currentTime; - }; + eckit::DateTime computeWinCreationTime(const eckit::DateTime& currentTime) const { return currentTime; }; eckit::DateTime computeWinEndTime(const eckit::DateTime& currentTime) const { auto ret = eckit::DateTime{updateWinEndTime(currentTime)}; @@ -45,34 +40,32 @@ class PeriodUpdater { protected: long span_; - void baseDump( const std::string& name, std::shared_ptr& IOmanager, const StatisticsOptions& opt ) const { + void baseDump(const std::string& name, std::shared_ptr& IOmanager, + const StatisticsOptions& opt) const { IOBuffer restartState{IOmanager->getBuffer(restartSize())}; restartState.zero(); serialize(restartState, IOmanager->getCurrentDir() + "/" + name + "_dump.txt", opt); - IOmanager->write(name, static_cast(1), restartSize() ); + IOmanager->write(name, static_cast(1), restartSize()); IOmanager->flush(); return; }; - void load( const std::string& name, std::shared_ptr& IOmanager, const StatisticsOptions& opt ) { + void load(const std::string& name, std::shared_ptr& IOmanager, const StatisticsOptions& opt) { IOBuffer restartState{IOmanager->getBuffer(restartSize())}; - IOmanager->read( name, restartSize() ); + IOmanager->read(name, restartSize()); deserialize(restartState, IOmanager->getCurrentDir() + "/" + name + "_load.txt", opt); restartState.zero(); return; }; private: - - size_t restartSize() const { - return static_cast(2); - } + size_t restartSize() const { return static_cast(2); } void serialize(IOBuffer& currState, const std::string& fname, const StatisticsOptions& opt) const { if (opt.debugRestart()) { std::ofstream outFile(fname); - //outFile << name() << std::endl; - //outFile << timeUnit() << std::endl; + // outFile << name() << std::endl; + // outFile << timeUnit() << std::endl; outFile << span_ << std::endl; outFile.close(); } @@ -93,7 +86,6 @@ class PeriodUpdater { } return; } - }; -} \ No newline at end of file +} // namespace multio::action \ No newline at end of file diff --git a/src/multio/action/statistics/synoptic-filters/AllTimesFilter.h b/src/multio/action/statistics/synoptic-filters/AllTimesFilter.h index ad83ee51..178b5ef7 100644 --- a/src/multio/action/statistics/synoptic-filters/AllTimesFilter.h +++ b/src/multio/action/statistics/synoptic-filters/AllTimesFilter.h @@ -7,9 +7,9 @@ namespace multio::action { class AllTimesFilter final : public SynopticFilter { public: - AllTimesFilter(){}; + AllTimesFilter() {}; - ~AllTimesFilter(){}; + ~AllTimesFilter() {}; size_t size() const { return static_cast(1); }; @@ -30,4 +30,3 @@ class AllTimesFilter final : public SynopticFilter { }; } // namespace multio::action - diff --git a/src/multio/action/statistics/synoptic-filters/DailyCustomFilter.h b/src/multio/action/statistics/synoptic-filters/DailyCustomFilter.h index 20c7fd40..bbdeccb3 100644 --- a/src/multio/action/statistics/synoptic-filters/DailyCustomFilter.h +++ b/src/multio/action/statistics/synoptic-filters/DailyCustomFilter.h @@ -10,7 +10,6 @@ #include "multio/config/ComponentConfiguration.h" - namespace multio::action { @@ -108,7 +107,7 @@ class DailyCustomFilter final : public SynopticFilter { public: DailyCustomFilter(const eckit::LocalConfiguration& compConf, const StatisticsConfiguration& cfg) : reqHours_{initDailyCustomFilter(compConf, cfg)}, reqHoursInv_{initReqHouorsInv(reqHours_)} {}; - ~DailyCustomFilter(){}; + ~DailyCustomFilter() {}; size_t size() const { return reqHours_.size(); }; @@ -136,4 +135,3 @@ class DailyCustomFilter final : public SynopticFilter { } // namespace multio::action - diff --git a/src/multio/action/statistics/synoptic-filters/DailyHoursFilter.h b/src/multio/action/statistics/synoptic-filters/DailyHoursFilter.h index e334e494..89c0e34c 100644 --- a/src/multio/action/statistics/synoptic-filters/DailyHoursFilter.h +++ b/src/multio/action/statistics/synoptic-filters/DailyHoursFilter.h @@ -3,13 +3,12 @@ #include "multio/action/statistics/synoptic-filters/Filter.h" - namespace multio::action { class DailyHoursFilter final : public SynopticFilter { public: - DailyHoursFilter(){}; - ~DailyHoursFilter(){}; + DailyHoursFilter() {}; + ~DailyHoursFilter() {}; size_t size() const { return static_cast(24); }; @@ -30,4 +29,3 @@ class DailyHoursFilter final : public SynopticFilter { }; } // namespace multio::action - diff --git a/src/multio/action/statistics/synoptic-filters/Filter.h b/src/multio/action/statistics/synoptic-filters/Filter.h index 6276f73d..f7d3de55 100644 --- a/src/multio/action/statistics/synoptic-filters/Filter.h +++ b/src/multio/action/statistics/synoptic-filters/Filter.h @@ -13,9 +13,9 @@ namespace multio::action { class SynopticFilter { public: - SynopticFilter(){}; + SynopticFilter() {}; - virtual ~SynopticFilter(){}; + virtual ~SynopticFilter() {}; virtual void fillMetadata(size_t idx, message::Metadata& metadata) const = 0; diff --git a/src/multio/api/c/multio_capi.cc b/src/multio/api/c/multio_capi.cc index 163f0d8c..274f154d 100644 --- a/src/multio/api/c/multio_capi.cc +++ b/src/multio/api/c/multio_capi.cc @@ -463,10 +463,10 @@ int multio_delete_handle(multio_handle_t* mio) { multio::message::Metadata md; md.set("flushKind", "end-of-simulation"); - - mio->dispatch(std::move(md), eckit::Buffer{0}, Message::Tag::Flush); - - // TODO add sleep + + mio->dispatch(std::move(md), eckit::Buffer{0}, Message::Tag::Flush); + + // TODO add sleep delete mio; }); #else @@ -507,13 +507,13 @@ int multio_close_connections(multio_handle_t* mio) { return wrapApiFunction( [mio]() { ASSERT(mio); - - multio::message::Metadata md; + + multio::message::Metadata md; md.set("flushKind", "end-of-simulation"); - - mio->dispatch(std::move(md), eckit::Buffer{0}, Message::Tag::Flush); - - mio->closeConnections(); + + mio->dispatch(std::move(md), eckit::Buffer{0}, Message::Tag::Flush); + + mio->closeConnections(); }, mio); #else @@ -700,7 +700,7 @@ int multio_write_field_buffer(multio_handle_t* mio, multio_metadata_t* md, multi md->md.acquire(); // Make sure metadata is not stored in a stateful container from last write md->md.modify().set("format", "raw"); - + if (byte_size == 4) { md->md.modify().set("precision", "single"); } diff --git a/src/multio/message/Metadata.h b/src/multio/message/Metadata.h index 0dba6d73..2d1da9d8 100644 --- a/src/multio/message/Metadata.h +++ b/src/multio/message/Metadata.h @@ -52,11 +52,11 @@ class Metadata : public BaseMetadata { using ConstIterator = typename MapType::const_iterator; // The single place we need to change to support lookups in parametrization. - // Important: The non-const version will copy values from the global parametrization dictionary + // Important: The non-const version will copy values from the global parametrization dictionary // to the local one - It will always return an iterator to the local dictionary. // This allows values to be modified in the local object directly. - // - // The const version will return either an iterator to the local object or the + // + // The const version will return either an iterator to the local object or the // global parametrization dictionary. // If keys are assumed to hold nested objects or large arrays which won't be modified anyway - // it is reasonable to explicitly const-cast the object to select the const lookup version. diff --git a/src/multio/message/MetadataMatcher.cc b/src/multio/message/MetadataMatcher.cc index f47ec518..0bc7c554 100644 --- a/src/multio/message/MetadataMatcher.cc +++ b/src/multio/message/MetadataMatcher.cc @@ -74,9 +74,9 @@ bool MatchKeys::matches(const Metadata& md) const { break; } if (kv.second.find(searchKey->second) == kv.second.end()) { - if (enforceSameKeyTypes_ && kv.second.size() > 0 ) { + if (enforceSameKeyTypes_ && kv.second.size() > 0) { // Note: NEMO needs the possibiity to use empty sets!!! - //if (kv.second.size() == 0) { + // if (kv.second.size() == 0) { // std::ostringstream oss; // oss << "[enforce-same-key-type] Matcher for key \"" << kv.first << "\" is an empty set"; // throw MetadataException(oss.str()); diff --git a/src/multio/message/Parametrization.h b/src/multio/message/Parametrization.h index 017ec658..b5ecfbd0 100644 --- a/src/multio/message/Parametrization.h +++ b/src/multio/message/Parametrization.h @@ -10,13 +10,13 @@ namespace multio::message { static std::string PARAMETRIZATION_PAYLOAD_KEY = "payloadKey"; /** Global singleton metadata object that contains information that may be send once at the beginning of a run - * - * Important: Key-value pairs are ment to be constant. It is possible to update a key with the same value multiple times. - * However, to enforce consistency a key can not be updated with a different value - this will throw an exception to - * notify about inconsistency. * - * This singleton object is shared accross multiple multio instances (e.g. IFS and NEMO). Hence it is assumed that - * all global key-value pairs of different models are exclusive to each other or contain the same value. + * Important: Key-value pairs are ment to be constant. It is possible to update a key with the same value multiple + * times. However, to enforce consistency a key can not be updated with a different value - this will throw an exception + * to notify about inconsistency. + * + * This singleton object is shared accross multiple multio instances (e.g. IFS and NEMO). Hence it is + * assumed that all global key-value pairs of different models are exclusive to each other or contain the same value. * */ class Parametrization { diff --git a/src/multio/tools/multio-feed.cc b/src/multio/tools/multio-feed.cc index c6685d7a..ef888922 100644 --- a/src/multio/tools/multio-feed.cc +++ b/src/multio/tools/multio-feed.cc @@ -70,12 +70,12 @@ class MetadataSetter : public eckit::LocalConfiguration { std::vector keys() { return eckit::LocalConfiguration::keys(); } }; -void parseStepRange(std::string const& stepRange, long& startStep, long& endStep ) { +void parseStepRange(std::string const& stepRange, long& startStep, long& endStep) { static const std::regex period_grammar("([0-9]+)-([0-9]+)"); std::smatch match; if (std::regex_match(stepRange, match, period_grammar)) { startStep = std::stol(match[1].str()); - endStep = std::stol(match[2].str()); + endStep = std::stol(match[2].str()); } else { throw eckit::SeriousBug("Wrong grammar in period definition : " + stepRange, Here()); @@ -152,14 +152,14 @@ void MultioFeed::init(const eckit::option::CmdArgs& args) { args.get("plans", configPath_); - if ( args.has("stepRange") ){ + if (args.has("stepRange")) { args.get("stepRange", stepRange_); } else { stepRange_ = "0-1000000000"; } - parseStepRange( stepRange_, minStep_, maxStep_ ); + parseStepRange(stepRange_, minStep_, maxStep_); if (!configPath_.empty()) { ::setenv("MULTIO_PLANS_FILE", configPath_.c_str(), 1); @@ -204,7 +204,7 @@ void MultioFeed::execute(const eckit::option::CmdArgs& args) { // metadata.set("step", metadataDetailed.getLong("step")); } - if (metadataDetailed.has("startStep") && metadataDetailed.has("endStep")){ + if (metadataDetailed.has("startStep") && metadataDetailed.has("endStep")) { long startStep = metadataDetailed.getLong("startStep"); long endStep = metadataDetailed.getLong("endStep"); if (startStep < minStep_ || endStep > maxStep_) { @@ -214,7 +214,7 @@ void MultioFeed::execute(const eckit::option::CmdArgs& args) { // metadata.set("endStep", metadataDetailed.getLong("endStep")); } - if (!metadataDetailed.has("startStep") && metadataDetailed.has("endStep")){ + if (!metadataDetailed.has("startStep") && metadataDetailed.has("endStep")) { long endStep = metadataDetailed.getLong("endStep"); if (endStep < minStep_ || endStep > maxStep_) { continue; @@ -337,13 +337,12 @@ void MultioFeed::execute(const eckit::option::CmdArgs& args) { } } - if (imultio_flush_()) { + if (imultio_flush_()) { ASSERT(false); } - } - if (imultio_flush_last_()) { + if (imultio_flush_last_()) { ASSERT(false); } diff --git a/src/multio/tools/multio-statistics-stress-test.cc b/src/multio/tools/multio-statistics-stress-test.cc index 3308f60b..5a864bbd 100644 --- a/src/multio/tools/multio-statistics-stress-test.cc +++ b/src/multio/tools/multio-statistics-stress-test.cc @@ -14,9 +14,9 @@ /// @date Oct 2019 +#include #include #include -#include #include "eckit/config/LocalConfiguration.h" #include "eckit/exception/Exceptions.h" @@ -69,7 +69,6 @@ class MultioStatisticsStressTester final : public multio::MultioTool { MultioStatisticsStressTester(int argc, char** argv); private: - void usage(const std::string& tool) const override { eckit::Log::info() << std::endl << "Usage: " << tool << " inputFile [options]" << std::endl; eckit::Log::info() << std::endl @@ -93,34 +92,30 @@ class MultioStatisticsStressTester final : public multio::MultioTool { long nValues_; long precisionRatio_; std::string configPath_ = ""; - }; -MultioStatisticsStressTester::MultioStatisticsStressTester(int argc, char** argv) :multio::MultioTool{argc, argv} ,nFlushFreq_{1},nSteps_{5},nParamID_{1},nLevels_{1}, nValues_{10000},precisionRatio_{100} - { +MultioStatisticsStressTester::MultioStatisticsStressTester(int argc, char** argv) : + multio::MultioTool{argc, argv}, + nFlushFreq_{1}, + nSteps_{5}, + nParamID_{1}, + nLevels_{1}, + nValues_{10000}, + precisionRatio_{100} { - options_.push_back( - new eckit::option::SimpleOption("nFlushFreq", - "number of steps between flush ")); + options_.push_back(new eckit::option::SimpleOption("nFlushFreq", "number of steps between flush ")); - options_.push_back( - new eckit::option::SimpleOption("nSteps", - "number of Steps to be generated ")); + options_.push_back(new eckit::option::SimpleOption("nSteps", "number of Steps to be generated ")); - options_.push_back( - new eckit::option::SimpleOption("nParams", - "number of param ID to be generated ")); - options_.push_back( - new eckit::option::SimpleOption("nLevels", - "number of levels to be generated")); + options_.push_back(new eckit::option::SimpleOption("nParams", "number of param ID to be generated ")); + options_.push_back(new eckit::option::SimpleOption("nLevels", "number of levels to be generated")); - options_.push_back( - new eckit::option::SimpleOption("nValues", - "number of values per message")); + options_.push_back(new eckit::option::SimpleOption("nValues", "number of values per message")); options_.push_back( new eckit::option::SimpleOption("precisionRatio", - "percentage of messages in double wrt single precision (80 means that 80% of messages will be in double precision)")); + "percentage of messages in double wrt single precision (80 means that " + "80% of messages will be in double precision)")); options_.push_back( new eckit::option::SimpleOption("plans", "Path to YAML/JSON file containing plans and actions.")); @@ -128,12 +123,12 @@ MultioStatisticsStressTester::MultioStatisticsStressTester(int argc, char** argv void MultioStatisticsStressTester::init(const eckit::option::CmdArgs& args) { - args.get("nFlushFreq", nFlushFreq_ ); - args.get("nSteps", nSteps_ ); - args.get("nParams", nParamID_ ); - args.get("nLevels", nLevels_ ); - args.get("nValues", nValues_ ); - args.get("precisionRatio", precisionRatio_ ); + args.get("nFlushFreq", nFlushFreq_); + args.get("nSteps", nSteps_); + args.get("nParams", nParamID_); + args.get("nLevels", nLevels_); + args.get("nValues", nValues_); + args.get("precisionRatio", precisionRatio_); args.get("plans", configPath_); @@ -151,89 +146,87 @@ void MultioStatisticsStressTester::execute(const eckit::option::CmdArgs& args) { eckit::Buffer data_d; eckit::Buffer data_f; - data_d.resize( nValues_*sizeof(double) ); - data_f.resize( nValues_*sizeof(float) ); + data_d.resize(nValues_ * sizeof(double)); + data_f.resize(nValues_ * sizeof(float)); double* vald = static_cast(data_d.data()); float* valf = static_cast(data_f.data()); - auto rndd = [](){ return (double)(rand()) / (double)(rand()); }; - auto rndf = [](){ return (float)(rand()) / (float)(rand()); }; - for ( int d=0; d(100*(static_cast(msgID) / static_cast(nParamID_*nLevels_))); - if ( ratio <= precisionRatio_ ){ - metadata.set("precision", "double"); - metadata.set("globalSize", data_d.size() / sizeof(double)); - size_t words = eckit::round(data_d.size(), sizeof(fortint)) / sizeof(fortint); - fortint iwords = static_cast(words); - - if (imultio_write_raw_(&metadata, reinterpret_cast(data_d.data()), &iwords)) { - ASSERT(false); - } + for (int i = 0; i < nSteps_; i++) { + + // Generic metadata (not enough for encoding) + metadata.set("gribEdition", "2"); + + metadata.set("domain", "g"); + metadata.set("expver", "hvi1"); + metadata.set("class", "rd"); + metadata.set("type", "fc"); + metadata.set("stream", "lwda"); + metadata.set("anoffset", 9); + + + metadata.set("levtype", "ml"); + metadata.set("gridType", "reduced_gg"); + metadata.set("name", "Logarithm of surface pressure"); + metadata.set("shortName", "lnsp"); + + // Time management + metadata.set("date", long(20200120)); + metadata.set("time", long(0)); + metadata.set("timeStep", long(3600)); + metadata.set("step-frequency", long(1)); + + + // Set the step idx + metadata.set("step", i); + metadata.set("stepId", i); + + long msgID = 0; + // Push + for (long j = 0; j < nParamID_; j++) { + metadata.set("paramId", j * 10); + metadata.set("param", j * 10); + for (long k = 0; k < nLevels_; k++) { + msgID++; + metadata.set("level", k * 100); + metadata.set("levelist", k * 100); + + long ratio + = static_cast(100 * (static_cast(msgID) / static_cast(nParamID_ * nLevels_))); + if (ratio <= precisionRatio_) { + metadata.set("precision", "double"); + metadata.set("globalSize", data_d.size() / sizeof(double)); + size_t words = eckit::round(data_d.size(), sizeof(fortint)) / sizeof(fortint); + fortint iwords = static_cast(words); + + if (imultio_write_raw_(&metadata, reinterpret_cast(data_d.data()), &iwords)) { + ASSERT(false); } - else { - metadata.set("precision", "single"); - metadata.set("globalSize", data_f.size() / sizeof(float)); - size_t words = eckit::round(data_f.size(), sizeof(fortint)) / sizeof(fortint); - fortint iwords = static_cast(words); - - if (imultio_write_raw_(&metadata, reinterpret_cast(data_f.data()), &iwords)) { - ASSERT(false); - } + } + else { + metadata.set("precision", "single"); + metadata.set("globalSize", data_f.size() / sizeof(float)); + size_t words = eckit::round(data_f.size(), sizeof(fortint)) / sizeof(fortint); + fortint iwords = static_cast(words); + + if (imultio_write_raw_(&metadata, reinterpret_cast(data_f.data()), &iwords)) { + ASSERT(false); } - } } + } - if ( (i+1)%nFlushFreq_ == 0 || nFlushFreq_==1 ){ - if (imultio_flush_()) { - ASSERT(false); - } + if ((i + 1) % nFlushFreq_ == 0 || nFlushFreq_ == 1) { + if (imultio_flush_()) { + ASSERT(false); } - + } } }