Skip to content

Commit 7628a1e

Browse files
MircoValentiniECMWFMircoValentiniECMWF
committed
Merge branch 'hotfix/2.1.10-new-restart-in-statistics' into develop
2 parents fd60257 + 4216b63 commit 7628a1e

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

61 files changed

+3648
-1216
lines changed

‎CMakeLists.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ ecbuild_add_option( FEATURE HEALPIX_EXAMPLES
102102

103103
### Definitions used by the output manager
104104
ecbuild_add_option( FEATURE OUTPUT_MANAGER
105-
DEFAULT ON
105+
DEFAULT ON
106106
DESCRIPTION "Build the output manager" )
107107

108108
ecbuild_add_option( FEATURE OUTPUT_MANAGER_ENCODER_REPORT

‎src/multio/action/statistics/CMakeLists.txt

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,21 @@
11

22
list( APPEND _statistics_sources
3-
StatisticsConfiguration.cc
4-
StatisticsConfiguration.h
3+
cfg/StatisticsConfiguration.cc
4+
cfg/StatisticsConfiguration.h
5+
cfg/StatisticsOptions.cc
6+
cfg/StatisticsOptions.h
7+
period-updaters/PeriodUpdater.h
8+
period-updaters/HourPeriodUpdater.h
9+
period-updaters/DayPeriodUpdater.h
10+
period-updaters/MonthPeriodUpdater.h
511
StatisticsIO.cc
612
StatisticsIO.h
713
io/FstreamIO.cc
814
io/FstreamIO.h
915
TimeUtils.cc
1016
TimeUtils.h
11-
PeriodUpdater.cc
12-
PeriodUpdater.h
17+
PeriodUpdaters.cc
18+
PeriodUpdaters.h
1319
OperationWindow.cc
1420
OperationWindow.h
1521
Operations.cc
@@ -25,10 +31,20 @@ list( APPEND _statistics_sources
2531
operations/Maximum.h
2632
operations/DeAccumulate.h
2733
operations/FixedWindowFluxAverage.h
34+
RemapParamID.cc
35+
RemapParamID.h
2836
TemporalStatistics.cc
2937
TemporalStatistics.h
3038
Statistics.cc
3139
Statistics.h
40+
# SynopticCollection.cc
41+
# SynopticCollection.h
42+
# SynopticFilters.cc
43+
# SynopticFilters.h
44+
# synoptic-filters/AllTimesFilter.h
45+
# synoptic-filters/DailyCustomFilter.h
46+
# synoptic-filters/DailyHoursFilter.h
47+
# synoptic-filters/Filter.h
3248
)
3349

3450
list( APPEND _statistics_libs
@@ -54,16 +70,15 @@ ecbuild_add_library(
5470

5571
TARGET multio-action-statistics
5672

57-
TYPE SHARED # Due to reliance on factory self registration this library cannot be static
58-
5973
SOURCES
6074
${_statistics_sources}
6175

76+
6277
PRIVATE_INCLUDES
6378
${ECKIT_INCLUDE_DIRS}
6479

6580
CONDITION
6681

6782
PUBLIC_LIBS
68-
${_statistics_libs}
83+
${_statistics_libs}
6984
)

‎src/multio/action/statistics/OperationWindow.cc

Lines changed: 129 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,9 @@ void yyyymmdd2ymd(uint64_t yyyymmdd, long& y, long& m, long& d) {
3333
}
3434

3535
void hhmmss2hms(uint64_t hhmmss, long& h, long& m, long& s) {
36-
h = static_cast<long>(hhmmss % 100);
36+
s = static_cast<long>(hhmmss % 100);
3737
m = static_cast<long>((hhmmss % 10000) / 100);
38-
s = static_cast<long>((hhmmss % 1000000) / 10000);
38+
h = static_cast<long>((hhmmss % 1000000) / 10000);
3939
if (s < 0 || s > 59) {
4040
throw eckit::SeriousBug("invalid seconds range", Here());
4141
}
@@ -57,7 +57,38 @@ eckit::DateTime yyyymmdd_hhmmss2DateTime(uint64_t yyyymmdd, uint64_t hhmmss) {
5757
} // namespace
5858

5959

60-
OperationWindow::OperationWindow(std::shared_ptr<StatisticsIO>& IOmanager, const StatisticsConfiguration& cfg) :
60+
OperationWindow make_window( const std::unique_ptr<PeriodUpdater>& periodUpdater, const StatisticsConfiguration& cfg) {
61+
eckit::DateTime epochPoint{cfg.epoch()};
62+
eckit::DateTime startPoint{periodUpdater->computeWinStartTime(cfg.winStart())};
63+
eckit::DateTime creationPoint{periodUpdater->computeWinCreationTime(cfg.winStart())};
64+
eckit::DateTime endPoint{periodUpdater->computeWinEndTime(startPoint)};
65+
long windowType = 0;
66+
if ( cfg.options().windowType() == "forward-offset" ){
67+
windowType = 0;
68+
}
69+
else if ( cfg.options().windowType() == "backward-offset" ) {
70+
windowType = 1;
71+
}
72+
else {
73+
std::ostringstream os;
74+
os << " Unknown window type: " << cfg.options().windowType() << std::endl;
75+
throw eckit::SeriousBug(os.str(), Here());
76+
};
77+
return OperationWindow{epochPoint, startPoint, creationPoint, endPoint, cfg.timeStep(), windowType};
78+
};
79+
80+
OperationWindow load_window( std::shared_ptr<StatisticsIO>& IOmanager, const StatisticsOptions& opt ) {
81+
IOmanager->pushDir( "operationWindow" );
82+
// std::ostringstream logos;
83+
// logos << " - Loading operationWindow from: " << IOmanager->getCurrentDir() << std::endl;
84+
// LOG_DEBUG_LIB(LibMultio) << logos.str() << std::endl;
85+
OperationWindow opwin{IOmanager, opt};
86+
IOmanager->popDir();
87+
return opwin;
88+
};
89+
90+
91+
OperationWindow::OperationWindow(std::shared_ptr<StatisticsIO>& IOmanager, const StatisticsOptions& opt) :
6192
epochPoint_{eckit::Date{0}, eckit::Time{0}},
6293
startPoint_{eckit::Date{0}, eckit::Time{0}},
6394
creationPoint_{eckit::Date{0}, eckit::Time{0}},
@@ -66,14 +97,15 @@ OperationWindow::OperationWindow(std::shared_ptr<StatisticsIO>& IOmanager, const
6697
endPoint_{eckit::Date{0}, eckit::Time{0}},
6798
lastFlush_{eckit::Date{0}, eckit::Time{0}},
6899
timeStepInSeconds_{0},
69-
count_{0} {
70-
load(IOmanager, cfg);
100+
count_{0},
101+
type_{0} {
102+
load(IOmanager, opt);
71103
return;
72104
}
73105

74106
OperationWindow::OperationWindow(const eckit::DateTime& epochPoint, const eckit::DateTime& startPoint,
75107
const eckit::DateTime& creationPoint, const eckit::DateTime& endPoint,
76-
long timeStepInSeconds) :
108+
long timeStepInSeconds, long windowType) :
77109
epochPoint_{epochPoint},
78110
startPoint_{startPoint},
79111
creationPoint_{creationPoint},
@@ -82,26 +114,28 @@ OperationWindow::OperationWindow(const eckit::DateTime& epochPoint, const eckit:
82114
endPoint_{endPoint},
83115
lastFlush_{epochPoint},
84116
timeStepInSeconds_{timeStepInSeconds},
85-
count_{0} {}
117+
count_{0},
118+
type_{windowType} {}
119+
86120

87121
long OperationWindow::count() const {
88122
return count_;
89123
}
90124

91-
void OperationWindow::load(std::shared_ptr<StatisticsIO>& IOmanager, const StatisticsConfiguration& cfg) {
125+
void OperationWindow::dump(std::shared_ptr<StatisticsIO>& IOmanager, const StatisticsOptions& opt) const {
92126
IOBuffer restartState{IOmanager->getBuffer(restartSize())};
93-
IOmanager->read("window", restartSize());
94-
deserialize(restartState);
95127
restartState.zero();
128+
serialize(restartState, IOmanager->getCurrentDir() + "/operationWindow_dump.txt", opt );
129+
IOmanager->write("operationWindow", static_cast<size_t>(16), restartSize() );
130+
IOmanager->flush();
96131
return;
97132
}
98133

99-
void OperationWindow::dump(std::shared_ptr<StatisticsIO>& IOmanager, const StatisticsConfiguration& cfg) const {
134+
void OperationWindow::load(std::shared_ptr<StatisticsIO>& IOmanager, const StatisticsOptions& opt) {
100135
IOBuffer restartState{IOmanager->getBuffer(restartSize())};
136+
IOmanager->read( "operationWindow", restartSize() );
137+
deserialize(restartState, IOmanager->getCurrentDir() + "/operationWindow_load.txt", opt);
101138
restartState.zero();
102-
serialize(restartState);
103-
IOmanager->write("window", restartSize());
104-
IOmanager->flush();
105139
return;
106140
}
107141

@@ -126,9 +160,32 @@ void OperationWindow::updateWindow(const eckit::DateTime& startPoint, const ecki
126160
return;
127161
}
128162

163+
std::string OperationWindow::windowType() const {
164+
if (type_ == 0) {
165+
return std::string{"forward-offset"};
166+
} else if (type_ == 1) {
167+
return std::string{"backward-offset"};
168+
} else {
169+
std::ostringstream os;
170+
os << *this << " Unknown window type " << std::endl;
171+
throw eckit::SeriousBug(os.str(), Here());
172+
}
173+
}
174+
129175

130176
bool OperationWindow::isWithin(const eckit::DateTime& dt) const {
131-
bool ret = gtLowerBound(dt, true) && leUpperBound(dt, false);
177+
bool ret;
178+
if ( type_ == 0 ) {
179+
ret = gtLowerBound(dt, false) && leUpperBound(dt, false);
180+
}
181+
else if ( type_ == 1 ) {
182+
ret = geLowerBound(dt, false) && ltUpperBound(dt, false);
183+
}
184+
else {
185+
std::ostringstream os;
186+
os << *this << " Unknown window type " << std::endl;
187+
throw eckit::SeriousBug(os.str(), Here());
188+
}
132189
LOG_DEBUG_LIB(LibMultio) << " ------ Is " << dt << " within " << *this << "? -- " << (ret ? "yes" : "no")
133190
<< std::endl;
134191
return ret;
@@ -143,6 +200,15 @@ bool OperationWindow::gtLowerBound(const eckit::DateTime& dt, bool throw_error)
143200
return dt > creationPoint_;
144201
};
145202

203+
bool OperationWindow::geLowerBound(const eckit::DateTime& dt, bool throw_error) const {
204+
if (throw_error && creationPoint_ > dt) {
205+
std::ostringstream os;
206+
os << *this << " : " << dt << " is outside of current period : lower Bound violation" << std::endl;
207+
throw eckit::SeriousBug(os.str(), Here());
208+
}
209+
return dt >= creationPoint_;
210+
};
211+
146212
bool OperationWindow::leUpperBound(const eckit::DateTime& dt, bool throw_error) const {
147213
// TODO: test without 1 second added. Now it should work
148214
if (throw_error && dt > endPoint()) {
@@ -153,6 +219,16 @@ bool OperationWindow::leUpperBound(const eckit::DateTime& dt, bool throw_error)
153219
return dt <= endPoint();
154220
};
155221

222+
bool OperationWindow::ltUpperBound(const eckit::DateTime& dt, bool throw_error) const {
223+
// TODO: test without 1 second added. Now it should work
224+
if (throw_error && dt >= endPoint()) {
225+
std::ostringstream os;
226+
os << *this << " : " << dt << " is outside of current period : upper Bound violation" << std::endl;
227+
throw eckit::SeriousBug(os.str(), Here());
228+
}
229+
return dt < endPoint();
230+
};
231+
156232
long OperationWindow::timeSpanInHours() const {
157233
return long(endPoint_ - creationPoint_) / 3600;
158234
}
@@ -356,8 +432,22 @@ long OperationWindow::lastFlushInSteps() const {
356432
return (lastFlush_ - epochPoint_) / timeStepInSeconds_;
357433
}
358434

359-
void OperationWindow::serialize(IOBuffer& currState) const {
360-
435+
void OperationWindow::serialize(IOBuffer& currState, const std::string& fname, const StatisticsOptions& opt) const {
436+
437+
if ( opt.debugRestart() ) {
438+
std::ofstream outFile(fname);
439+
outFile << "epochPoint_ :: " << epochPoint_ << std::endl;
440+
outFile << "startPoint_ :: " << startPoint_ << std::endl;
441+
outFile << "endPoint_ :: " << endPoint_ << std::endl;
442+
outFile << "creationPoint_ :: " << creationPoint_ << std::endl;
443+
outFile << "prevPoint_ :: " << prevPoint_ << std::endl;
444+
outFile << "currPoint_ :: " << currPoint_ << std::endl;
445+
outFile << "lastFlush_ :: " << lastFlush_ << std::endl;
446+
outFile << "timeStepInSeconds_ :: " << timeStepInSeconds_ << std::endl;
447+
outFile << "count_ :: " << count_ << std::endl;
448+
outFile << "type_ :: " << type_ << std::endl;
449+
outFile.close();
450+
}
361451

362452
currState[0] = static_cast<std::uint64_t>(epochPoint_.date().yyyymmdd());
363453
currState[1] = static_cast<std::uint64_t>(epochPoint_.time().hhmmss());
@@ -377,18 +467,19 @@ void OperationWindow::serialize(IOBuffer& currState) const {
377467
currState[10] = static_cast<std::uint64_t>(currPoint_.date().yyyymmdd());
378468
currState[11] = static_cast<std::uint64_t>(currPoint_.time().hhmmss());
379469

380-
currState[12] = static_cast<std::uint64_t>(currPoint_.date().yyyymmdd());
381-
currState[13] = static_cast<std::uint64_t>(currPoint_.time().hhmmss());
470+
currState[12] = static_cast<std::uint64_t>(lastFlush_.date().yyyymmdd());
471+
currState[13] = static_cast<std::uint64_t>(lastFlush_.time().hhmmss());
382472

383473
currState[14] = static_cast<std::uint64_t>(timeStepInSeconds_);
384474
currState[15] = static_cast<std::uint64_t>(count_);
475+
currState[16] = static_cast<std::uint64_t>(type_);
385476

386477
currState.computeChecksum();
387478

388479
return;
389480
}
390481

391-
void OperationWindow::deserialize(const IOBuffer& currState) {
482+
void OperationWindow::deserialize(const IOBuffer& currState, const std::string& fname, const StatisticsOptions& opt) {
392483

393484
currState.checkChecksum();
394485
epochPoint_ = yyyymmdd_hhmmss2DateTime(static_cast<long>(currState[0]), static_cast<long>(currState[1]));
@@ -400,16 +491,32 @@ void OperationWindow::deserialize(const IOBuffer& currState) {
400491
lastFlush_ = yyyymmdd_hhmmss2DateTime(static_cast<long>(currState[12]), static_cast<long>(currState[13]));
401492
timeStepInSeconds_ = static_cast<long>(currState[14]);
402493
count_ = static_cast<long>(currState[15]);
494+
type_ = static_cast<long>(currState[16]);
495+
496+
if ( opt.debugRestart() ) {
497+
std::ofstream outFile(fname);
498+
outFile << "epochPoint_ :: " << epochPoint_ << std::endl;
499+
outFile << "startPoint_ :: " << startPoint_ << std::endl;
500+
outFile << "endPoint_ :: " << endPoint_ << std::endl;
501+
outFile << "creationPoint_ :: " << creationPoint_ << std::endl;
502+
outFile << "prevPoint_ :: " << prevPoint_ << std::endl;
503+
outFile << "currPoint_ :: " << currPoint_ << std::endl;
504+
outFile << "lastFlush_ :: " << lastFlush_ << std::endl;
505+
outFile << "timeStepInSeconds_ :: " << timeStepInSeconds_ << std::endl;
506+
outFile << "count_ :: " << count_ << std::endl;
507+
outFile << "type_ :: " << type_ << std::endl;
508+
outFile.close();
509+
}
403510

404511
return;
405512
}
406513

407514
size_t OperationWindow::restartSize() const {
408-
return static_cast<size_t>(17);
515+
return static_cast<size_t>(18);
409516
}
410517

411518
void OperationWindow::print(std::ostream& os) const {
412-
os << "MovingWindow(" << startPoint_ << " to " << endPoint() << ")";
519+
os << "OperationWindow(" << startPoint_ << " to " << endPoint() << ")";
413520
}
414521

415522
std::ostream& operator<<(std::ostream& os, const OperationWindow& a) {

‎src/multio/action/statistics/OperationWindow.h

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,10 @@
55

66
#include "eckit/types/DateTime.h"
77

8-
#include "StatisticsConfiguration.h"
8+
9+
#include "multio/action/statistics/cfg/StatisticsOptions.h"
10+
#include "multio/action/statistics/cfg/StatisticsConfiguration.h"
11+
#include "multio/action/statistics/period-updaters/PeriodUpdater.h"
912
#include "StatisticsIO.h"
1013
#include "multio/message/Message.h"
1114
#include "multio/util/DateTime.h"
@@ -14,22 +17,26 @@ namespace multio::action {
1417

1518
class OperationWindow {
1619
public:
17-
OperationWindow(std::shared_ptr<StatisticsIO>& IOmanager, const StatisticsConfiguration& cfg);
20+
OperationWindow(std::shared_ptr<StatisticsIO>& IOmanager, const StatisticsOptions& opt);
1821

1922
OperationWindow(const eckit::DateTime& epochPoint, const eckit::DateTime& startPoint,
20-
const eckit::DateTime& creationPoint, const eckit::DateTime& endPoint, long timeStepInSeconds);
23+
const eckit::DateTime& creationPoint, const eckit::DateTime& endPoint,
24+
long timeStepInSeconds, long windowType);
2125

2226
long count() const;
2327

2428
void updateData(const eckit::DateTime& currentPoint);
2529
void updateWindow(const eckit::DateTime& startPoint, const eckit::DateTime& endPoint);
2630

27-
void dump(std::shared_ptr<StatisticsIO>& IOmanager, const StatisticsConfiguration& cfg) const;
28-
void load(std::shared_ptr<StatisticsIO>& IOmanager, const StatisticsConfiguration& cfg);
31+
void dump(std::shared_ptr<StatisticsIO>& IOmanager, const StatisticsOptions& opt) const;
32+
void load(std::shared_ptr<StatisticsIO>& IOmanager, const StatisticsOptions& opt );
2933

34+
std::string windowType() const;
3035
bool isWithin(const eckit::DateTime& dt) const;
3136
bool gtLowerBound(const eckit::DateTime& dt, bool throw_error) const;
37+
bool geLowerBound(const eckit::DateTime& dt, bool throw_error) const;
3238
bool leUpperBound(const eckit::DateTime& dt, bool throw_error) const;
39+
bool ltUpperBound(const eckit::DateTime& dt, bool throw_error) const;
3340

3441
long timeSpanInSeconds() const;
3542
long timeSpanInHours() const;
@@ -107,12 +114,17 @@ class OperationWindow {
107114

108115
long timeStepInSeconds_;
109116
long count_;
117+
long type_;
110118

111-
void serialize(IOBuffer& currState) const;
112-
void deserialize(const IOBuffer& currState);
119+
void serialize(IOBuffer& currState, const std::string& fname, const StatisticsOptions& opt) const;
120+
void deserialize(const IOBuffer& currState, const std::string& fname, const StatisticsOptions& opt);
113121

114122
void print(std::ostream& os) const;
115123
friend std::ostream& operator<<(std::ostream& os, const OperationWindow& a);
116124
};
117125

126+
OperationWindow make_window( const std::unique_ptr<PeriodUpdater>& periodUpdater, const StatisticsConfiguration& cfg);
127+
OperationWindow load_window( std::shared_ptr<StatisticsIO>& IOmanager, const StatisticsOptions& opt );
128+
118129
} // namespace multio::action
130+

0 commit comments

Comments
 (0)