statistics-mtg2: accumulate the accumulated parameters under their own paramId, and from the forecast start

share/multio/param-mappings/statistics_param_mappings.yaml

@@ -4866,3 +4866,87 @@
     paramOut: 266909
   - typeOfStatisticalProcessing: 6
     paramOut: 267909
+
+# Manually added mappings: accumulation of the IFS accumulated parameters.
+# The input is the per-step increment of an accumulated parameter, sent under the paramId of the
+# accumulation itself, so the accumulation keeps its paramId (paramIn == paramOut). The values are
+# not converted and must already be in the units of the paramId: kg m**-2 (water: 228228, 228143,
+# 3062, 228144, 260259, 231002, 231010, 231012), J m**-2 (146, 147, 176, 177, 169, 175, 178, 179,
+# 212) or N m**-2 s (180, 181). The generator above does not produce these entries: keep them when
+# this file is regenerated.
+- paramIn: 228228
+  mappings:
+  - typeOfStatisticalProcessing: 1
+    paramOut: 228228
+- paramIn: 228143
+  mappings:
+  - typeOfStatisticalProcessing: 1
+    paramOut: 228143
+- paramIn: 3062
+  mappings:
+  - typeOfStatisticalProcessing: 1
+    paramOut: 3062
+- paramIn: 228144
+  mappings:
+  - typeOfStatisticalProcessing: 1
+    paramOut: 228144
+- paramIn: 260259
+  mappings:
+  - typeOfStatisticalProcessing: 1
+    paramOut: 260259
+- paramIn: 231002
+  mappings:
+  - typeOfStatisticalProcessing: 1
+    paramOut: 231002
+- paramIn: 231010
+  mappings:
+  - typeOfStatisticalProcessing: 1
+    paramOut: 231010
+- paramIn: 231012
+  mappings:
+  - typeOfStatisticalProcessing: 1
+    paramOut: 231012
+- paramIn: 146
+  mappings:
+  - typeOfStatisticalProcessing: 1
+    paramOut: 146
+- paramIn: 147
+  mappings:
+  - typeOfStatisticalProcessing: 1
+    paramOut: 147
+- paramIn: 176
+  mappings:
+  - typeOfStatisticalProcessing: 1
+    paramOut: 176
+- paramIn: 177
+  mappings:
+  - typeOfStatisticalProcessing: 1
+    paramOut: 177
+- paramIn: 169
+  mappings:
+  - typeOfStatisticalProcessing: 1
+    paramOut: 169
+- paramIn: 175
+  mappings:
+  - typeOfStatisticalProcessing: 1
+    paramOut: 175
+- paramIn: 178
+  mappings:
+  - typeOfStatisticalProcessing: 1
+    paramOut: 178
+- paramIn: 179
+  mappings:
+  - typeOfStatisticalProcessing: 1
+    paramOut: 179
+- paramIn: 212
+  mappings:
+  - typeOfStatisticalProcessing: 1
+    paramOut: 212
+- paramIn: 180
+  mappings:
+  - typeOfStatisticalProcessing: 1
+    paramOut: 180
+- paramIn: 181
+  mappings:
+  - typeOfStatisticalProcessing: 1
+    paramOut: 181

src/multio/action/statistics-mtg2/CONTEXT.md

@@ -110,6 +110,26 @@ For a newly created forward-offset window, a message exactly on the nominal lowe
 
 This lower-bound behavior is structural and must not depend on `initial-condition-present`.
 
+## Forecast-Start Window Origin
+
+```yaml
+output-frequency: "3h"
+options:
+  window-origin: forecast-start   # default: window-start
+```
+
+The default `window-start` is the behaviour described above: a calendar window of `output-frequency` that is emitted when a field outside it arrives and then reset.
+
+With `forecast-start` the statistic accumulates from the simulation start (the first-step flush) and is never reset; `output-frequency` is only the emission cadence:
+
+- the window is `(simulation start, next emission point]`; a sample on the emission point is emitted immediately (no later field or flush needed), and the end then moves to the next emission point;
+- the emitted `step` is the absolute current step, `timespan` is `fs` when the window starts at the forecast epoch (otherwise the elapsed duration), and `misc-distanceFromPreviousStepInSeconds` is the elapsed length;
+- completeness is judged against the elapsed span (first sample seen, every sample at its declared distance from the previous one); a missing sample makes all later emissions of the field incomplete (suppressed unless `emit-incomplete-statistics`);
+- a field after the next emission point is an error: the input must contain every emission point;
+- a `last-step` flush emits the open tail (samples after the last emission) once, a run ending on an emission point has no tail;
+- `lastFlush_` of the window is the last emitted point; the restart format is unchanged, the origin comes from the options when a window is loaded, and a restored window keeps the original forecast start (a run without restored state accumulates from its own start);
+- accepted: operations `accumulate`, `average`, `maximum`, `minimum` (their `compute` leaves the state untouched), `output-frequency` in hours, `window-type: forward-offset`, instantaneous input only, simulation start on a whole hour.
+
 ## Initial Condition Option
 
 The active option is:

src/multio/action/statistics-mtg2/OperationWindow.cc

@@ -62,6 +62,24 @@ eckit::DateTime yyyymmdd_hhmmss2DateTime(uint64_t yyyymmdd, uint64_t hhmmss) {
 OperationWindow make_window(const std::unique_ptr<PeriodUpdater>& periodUpdater, const StatisticsConfiguration& cfg,
                             const eckit::DateTime& simulationStart) {
     eckit::DateTime epochPoint{cfg.epoch()};
+
+    if (cfg.options().windowOrigin() == WindowOrigin::ForecastStart) {
+        // The window opens exactly at the simulation start and ends at the first emission point. It is never advanced
+        // by a field: later emission points are reached by OperationWindow::extendEnd.
+        if (simulationStart.time().minutes() != 0 || simulationStart.time().seconds() != 0) {
+            std::ostringstream os;
+            os << "window-origin forecast-start requires a simulation start on a whole hour, got " << simulationStart;
+            throw eckit::UserError{os.str(), Here()};
+        }
+        eckit::DateTime endPoint{periodUpdater->computeWinEndTime(simulationStart)};
+        return OperationWindow{epochPoint,
+                               simulationStart,
+                               eckit::DateTime{cfg.curr()},
+                               endPoint,
+                               cfg.options().windowType(),
+                               WindowOrigin::ForecastStart};
+    }
+
     eckit::DateTime startPoint{periodUpdater->computeWinStartTime(simulationStart)};
     eckit::DateTime endPoint{periodUpdater->computeWinEndTime(startPoint)};
 
@@ -99,6 +117,7 @@ OperationWindow::OperationWindow(std::shared_ptr<StatisticsIO>& IOmanager, const
     count_{0},
     counts_{},
     windowType_{WindowType::ForwardOffset},
+    windowOrigin_{opt.windowOrigin()},
     firstPoint_{},
     declaredDistanceHistogram_{},
     observedDistanceHistogram_{},
@@ -110,7 +129,7 @@ OperationWindow::OperationWindow(std::shared_ptr<StatisticsIO>& IOmanager, const
 
 OperationWindow::OperationWindow(const eckit::DateTime& epochPoint, const eckit::DateTime& startPoint,
                                  const eckit::DateTime& creationPoint, const eckit::DateTime& endPoint,
-                                 WindowType windowType) :
+                                 WindowType windowType, WindowOrigin windowOrigin) :
     epochPoint_{epochPoint},
     startPoint_{startPoint},
     creationPoint_{creationPoint},
@@ -121,6 +140,7 @@ OperationWindow::OperationWindow(const eckit::DateTime& epochPoint, const eckit:
     count_{0},
     counts_{},
     windowType_{windowType},
+    windowOrigin_{windowOrigin},
     firstPoint_{},
     declaredDistanceHistogram_{},
     observedDistanceHistogram_{},
@@ -221,6 +241,15 @@ void OperationWindow::updateWindow(const eckit::DateTime& startPoint, const ecki
     return;
 }
 
+void OperationWindow::extendEnd(const eckit::DateTime& endPoint) {
+    if (endPoint <= endPoint_) {
+        std::ostringstream os;
+        os << *this << " : new end " << endPoint << " does not extend the window end " << endPoint_ << std::endl;
+        throw eckit::SeriousBug(os.str(), Here());
+    }
+    endPoint_ = endPoint;
+}
+
 bool OperationWindow::isWithin(const eckit::DateTime& dt) const {
     bool ret;
     if (windowType_ == WindowType::ForwardOffset) {
@@ -283,6 +312,18 @@ long OperationWindow::timeSpanInSeconds() const {
     return long(endPoint_ - startPoint_);
 }
 
+long OperationWindow::emissionSpanInSeconds() const {
+    return windowOrigin_ == WindowOrigin::ForecastStart ? long(currPoint_ - startPoint_) : timeSpanInSeconds();
+}
+
+bool OperationWindow::hasUnemittedSamples() const {
+    return count_ > 0 && currPoint_ > lastFlush_;
+}
+
+WindowOrigin OperationWindow::windowOrigin() const {
+    return windowOrigin_;
+}
+
 long OperationWindow::lastPointsDiffInSeconds() const {
     return long(currPoint_ - prevPoint_);
 }
@@ -401,6 +442,10 @@ bool OperationWindow::isComplete() const {
     if (!firstPoint_ || !contiguous_) {
         return false;
     }
+    if (windowOrigin_ == WindowOrigin::ForecastStart) {
+        // Judged against the elapsed span (start .. current point), not against the next emission point
+        return true;
+    }
     if (windowType_ == WindowType::ForwardOffset) {
         return currPoint_ == endPoint_;
     }
@@ -452,7 +497,7 @@ const std::map<std::int64_t, std::size_t>& OperationWindow::declaredDistanceHist
 
 std::map<std::int64_t, std::size_t> OperationWindow::observedDistanceHistogram() const {
     auto histogram = observedDistanceHistogram_;
-    if (firstPoint_) {
+    if (firstPoint_ && windowOrigin_ != WindowOrigin::ForecastStart) {
         const auto trailingDistance = static_cast<std::int64_t>(endPoint_ - currPoint_);
         if (trailingDistance > 0) {
             histogram[trailingDistance]++;

src/multio/action/statistics-mtg2/OperationWindow.h

@@ -22,7 +22,8 @@ class OperationWindow {
     OperationWindow(std::shared_ptr<StatisticsIO>& IOmanager, const StatisticsOptions& opt);
 
     OperationWindow(const eckit::DateTime& epochPoint, const eckit::DateTime& startPoint,
-                    const eckit::DateTime& creationPoint, const eckit::DateTime& endPoint, WindowType windowType);
+                    const eckit::DateTime& creationPoint, const eckit::DateTime& endPoint, WindowType windowType,
+                    WindowOrigin windowOrigin = WindowOrigin::WindowStart);
 
     long count() const;
     const std::vector<long>& counts() const;
@@ -33,6 +34,9 @@ class OperationWindow {
     void updateData(const eckit::DateTime& currentPoint, std::int64_t distanceFromPreviousStepInSeconds);
     void updateWindow(const eckit::DateTime& startPoint, const eckit::DateTime& endPoint);
 
+    /// Forecast-start windows only: move the end to the next emission point without touching the accumulated state
+    void extendEnd(const eckit::DateTime& endPoint);
+
     void dump(std::shared_ptr<StatisticsIO>& IOmanager, const StatisticsOptions& opt) const;
     void load(std::shared_ptr<StatisticsIO>& IOmanager, const StatisticsOptions& opt);
 
@@ -44,6 +48,9 @@ class OperationWindow {
 
     long timeSpanInSeconds() const;
     long timeSpanInHours() const;
+    /// Length reported with an emission: the nominal window length, or for a forecast-start window the elapsed
+    /// length from the window start to the current point
+    long emissionSpanInSeconds() const;
     long lastPointsDiffInSeconds() const;
 
     util::DateTimeDiff lastPointsDiff() const;
@@ -83,6 +90,9 @@ class OperationWindow {
     bool isUniform() const;
     std::string incompleteReason() const;
     WindowType windowType() const;
+    WindowOrigin windowOrigin() const;
+    /// Forecast-start windows only: samples have been received since the last emission
+    bool hasUnemittedSamples() const;
     std::int64_t lastDeclaredDistance() const;
     const std::map<std::int64_t, std::size_t>& declaredDistanceHistogram() const;
     std::map<std::int64_t, std::size_t> observedDistanceHistogram() const;
@@ -91,6 +101,7 @@ class OperationWindow {
 
     std::string stepRangeInHours(const eckit::DateTime& refPoint) const;
 
+    /// Records the current point as the last emitted point
     void updateFlush();
 
     size_t restartSize() const;
@@ -107,6 +118,8 @@ class OperationWindow {
     long count_;
     mutable std::vector<long> counts_;
     WindowType windowType_;
+    // Not part of the restart state: taken from the options when a window is created or loaded
+    WindowOrigin windowOrigin_;
     std::optional<eckit::DateTime> firstPoint_;
     std::map<std::int64_t, std::size_t> declaredDistanceHistogram_;
     std::map<std::int64_t, std::size_t> observedDistanceHistogram_;

src/multio/action/statistics-mtg2/PeriodUpdaters.h

@@ -15,6 +15,9 @@
 
 namespace multio::action::statistics_mtg2 {
 
+/// Split an output frequency such as "3h" into its span (3) and its period kind ("h")
+void parsePeriodGrammar(std::string const& output_freq, long& span, std::string& periodKind);
+
 std::unique_ptr<PeriodUpdater> make_period_updater(std::string const& output_freq, const StatisticsConfiguration& cfg);
 std::unique_ptr<PeriodUpdater> load_period_updater(std::shared_ptr<StatisticsIO>& IOmanager,
                                                    const StatisticsOptions& opt);