TransactionDetails data reduction with lag and lead and export to csv

src/main/java/de/rwth/idsg/steve/repository/TransactionRepository.java

@@ -33,12 +33,15 @@
     List<Transaction> getTransactions(TransactionQueryForm form);
 
     void writeTransactionsCSV(TransactionQueryForm form, Writer writer);
+
+    void writeTransactionsDetailsCSV(int transactionPk, Writer writer,
+            boolean dataReduction, boolean firstArrivingMeterValueIfMultiple);
 
     List<Integer> getActiveTransactionIds(String chargeBoxId);
 
-    TransactionDetails getDetails(int transactionPk, boolean firstArrivingMeterValueIfMultiple);
+    TransactionDetails getDetails(int transactionPk, boolean dataReduction, boolean firstArrivingMeterValueIfMultiple);
 
     default TransactionDetails getDetails(int transactionPk) {
-        return getDetails(transactionPk, true);
+        return getDetails(transactionPk, true, true);
     }
 }

src/main/java/de/rwth/idsg/steve/repository/impl/TransactionRepositoryImpl.java

@@ -16,40 +16,43 @@
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 package de.rwth.idsg.steve.repository.impl;
 
 import de.rwth.idsg.steve.SteveException;
 import de.rwth.idsg.steve.repository.TransactionRepository;
 import de.rwth.idsg.steve.repository.dto.Transaction;
 import de.rwth.idsg.steve.repository.dto.TransactionDetails;
 import de.rwth.idsg.steve.utils.DateTimeUtils;
 import de.rwth.idsg.steve.web.dto.TransactionQueryForm;
 import jooq.steve.db.enums.TransactionStopEventActor;
 import jooq.steve.db.tables.records.ConnectorMeterValueRecord;
 import jooq.steve.db.tables.records.TransactionStartRecord;
 import org.joda.time.DateTime;
 import org.jooq.Condition;
 import org.jooq.DSLContext;
-import org.jooq.Field;
+//import org.jooq.Field;
 import org.jooq.Record12;
 import org.jooq.Record9;
 import org.jooq.RecordMapper;
 import org.jooq.SelectQuery;
 import org.jooq.Table;
 import org.jooq.impl.DSL;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Repository;
 
 import java.io.Writer;
 import java.util.List;
 
 import static de.rwth.idsg.steve.utils.CustomDSL.date;
 import static jooq.steve.db.tables.ChargeBox.CHARGE_BOX;
 import static jooq.steve.db.tables.Connector.CONNECTOR;
 import static jooq.steve.db.tables.ConnectorMeterValue.CONNECTOR_METER_VALUE;
 import static jooq.steve.db.tables.OcppTag.OCPP_TAG;
 import static jooq.steve.db.tables.Transaction.TRANSACTION;
 import static jooq.steve.db.tables.TransactionStart.TRANSACTION_START;
+import org.jooq.Record11;
+import org.jooq.Record8;
+//import static org.jooq.impl.DSL.*;
 
 /**
  * @author Sevket Goekay <[email protected]>
@@ -77,6 +80,28 @@
                             .formatCSV(writer);
     }
 
+    @Override
+    public void writeTransactionsDetailsCSV(int transactionPk, Writer writer, 
+                    boolean dataReduction, boolean firstArrivingMeterValueIfMultiple) {
+        // write a few information about the transaction
+        TransactionQueryForm form = new TransactionQueryForm();
+        form.setTransactionPk(transactionPk);
+        form.setType(TransactionQueryForm.QueryType.ALL);
+        form.setPeriodType(TransactionQueryForm.QueryPeriodType.ALL);
+        //getInternalCSV(form).fetch().formatCSV(writer);
+        Record12<Integer, String, Integer, String, DateTime, String, DateTime,
+                    String, String, Integer, Integer, TransactionStopEventActor>
+                res = getInternal(form).fetchOne();
+        if (res == null) {
+            throw new SteveException("There is no transaction with id '%s'", transactionPk);
+        }
+        res.formatCSV(writer);
+        Transaction transaction = new TransactionMapper().map(res); //.map(new TransactionMapper());
+        TransactionStartRecord nextTx = null;
+        getDetailsQuery(transaction, nextTx, transactionPk, dataReduction, firstArrivingMeterValueIfMultiple)
+                .fetch().formatCSV(writer);
+    }
+
     @Override
     public List<Integer> getActiveTransactionIds(String chargeBoxId) {
         return ctx.select(TRANSACTION.TRANSACTION_PK)
@@ -89,39 +114,64 @@
     }
 
     @Override
-    public TransactionDetails getDetails(int transactionPk, boolean firstArrivingMeterValueIfMultiple) {
-
+    public TransactionDetails getDetails(int transactionPk,
+            boolean dataReduction, boolean firstArrivingMeterValueIfMultiple) {
         // -------------------------------------------------------------------------
         // Step 1: Collect general data about transaction
         // -------------------------------------------------------------------------
-
+        //
         TransactionQueryForm form = new TransactionQueryForm();
         form.setTransactionPk(transactionPk);
         form.setType(TransactionQueryForm.QueryType.ALL);
         form.setPeriodType(TransactionQueryForm.QueryPeriodType.ALL);
 
-        Record12<Integer, String, Integer, String, DateTime, String, DateTime, String, String, Integer, Integer, TransactionStopEventActor>
-                transaction = getInternal(form).fetchOne();
+        Record12<Integer, String, Integer, String, DateTime, String, DateTime,
+                String, String, Integer, Integer, TransactionStopEventActor>
+                res = getInternal(form).fetchOne();
 
-        if (transaction == null) {
+        if (res == null) {
             throw new SteveException("There is no transaction with id '%s'", transactionPk);
         }
+        //Transaction transaction = getInternal(form).fetchOne(new TransactionMapper());
+        TransactionMapper mapper = new TransactionMapper();
+        Transaction transaction = mapper.map(res);
+        TransactionStartRecord nextTx = null;
+        List<TransactionDetails.MeterValues> values =
+                getDetailsQuery(transaction, nextTx, transactionPk, dataReduction, firstArrivingMeterValueIfMultiple)
+                   .fetch()
+                   .map(r -> TransactionDetails.MeterValues.builder()
+                                                           .valueTimestamp(r.value1())
+                                                           .value(r.value2())
+                                                           .readingContext(r.value3())
+                                                           .format(r.value4())
+                                                           .measurand(r.value5())
+                                                           .location(r.value6())
+                                                           .unit(r.value7())
+                                                           .phase(r.value8())
+                                                           .build());
+
+        return new TransactionDetails(transaction, values, nextTx);
+    }
 
-        DateTime startTimestamp = transaction.value5();
-        DateTime stopTimestamp = transaction.value7();
-        String stopValue = transaction.value8();
-        String chargeBoxId = transaction.value2();
-        int connectorId = transaction.value3();
-
+    private SelectQuery<Record8<DateTime, String, String, String, String, String, String, String>>
+        getDetailsQuery(Transaction transaction, TransactionStartRecord nextTx, int transactionPk,
+                boolean dataReduction, boolean firstArrivingMeterValueIfMultiple) {
+        // -------------------------------------------------------------------------
+        // Step 1a: Collect general data about transaction
+        // -------------------------------------------------------------------------
+        //
+        DateTime startTimestamp = transaction.getStartTimestamp();
+        DateTime stopTimestamp = transaction.getStopTimestamp();
+        String stopValue = transaction.getStopValue();
+        String chargeBoxId = transaction.getChargeBoxId();
+        int connectorId = transaction.getConnectorId();
         // -------------------------------------------------------------------------
         // Step 2: Collect intermediate meter values
         // -------------------------------------------------------------------------
-
+        //
         Condition timestampCondition;
-        TransactionStartRecord nextTx = null;
-
+        //TransactionStartRecord nextTx = null;
         if (stopTimestamp == null && stopValue == null) {
-
             // https://github.com/steve-community/steve/issues/97
             //
             // handle "zombie" transaction, for which we did not receive any StopTransaction. if we do not handle it,
@@ -175,55 +225,94 @@
         // UNION removes all duplicate records
         //
         Table<ConnectorMeterValueRecord> t1 = transactionQuery.union(timestampQuery).asTable("t1");
+        if (!dataReduction) {
+            return ctx.select(
+                    t1.field(2, DateTime.class),
+                    t1.field(3, String.class),
+                    t1.field(4, String.class),
+                    t1.field(5, String.class),
+                    t1.field(6, String.class),
+                    t1.field(7, String.class),
+                    t1.field(8, String.class),
+                    t1.field(9, String.class))
+                    .from(t1)
+                    //.orderBy(innerTable.field(2))
+                    .getQuery();
+        }
 
         // -------------------------------------------------------------------------
         // Step 3: Charging station might send meter vales at fixed intervals (e.g.
         // every 15 min) regardless of the fact that connector's meter value did not
         // change (e.g. vehicle is fully charged, but cable is still connected). This
-        // yields multiple entries in db with the same value but different timestamp.
+        // yields multiple entries in db with the same value but different timestamps.
         // We are only interested in the first (or last) arriving entry.
         // -------------------------------------------------------------------------
 
-        Field<DateTime> dateTimeField;
+        //Field<DateTime> dateTimeField;
+        Table<Record11<Integer, Integer, DateTime, String, String, String,
+                String, String, String, String, Double>> innerTable;
         if (firstArrivingMeterValueIfMultiple) {
-            dateTimeField = DSL.min(t1.field(2, DateTime.class)).as("min");
+            //dateTimeField = DSL.min(t1.field(2, DateTime.class)).as("min");
+             innerTable =
+                 ctx.select(t1.field(0, Integer.class),
+                        t1.field(1, Integer.class),
+                        t1.field(2, DateTime.class),
+                        t1.field(3, String.class),
+                        t1.field(4, String.class),
+                        t1.field(5, String.class),
+                        t1.field(6, String.class),
+                        t1.field(7, String.class),
+                        t1.field(8, String.class),
+                        t1.field(9, String.class),
+                        //lag(t1.field("value"))
+                        //        .over(partitionBy(t1.field("measurand"),t1.field("phase"))
+                        //                .orderBy(t1.field("value_timestamp"))).cast(String.class).as("lagVal"),
+                        t1.field("value", Double.class).sub(DSL.lag(t1.field("value", Double.class))
+                                .over(DSL.partitionBy(t1.field("measurand"), t1.field("phase"))
+                                        .orderBy(t1.field("value_timestamp")))).as("diff"))
+                        .from(t1)
+                   .where(t1.field("transaction_pk", Integer.class).eq(transactionPk))
+                        .and((t1.field("format", String.class).eq("raw")).or(t1.field("format", String.class).isNull()))
+                        .asTable();
         } else {
-            dateTimeField = DSL.max(t1.field(2, DateTime.class)).as("max");
-        }
-
-        List<TransactionDetails.MeterValues> values =
-                ctx.select(
-                        dateTimeField,
+            //dateTimeField = DSL.max(t1.field(2, DateTime.class)).as("max");
+            innerTable =
+                 ctx.select(t1.field(0, Integer.class),
+                        t1.field(1, Integer.class),
+                        t1.field(2, DateTime.class),
                         t1.field(3, String.class),
                         t1.field(4, String.class),
                         t1.field(5, String.class),
                         t1.field(6, String.class),
                         t1.field(7, String.class),
                         t1.field(8, String.class),
-                        t1.field(9, String.class))
+                        t1.field(9, String.class),
+                        //lead(t1.field("value"))
+                        //        .over(partitionBy(t1.field("measurand"),t1.field("phase"))
+                        //                .orderBy(t1.field("value_timestamp"))).cast(String.class).as("leadVal"),
+                        t1.field("value", Double.class).sub(DSL.lead(t1.field("value", Double.class))
+                                .over(DSL.partitionBy(t1.field("measurand"), t1.field("phase"))
+                                        .orderBy(t1.field("value_timestamp")))).as("diff"))
                    .from(t1)
-                   .groupBy(
-                           t1.field(3),
-                           t1.field(4),
-                           t1.field(5),
-                           t1.field(6),
-                           t1.field(7),
-                           t1.field(8),
-                           t1.field(9))
-                   .orderBy(dateTimeField)
-                   .fetch()
-                   .map(r -> TransactionDetails.MeterValues.builder()
-                                                           .valueTimestamp(r.value1())
-                                                           .value(r.value2())
-                                                           .readingContext(r.value3())
-                                                           .format(r.value4())
-                                                           .measurand(r.value5())
-                                                           .location(r.value6())
-                                                           .unit(r.value7())
-                                                           .phase(r.value8())
-                                                           .build());
+                   .where(t1.field("transaction_pk", Integer.class).eq(transactionPk))
+                        .and((t1.field("format", String.class).eq("raw")).or(t1.field("format", String.class).isNull()))
+                        .asTable();
+        }
 
-        return new TransactionDetails(new TransactionMapper().map(transaction), values, nextTx);
+        return ctx.select(
+                        innerTable.field(2, DateTime.class),
+                        innerTable.field(3, String.class),
+                        innerTable.field(4, String.class),
+                        innerTable.field(5, String.class),
+                        innerTable.field(6, String.class),
+                        innerTable.field(7, String.class),
+                        innerTable.field(8, String.class),
+                        innerTable.field(9, String.class))
+                   .from(innerTable)
+                    .where(innerTable.field(10, Double.class).ne(0.0))
+                        .or(innerTable.field(10, Double.class).isNull())
+                   //.orderBy(innerTable.field(2))
+                .getQuery();
     }
 
     // -------------------------------------------------------------------------

src/main/java/de/rwth/idsg/steve/service/TransactionStopService.java

@@ -57,7 +57,7 @@ public void stop(List<Integer> transactionPkList) {
     }
 
     public void stop(Integer transactionPk) {
-        TransactionDetails thisTxDetails = transactionRepository.getDetails(transactionPk, false);
+        TransactionDetails thisTxDetails = transactionRepository.getDetails(transactionPk, true, false);
         Transaction thisTx = thisTxDetails.getTransaction();
 
         // early exit, if transaction is already stopped

src/main/java/de/rwth/idsg/steve/web/controller/TransactionsReservationsController.java

@@ -25,6 +25,7 @@
 import de.rwth.idsg.steve.service.OcppTagService;
 import de.rwth.idsg.steve.service.TransactionStopService;
 import de.rwth.idsg.steve.web.dto.ReservationQueryForm;
+import de.rwth.idsg.steve.web.dto.TransactionDetailQueryForm;
 import de.rwth.idsg.steve.web.dto.TransactionQueryForm;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Controller;
@@ -64,6 +65,7 @@
     private static final String TRANSACTIONS_PATH = "/transactions";
     private static final String TRANSACTION_STOP_PATH = "/transactions/stop/{transactionPk}";
     private static final String TRANSACTIONS_DETAILS_PATH = "/transactions/details/{transactionPk}";
+    private static final String TRANSACTIONS_DETAILS_XLS_PATH = "/transactions/detailsxls"; 
     private static final String TRANSACTIONS_QUERY_PATH = "/transactions/query";
     private static final String RESERVATIONS_PATH = "/reservations";
     private static final String RESERVATIONS_QUERY_PATH = "/reservations/query";
@@ -90,6 +92,8 @@
 
     @RequestMapping(value = TRANSACTIONS_DETAILS_PATH)
     public String getTransactionDetails(@PathVariable("transactionPk") int transactionPk, Model model) {
+        TransactionDetailQueryForm params = new TransactionDetailQueryForm();
+        model.addAttribute(PARAMS, params);
         model.addAttribute("details", transactionRepository.getDetails(transactionPk));
         return "data-man/transactionDetails";
     }
@@ -121,11 +125,30 @@
         }
     }
 
+    @RequestMapping(value = TRANSACTIONS_DETAILS_XLS_PATH)
+    public String getTransactionsDetailXLS(@Valid @ModelAttribute(PARAMS) TransactionDetailQueryForm params,
+                                       BindingResult result, Model model,
+                                       HttpServletResponse response) throws IOException {
+        if (result.hasErrors()) {
+            return null;
+        }
+        int transactionPk = params.getTransactionPk();
+        String fileName = "transaction" + transactionPk + "_MeterValues.csv";
+        String headerKey = "Content-Disposition";
+        String headerValue = String.format("attachment; filename=\"%s\"", fileName);
+        response.setContentType("text/csv");
+        response.setHeader(headerKey, headerValue);
+        transactionRepository.writeTransactionsDetailsCSV(transactionPk,
+                                                          response.getWriter(),
+                                                          params.isDataReduction(),
+                                                          params.isFirstArrivingMeterValueIfMultiple());
+        return null;
+    }
+
     @RequestMapping(value = RESERVATIONS_PATH)
     public String getReservations(Model model) {
         ReservationQueryForm params = new ReservationQueryForm();
         initResList(model);
-
         model.addAttribute("reservList", reservationRepository.getReservations(params));
         model.addAttribute(PARAMS, params);
         return "data-man/reservations";
@@ -137,7 +160,6 @@
         if (!result.hasErrors()) {
             model.addAttribute("reservList", reservationRepository.getReservations(params));
         }
-
         initResList(model);
         model.addAttribute(PARAMS, params);
         return "data-man/reservations";

src/main/java/de/rwth/idsg/steve/web/dto/TransactionDetailQueryForm.java

@@ -0,0 +1,46 @@
+/*
+ * SteVe - SteckdosenVerwaltung - https://github.com/steve-community/steve
+ * Copyright (C) 2013-2023 SteVe Community Team
+ * All Rights Reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+package de.rwth.idsg.steve.web.dto;
+
+
+import lombok.Getter;
+import lombok.Setter;
+import lombok.ToString;
+
+/**
+ * @author Frank Brosi
+ * @since 31.05.2023
+ */
+@Getter
+@Setter
+@ToString(callSuper = true)
+public class TransactionDetailQueryForm extends QueryForm {
+
+    // Internal database Id
+    private Integer transactionPk;
+
+    private boolean returnCSV = false;
+    private boolean dataReduction = true;
+    private boolean firstArrivingMeterValueIfMultiple = true;
+
+    public boolean isTransactionPkSet() {
+        return transactionPk != null;
+    }
+
+}