Linearize timer overflows

.github/workflows/pythonapp.yml

@@ -36,6 +36,8 @@ jobs:
       - uses: actions/upload-artifact@v4
         with:
           path: ./wheelhouse/*.whl
+          include-hidden-files: true
+          retention-days: 1
 
   build_sdist:
     name: Build source distribution
@@ -51,6 +53,8 @@ jobs:
       - uses: actions/upload-artifact@v4
         with:
           path: dist/*.tar.gz
+          include-hidden-files: true
+          retention-days: 1
   upload_pypi:
     needs: [build_wheels] # , build_sdist
     runs-on: ubuntu-latest
@@ -60,8 +64,12 @@ jobs:
     steps:
       - uses: actions/[email protected]
         with:
-          name: artifact
+          # name: artifact
           path: dist
+          merge-multiple: true
+          # path: .vitest-reports
+          pattern: *
+          # merge-multiple: true
 
       - uses: pypa/[email protected]
         with:

docs/linux_installation.md

@@ -0,0 +1,92 @@
+There are several things to consider:
+- minimum required Python version is currently 3.8.
+- Eigentlich ist this document about installation of pyXCP on Linux, aber das folgende trifft auch die Installation von Python packages on modern Linux distros in general.
+... trift auf moderne Linux distros im allgemeinen zu.
+
+Wir starten gleich mit dem Fazit:
+The upshort: Use virtual environments (???LINK) when ever possible!
+
+You may want to start with `pip install pyxcp`, but this ????,
+??? but you may run into problems???
+so read on.
+
+For the sake of reproduceability, lets start with a `Dockerfile`:
+
+``` Docker
+FROM ubuntu:24.04
+
+RUN apt update
+RUN apt upgrade -y
+RUN apt install -y pkg-config
+RUN apt install -y gcc git cmake libssl-dev python3 python3-pip python3-venv python3-poetry pipx rustc cargo libffi-dev
+RUN apt install -y zsh nano tmux
+
+CMD ["/bin/zsh"]
+```
+Note
+----
+Depending on your distro there seem to be two main differences (besides the package manager):
+- `libssl-dev` vs. `openssl-devel`
+- `libffi` vs. `libffi-dev`
+
+
+```shell
+ find /usr -name "libpython*.so"
+```
+
+``` shell
+raspi% find /usr -name "libpython*.so"
+/usr/lib/aarch64-linux-gnu/libpeas-1.0/loaders/libpython3loader.so
+/usr/lib/aarch64-linux-gnu/libpython3.11.so
+/usr/lib/aarch64-linux-gnu/libpython3.11d.so
+/usr/lib/python3.11/config-3.11d-aarch64-linux-gnu/libpython3.11.so
+/usr/lib/python3.11/config-3.11d-aarch64-linux-gnu/libpython3.11d.so
+/usr/lib/python3.11/config-3.11-aarch64-linux-gnu/libpython3.11.so
+find: ‘/usr/share/polkit-1/rules.d’: Keine Berechtigung
+```
+
+```shell
+ find /usr -name "libpython*.a"
+```
+
+```shell
+raspi% find /usr -name "libpython*.a"
+/usr/lib/aarch64-linux-gnu/libpython3.11.a
+/usr/lib/aarch64-linux-gnu/libpython3.11d.a
+/usr/lib/python3.11/config-3.11d-aarch64-linux-gnu/libpython3.11d.a
+/usr/lib/python3.11/config-3.11-aarch64-linux-gnu/libpython3.11.a
+/usr/lib/python3.11/config-3.11-aarch64-linux-gnu/libpython3.11-pic.a
+find: ‘/usr/share/polkit-1/rules.d’: Keine Berechtigung
+
+```
+
+ find /usr -name "libpython*.a"
+
+
+After building an image and running a container from it you may run
+``` shell
+pipx install pyxcp
+```
+and you're done.
+
+![Screenshot](./images/docker01.png "Screenshot / Docker")
+
+But wait, it's probably not that easy on your Linux box!
+
+- can be confusing at best and outright break the entire underlying operating system at worst.
+
+While this could be brute-forced away with
+`pip install --break-system-packages --ignore-installed  <package>`
+~/.config/pip/pip.conf:
+[global]
+break-system-packages = true
+and in other situations
+`pip install --user  <package>`
+may work, the user is highly advised to use `pipx` as package installer.
+
+This is really nice, if one is installing .... like `csvkit`
+
+See:
+- [externally managed environments](https://packaging.python.org/en/latest/specifications/externally-managed-environments/#externally-managed-environments)
+- [PEP-668 (historical)](https://peps.python.org/pep-0668/)
+

docs/tutorial.md

@@ -0,0 +1,9 @@
+Tutorial
+========
+
+
+* Python
+    - 2.4
+
+.. code-block:: python
+    import sysconfig

pyxcp/recorder/unfolder.hpp

@@ -7,6 +7,7 @@
 #include <charconv>
 #include <cstring>
 #include <iostream>
+#include <limits>
 #include <map>
 #if __has_include(<stdfloat>)
     #include <stdfloat>
@@ -637,6 +638,33 @@ struct MeasurementParameters {
         return m_first_pids;
     }
 
+#undef max  // Thanks to Windows.
+
+    auto get_overflow_value() const {
+        std::uint64_t ov_value{};
+        switch (m_ts_size) {
+            case 1:
+                ov_value = std::numeric_limits<std::uint8_t>::max();
+                break;
+            case 2:
+                ov_value = std::numeric_limits<std::uint16_t>::max();
+                break;
+            case 4:
+                ov_value = std::numeric_limits<std::uint32_t>::max();
+                break;
+//            case 8:
+//                ov_value = std::numeric_limits<std::uint64_t>::max() + 1;
+//                break;
+            default:
+                throw std::runtime_error("Invalid timestamp size");
+        }
+        ov_value++;
+        std::cout << "TS-V: " << ov_value << " scale: " << m_ts_scale_factor << std::endl;
+        ov_value = static_cast<std::uint64_t>(ov_value * m_ts_scale_factor);
+        std::cout << "OVRF-Value: " << ov_value << std::endl;
+        return ov_value;
+    }
+
     std::uint8_t               m_byte_order;
     std::uint8_t               m_id_field_size;
     bool                       m_timestamps_supported;
@@ -1050,6 +1078,9 @@ class DAQPolicyBase {
     }
 
     virtual void set_parameters(const MeasurementParameters& params) noexcept {
+        m_overflow_value = params.get_overflow_value();
+        m_overflow_counter = 0ULL;
+        std::cout << "Overflow value: " << m_overflow_value << ", Overflow counter: " << m_overflow_counter << std::endl;
         initialize();
     }
 
@@ -1058,6 +1089,9 @@ class DAQPolicyBase {
     virtual void initialize() = 0;
 
     virtual void finalize() = 0;
+ private:
+    std::uint64_t m_overflow_value{};
+    std::uint64_t m_overflow_counter{};
 };
 
 class DaqRecorderPolicy : public DAQPolicyBase {
@@ -1164,6 +1198,38 @@ struct ValueHolder {
     std::any m_value;
 };
 
+class Overflow {
+public:
+
+    Overflow(std::uint64_t overflow_value) : m_overflow_value(overflow_value), m_overflow_counter(0ULL), m_previous_timestamp(0ULL) {
+    }
+
+    auto get_previous_timestamp() const noexcept {
+        return m_previous_timestamp;
+    }
+
+    void set_previous_timestamp(std::uint64_t timestamp) noexcept {
+        m_previous_timestamp = timestamp;
+    }
+
+    void inc_overflow_counter() noexcept {
+        m_overflow_counter++;
+    }
+
+    auto get_overflow_counter() const noexcept {
+        return m_overflow_counter;
+    }
+
+    auto get_value() const noexcept {
+        return m_overflow_value * m_overflow_counter;
+    }
+
+private:
+    std::uint64_t m_overflow_value{};
+    std::uint64_t m_overflow_counter{};
+    std::uint64_t m_previous_timestamp{};
+};
+
 class XcpLogFileDecoder {
    public:
 
@@ -1172,6 +1238,11 @@ class XcpLogFileDecoder {
         if (metadata != "") {
             auto des  = Deserializer(metadata);
             m_params  = des.run();
+
+            for (auto idx=0; idx < m_params.get_daq_lists().size(); ++idx) {
+                m_overflows.emplace_back(Overflow(m_params.get_overflow_value()));
+            }
+
             m_decoder = std::make_unique<DAQProcessor>(m_params);
         } else {
             // cannot proceed!!!
@@ -1215,7 +1286,18 @@ class XcpLogFileDecoder {
                 auto result = m_decoder->feed(timestamp, str_data);
                 if (result) {
                     const auto& [daq_list, ts0, ts1, meas] = *result;
-                    on_daq_list(daq_list, ts0, ts1, meas);
+
+                    auto& overflow = m_overflows[daq_list];
+
+                    if (overflow.get_previous_timestamp() > ts1) {
+                        overflow.inc_overflow_counter();
+                        // Maybe on debug-level?
+                        // std::cout << "Overflow detected, counter: " << overflow.get_overflow_counter() << " " << overflow.get_previous_timestamp() << " " << ts1 << std::endl;
+                    }
+
+                    on_daq_list(daq_list, ts0, ts1 + overflow.get_value(), meas);
+
+                    overflow.set_previous_timestamp(ts1);
                 }
             }
         }
@@ -1244,6 +1326,7 @@ class XcpLogFileDecoder {
     XcpLogFileReader              m_reader;
     std::unique_ptr<DAQProcessor> m_decoder;
     MeasurementParameters         m_params;
+    std::vector<Overflow>           m_overflows;
 };
 
 #endif  // RECORDER_UNFOLDER_HPP