Merge branch 'develop'

MobileTeleSystems · Jan 27, 2025 · 26379f1 · 26379f1
2 parents bd14cd5 + c3878b7
commit 26379f1
Show file tree

Hide file tree

Showing 63 changed files with 852 additions and 97 deletions.
diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml
@@ -7,7 +7,7 @@ on:
       - master
 
 env:
-  DEFAULT_PYTHON: '3.12'
+  DEFAULT_PYTHON: '3.13'
 
 permissions:
   contents: read

diff --git a/.github/workflows/dev-release.yml b/.github/workflows/dev-release.yml
@@ -9,7 +9,7 @@ on:
   workflow_dispatch:
 
 env:
-  DEFAULT_PYTHON: '3.12'
+  DEFAULT_PYTHON: '3.13'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -6,7 +6,7 @@ on:
       - '[0-9]+.[0-9]+.[0-9]+'
 
 env:
-  DEFAULT_PYTHON: '3.12'
+  DEFAULT_PYTHON: '3.13'
 
 jobs:
   release:
@@ -110,4 +110,5 @@ jobs:
           name: ${{ steps.release-name.outputs.name }}
           body_path: changelog.md
           files: |
-            dist/*
+            dist/*.tar.gz
+            dist/*.whl
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -13,7 +13,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  DEFAULT_PYTHON: '3.12'
+  DEFAULT_PYTHON: '3.13'
 
 jobs:
   tests:
@@ -22,9 +22,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.7', '3.12']
-        pydantic-version: ['1', '2']
-        os: [ubuntu-latest]
+        include:
+          - os: ubuntu-latest
+            python-version: '3.13'
+            pydantic-version: '2'
+          - os: ubuntu-22.04
+            python-version: '3.7'
+            pydantic-version: '1'
 
     steps:
       - name: Checkout code
@@ -116,7 +120,7 @@ jobs:
         run: ./combine_coverage.sh
 
       - name: Check coverage
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           file: ./reports/coverage.xml

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -44,7 +44,7 @@ repos:
         args:
           - --license-filepath
           - .spdx-license-header.txt
-          - --use-current-year
+          - --allow-past-years
           - --no-extra-eol
 
   - repo: https://github.com/codespell-project/codespell
@@ -78,7 +78,7 @@ repos:
       - id: text-unicode-replacement-char
 
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.19.0
+    rev: v3.19.1
     hooks:
       - id: pyupgrade
         args: [--py37-plus, --keep-runtime-typing]
@@ -102,7 +102,7 @@ repos:
           - black==24.4.2
 
   - repo: https://github.com/pycqa/bandit
-    rev: 1.7.10
+    rev: 1.8.2
     hooks:
       - id: bandit
         args:

diff --git a/.readthedocs.yml b/.readthedocs.yml
@@ -1,9 +1,12 @@
 version: 2
 
+sphinx:
+  configuration: docs/conf.py
+
 build:
   os: ubuntu-22.04
   tools:
-    python: '3.12'
+    python: '3.13'
   jobs:
     post_checkout:
       - git fetch --unshallow || true

diff --git a/.spdx-license-header.txt b/.spdx-license-header.txt
@@ -1,2 +1,2 @@
-SPDX-FileCopyrightText: 2021-2024 MTS PJSC
+SPDX-FileCopyrightText: 2021-2025 MTS PJSC
 SPDX-License-Identifier: Apache-2.0
diff --git a/LICENSE.txt b/LICENSE.txt
@@ -1,4 +1,4 @@
-Copyright 2021-2024 MTS PJSC.  All rights reserved.
+Copyright 2021-2025 MTS PJSC.  All rights reserved.
 
                                  Apache License
                            Version 2.0, January 2004

diff --git a/README.rst b/README.rst
@@ -3,24 +3,35 @@
 ETL Entities
 ============
 
-|Repo Status| |PyPI| |PyPI License| |PyPI Python Version|
-|Documentation| |Build Status| |Coverage| |pre-commit.ci|
+|Repo Status| |PyPI Latest Release| |PyPI License| |PyPI Python Version| |PyPI Downloads|
+|Documentation| |CI Status| |Test Coverage| |pre-commit.ci Status|
 
 .. |Repo Status| image:: https://www.repostatus.org/badges/latest/active.svg
+    :alt: Repo status - Active
     :target: https://github.com/MobileTeleSystems/etl-entities
-.. |PyPI| image:: https://img.shields.io/pypi/v/etl-entities
+.. |PyPI Latest Release| image:: https://img.shields.io/pypi/v/etl-entities
+    :alt: PyPI - Latest Release
     :target: https://pypi.org/project/etl-entities/
 .. |PyPI License| image:: https://img.shields.io/pypi/l/etl-entities.svg
+    :alt: PyPI - License
     :target: https://github.com/MobileTeleSystems/etl-entities/blob/develop/LICENSE.txt
 .. |PyPI Python Version| image:: https://img.shields.io/pypi/pyversions/etl-entities.svg
-    :target: https://badge.fury.io/py/etl-entities
+    :alt: PyPI - Python Version
+    :target: https://pypi.org/project/etl-entities/
+.. |PyPI Downloads| image:: https://img.shields.io/pypi/dm/etl-entities
+    :alt: PyPI - Downloads
+    :target: https://pypi.org/project/etl-entities/
 .. |Documentation| image:: https://readthedocs.org/projects/etl-entities/badge/?version=stable
+    :alt: Documentation - ReadTheDocs
     :target: https://etl-entities.readthedocs.io/
-.. |Build Status| image:: https://github.com/MobileTeleSystems/etl-entities/workflows/Tests/badge.svg
+.. |CI Status| image:: https://github.com/MobileTeleSystems/etl-entities/workflows/Tests/badge.svg
+    :alt: Github Actions - latest CI build status
     :target: https://github.com/MobileTeleSystems/etl-entities/actions
-.. |Coverage| image:: https://codecov.io/gh/MobileTeleSystems/etl-entities/branch/develop/graph/badge.svg?token=RIO8URKNZJ
+.. |Test Coverage| image:: https://codecov.io/gh/MobileTeleSystems/etl-entities/branch/develop/graph/badge.svg?token=RIO8URKNZJ
+    :alt: Test coverage - percent
     :target: https://codecov.io/gh/MobileTeleSystems/etl-entities
-.. |pre-commit.ci| image:: https://results.pre-commit.ci/badge/github/MobileTeleSystems/etl-entities/develop.svg
+.. |pre-commit.ci Status| image:: https://results.pre-commit.ci/badge/github/MobileTeleSystems/etl-entities/develop.svg
+    :alt: pre-commit.ci - status
     :target: https://results.pre-commit.ci/latest/github/MobileTeleSystems/etl-entities/develop
 
 What is ETL Entities?
@@ -35,6 +46,7 @@ Currently implemented:
     * ``ColumnDateHWM``
     * ``ColumnDateTimeHWM``
     * ``FileListHWM``
+    * ``FileModifiedTimeHWM``
     * ``KeyValueIntHWM``
 
 * HWM Store classes:

diff --git a/codecov.yml b/codecov.yml
@@ -2,5 +2,5 @@ coverage:
   status:
     project:
       default:
-        target: 94%
+        target: 92%
         threshold: 1%
diff --git a/docs/changelog/2.5.0.rst b/docs/changelog/2.5.0.rst
@@ -0,0 +1,13 @@
+2.5.0 (2025-01-27)
+==================
+
+Features
+--------
+
+- Implement ``FileModifiedTimeHWM``, HWM based on file modification time. (:github:pull:`109`)
+
+
+Improvements
+------------
+
+- Add compatibility with ``Python 3.13`` (:github:pull:`106`)
diff --git a/docs/changelog/index.rst b/docs/changelog/index.rst
@@ -3,6 +3,7 @@
     :caption: Changelog
 
     DRAFT
+    2.5.0
     2.4.0
     2.3.1
     2.3.0

diff --git a/docs/conf.py b/docs/conf.py
@@ -25,7 +25,7 @@
 # -- Project information -----------------------------------------------------
 
 project = "etl-entities"
-copyright = "2021-2024 MTS PJSC"
+copyright = "2021-2025 MTS PJSC"
 author = "DataOps.ETL"
 
 # The version info for the project you're documenting, acts as replacement for

diff --git a/docs/hwm/file/file_mtime_hwm.rst b/docs/hwm/file/file_mtime_hwm.rst
@@ -0,0 +1,7 @@
+File Modified Time HWM
+===========================
+
+.. currentmodule:: etl_entities.hwm.file.file_mtime_hwm
+
+.. autoclass:: FileModifiedTimeHWM
+    :members: name, set_value, dict, json, copy, covers, update
diff --git a/docs/hwm/file/index.rst b/docs/hwm/file/index.rst
@@ -8,6 +8,7 @@ File HWM
     :caption: HWM classes
 
     file_list_hwm
+    file_mtime_hwm
 
 What is File HWM?
 -----------------
@@ -40,18 +41,13 @@ This technique is called ``High WaterMark`` or ``HWM`` for short.
 It is used by different `strategies <https://onetl.readthedocs.io/en/latest/strategy/index.html#strategy>`_ to implement some complex logic
 of filtering source data.
 
-
 Supported types
 ---------------
 
 There are a several ways to track HWM value:
 
-    * Save the entire file list, and then select only files not present in this list
-      (``file_list``)
-    * Save max modified time of all files, and then select only files with ``modified_time``
+    * Save list o file paths, and then select only files not present in this list - :obj:`FileListHWM`
+    * Save max modified time of all files, and then select only files with modified time (``file.stat().st_mtime``) - :obj:`FileModifiedTimeHWM`
       higher than this value
     * If file name contains some incrementing value, e.g. id or datetime,
-      parse it and save max value of all files, then select only files with higher value
-    * and so on
-
-Currently the only HWM type implemented for files is ``file_list``. Other ones can be implemented on-demand
+      parse it and save max value of all files, then select only files with higher value - not implemented for now.
diff --git a/etl_entities/VERSION b/etl_entities/VERSION
@@ -1 +1 @@
-2.4.0
+2.5.0
diff --git a/etl_entities/__init__.py b/etl_entities/__init__.py
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2021-2024 MTS PJSC
+# SPDX-FileCopyrightText: 2021-2025 MTS PJSC
 # SPDX-License-Identifier: Apache-2.0
 import os
 

diff --git a/etl_entities/entity.py b/etl_entities/entity.py
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2021-2024 MTS PJSC
+# SPDX-FileCopyrightText: 2021-2025 MTS PJSC
 # SPDX-License-Identifier: Apache-2.0
 # isort: skip_file
 

diff --git a/etl_entities/hwm/__init__.py b/etl_entities/hwm/__init__.py
@@ -1,11 +1,12 @@
-# SPDX-FileCopyrightText: 2021-2024 MTS PJSC
+# SPDX-FileCopyrightText: 2021-2025 MTS PJSC
 # SPDX-License-Identifier: Apache-2.0
 from etl_entities.hwm.column.column_hwm import ColumnHWM
 from etl_entities.hwm.column.date_hwm import ColumnDateHWM
 from etl_entities.hwm.column.datetime_hwm import ColumnDateTimeHWM
 from etl_entities.hwm.column.int_hwm import ColumnIntHWM
 from etl_entities.hwm.file.file_hwm import FileHWM
 from etl_entities.hwm.file.file_list_hwm import FileListHWM
+from etl_entities.hwm.file.file_mtime_hwm import FileModifiedTimeHWM
 from etl_entities.hwm.hwm import HWM
 from etl_entities.hwm.hwm_type_registry import HWMTypeRegistry, register_hwm_type
 from etl_entities.hwm.key_value.key_value_hwm import KeyValueHWM
@@ -19,6 +20,7 @@
     "ColumnIntHWM",
     "FileHWM",
     "FileListHWM",
+    "FileModifiedTimeHWM",
     "KeyValueHWM",
     "KeyValueIntHWM",
     "HWMTypeRegistry",

diff --git a/etl_entities/hwm/column/__init__.py b/etl_entities/hwm/column/__init__.py
@@ -1,2 +1,2 @@
-# SPDX-FileCopyrightText: 2021-2024 MTS PJSC
+# SPDX-FileCopyrightText: 2021-2025 MTS PJSC
 # SPDX-License-Identifier: Apache-2.0
diff --git a/etl_entities/hwm/column/column_hwm.py b/etl_entities/hwm/column/column_hwm.py
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2021-2024 MTS PJSC
+# SPDX-FileCopyrightText: 2021-2025 MTS PJSC
 # SPDX-License-Identifier: Apache-2.0
 from __future__ import annotations
 
@@ -164,10 +164,7 @@ def update(self: ColumnHWMType, value: ColumnValueType) -> ColumnHWMType:
         2
         """
 
-        if self.value is None:
-            return self.set_value(value)
-
-        if self.value < value:  # type: ignore[operator]
+        if self.value is None or self.value < value:  # type: ignore[operator]
             return self.set_value(value)
 
         return self

diff --git a/etl_entities/hwm/column/date_hwm.py b/etl_entities/hwm/column/date_hwm.py
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2021-2024 MTS PJSC
+# SPDX-FileCopyrightText: 2021-2025 MTS PJSC
 # SPDX-License-Identifier: Apache-2.0
 from __future__ import annotations
 
@@ -18,7 +18,7 @@
 
 @register_hwm_type("column_date")
 class ColumnDateHWM(ColumnHWM[date]):
-    """Date HWM type
+    """HWM based on tracking latest column value of type :obj:`datetime.date`.
 
     Parameters
     ----------

diff --git a/etl_entities/hwm/column/datetime_hwm.py b/etl_entities/hwm/column/datetime_hwm.py
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2021-2024 MTS PJSC
+# SPDX-FileCopyrightText: 2021-2025 MTS PJSC
 # SPDX-License-Identifier: Apache-2.0
 from __future__ import annotations
 
@@ -18,8 +18,7 @@
 
 @register_hwm_type("column_datetime")
 class ColumnDateTimeHWM(ColumnHWM[datetime]):
-    """DateTime HWM type
-
+    """HWM based on tracking latest column value of type :obj:`datetime.datetime`.
 
     Parameters
     ----------

diff --git a/etl_entities/hwm/column/int_hwm.py b/etl_entities/hwm/column/int_hwm.py
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2021-2024 MTS PJSC
+# SPDX-FileCopyrightText: 2021-2025 MTS PJSC
 # SPDX-License-Identifier: Apache-2.0
 from __future__ import annotations
 
@@ -16,7 +16,7 @@
 
 @register_hwm_type("column_int")
 class ColumnIntHWM(ColumnHWM[int]):
-    """Integer HWM type
+    """HWM based on tracking latest column value of type :obj:`int`.
 
     Parameters
     ----------

diff --git a/etl_entities/hwm/file/__init__.py b/etl_entities/hwm/file/__init__.py
@@ -1,2 +1,2 @@
-# SPDX-FileCopyrightText: 2021-2024 MTS PJSC
+# SPDX-FileCopyrightText: 2021-2025 MTS PJSC
 # SPDX-License-Identifier: Apache-2.0
diff --git a/etl_entities/hwm/file/file_hwm.py b/etl_entities/hwm/file/file_hwm.py
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2021-2024 MTS PJSC
+# SPDX-FileCopyrightText: 2021-2025 MTS PJSC
 # SPDX-License-Identifier: Apache-2.0
 from __future__ import annotations
 

diff --git a/etl_entities/hwm/file/file_list_hwm.py b/etl_entities/hwm/file/file_list_hwm.py
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2021-2024 MTS PJSC
+# SPDX-FileCopyrightText: 2021-2025 MTS PJSC
 # SPDX-License-Identifier: Apache-2.0
 from __future__ import annotations
 
@@ -21,7 +21,7 @@
 
 @register_hwm_type("file_list")
 class FileListHWM(FileHWM[FileListType]):
-    """File List HWM type
+    """HWM based on tracking list of file names.
 
     Parameters
     ----------
@@ -72,10 +72,10 @@ def covers(self, value: str | os.PathLike) -> bool:  # type: ignore
         --------
 
         >>> from etl_entities.hwm import FileListHWM
-        >>> hwm = FileListHWM(value={"/some/path.py"}, name="my_hwm")
-        >>> hwm.covers("/some/path.py")  # path in HWM
+        >>> hwm = FileListHWM(value={"/some/old_file.py"}, name="my_hwm")
+        >>> hwm.covers("/some/old_file.py")  # path in HWM
         True
-        >>> hwm.covers("/another/path.py")  # path not in HWM
+        >>> hwm.covers("/some/new_file.py")  # path not in HWM
         False
         """
 
@@ -90,7 +90,7 @@ def update(self: FileListHWMType, value: str | os.PathLike | Iterable[str | os.P
 
         Returns
         -------
-        result : FileHWM
+        result : FileListHWM
 
             Self
-Original file line number
+Diff line change
@@ Expand Up / @@ -3,6 +3,7 @@ @@
         :caption: Changelog
         DRAFT
+.5.0
 .4.0
 .3.1
 .3.0
@@ Expand Down @@