From 16429a61497ea68ea2b174d02e16eca35adb841f Mon Sep 17 00:00:00 2001 From: ccp_zeulix Date: Wed, 22 May 2024 12:57:49 +0000 Subject: [PATCH] Version 1.2.0 - "Instance" support ### Added - The `logging` package to the `ccptools.structs._base` - Methods for casting between Datetime and timestamp (number of seconds since UNIX Epoch as a float) that work even on Windows when the built in `datetime.timestamp()` and `datetime.fromtimestamp()` methods fail for negative values and more - Methods for casting between Datetime and "instance" (number of milliseconds since UNIX Epoch as an int) ### Changed - How `any_to_datetime` handles "ambiguous" numeric values when deciding between "timestamp", "instance" and "filetime" - How `any_to_datetime` handles strings such that if a given string is a simple int or float, it's cast and treated as such ### Removed - The `utc` argument from `any_to_datetime` --- CHANGELOG.md | 26 ++++++ README.md | 1 + ccptools/__init__.py | 2 +- ccptools/dtu/casting/__init__.py | 2 + ccptools/dtu/casting/_any.py | 90 ++++++++++++++++--- ccptools/dtu/casting/_instant.py | 31 +++++++ ccptools/dtu/casting/_timestamp.py | 50 +++++++++++ ccptools/structs/_base.py | 2 + tests/datetimeutils/test_datetimeutils.py | 1 - tests/datetimeutils/test_instant.py | 9 ++ .../test_legacy_datetimeutils.py | 1 - 11 files changed, 198 insertions(+), 17 deletions(-) create mode 100644 ccptools/dtu/casting/_instant.py create mode 100644 ccptools/dtu/casting/_timestamp.py create mode 100644 tests/datetimeutils/test_instant.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 5078be3..cf07a8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,32 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.2.0] - 2024-22-05 + +### Added + +- The `logging` package to the `ccptools.structs._base` +- Methods for casting between Datetime and timestamp (number of seconds since + UNIX Epoch as a float) that work even on Windows when the built in + `datetime.timestamp()` and `datetime.fromtimestamp()` methods fail for + negative values and more +- Methods for casting between Datetime and "instance" (number of milliseconds + since UNIX Epoch as an int) + + +### Changed + +- How `any_to_datetime` handles "ambiguous" numeric values when deciding between + "timestamp", "instance" and "filetime" +- How `any_to_datetime` handles strings such that if a given string is a simple + int or float, it's cast and treated as such + + +### Removed + +- The `utc` argument from `any_to_datetime` + + ## [1.1.0] - 2024-04-08 ### Added diff --git a/README.md b/README.md index b9b1aae..cffd745 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,7 @@ import abc # For interfaces (Abstract Base Classes) import dataclasses # For dataclass structs import decimal # Used whenever we're handling money import enum # Also used for struct creation +import logging # Used pretty much everywhere import re # Used surprisingly frequently import time # Very commonly used ``` diff --git a/ccptools/__init__.py b/ccptools/__init__.py index 6174f8e..3d201c6 100644 --- a/ccptools/__init__.py +++ b/ccptools/__init__.py @@ -1,4 +1,4 @@ -__version__ = '1.1.0' +__version__ = '1.2.0' __author__ = 'Thordur Matthiasson ' __license__ = 'MIT License' diff --git a/ccptools/dtu/casting/__init__.py b/ccptools/dtu/casting/__init__.py index ea663ed..02052f3 100644 --- a/ccptools/dtu/casting/__init__.py +++ b/ccptools/dtu/casting/__init__.py @@ -1,3 +1,5 @@ from ._filetime import * from ._string import * +from ._timestamp import * +from ._instant import * from ._any import * diff --git a/ccptools/dtu/casting/_any.py b/ccptools/dtu/casting/_any.py index bf26ecd..91893a0 100644 --- a/ccptools/dtu/casting/_any.py +++ b/ccptools/dtu/casting/_any.py @@ -1,15 +1,26 @@ __all__ = [ 'any_to_datetime', ] +import warnings from ccptools.dtu.structs import * from ccptools._common import * from ccptools.dtu.casting._filetime import * from ccptools.dtu.casting._string import * - +from ccptools.dtu.casting._timestamp import * +from ccptools.dtu.casting._instant import * _NOT_SUPPLIED = object() -_FILETIME_THRESHOLD = 29999999999 + +_now = Datetime.now() +_1000_years_plus = _now.replace(_now.year + 1000) +_1000_years_minus = _now.replace(_now.year - 1000) + +_TIMESTAMP_MIN_RANGE = datetime_to_timestamp(_1000_years_minus) +_TIMESTAMP_MAX_RANGE = datetime_to_timestamp(_1000_years_plus) + +_INSTANT_MIN_RANGE = datetime_to_instant(_1000_years_minus) +_INSTANT_MAX_RANGE = datetime_to_instant(_1000_years_plus) _REVERSE_DATETIME_REXEX = re.compile(r'(?P3[01]|[012]?\d)[- /.,\\](?P1[012]|0?\d)[- /.,\\]' r'(?P[12][0189]\d{2})(?:[ @Tt]{0,1}(?:(?P[2][0-3]|[01]?\d)[ .:,]' @@ -22,31 +33,67 @@ def any_to_datetime(temporal_object: T_TEMPORAL_VALUE, - default: Any = _NOT_SUPPLIED, - utc: bool = True) -> Union[Datetime, Any]: + default: Any = _NOT_SUPPLIED) -> Union[Datetime, Any]: """Turns datetime, date, Windows filetime and posix time into a python datetime if possible. By default, returns the same input value on failed casting but another default return value can be given. - This function is mostly timezone naive, but it can be instructed to correct - for the local timezone in cases where it gets a UNIX timestamp and - generates a datetime object from that. The default behaviour is to use UTC. + This function is timezone naive. + + If given a number the following trickery is performed: + + - If the number, treated as a timestamp, represents a datetime value that is + within 1000 years of now (past or future) it will be treated as a timestamp + because timestamps are the most commonly used numerical representations of + datetimes + - If the number is outside that range, we'll check if the number would be + within 1000 years of now if treated as an instant. + - Otherwise, we assume that such a large number must be a filetime + + This does mean that there are certain cases that will yield incorrect + results, including: + + - Timestamps more than 1000 years in the past or future (they'll be + treated as instants or filetimes) + - Instants within a couple of years of 1970 (1969-01-20 to 1971-01-22 at + the time of this writing) will be treated as timestamps + - Filetimes for the years 1600-1601 might get treated as instants or + timestamps + + Concerning strings, there are also potential pitfalls if casting US + formatted YYYY-DD-MM strings, as this method will FIRST assume a standard + ISO format and only try the US one if that fails, so US formatted strings + with days between 1 and 12 will be treated as ISO and have their day and + month numbers switched. + + Note: This is a "best-guess" method and if these edge cases are + unacceptable, you should totally not be using it, and instead, know exactly + what format your data is in and use the appropriate specific casting method. """ if default == _NOT_SUPPLIED: default = temporal_object try: if isinstance(temporal_object, Datetime): return temporal_object + if isinstance(temporal_object, Date): return datetime.datetime.combine(temporal_object, Time()) + if isinstance(temporal_object, (float, int)): - if temporal_object > _FILETIME_THRESHOLD: # Most likely Windows FILETIME - return filetime_to_datetime(temporal_object) - else: # Might be Unix timestamp - if utc: - return Datetime.utcfromtimestamp(temporal_object) - else: - return Datetime.fromtimestamp(temporal_object) + if _TIMESTAMP_MIN_RANGE < temporal_object < _TIMESTAMP_MAX_RANGE: + # This range means that the number, if treated as a timestamp, + # represents a datetime within 1000 years to/from now so it's the + # most likely bet! + return timestamp_to_datetime(temporal_object) + + if _INSTANT_MIN_RANGE < temporal_object < _INSTANT_MAX_RANGE: + # This range means that the number, if treated as an instant, + # represents a datetime within 1000 years to/from now so it's the + # second most likely bet! + return timestamp_to_datetime(temporal_object) + + # This number is so large that it's most likely a filetime! + return filetime_to_datetime(temporal_object) if isinstance(temporal_object, bytes): try: @@ -55,14 +102,29 @@ def any_to_datetime(temporal_object: T_TEMPORAL_VALUE, return temporal_object if isinstance(temporal_object, str): + # Is this a number in string format? + try: + # Let's just try and pass this through the int caster, if that works, we evaluate it again as an int + return any_to_datetime(int(temporal_object)) + except (TypeError, ValueError): + pass + + try: + # Let's just try and pass this through the float caster, if that works, we evaluate it again as an int + return any_to_datetime(float(temporal_object)) + except (TypeError, ValueError): + pass + # First we'll try the day-month-year pattern value = regex_to_datetime(temporal_object, _REVERSE_DATETIME_REXEX) if value: return value + # Then the month-day-year pattern value = regex_to_datetime(temporal_object, _REVERSE_US_DATETIME_REXEX) if value: return value + # How'bout good old ISO year-month-day then? :D value = isostr_to_datetime(temporal_object) if value: diff --git a/ccptools/dtu/casting/_instant.py b/ccptools/dtu/casting/_instant.py new file mode 100644 index 0000000..e5ef8fd --- /dev/null +++ b/ccptools/dtu/casting/_instant.py @@ -0,0 +1,31 @@ +__all__ = [ + 'instant_to_datetime', + 'datetime_to_instant', +] +from ccptools.dtu.structs import * +from ._timestamp import * + + +def instant_to_datetime(milliseconds_since_epoch: T_NUMBER, minmax_on_fail: bool = False) -> Datetime: + """Converts an integer representing milliseconds since the Unix epoch + (January 1, 1970) to a Python datetime object. + + :param milliseconds_since_epoch: Milliseconds since Unix epoch (January 1, 1970). + :param minmax_on_fail: If True, will return the minimum or maximum possible + value of Datetime in case of overflow (positive or + negative) + :return: A Python Datetime + """ + return timestamp_to_datetime(milliseconds_since_epoch / 1000., minmax_on_fail) + + +def datetime_to_instant(dt: T_DATE_VALUE) -> int: + """Converts a Python datetime object to the number of milliseconds since + Unix epoch (January 1, 1970). + + If given a date only, it will assume a time of 00:00:00.000000. + + :param dt: Python datetime (or date). + :return: Number of milliseconds since Unix epoch (January 1, 1970) + """ + return int(datetime_to_timestamp(dt) * 1000) diff --git a/ccptools/dtu/casting/_timestamp.py b/ccptools/dtu/casting/_timestamp.py new file mode 100644 index 0000000..9aa132c --- /dev/null +++ b/ccptools/dtu/casting/_timestamp.py @@ -0,0 +1,50 @@ +__all__ = [ + 'timestamp_to_datetime', + 'datetime_to_timestamp', +] +from ccptools.dtu.structs import * +import calendar + + +def timestamp_to_datetime(seconds_since_epoch: T_NUMBER, minmax_on_fail: bool = False) -> Datetime: + """Converts an int or float representing seconds since the Unix epoch + (January 1, 1970) to a Python datetime object. + + :param seconds_since_epoch: Seconds since Unix epoch (January 1, 1970). + :param minmax_on_fail: If True, will return the minimum or maximum possible + value of Datetime in case of overflow (positive or + negative) + :return: A Python Datetime + """ + try: + return Datetime.fromtimestamp(seconds_since_epoch) + except OSError: + try: + return Datetime(1970, 1, 1, 0, 0, 0, 0) + TimeDelta(seconds=seconds_since_epoch) + except OverflowError: + if minmax_on_fail: + if seconds_since_epoch > 0: + return Datetime.max + else: + return Datetime.min + else: + raise + + +def datetime_to_timestamp(dt: T_DATE_VALUE) -> float: + """Converts a Python datetime object to the number of seconds since + Unix epoch (January 1, 1970) as a float, including fractional seconds. + + If given a date only, it will assume a time of 00:00:00.000000. + + :param dt: Python datetime (or date). + :return: Number of seconds since Unix epoch (January 1, 1970) + """ + if not isinstance(dt, Datetime) and isinstance(dt, Date): + dt = Datetime.combine(dt, Time(0, 0, 0, 0)) + # TODO(thordurm@ccpgames.com>) 2024-05-22: HANDLE SECONDS!!! + + int_part = float(calendar.timegm(dt.utctimetuple())) + if dt.microsecond: + int_part += dt.microsecond / 1000000.0 + return int_part diff --git a/ccptools/structs/_base.py b/ccptools/structs/_base.py index b0bafa4..f43756d 100644 --- a/ccptools/structs/_base.py +++ b/ccptools/structs/_base.py @@ -15,5 +15,7 @@ import dataclasses import decimal import enum +import logging import re import time + diff --git a/tests/datetimeutils/test_datetimeutils.py b/tests/datetimeutils/test_datetimeutils.py index f532ea8..5cdba79 100644 --- a/tests/datetimeutils/test_datetimeutils.py +++ b/tests/datetimeutils/test_datetimeutils.py @@ -21,7 +21,6 @@ def assertDefault(value): assertSame(300117804, (1979, 7, 6, 14, 3, 24)) assertSame(300117804.321321, (1979, 7, 6, 14, 3, 24, 321321)) assertSame(1570875489.134, (2019, 10, 12, 10, 18, 9, 134000)) - assertSame(100000000000, (1601, 1, 1, 2, 46, 40)) assertSame(None, None) assertSame('2013-06-10T12:13:14', (2013, 6, 10, 12, 13, 14)) diff --git a/tests/datetimeutils/test_instant.py b/tests/datetimeutils/test_instant.py new file mode 100644 index 0000000..d45047b --- /dev/null +++ b/tests/datetimeutils/test_instant.py @@ -0,0 +1,9 @@ +import unittest +from ccptools.dtu.structs import * +from ccptools.dtu.casting import * + + +class TestInstant(unittest.TestCase): + def test_instant_to_datetime(self): + _dt = Datetime(2024, 5, 22, 10, 37, 54, 123000) + self.assertEqual(_dt, instant_to_datetime(1716374274123)) diff --git a/tests/datetimeutils/test_legacy_datetimeutils.py b/tests/datetimeutils/test_legacy_datetimeutils.py index f27a5f7..f771404 100644 --- a/tests/datetimeutils/test_legacy_datetimeutils.py +++ b/tests/datetimeutils/test_legacy_datetimeutils.py @@ -46,7 +46,6 @@ def assertDefault(value): assertSame(300117804, (1979, 7, 6, 14, 3, 24)) assertSame(300117804.321321, (1979, 7, 6, 14, 3, 24, 321321)) assertSame(1570875489.134, (2019, 10, 12, 10, 18, 9, 134000)) - assertSame(100000000000, (1601, 1, 1, 2, 46, 40)) assertSame(None, None) assertSame('2013-06-10T12:13:14', (2013, 6, 10, 12, 13, 14))