From a335e299235cce94272f51f3dc8c0e8d4aa6bc21 Mon Sep 17 00:00:00 2001 From: russellwheatley Date: Tue, 20 Jun 2023 12:01:58 +0100 Subject: [PATCH 1/9] fix: timestamp conversion to handle nanosecond decimal --- src/firebase_functions/firestore_fn.py | 14 +++++--- src/firebase_functions/private/util.py | 46 ++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/src/firebase_functions/firestore_fn.py b/src/firebase_functions/firestore_fn.py index b8ae498..d81a7c8 100644 --- a/src/firebase_functions/firestore_fn.py +++ b/src/firebase_functions/firestore_fn.py @@ -111,10 +111,16 @@ def _firestore_endpoint_handler( event_namespace = event_attributes["namespace"] event_document = event_attributes["document"] event_database = event_attributes["database"] - event_time = _dt.datetime.strptime( - event_attributes["time"], - "%Y-%m-%dT%H:%M:%S.%f%z", - ) + + time = event_attributes["time"] + is_nanoseconds = _util.is_nanoseconds_timestamp(time) + + if (is_nanoseconds): + event_time = _util.nanoseconds_timestamp_conversion(time) + + else: + event_time = _util.microsecond_timestamp_conversion(time) + event_time = _util.nanoseconds_timestamp_conversion(time) if _DEFAULT_APP_NAME not in _apps: initialize_app() diff --git a/src/firebase_functions/private/util.py b/src/firebase_functions/private/util.py index 49bf264..f77a7b7 100644 --- a/src/firebase_functions/private/util.py +++ b/src/firebase_functions/private/util.py @@ -19,6 +19,7 @@ import json as _json import typing as _typing import dataclasses as _dataclasses +import datetime as _dt import enum as _enum from flask import Request as _Request from functions_framework import logging as _logging @@ -308,3 +309,48 @@ def firebase_config() -> None | FirebaseConfig: f'FIREBASE_CONFIG JSON string "{json_str}" is not valid json. {err}' ) from err return FirebaseConfig(storage_bucket=json_data.get("storageBucket")) + + +def nanoseconds_timestamp_conversion(time: str) -> _dt.datetime: + """Converts a nanosecond timestamp and returns a datetime object of the current time in UTC""" + + # Split the string into date-time and nanoseconds + s_datetime, s_ns = time.split(".") + + # Split the nanoseconds from the timezone specifier ('Z') + s_ns, _ = s_ns.split("Z") + + # Only take the first 6 digits of the nanoseconds + s_ns = s_ns[:6] + + # Put the string back together + s_processed = f"{s_datetime}.{s_ns}Z" + + # Now parse the date-time string + event_time = _dt.datetime.strptime(s_processed, "%Y-%m-%dT%H:%M:%S.%fZ") + + # strptime assumes local time, we know it's UTC and so: + event_time = event_time.replace(tzinfo=_dt.timezone.utc) + + return event_time + + +def is_nanoseconds_timestamp(time: str) -> bool: + """Return a bool which indicates if the timestamp is in nanoseconds""" + # Split the string into date-time and fraction of second + _, s_fraction = time.split(".") + + # Split the fraction from the timezone specifier ('Z' or 'z') + s_fraction, _ = s_fraction.split( + "Z") if "Z" in s_fraction else s_fraction.split("z") + + # If the fraction is 9 digits long, it's a nanosecond timestamp + return len(s_fraction) == 9 + + +def microsecond_timestamp_conversion(time: str) -> _dt.datetime: + """Converts a microsecond timestamp and returns a datetime object of the current time in UTC""" + return _dt.datetime.strptime( + time, + "%Y-%m-%dT%H:%M:%S.%f%z", + ) From 1ec71de6da89695bc7543911a9a8bbf1011d36aa Mon Sep 17 00:00:00 2001 From: russellwheatley Date: Tue, 20 Jun 2023 12:02:09 +0100 Subject: [PATCH 2/9] test: nanosecond conversion --- tests/test_util.py | 65 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/tests/test_util.py b/tests/test_util.py index e13fe1c..267eccc 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -15,7 +15,8 @@ Internal utils tests. """ from os import environ, path -from firebase_functions.private.util import firebase_config +from firebase_functions.private.util import firebase_config, microsecond_timestamp_conversion, nanoseconds_timestamp_conversion, is_nanoseconds_timestamp +import datetime as _dt test_bucket = "python-functions-testing.appspot.com" test_config_file = path.join(path.dirname(path.realpath(__file__)), @@ -40,3 +41,65 @@ def test_firebase_config_loads_from_env_file(): environ["FIREBASE_CONFIG"] = test_config_file assert firebase_config().storage_bucket == test_bucket, ( "Failure, firebase_config did not load from env variable.") + + +def test_microsecond_conversion(): + """ + Testing microsecond_timestamp_conversion works as intended + """ + timestamps = [ + ("2023-06-20T10:15:22.396358Z", "2023-06-20T10:15:22.396358Z"), + ("2021-02-20T11:23:45.987123Z", "2021-02-20T11:23:45.987123Z"), + ("2022-09-18T09:15:38.246824Z", "2022-09-18T09:15:38.246824Z"), + ("2010-09-18T09:15:38.246824Z", "2010-09-18T09:15:38.246824Z"), + ] + + for input_timestamp, expected_output in timestamps: + expected_datetime = _dt.datetime.strptime(expected_output, + "%Y-%m-%dT%H:%M:%S.%fZ") + expected_datetime = expected_datetime.replace(tzinfo=_dt.timezone.utc) + assert microsecond_timestamp_conversion( + input_timestamp) == expected_datetime + + +def test_nanosecond_conversion(): + """ + Testing nanoseconds_timestamp_conversion works as intended + """ + timestamps = [ + ("2023-01-01T12:34:56.123456789Z", "2023-01-01T12:34:56.123456Z"), + ("2023-02-14T14:37:52.987654321Z", "2023-02-14T14:37:52.987654Z"), + ("2023-03-21T06:43:58.564738291Z", "2023-03-21T06:43:58.564738Z"), + ("2023-08-15T22:22:22.222222222Z", "2023-08-15T22:22:22.222222Z"), + ] + + for input_timestamp, expected_output in timestamps: + expected_datetime = _dt.datetime.strptime(expected_output, + "%Y-%m-%dT%H:%M:%S.%fZ") + expected_datetime = expected_datetime.replace(tzinfo=_dt.timezone.utc) + assert nanoseconds_timestamp_conversion( + input_timestamp) == expected_datetime + + +def test_is_nanoseconds_timestamp(): + """ + Testing is_nanoseconds_timestamp works as intended + """ + microsecond_timestamp1 = "2023-06-20T10:15:22.396358Z" + microsecond_timestamp2 = "2021-02-20T11:23:45.987123Z" + microsecond_timestamp3 = "2022-09-18T09:15:38.246824Z" + microsecond_timestamp4 = "2010-09-18T09:15:38.246824Z" + + nanosecond_timestamp1 = "2023-01-01T12:34:56.123456789Z" + nanosecond_timestamp2 = "2023-02-14T14:37:52.987654321Z" + nanosecond_timestamp3 = "2023-03-21T06:43:58.564738291Z" + nanosecond_timestamp4 = "2023-08-15T22:22:22.222222222Z" + + assert is_nanoseconds_timestamp(microsecond_timestamp1) is False + assert is_nanoseconds_timestamp(microsecond_timestamp2) is False + assert is_nanoseconds_timestamp(microsecond_timestamp3) is False + assert is_nanoseconds_timestamp(microsecond_timestamp4) is False + assert is_nanoseconds_timestamp(nanosecond_timestamp1) is True + assert is_nanoseconds_timestamp(nanosecond_timestamp2) is True + assert is_nanoseconds_timestamp(nanosecond_timestamp3) is True + assert is_nanoseconds_timestamp(nanosecond_timestamp4) is True From 8b8baabbd26e24c2fb5d157cb244dc3797a4e11b Mon Sep 17 00:00:00 2001 From: russellwheatley Date: Tue, 20 Jun 2023 12:14:36 +0100 Subject: [PATCH 3/9] format --- src/firebase_functions/firestore_fn.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/firebase_functions/firestore_fn.py b/src/firebase_functions/firestore_fn.py index d81a7c8..06a6949 100644 --- a/src/firebase_functions/firestore_fn.py +++ b/src/firebase_functions/firestore_fn.py @@ -117,7 +117,6 @@ def _firestore_endpoint_handler( if (is_nanoseconds): event_time = _util.nanoseconds_timestamp_conversion(time) - else: event_time = _util.microsecond_timestamp_conversion(time) event_time = _util.nanoseconds_timestamp_conversion(time) From f72b9a7ba62058201e0aca583d3263821b7dd3ca Mon Sep 17 00:00:00 2001 From: russellwheatley Date: Tue, 20 Jun 2023 12:16:17 +0100 Subject: [PATCH 4/9] fix linting issues --- src/firebase_functions/firestore_fn.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/firebase_functions/firestore_fn.py b/src/firebase_functions/firestore_fn.py index 06a6949..68e05fb 100644 --- a/src/firebase_functions/firestore_fn.py +++ b/src/firebase_functions/firestore_fn.py @@ -18,7 +18,6 @@ import dataclasses as _dataclass import functools as _functools import typing as _typing -import datetime as _dt import google.events.cloud.firestore as _firestore import google.cloud.firestore_v1 as _firestore_v1 import firebase_functions.private.util as _util @@ -115,7 +114,7 @@ def _firestore_endpoint_handler( time = event_attributes["time"] is_nanoseconds = _util.is_nanoseconds_timestamp(time) - if (is_nanoseconds): + if is_nanoseconds: event_time = _util.nanoseconds_timestamp_conversion(time) else: event_time = _util.microsecond_timestamp_conversion(time) From 62572c08ca8247c0b2aa9c0a2f9117a418344f1b Mon Sep 17 00:00:00 2001 From: russellwheatley Date: Tue, 20 Jun 2023 12:24:17 +0100 Subject: [PATCH 5/9] chore: remove duplicate call --- src/firebase_functions/firestore_fn.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/firebase_functions/firestore_fn.py b/src/firebase_functions/firestore_fn.py index 68e05fb..eeed3d4 100644 --- a/src/firebase_functions/firestore_fn.py +++ b/src/firebase_functions/firestore_fn.py @@ -118,7 +118,6 @@ def _firestore_endpoint_handler( event_time = _util.nanoseconds_timestamp_conversion(time) else: event_time = _util.microsecond_timestamp_conversion(time) - event_time = _util.nanoseconds_timestamp_conversion(time) if _DEFAULT_APP_NAME not in _apps: initialize_app() From d69c5da7c5cff9722fee59abde046062a4d14e84 Mon Sep 17 00:00:00 2001 From: russellwheatley Date: Wed, 21 Jun 2023 13:12:24 +0100 Subject: [PATCH 6/9] chore: make PR review changes --- src/firebase_functions/firestore_fn.py | 2 +- src/firebase_functions/private/util.py | 28 +++++++++----------------- tests/test_util.py | 18 ++++++++--------- 3 files changed, 19 insertions(+), 29 deletions(-) diff --git a/src/firebase_functions/firestore_fn.py b/src/firebase_functions/firestore_fn.py index eeed3d4..9f1d648 100644 --- a/src/firebase_functions/firestore_fn.py +++ b/src/firebase_functions/firestore_fn.py @@ -112,7 +112,7 @@ def _firestore_endpoint_handler( event_database = event_attributes["database"] time = event_attributes["time"] - is_nanoseconds = _util.is_nanoseconds_timestamp(time) + is_nanoseconds = _util.is_precision_timestamp(time) if is_nanoseconds: event_time = _util.nanoseconds_timestamp_conversion(time) diff --git a/src/firebase_functions/private/util.py b/src/firebase_functions/private/util.py index f77a7b7..3c7ad69 100644 --- a/src/firebase_functions/private/util.py +++ b/src/firebase_functions/private/util.py @@ -314,28 +314,18 @@ def firebase_config() -> None | FirebaseConfig: def nanoseconds_timestamp_conversion(time: str) -> _dt.datetime: """Converts a nanosecond timestamp and returns a datetime object of the current time in UTC""" - # Split the string into date-time and nanoseconds - s_datetime, s_ns = time.split(".") - - # Split the nanoseconds from the timezone specifier ('Z') - s_ns, _ = s_ns.split("Z") - - # Only take the first 6 digits of the nanoseconds - s_ns = s_ns[:6] - - # Put the string back together - s_processed = f"{s_datetime}.{s_ns}Z" - - # Now parse the date-time string - event_time = _dt.datetime.strptime(s_processed, "%Y-%m-%dT%H:%M:%S.%fZ") - - # strptime assumes local time, we know it's UTC and so: - event_time = event_time.replace(tzinfo=_dt.timezone.utc) + # Separate the date and time part from the nanoseconds. + datetime_str, nanosecond_str = time.replace("Z", "").split(".") + # Parse the date and time part of the string. + event_time = _dt.datetime.strptime(datetime_str, "%Y-%m-%dT%H:%M:%S") + # Add the microseconds and timezone. + event_time = event_time.replace(microsecond=int(nanosecond_str[:6]), + tzinfo=_dt.timezone.utc) return event_time -def is_nanoseconds_timestamp(time: str) -> bool: +def is_precision_timestamp(time: str) -> bool: """Return a bool which indicates if the timestamp is in nanoseconds""" # Split the string into date-time and fraction of second _, s_fraction = time.split(".") @@ -345,7 +335,7 @@ def is_nanoseconds_timestamp(time: str) -> bool: "Z") if "Z" in s_fraction else s_fraction.split("z") # If the fraction is 9 digits long, it's a nanosecond timestamp - return len(s_fraction) == 9 + return len(s_fraction) > 6 def microsecond_timestamp_conversion(time: str) -> _dt.datetime: diff --git a/tests/test_util.py b/tests/test_util.py index 267eccc..10ede45 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -15,7 +15,7 @@ Internal utils tests. """ from os import environ, path -from firebase_functions.private.util import firebase_config, microsecond_timestamp_conversion, nanoseconds_timestamp_conversion, is_nanoseconds_timestamp +from firebase_functions.private.util import firebase_config, microsecond_timestamp_conversion, nanoseconds_timestamp_conversion, is_precision_timestamp import datetime as _dt test_bucket = "python-functions-testing.appspot.com" @@ -95,11 +95,11 @@ def test_is_nanoseconds_timestamp(): nanosecond_timestamp3 = "2023-03-21T06:43:58.564738291Z" nanosecond_timestamp4 = "2023-08-15T22:22:22.222222222Z" - assert is_nanoseconds_timestamp(microsecond_timestamp1) is False - assert is_nanoseconds_timestamp(microsecond_timestamp2) is False - assert is_nanoseconds_timestamp(microsecond_timestamp3) is False - assert is_nanoseconds_timestamp(microsecond_timestamp4) is False - assert is_nanoseconds_timestamp(nanosecond_timestamp1) is True - assert is_nanoseconds_timestamp(nanosecond_timestamp2) is True - assert is_nanoseconds_timestamp(nanosecond_timestamp3) is True - assert is_nanoseconds_timestamp(nanosecond_timestamp4) is True + assert is_precision_timestamp(microsecond_timestamp1) is False + assert is_precision_timestamp(microsecond_timestamp2) is False + assert is_precision_timestamp(microsecond_timestamp3) is False + assert is_precision_timestamp(microsecond_timestamp4) is False + assert is_precision_timestamp(nanosecond_timestamp1) is True + assert is_precision_timestamp(nanosecond_timestamp2) is True + assert is_precision_timestamp(nanosecond_timestamp3) is True + assert is_precision_timestamp(nanosecond_timestamp4) is True From 86efe31ce5702c0b04219ca0c4ec02fe26d32eb3 Mon Sep 17 00:00:00 2001 From: russellwheatley Date: Wed, 21 Jun 2023 13:18:36 +0100 Subject: [PATCH 7/9] format: space --- src/firebase_functions/private/util.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/firebase_functions/private/util.py b/src/firebase_functions/private/util.py index 92237c8..9ee721a 100644 --- a/src/firebase_functions/private/util.py +++ b/src/firebase_functions/private/util.py @@ -344,6 +344,8 @@ def microsecond_timestamp_conversion(time: str) -> _dt.datetime: time, "%Y-%m-%dT%H:%M:%S.%f%z", ) + + def normalize_path(path: str) -> str: """ Normalize a path string to a consistent format. From c3f20f07d769d1a77839cbe048da8f6896350695 Mon Sep 17 00:00:00 2001 From: russellwheatley Date: Mon, 26 Jun 2023 14:23:58 +0100 Subject: [PATCH 8/9] chore: PR review --- src/firebase_functions/private/util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/firebase_functions/private/util.py b/src/firebase_functions/private/util.py index 9ee721a..6957a6f 100644 --- a/src/firebase_functions/private/util.py +++ b/src/firebase_functions/private/util.py @@ -315,7 +315,7 @@ def nanoseconds_timestamp_conversion(time: str) -> _dt.datetime: """Converts a nanosecond timestamp and returns a datetime object of the current time in UTC""" # Separate the date and time part from the nanoseconds. - datetime_str, nanosecond_str = time.replace("Z", "").split(".") + datetime_str, nanosecond_str = time.replace("Z", "").replace("z", "").split(".") # Parse the date and time part of the string. event_time = _dt.datetime.strptime(datetime_str, "%Y-%m-%dT%H:%M:%S") # Add the microseconds and timezone. @@ -334,7 +334,7 @@ def is_precision_timestamp(time: str) -> bool: s_fraction, _ = s_fraction.split( "Z") if "Z" in s_fraction else s_fraction.split("z") - # If the fraction is 9 digits long, it's a nanosecond timestamp + # If the fraction is more than 6 digits long, it's a nanosecond timestamp return len(s_fraction) > 6 From 8e12c5fcaf3e4b15503a357546c37fa2f382e599 Mon Sep 17 00:00:00 2001 From: russellwheatley Date: Mon, 26 Jun 2023 14:25:20 +0100 Subject: [PATCH 9/9] format --- src/firebase_functions/private/util.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/firebase_functions/private/util.py b/src/firebase_functions/private/util.py index 6957a6f..9c521e9 100644 --- a/src/firebase_functions/private/util.py +++ b/src/firebase_functions/private/util.py @@ -315,7 +315,8 @@ def nanoseconds_timestamp_conversion(time: str) -> _dt.datetime: """Converts a nanosecond timestamp and returns a datetime object of the current time in UTC""" # Separate the date and time part from the nanoseconds. - datetime_str, nanosecond_str = time.replace("Z", "").replace("z", "").split(".") + datetime_str, nanosecond_str = time.replace("Z", "").replace("z", + "").split(".") # Parse the date and time part of the string. event_time = _dt.datetime.strptime(datetime_str, "%Y-%m-%dT%H:%M:%S") # Add the microseconds and timezone.