Made datetime attribute insert and match time-zone invariant (#301)

## What is the goal of this PR?

Enables the BDD tests to check timezone-invariance of inserting and reading datetime attributes.

## What are the changes implemented in this PR?

- During deserialization of datetime, we incorrectly used `fromtimestamp` function. This function is environment timezone-sensitive. The correct function to use is `utcfromtimestamp` which converts the milliseconds since epoch to UTC datetime object. This conforms to the serialization process: we convert datetime object to milliseconds since epoch assuming the datetime is in UTC.
- Added step which sets environment timezone to `time_zone_label`
- Fixed parsing of class `AttributeMatcher` and `ValueMatcher`.


Fixes #298.

Author: shiladitya-mukherjee

.factory/automation.yml

@@ -113,6 +113,7 @@ build:
       type: foreground
       command: |
         export PATH="$HOME/.local/bin:$PATH"
+        sudo apt-get update
         sudo apt install python3-pip -y
         python3 -m pip install -U pip
         python3 -m pip install -r requirements_dev.txt

dependencies/vaticle/repositories.bzl

@@ -39,5 +39,6 @@ def vaticle_typedb_behaviour():
     git_repository(
         name = "vaticle_typedb_behaviour",
         remote = "https://github.com/vaticle/typedb-behaviour",
-        commit = "767bf98fef7383addf42a1ae6e97a44874bb4f0b" # sync-marker: do not remove this comment, this is used for sync-dependencies by @vaticle_typedb_behaviour
+        commit = "90b4addcd71a398e9e52e322021c49310e66a47a" # sync-marker: do not remove this comment, this is used for sync-dependencies by @vaticle_typedb_behaviour
     )
+

tests/behaviour/concept/serialization/json/BUILD

@@ -40,7 +40,7 @@ typedb_behaviour_py_test(
         "//tests/behaviour/connection:steps",
         "//tests/behaviour/connection/database:steps",
         "//tests/behaviour/connection/session:steps",
-        "//tests/behaviour/connection/transaction:steps",
+        "//tests/behaviour/connection/transaction:steps"
     ],
     deps = [
         "//:client_python",

tests/behaviour/concept/thing/attribute/BUILD

@@ -43,6 +43,7 @@ typedb_behaviour_py_test(
         "//tests/behaviour/connection/database:steps",
         "//tests/behaviour/connection/session:steps",
         "//tests/behaviour/connection/transaction:steps",
+        "//tests/behaviour/util:util_steps"
     ],
     deps = [
         "//:client_python",

tests/behaviour/concept/thing/attribute/attribute_steps.py

@@ -149,3 +149,4 @@ def step_impl(context: Context, var: str, value: str):
 @step("attribute {var:Var} has datetime value: {value:DateTime}")
 def step_impl(context: Context, var: str, value: datetime):
     assert_that(context.get(var).as_attribute().get_value(), is_(value))
+

tests/behaviour/config/parameters.py

@@ -58,11 +58,18 @@ def parseWords(text):
 
 
 @parse.with_pattern(r"\d\d\d\d-\d\d-\d\d(?: \d\d:\d\d:\d\d)?")
+def parse_datetime_pattern(text: str) -> datetime:
+    return parse_datetime(str)
+
+
 def parse_datetime(text: str) -> datetime:
     try:
-        return datetime.strptime(text, "%Y-%m-%d %H:%M:%S")
+        return datetime.strptime(text, "%Y-%m-%dT%H:%M:%S")
     except ValueError:
-        return datetime.strptime(text, "%Y-%m-%d")
+        try:
+            return datetime.strptime(text, "%Y-%m-%d %H:%M:%S")
+        except ValueError:
+            return datetime.strptime(text, "%Y-%m-%d")
 
 
 register_type(DateTime=parse_datetime)

tests/behaviour/typeql/language/insert/BUILD

@@ -34,6 +34,7 @@ typedb_behaviour_py_test(
         "//tests/behaviour/connection/session:steps",
         "//tests/behaviour/connection/transaction:steps",
         "//tests/behaviour/typeql:steps",
+        "//tests/behaviour/util:util_steps"
     ],
     deps = [
         "//:client_python",

tests/behaviour/typeql/typeql_steps.py

@@ -225,7 +225,7 @@ class AttributeMatcher(ConceptMatcher, ABC):
 
     def __init__(self, type_and_value: str):
         self.type_and_value = type_and_value
-        s = type_and_value.split(":")
+        s = type_and_value.split(":", 1)
         assert_that(s, has_length(2),
                     "[%s] is not a valid attribute identifier. It should have format \"type_label:value\"." % type_and_value)
         self.type_label, self.value_string = s
@@ -284,7 +284,7 @@ class ValueMatcher(ConceptMatcher):
 
     def __init__(self, value_type_and_value: str):
         self.value_type_and_value = value_type_and_value
-        s = value_type_and_value.split(":")
+        s = value_type_and_value.split(":", 1)
         assert_that(s, has_length(2),
                     "[%s] is not a valid identifier. It should have format \"value_type:value\"." % value_type_and_value)
         self.value_type_name, self.value_string = s

tests/behaviour/util/util_steps.py

@@ -25,6 +25,13 @@
 
 from tests.behaviour.context import Context
 
+import os
+import time
+
+@step("set time-zone is: {time_zone_label}")
+def step_impl(context: Context, time_zone_label: str):
+    os.environ["TZ"] = time_zone_label
+    time.tzset()
 
 @step("wait {seconds} seconds")
 def step_impl(context: Context, seconds: str):

tool/behave_rule.bzl

@@ -118,7 +118,7 @@ def _rule_implementation(ctx):
     cmd += " && rm -rf " + steps_out_dir
     cmd += " && mkdir " + steps_out_dir + " && "
     cmd += " && ".join(["cp %s %s" % (step_file.path, steps_out_dir) for step_file in ctx.files.steps])
-    cmd += " && behave %s --no-capture -D port=$PORT && export RESULT=0 || export RESULT=1" % feats_dir
+    cmd += " && python3 -m behave %s --no-capture -D port=$PORT && export RESULT=0 || export RESULT=1" % feats_dir
     cmd += """
            echo Tests concluded with exit value $RESULT
            echo Stopping server.

typedb/common/exception.py

@@ -116,7 +116,7 @@ def __init__(self, code: int, message: str):
 NONEXISTENT_EXPLAINABLE_CONCEPT = ConceptErrorMessage(10, "The concept identified by '%s' is not explainable.")
 NONEXISTENT_EXPLAINABLE_OWNERSHIP = ConceptErrorMessage(11, "The ownership by owner '%s' of attribute '%s' is not explainable.")
 GET_HAS_WITH_MULTIPLE_FILTERS = ConceptErrorMessage(12, "Only one filter can be applied at a time to get_has. The possible filters are: [attribute_type, attribute_types, annotations]")
-
+UNSUPPORTED_TIMEZONE_INFORMATION = ConceptErrorMessage(13, "A date-time attribute cannot accept timezone aware datetime objects.")
 
 class QueryErrorMessage(ErrorMessage):
 

typedb/concept/proto/concept_proto_builder.py

@@ -25,7 +25,7 @@
 from typedb.api.concept.type.role_type import RoleType
 from typedb.api.concept.type.thing_type import ThingType, Annotations, Annotation
 from typedb.api.concept.type.type import Type
-from typedb.common.exception import TypeDBClientException, BAD_ENCODING, BAD_ANNOTATION
+from typedb.common.exception import TypeDBClientException, BAD_ENCODING, BAD_ANNOTATION, UNSUPPORTED_TIMEZONE_INFORMATION
 from typedb.common.rpc.request_builder import proto_role_type, proto_thing_type, byte_string
 
 
@@ -83,6 +83,8 @@ def string_value(value: str):
 
 
 def datetime_value(value: datetime):
+    if value.tzinfo is not None:
+        raise TypeDBClientException.of(UNSUPPORTED_TIMEZONE_INFORMATION)
     value_proto = concept_proto.ConceptValue()
     value_proto.date_time = int((value - datetime(1970, 1, 1)).total_seconds() * 1000)
     return value_proto

typedb/concept/thing/attribute.py

@@ -211,7 +211,7 @@ def __init__(self, iid: str, is_inferred: bool, type_: DateTimeAttributeType, va
 
     @staticmethod
     def of(thing_proto: concept_proto.Thing):
-        return _DateTimeAttribute(concept_proto_reader.iid(thing_proto.iid), thing_proto.inferred, concept_proto_reader.attribute_type(thing_proto.type), datetime.fromtimestamp(float(thing_proto.value.date_time) / 1000.0))
+        return _DateTimeAttribute(concept_proto_reader.iid(thing_proto.iid), thing_proto.inferred, concept_proto_reader.attribute_type(thing_proto.type), datetime.utcfromtimestamp(float(thing_proto.value.date_time) / 1000.0))
 
     def get_type(self) -> "DateTimeAttributeType":
         return self._type

typedb/concept/value/value.py

@@ -111,7 +111,7 @@ def __init__(self, value: datetime):
 
     @staticmethod
     def of(value_proto: concept_proto.Value):
-        return _DateTimeValue(datetime.fromtimestamp(float(value_proto.value.date_time) / 1000.0))
+        return _DateTimeValue(datetime.utcfromtimestamp(float(value_proto.value.date_time) / 1000.0))
 
     def get_value(self):
         return self._value