TDL-18879: Clear offset when max_skip error is encountered and return 0 if skip is greater than 250k in the current state.

tap_closeio/streams.py

@@ -12,6 +12,9 @@
 
 LOGGER = singer.get_logger()
 
+# default date date window size in days
+DATE_WINDOW_SIZE = 5
+
 PATHS = {
     IDS.CUSTOM_FIELDS: "/custom_fields/lead/",
     IDS.LEADS: "/lead/",
@@ -168,15 +171,15 @@ def sync_activities(ctx):
 
     try:
         # get date window from config
-        date_window = int(ctx.config.get("date_window", 15))
-        # if date_window is 0, '0' or None, then set default window size of 15 days
+        date_window = int(ctx.config.get("date_window", DATE_WINDOW_SIZE))
+        # if date_window is 0, '0' or None, then set default window size of 5 days
         if not date_window:
-            LOGGER.warning("Invalid value of date window is passed: \'{}\', using default window size of 15 days.".format(ctx.config.get("date_window")))
-            date_window = 15
+            LOGGER.warning("Invalid value of date window is passed: \'{}\', using default window size of 5 days.".format(ctx.config.get("date_window")))
+            date_window = DATE_WINDOW_SIZE
     except ValueError:
-        LOGGER.warning("Invalid value of date window is passed: \'{}\', using default window size of 15 days.".format(ctx.config.get("date_window")))
+        LOGGER.warning("Invalid value of date window is passed: \'{}\', using default window size of 5 days.".format(ctx.config.get("date_window")))
         # In case of empty string(''), use default window
-        date_window = 15
+        date_window = DATE_WINDOW_SIZE
 
     LOGGER.info("Using offset seconds {}".format(offset_secs))
     start_date -= timedelta(seconds=offset_secs)

tests/unittests/test_activity_stream_date_window.py

@@ -9,12 +9,12 @@ class TestActivityStreamDateWindow(unittest.TestCase):
 
     def test_activity_stream_default_date_window(self, mocked_paginated_sync):
         """
-            Test case to verify we are calling Activity API in 15 days window (default) when no "date_window" is passed in the config
+            Test case to verify we are calling Activity API in 5 days window (default) when no "date_window" is passed in the config
         """
         # now date
         now_date = datetime.now()
         config = {
-            "start_date": (now_date - timedelta(days=40)).strftime("%Y-%m-%d/"), # set date 40 days later than now
+            "start_date": (now_date - timedelta(days=40)).strftime("%Y-%m-%d"), # set date 40 days later than now
             "api_key": "test_API_key"
         }
         state = {}
@@ -25,13 +25,14 @@ def test_activity_stream_default_date_window(self, mocked_paginated_sync):
         # function call
         sync_activities(ctx)
 
-        # verify we called 'paginated_sync' 3 times as the start date is 40 days later and default date window is 15 days
-        self.assertEqual(mocked_paginated_sync.call_count, 3)
+        # verify we called 'paginated_sync' 3 times as the start date is 40 days later and default date window is 5 days
+        # as the default lookback window of 1 day is used, as a result, there will be additional 1 API call
+        self.assertEqual(mocked_paginated_sync.call_count, 9)
 
     @mock.patch("tap_closeio.streams.LOGGER.warning")
     def test_activity_stream_empty_string_date_window(self, mocked_logger_warning, mocked_paginated_sync):
         """
-            Test case to verify we are calling Activity API in 15 days window (default) when empty string "date_window" is passed in the config
+            Test case to verify we are calling Activity API in 5 days window (default) when empty string "date_window" is passed in the config
         """
         # now date
         now_date = datetime.now()
@@ -49,14 +50,15 @@ def test_activity_stream_empty_string_date_window(self, mocked_logger_warning, m
         sync_activities(ctx)
 
         # verify we called 'paginated_sync' 14 times as the start date is 40 days later and we have set default date window
-        self.assertEqual(mocked_paginated_sync.call_count, 3)
+        # as the default lookback window of 1 day is used, as a result, there will be additional 1 API call
+        self.assertEqual(mocked_paginated_sync.call_count, 9)
         # verify warning is raised for invalid date window
-        mocked_logger_warning.assert_called_with("Invalid value of date window is passed: '', using default window size of 15 days.")
+        mocked_logger_warning.assert_called_with("Invalid value of date window is passed: '', using default window size of 5 days.")
 
     @mock.patch("tap_closeio.streams.LOGGER.warning")
     def test_activity_stream_0_date_window(self, mocked_logger_warning, mocked_paginated_sync):
         """
-            Test case to verify we are calling Activity API in 15 days window (default) when int 0 "date_window" is passed in the config
+            Test case to verify we are calling Activity API in 5 days window (default) when int 0 "date_window" is passed in the config
         """
         # now date
         now_date = datetime.now()
@@ -74,14 +76,15 @@ def test_activity_stream_0_date_window(self, mocked_logger_warning, mocked_pagin
         sync_activities(ctx)
 
         # verify we called 'paginated_sync' 14 times as the start date is 40 days later and we have set default date window
-        self.assertEqual(mocked_paginated_sync.call_count, 3)
+        # as the default lookback window of 1 day is used, as a result, there will be additional 1 API call
+        self.assertEqual(mocked_paginated_sync.call_count, 9)
         # verify warning is raised for invalid date window
-        mocked_logger_warning.assert_called_with("Invalid value of date window is passed: '0', using default window size of 15 days.")
+        mocked_logger_warning.assert_called_with("Invalid value of date window is passed: '0', using default window size of 5 days.")
 
     @mock.patch("tap_closeio.streams.LOGGER.warning")
     def test_activity_stream_string_0_date_window(self, mocked_logger_warning, mocked_paginated_sync):
         """
-            Test case to verify we are calling Activity API in 15 days window (default) when string 0 "date_window" is passed in the config
+            Test case to verify we are calling Activity API in 5 days window (default) when string 0 "date_window" is passed in the config
         """
         # now date
         now_date = datetime.now()
@@ -99,9 +102,10 @@ def test_activity_stream_string_0_date_window(self, mocked_logger_warning, mocke
         sync_activities(ctx)
 
         # verify we called 'paginated_sync' 14 times as the start date is 40 days later and we have set default date window
-        self.assertEqual(mocked_paginated_sync.call_count, 3)
+        # as the default lookback window of 1 day is used, as a result, there will be additional 1 API call
+        self.assertEqual(mocked_paginated_sync.call_count, 9)
         # verify warning is raised for invalid date window
-        mocked_logger_warning.assert_called_with("Invalid value of date window is passed: '0', using default window size of 15 days.")
+        mocked_logger_warning.assert_called_with("Invalid value of date window is passed: '0', using default window size of 5 days.")
 
     def test_activity_stream_configurable_date_window(self, mocked_paginated_sync):
         """