Skip to content

Commit

Permalink
Changes:
Browse files Browse the repository at this point in the history
1) Set max threads back to 20.
2) Fixed bug in Date windowing implementation.
  • Loading branch information
shantanu73 committed Mar 6, 2024
1 parent 859d923 commit db922e5
Showing 1 changed file with 5 additions and 6 deletions.
11 changes: 5 additions & 6 deletions tap_mambu/tap_generators/multithreaded_offset_generator.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ def _init_config(self):
self.end_of_file = False
self.fetch_batch_thread = None
self.last_batch_set = set()
self.max_threads = 1
self.max_threads = 20

@staticmethod
def check_and_get_set_reunion(a: set, b: set, lower_limit: int):
Expand Down Expand Up @@ -130,7 +130,7 @@ def _all_fetch_batch_steps(self):
end = datetime.strptime(end_datetime, '%Y-%m-%d').date()
temp = start + timedelta(days=self.date_window_size)
stop_iteration = True
while temp < end:
while start < end:
if stop_iteration:
self.offset = 0
self.modify_request_params(start, temp)
Expand All @@ -139,10 +139,9 @@ def _all_fetch_batch_steps(self):
if not final_buffer or stop_iteration:
start = temp
temp = start + timedelta(days=self.date_window_size)
self.offset = 0
self.modify_request_params(start, end)
final_buffer, stop_iteration = self.collect_batches(self.queue_batches())
self.preprocess_batches(final_buffer)
else:
final_buffer, stop_iteration = self.collect_batches(self.queue_batches())
self.preprocess_batches(final_buffer)
if not final_buffer or stop_iteration:
return False
return True
Expand Down

0 comments on commit db922e5

Please sign in to comment.