15
15
import pytz
16
16
from airbyte_cdk .models import SyncMode
17
17
from airbyte_cdk .sources import AbstractSource
18
- from airbyte_cdk .sources .streams import Stream
18
+ from airbyte_cdk .sources .streams import Stream , CheckpointMixin
19
19
from airbyte_cdk .sources .streams .http import HttpStream
20
20
from airbyte_cdk .sources .streams .core import StreamData
21
21
@@ -216,12 +216,15 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp
216
216
expenses ["toDate" ] = self .month_end .strftime ("%Y-%m-%d" )
217
217
combined_string = f"{ self .tenantid } { expenses ['fromDate' ]} { expenses ['toDate' ]} "
218
218
id_hash = hashlib .sha256 (combined_string .encode ())
219
- return [{"data" : expenses , "id" : id_hash .hexdigest ()}]
219
+ return [{"data" : expenses , "id" : id_hash .hexdigest (), "toDate" : expenses [ "toDate" ] }]
220
220
221
221
222
- class MgramsevaTenantExpenses (MgramsevaStream ):
222
+ class MgramsevaTenantExpenses (MgramsevaStream , CheckpointMixin ):
223
223
"""object for tenant payments"""
224
224
225
+ cursor_field = "toDate"
226
+ _cursor_value = None
227
+
225
228
def __init__ (
226
229
self , headers : dict , request_info : dict , user_request : dict , tenantid_list : list , fromdate : datetime , todate : datetime , ** kwargs
227
230
): # pylint: disable=super-init-not-called
@@ -234,8 +237,20 @@ def __init__(
234
237
self .request_info = request_info
235
238
self .user_request = user_request
236
239
self .tenantid_list = tenantid_list
237
- self .fromdate = fromdate
238
- self .todate = todate
240
+ self .fromdate = fromdate .replace (hour = 0 , minute = 0 , second = 0 , microsecond = 0 )
241
+ self .todate = todate .replace (hour = 0 , minute = 0 , second = 0 , microsecond = 0 )
242
+
243
+ @property
244
+ def state (self ) -> Mapping [str , Any ]:
245
+ if self ._cursor_value :
246
+ return {self .cursor_field : str (self ._cursor_value )}
247
+ else :
248
+ return {self .cursor_field : str (self .fromdate )}
249
+
250
+ @state .setter
251
+ def state (self , value : Mapping [str , Any ]):
252
+ if self .cursor_field in value :
253
+ self ._cursor_value = value [self .cursor_field ]
239
254
240
255
def read_records (
241
256
self ,
@@ -248,25 +263,30 @@ def read_records(
248
263
249
264
for tenantid in self .tenantid_list :
250
265
251
- month_start = self .fromdate . replace ( day = 1 )
266
+ state_value = self .state [ self . cursor_field ]
252
267
253
- while month_start < self . todate :
268
+ month_start = datetime . fromisoformat ( state_value ). replace ( day = 1 )
254
269
255
- next_month_start = month_start + relativedelta ( months = 1 ) - timedelta ( milliseconds = 1 )
270
+ to_date = self . todate . replace ( day = 1 )
256
271
272
+ while month_start < to_date :
273
+
274
+ next_month_start = month_start + relativedelta (months = 1 )
257
275
stream = MgramsevaTenantExpense (
258
276
"echallan-services/eChallan/v1/_expenseDashboard" ,
259
277
self .headers ,
260
278
self .request_info ,
261
279
self .user_request ,
262
280
tenantid ,
263
281
month_start ,
264
- next_month_start ,
282
+ next_month_start - timedelta ( milliseconds = 1 ) ,
265
283
"ExpenseDashboard" ,
266
284
)
267
- yield from stream .read_records (sync_mode , cursor_field , stream_slice , stream_state )
268
-
285
+ for record in stream .read_records (sync_mode , cursor_field , stream_slice , stream_state ):
286
+ yield record
269
287
month_start = next_month_start
288
+
289
+ self ._cursor_value = str (self .todate .replace (day = 1 ))
270
290
271
291
272
292
class MgramsevaPayments (MgramsevaStream ):
0 commit comments