File tree Expand file tree Collapse file tree 3 files changed +7
-7
lines changed Expand file tree Collapse file tree 3 files changed +7
-7
lines changed Original file line number Diff line number Diff line change 11description : " Scrape NTD endpoints from DOT API monthly"
2- schedule_interval : " 0 11 1 * *" # 11am GMT first day of every month
2+ schedule_interval : " 0 11 1 * *" # 11am UTC first day of every month
33tags :
44 - all_gusty_features
55default_args :
Original file line number Diff line number Diff line change 11description : " Scrape tables from DOT Ridership XLSX file daily"
2- schedule_interval : " 0 10 * * *" # 10am GMT every day
2+ schedule_interval : " 0 10 * * *" # 10am UTC every day
33tags :
44 - all_gusty_features
55default_args :
Original file line number Diff line number Diff line change @@ -69,7 +69,7 @@ def fetch_from_ntd_api(self):
6969 raise
7070
7171
72- class CSVExtract (NtdDataProductAPIExtract ):
72+ class JSONExtract (NtdDataProductAPIExtract ):
7373 bucket = API_BUCKET
7474
7575
@@ -90,12 +90,12 @@ def __init__(
9090 self .root_url = root_url
9191 self .endpoint_id = endpoint_id
9292 self .file_format = file_format
93- """An operator that downloads all data from a NTD API
94- and saves it as one CSV file hive-partitioned by date in Google Cloud
93+ """An operator that extracts and saves JSON data from the NTD API
94+ and saves it as one JSONL file, hive-partitioned by date in Google Cloud
9595 """
9696
97- # Save CSV files to the bucket
98- self .extract = CSVExtract (
97+ # Save JSONL files to the bucket
98+ self .extract = JSONExtract (
9999 year = self .year ,
100100 product = self .product + "/" + self .year ,
101101 root_url = self .root_url ,
You can’t perform that action at this time.
0 commit comments