File tree 3 files changed +7
-7
lines changed
3 files changed +7
-7
lines changed Original file line number Diff line number Diff line change 1
1
description : " Scrape NTD endpoints from DOT API monthly"
2
- schedule_interval : " 0 11 1 * *" # 11am GMT first day of every month
2
+ schedule_interval : " 0 11 1 * *" # 11am UTC first day of every month
3
3
tags :
4
4
- all_gusty_features
5
5
default_args :
Original file line number Diff line number Diff line change 1
1
description : " Scrape tables from DOT Ridership XLSX file daily"
2
- schedule_interval : " 0 10 * * *" # 10am GMT every day
2
+ schedule_interval : " 0 10 * * *" # 10am UTC every day
3
3
tags :
4
4
- all_gusty_features
5
5
default_args :
Original file line number Diff line number Diff line change @@ -69,7 +69,7 @@ def fetch_from_ntd_api(self):
69
69
raise
70
70
71
71
72
- class CSVExtract (NtdDataProductAPIExtract ):
72
+ class JSONExtract (NtdDataProductAPIExtract ):
73
73
bucket = API_BUCKET
74
74
75
75
@@ -90,12 +90,12 @@ def __init__(
90
90
self .root_url = root_url
91
91
self .endpoint_id = endpoint_id
92
92
self .file_format = file_format
93
- """An operator that downloads all data from a NTD API
94
- and saves it as one CSV file hive-partitioned by date in Google Cloud
93
+ """An operator that extracts and saves JSON data from the NTD API
94
+ and saves it as one JSONL file, hive-partitioned by date in Google Cloud
95
95
"""
96
96
97
- # Save CSV files to the bucket
98
- self .extract = CSVExtract (
97
+ # Save JSONL files to the bucket
98
+ self .extract = JSONExtract (
99
99
year = self .year ,
100
100
product = self .product + "/" + self .year ,
101
101
root_url = self .root_url ,
You can’t perform that action at this time.
0 commit comments