Skip to content

Commit c1f84bc

Browse files
correct inaccurate NTD extract object name (#3464)
* fix inaccurate NTD object extract name * clarify description * revise time zone name
1 parent d3cffd6 commit c1f84bc

File tree

3 files changed

+7
-7
lines changed

3 files changed

+7
-7
lines changed

Diff for: airflow/dags/sync_ntd_data_api/METADATA.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
description: "Scrape NTD endpoints from DOT API monthly"
2-
schedule_interval: "0 11 1 * *" # 11am GMT first day of every month
2+
schedule_interval: "0 11 1 * *" # 11am UTC first day of every month
33
tags:
44
- all_gusty_features
55
default_args:

Diff for: airflow/dags/sync_ntd_data_xlsx/METADATA.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
description: "Scrape tables from DOT Ridership XLSX file daily"
2-
schedule_interval: "0 10 * * *" # 10am GMT every day
2+
schedule_interval: "0 10 * * *" # 10am UTC every day
33
tags:
44
- all_gusty_features
55
default_args:

Diff for: airflow/plugins/operators/scrape_ntd_api.py

+5-5
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ def fetch_from_ntd_api(self):
6969
raise
7070

7171

72-
class CSVExtract(NtdDataProductAPIExtract):
72+
class JSONExtract(NtdDataProductAPIExtract):
7373
bucket = API_BUCKET
7474

7575

@@ -90,12 +90,12 @@ def __init__(
9090
self.root_url = root_url
9191
self.endpoint_id = endpoint_id
9292
self.file_format = file_format
93-
"""An operator that downloads all data from a NTD API
94-
and saves it as one CSV file hive-partitioned by date in Google Cloud
93+
"""An operator that extracts and saves JSON data from the NTD API
94+
and saves it as one JSONL file, hive-partitioned by date in Google Cloud
9595
"""
9696

97-
# Save CSV files to the bucket
98-
self.extract = CSVExtract(
97+
# Save JSONL files to the bucket
98+
self.extract = JSONExtract(
9999
year=self.year,
100100
product=self.product + "/" + self.year,
101101
root_url=self.root_url,

0 commit comments

Comments
 (0)