-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathget_forecasts.py
165 lines (153 loc) · 8.88 KB
/
get_forecasts.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
import os
import json
import time
from dotenv import load_dotenv
import get_endpoints as get_endpoints
import utils as utils
import logging
def get_forecasts(default_credential, endpoints):
'''Get forecast data for ski area locations, save to blob, return list of blob names
Args:
endpoints (dict): Dictionary of endpoints for each location
Returns:
forecast_blobs (dict): Dict of location:blob names for retrieved forecasts'''
# Load environment variables
load_dotenv(".env")
locations = json.loads(os.getenv("LOCATIONS"))
PURPOSE = os.getenv("PURPOSE")
EMAIL = os.getenv("EMAIL")
# API url for location metadata, header for requests
header = {'User-Agent' : (f'{PURPOSE}, {EMAIL}')}
# Define parameters
func_account_url = os.getenv('BLOB_ACCOUNT_URL')
container_name = "skiforecast"
# Fetch forecast data
# Get forecast data for each location, confirm successful download, save raw as .json
fails = {} # Accumulate fails in dictionary {location: (HTTPStatus, HTTPError)}
resolved = {} # Accumulate resolved fails in dictionary {location: 'location'}
forecast_blobs = {} # Accumulate forecast blob names in list
for location in locations.keys():
location_details = locations[location]
endpoint = endpoints[location]
blob_name = f'{location}_gridData.json'
forecast = utils.GridData(location, location_details, endpoint, header)
data = forecast.get_forecast()
response = forecast.get_status()
if response[1] == False:
utils.writeblob(blob_name, data, container_name, func_account_url, default_credential)
forecast_blobs[location] = f'{location}_gridData.json'
elif response[1] == True:
fails[location] = response
location = None
location_details = None
endpoint = None
# Handle missed endpoints
# Pass if all endpoints are found
if len(fails) == 0:
logging.info(f'\n\nFAILS: {fails}\n\n')
pass
# If endpoints are missing, attempt to resolve
elif len(fails) > 0:
logging.info(f'\n\nFAILS: {fails}\n\n')
for location in fails.keys():
logging.info(f'\n\nATTEMPTING TO RESOLVE: {location}\n\n')
http_status = fails[location][0]
http_error = fails[location][1]
location_details = locations[location]
endpoint = endpoints[location]
blob_name = f'{location}_gridData.json'
if http_status == None and http_error == True:
logging.info(f'\n\nFETCHING NEW ENDPOINTS\n\n')
ep = get_endpoints.get_endpoints()
forecast = utils.GridData(location, location_details, endpoint, header)
data = forecast.get_forecast()
response = forecast.get_status()
if response == (200, False):
resolved[location] = location # Add location to list of resolved endpoints
utils.writeblob(blob_name, data, container_name, func_account_url, default_credential)
forecast_blobs[location] = f'{location}_gridData.json'
logging.info(f'\n\nRESOLVED -- LOCATION: {location}, RESPONSE: {response}\n\n')
elif response[1] == True:
time.sleep(0.2)
fails[location] = response # Update fails list with new response
forecast = utils.GridData(location, location_details, endpoint, header)
data = forecast.get_forecast()
response = forecast.get_status()
if response == (200, False):
resolved[location] = location # Add location to list of resolved endpoints
utils.writeblob(blob_name, data, container_name, func_account_url, default_credential)
forecast_blobs[location] = f'{location}_gridData.json'
logging.info(f'\n\nRESOLVED -- LOCATION: {location}, RESPONSE: {response}\n\n')
if response[1] == True:
logging.info(f'\n\nCOULD NOT RESOLVE -- LOCATION: {location}, RESPONSE: {response}\n\n')
elif http_status != None and ((300 <= http_status < 500) and http_error == True):
logging.info(f'\n\nFETCHING NEW ENDPOINTS\n\n')
ep = get_endpoints.get_endpoints()
forecast = utils.GridData(location, location_details, endpoint, header)
data = forecast.get_forecast()
response = forecast.get_status()
if response == (200, False):
resolved[location] = location # Add location to list of resolved endpoints
utils.writeblob(blob_name, data, container_name, func_account_url, default_credential)
forecast_blobs[location] = f'{location}_gridData.json'
logging.info(f'\n\nRESOLVED -- LOCATION: {location}, RESPONSE: {response}\n\n')
elif response[1] == True:
time.sleep(0.2)
fails[location] = response # Update fails list with new response
forecast = utils.GridData(location, location_details, endpoint, header)
data = forecast.get_forecast()
response = forecast.get_status()
if response == (200, False):
resolved[location] = location # Add location to list of resolved endpoints
utils.writeblob(blob_name, data, container_name, func_account_url, default_credential)
forecast_blobs[location] = f'{location}_gridData.json'
logging.info(f'\n\nRESOLVED -- LOCATION: {location}, RESPONSE: {response}\n\n')
if response[1] == True:
logging.info(f'\n\nCOULD NOT RESOLVE -- LOCATION: {location}, RESPONSE: {response}\n\n')
elif http_status != None and ((500 <= http_status < 600) and http_error == True):
forecast = utils.GridData(location, location_details, endpoint, header)
data = forecast.get_forecast()
response = forecast.get_status()
if response == (200, False):
resolved[location] = location # Add location to list of resolved endpoints
utils.writeblob(blob_name, data, container_name, func_account_url, default_credential)
forecast_blobs[location] = f'{location}_gridData.json'
logging.info(f'\n\nRESOLVED -- LOCATION: {location}, RESPONSE: {response}\n\n')
elif response[1] == True:
time.sleep(0.2)
fails[location] = response # Update fails list with new response
forecast = utils.GridData(location, location_details, endpoint, header)
data = forecast.get_forecast()
response = forecast.get_status()
if response == (200, False):
resolved[location] = location # Add location to list of resolved endpoints
utils.writeblob(blob_name, data, container_name, func_account_url, default_credential)
forecast_blobs[location] = f'{location}_gridData.json'
logging.info(f'\n\nRESOLVED -- LOCATION: {location}, RESPONSE: {response}\n\n')
elif response[1] == True:
time.sleep(0.2)
fails[location] = response # Update fails list with new response
forecast = utils.GridData(location, location_details, endpoint, header)
data = forecast.get_forecast()
response = forecast.get_status()
if response == (200, False):
resolved[location] = location # Add location to list of resolved endpoints
utils.writeblob(blob_name, data, container_name, func_account_url, default_credential)
forecast_blobs[location] = f'{location}_gridData.json'
logging.info(f'\n\nRESOLVED -- LOCATION: {location}, RESPONSE: {response}\n\n')
if response[1] == True:
logging.info(f'\n\nCOULD NOT RESOLVE -- LOCATION: {location}, RESPONSE: {response}\n\n')
location = None
location_details = None
endpoint = None
# Remove resolved fails from fails list
for key in resolved.keys():
if key in fails.keys():
del fails[key]
# Log unresolved fails
if len(fails) != 0:
for key in resolved.keys():
if key in fails.keys():
del fails[key]
logging.info(f'\n\nUNRESOLVED FAILS: {fails}\n\n')
return forecast_blobs