1
+ """
2
+ A number of "plugins" for SubstrateInterface (and AsyncSubstrateInterface). At initial creation, it contains only
3
+ Retry (sync and async versions).
4
+ """
5
+
1
6
import asyncio
2
7
import logging
8
+ import socket
3
9
from functools import partial
4
10
from itertools import cycle
5
11
from typing import Optional
12
+
6
13
from websockets .exceptions import ConnectionClosed
7
14
8
- from async_substrate_interface .async_substrate import AsyncSubstrateInterface
15
+ from async_substrate_interface .async_substrate import AsyncSubstrateInterface , Websocket
9
16
from async_substrate_interface .errors import MaxRetriesExceeded
10
17
from async_substrate_interface .sync_substrate import SubstrateInterface
11
18
69
76
70
77
71
78
class RetrySyncSubstrate (SubstrateInterface ):
79
+ """
80
+ A subclass of SubstrateInterface that allows for handling chain failures by using backup chains. If a sustained
81
+ network failure is encountered on a chain endpoint, the object will initialize a new connection on the next chain in
82
+ the `fallback_chains` list. If the `retry_forever` flag is set, upon reaching the last chain in `fallback_chains`,
83
+ the connection will attempt to iterate over the list (starting with `url`) again.
84
+
85
+ E.g.
86
+ ```
87
+ substrate = RetrySyncSubstrate(
88
+ "wss://entrypoint-finney.opentensor.ai:443",
89
+ fallback_chains=["ws://127.0.0.1:9946"]
90
+ )
91
+ ```
92
+ In this case, if there is a failure on entrypoint-finney, the connection will next attempt to hit localhost. If this
93
+ also fails, a `MaxRetriesExceeded` exception will be raised.
94
+
95
+ ```
96
+ substrate = RetrySyncSubstrate(
97
+ "wss://entrypoint-finney.opentensor.ai:443",
98
+ fallback_chains=["ws://127.0.0.1:9946"],
99
+ retry_forever=True
100
+ )
101
+ ```
102
+ In this case, rather than a MaxRetriesExceeded exception being raised upon failure of the second chain (localhost),
103
+ the object will again being to initialize a new connection on entrypoint-finney, and then localhost, and so on and
104
+ so forth.
105
+ """
106
+
72
107
def __init__ (
73
108
self ,
74
109
url : str ,
@@ -117,6 +152,7 @@ def __init__(
117
152
raise ConnectionError (
118
153
f"Unable to connect at any chains specified: { [url ] + fallback_chains } "
119
154
)
155
+ # "connect" is only used by SubstrateInterface, not AsyncSubstrateInterface
120
156
retry_methods = ["connect" ] + RETRY_METHODS
121
157
self ._original_methods = {
122
158
method : getattr (self , method ) for method in retry_methods
@@ -125,33 +161,19 @@ def __init__(
125
161
setattr (self , method , partial (self ._retry , method ))
126
162
127
163
def _retry (self , method , * args , ** kwargs ):
164
+ method_ = self ._original_methods [method ]
128
165
try :
129
- method_ = self ._original_methods [method ]
130
166
return method_ (* args , ** kwargs )
131
167
except (MaxRetriesExceeded , ConnectionError , EOFError , ConnectionClosed ) as e :
132
168
try :
133
169
self ._reinstantiate_substrate (e )
134
- method_ = self ._original_methods [method ]
135
170
return method_ (* args , ** kwargs )
136
171
except StopIteration :
137
172
logger .error (
138
173
f"Max retries exceeded with { self .url } . No more fallback chains."
139
174
)
140
175
raise MaxRetriesExceeded
141
176
142
- def _retry_property (self , property_ ):
143
- try :
144
- return getattr (self , property_ )
145
- except (MaxRetriesExceeded , ConnectionError , EOFError , ConnectionClosed ) as e :
146
- try :
147
- self ._reinstantiate_substrate (e )
148
- return self ._retry_property (property_ )
149
- except StopIteration :
150
- logger .error (
151
- f"Max retries exceeded with { self .url } . No more fallback chains."
152
- )
153
- raise MaxRetriesExceeded
154
-
155
177
def _reinstantiate_substrate (self , e : Optional [Exception ] = None ) -> None :
156
178
next_network = next (self .fallback_chains )
157
179
self .ws .close ()
@@ -170,6 +192,34 @@ def _reinstantiate_substrate(self, e: Optional[Exception] = None) -> None:
170
192
171
193
172
194
class RetryAsyncSubstrate (AsyncSubstrateInterface ):
195
+ """
196
+ A subclass of AsyncSubstrateInterface that allows for handling chain failures by using backup chains. If a
197
+ sustained network failure is encountered on a chain endpoint, the object will initialize a new connection on
198
+ the next chain in the `fallback_chains` list. If the `retry_forever` flag is set, upon reaching the last chain
199
+ in `fallback_chains`, the connection will attempt to iterate over the list (starting with `url`) again.
200
+
201
+ E.g.
202
+ ```
203
+ substrate = RetryAsyncSubstrate(
204
+ "wss://entrypoint-finney.opentensor.ai:443",
205
+ fallback_chains=["ws://127.0.0.1:9946"]
206
+ )
207
+ ```
208
+ In this case, if there is a failure on entrypoint-finney, the connection will next attempt to hit localhost. If this
209
+ also fails, a `MaxRetriesExceeded` exception will be raised.
210
+
211
+ ```
212
+ substrate = RetryAsyncSubstrate(
213
+ "wss://entrypoint-finney.opentensor.ai:443",
214
+ fallback_chains=["ws://127.0.0.1:9946"],
215
+ retry_forever=True
216
+ )
217
+ ```
218
+ In this case, rather than a MaxRetriesExceeded exception being raised upon failure of the second chain (localhost),
219
+ the object will again being to initialize a new connection on entrypoint-finney, and then localhost, and so on and
220
+ so forth.
221
+ """
222
+
173
223
def __init__ (
174
224
self ,
175
225
url : str ,
@@ -212,62 +262,53 @@ def __init__(
212
262
for method in RETRY_METHODS :
213
263
setattr (self , method , partial (self ._retry , method ))
214
264
215
- def _reinstantiate_substrate (self , e : Optional [Exception ] = None ) -> None :
265
+ async def _reinstantiate_substrate (self , e : Optional [Exception ] = None ) -> None :
216
266
next_network = next (self .fallback_chains )
217
267
if e .__class__ == MaxRetriesExceeded :
218
268
logger .error (
219
269
f"Max retries exceeded with { self .url } . Retrying with { next_network } ."
220
270
)
221
271
else :
222
- print (f"Connection error. Trying again with { next_network } " )
223
- super ().__init__ (
224
- url = next_network ,
225
- ss58_format = self .ss58_format ,
226
- type_registry = self .type_registry ,
227
- use_remote_preset = self .use_remote_preset ,
228
- chain_name = self .chain_name ,
229
- _mock = self ._mock ,
230
- retry_timeout = self .retry_timeout ,
231
- max_retries = self .max_retries ,
272
+ logger .error (f"Connection error. Trying again with { next_network } " )
273
+ try :
274
+ await self .ws .shutdown ()
275
+ except AttributeError :
276
+ pass
277
+ if self ._forgettable_task is not None :
278
+ self ._forgettable_task : asyncio .Task
279
+ self ._forgettable_task .cancel ()
280
+ try :
281
+ await self ._forgettable_task
282
+ except asyncio .CancelledError :
283
+ pass
284
+ self .chain_endpoint = next_network
285
+ self .url = next_network
286
+ self .ws = Websocket (
287
+ next_network ,
288
+ options = {
289
+ "max_size" : self .ws_max_size ,
290
+ "write_limit" : 2 ** 16 ,
291
+ },
232
292
)
233
- self ._original_methods = {
234
- method : getattr (self , method ) for method in RETRY_METHODS
235
- }
236
- for method in RETRY_METHODS :
237
- setattr (self , method , partial (self ._retry , method ))
293
+ self ._initialized = False
294
+ self ._initializing = False
295
+ await self .initialize ()
238
296
239
297
async def _retry (self , method , * args , ** kwargs ):
298
+ method_ = self ._original_methods [method ]
240
299
try :
241
- method_ = self ._original_methods [method ]
242
300
return await method_ (* args , ** kwargs )
243
301
except (
244
302
MaxRetriesExceeded ,
245
303
ConnectionError ,
246
- ConnectionRefusedError ,
304
+ ConnectionClosed ,
247
305
EOFError ,
306
+ socket .gaierror ,
248
307
) as e :
249
308
try :
250
- self ._reinstantiate_substrate (e )
251
- await self .initialize ()
252
- method_ = getattr (self , method )
253
- if asyncio .iscoroutinefunction (method_ ):
254
- return await method_ (* args , ** kwargs )
255
- else :
256
- return method_ (* args , ** kwargs )
257
- except StopIteration :
258
- logger .error (
259
- f"Max retries exceeded with { self .url } . No more fallback chains."
260
- )
261
- raise MaxRetriesExceeded
262
-
263
- async def _retry_property (self , property_ ):
264
- try :
265
- return await getattr (self , property_ )
266
- except (MaxRetriesExceeded , ConnectionError , ConnectionRefusedError ) as e :
267
- try :
268
- self ._reinstantiate_substrate (e )
269
- return await self ._retry_property (property_ )
270
- except StopIteration :
309
+ await self ._reinstantiate_substrate (e )
310
+ return await method_ (* args , ** kwargs )
311
+ except StopAsyncIteration :
271
312
logger .error (
272
313
f"Max retries exceeded with { self .url } . No more fallback chains."
273
314
)
0 commit comments