Skip to content

Commit

Permalink
Add retries to download functions
Browse files Browse the repository at this point in the history
To mitigate occasional connection errors

#58
  • Loading branch information
tillywoodfield committed Mar 5, 2024
1 parent dd2e7ab commit 83655e1
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 0 deletions.
6 changes: 6 additions & 0 deletions iatikit/utils/download.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,15 @@
import zipfile

import requests
from requests.exceptions import ConnectionError
from retry import retry

from ..standard.codelist import CodelistSet
from .config import CONFIG
from . import helpers


@retry(ConnectionError, tries=3, delay=1, backoff=2)
def data():
path = CONFIG['paths']['registry']
# downloads from https://iati-data-dump.codeforiati.org
Expand All @@ -38,6 +41,7 @@ def data():
_unlink(zip_filepath)


@retry(ConnectionError, tries=3, delay=1, backoff=2)
def metadata():
logging.getLogger(__name__).info(
'Downloading metadata from the IATI registry...')
Expand Down Expand Up @@ -127,6 +131,7 @@ def _get_codelist_mappings(versions):
json.dump(organisation_mappings, handler)


@retry(ConnectionError, tries=3, delay=1, backoff=2)
def codelists():
def get_list_of_codelists(version):
if version in _VERY_OLD_IATI_VERSIONS:
Expand Down Expand Up @@ -218,6 +223,7 @@ def get_codelist(codelist_name, version):
_get_codelist_mappings(all_versions)


@retry(ConnectionError, tries=3, delay=1, backoff=2)
def schemas():
path = join(CONFIG['paths']['standard'], 'schemas')
shutil.rmtree(path, ignore_errors=True)
Expand Down
1 change: 1 addition & 0 deletions requirements.txt
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
retry
-e .

0 comments on commit 83655e1

Please sign in to comment.