From a7aee951fbc0456e9f71d6b9a4e495f40b3039f1 Mon Sep 17 00:00:00 2001 From: lbugnon Date: Wed, 11 Nov 2020 23:01:39 -0300 Subject: [PATCH 1/5] Rebase --- wfdb/io/download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wfdb/io/download.py b/wfdb/io/download.py index 22e9e958..92e502bf 100644 --- a/wfdb/io/download.py +++ b/wfdb/io/download.py @@ -262,7 +262,7 @@ def get_record_list(db_dir, records='all'): """ # Full url PhysioNet database - if '/' not in db_dir: + if os.sep not in db_dir: db_url = posixpath.join(config.db_index_url, db_dir, record.get_version(db_dir)) else: db_url = posixpath.join(config.db_index_url, db_dir) From 980f9bbf51cfc8bc1626b088c4bdd89925304459 Mon Sep 17 00:00:00 2001 From: lbugnon Date: Thu, 12 Nov 2020 23:41:59 -0300 Subject: [PATCH 2/5] replace os.sep with / to fix downloads in windows --- wfdb/io/download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wfdb/io/download.py b/wfdb/io/download.py index 92e502bf..64f309b8 100644 --- a/wfdb/io/download.py +++ b/wfdb/io/download.py @@ -262,7 +262,7 @@ def get_record_list(db_dir, records='all'): """ # Full url PhysioNet database - if os.sep not in db_dir: + if "/" not in db_dir: db_url = posixpath.join(config.db_index_url, db_dir, record.get_version(db_dir)) else: db_url = posixpath.join(config.db_index_url, db_dir) From 15f06678485c1eecc16eb9d3f418ec1ef799dd6a Mon Sep 17 00:00:00 2001 From: lbugnon Date: Thu, 12 Nov 2020 18:43:52 -0300 Subject: [PATCH 3/5] making the use of multiprocess optional --- wfdb/io/download.py | 5 +++-- wfdb/io/record.py | 19 +++++++++++++------ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/wfdb/io/download.py b/wfdb/io/download.py index 64f309b8..12f2284f 100644 --- a/wfdb/io/download.py +++ b/wfdb/io/download.py @@ -513,8 +513,9 @@ def dl_files(db, dl_dir, files, keep_subdirs=True, overwrite=False): print('Downloading files...') # Create multiple processes to download files. # Limit to 2 connections to avoid overloading the server - pool = multiprocessing.Pool(processes=2) - pool.map(dl_pn_file, dl_inputs) + #pool = multiprocessing.Pool(processes=2) + #pool.map(dl_pn_file, dl_inputs) + dl_pn_file(dl_inputs) print('Finished downloading files') return diff --git a/wfdb/io/record.py b/wfdb/io/record.py index a53ad4fa..dd779989 100644 --- a/wfdb/io/record.py +++ b/wfdb/io/record.py @@ -4469,7 +4469,7 @@ def is_monotonic(full_list): def dl_database(db_dir, dl_dir, records='all', annotators='all', - keep_subdirs=True, overwrite=False): + keep_subdirs=True, overwrite=False, use_multiprocess=True): """ Download WFDB record (and optionally annotation) files from a PhysioNet database. The database must contain a 'RECORDS' file in @@ -4509,6 +4509,8 @@ def dl_database(db_dir, dl_dir, records='all', annotators='all', file is smaller, the file will be assumed to be partially downloaded and the remaining bytes will be downloaded and appended. + use_multiprocess : bool, optional + If True, multiprocess package is used to download files. Returns ------- @@ -4525,7 +4527,8 @@ def dl_database(db_dir, dl_dir, records='all', annotators='all', db_dir = posixpath.join(dir_list[0], get_version(dir_list[0]), *dir_list[1:]) else: db_dir = posixpath.join(db_dir, get_version(db_dir)) - db_url = posixpath.join(download.PN_CONTENT_URL, db_dir) + '/' + + db_url = posixpath.join(download.PN_CONTENT_URL, db_dir) # Check if the database is valid _url.openurl(db_url, check_access=True) @@ -4596,10 +4599,14 @@ def dl_database(db_dir, dl_dir, records='all', annotators='all', download.make_local_dirs(dl_dir, dl_inputs, keep_subdirs) print('Downloading files...') - # Create multiple processes to download files. - # Limit to 2 connections to avoid overloading the server - pool = multiprocessing.Pool(processes=2) - pool.map(download.dl_pn_file, dl_inputs) + if use_multiprocess: + # Create multiple processes to download files. + # Limit to 2 connections to avoid overloading the server + pool = multiprocessing.Pool(processes=2) + pool.map(download.dl_pn_file, dl_inputs) + else: + for input in dl_inputs: + download.dl_pn_file(input) print('Finished downloading files') return From 30d7d390451b17f7335b3961358b6d289a9bef23 Mon Sep 17 00:00:00 2001 From: lbugnon Date: Wed, 7 Jul 2021 12:07:52 -0300 Subject: [PATCH 4/5] Remove comments --- wfdb/io/download.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/wfdb/io/download.py b/wfdb/io/download.py index 12f2284f..e2a36868 100644 --- a/wfdb/io/download.py +++ b/wfdb/io/download.py @@ -511,10 +511,6 @@ def dl_files(db, dl_dir, files, keep_subdirs=True, overwrite=False): make_local_dirs(dl_dir, dl_inputs, keep_subdirs) print('Downloading files...') - # Create multiple processes to download files. - # Limit to 2 connections to avoid overloading the server - #pool = multiprocessing.Pool(processes=2) - #pool.map(dl_pn_file, dl_inputs) dl_pn_file(dl_inputs) print('Finished downloading files') From ab3f0887bdac18b7d2dba73bd02490a4b7271c72 Mon Sep 17 00:00:00 2001 From: lbugnon Date: Fri, 1 Oct 2021 02:24:13 -0300 Subject: [PATCH 5/5] fix style --- wfdb/io/download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wfdb/io/download.py b/wfdb/io/download.py index e2a36868..9ac0085d 100644 --- a/wfdb/io/download.py +++ b/wfdb/io/download.py @@ -262,7 +262,7 @@ def get_record_list(db_dir, records='all'): """ # Full url PhysioNet database - if "/" not in db_dir: + if '/' not in db_dir: db_url = posixpath.join(config.db_index_url, db_dir, record.get_version(db_dir)) else: db_url = posixpath.join(config.db_index_url, db_dir)