Skip to content

Commit

Permalink
allow only one thread in pbar_callback
Browse files Browse the repository at this point in the history
This commit adds a lock to limit the number
of threads in `pbar_callback` (a local function
in `datalad.downloaders.s3.S3DownloadSession.download`)
to one. This should prevent faulty commands in
the "datalad" git annex special remote.
  • Loading branch information
christian-monch committed Jul 24, 2024
1 parent 02ba927 commit 06de1a8
Showing 1 changed file with 8 additions and 5 deletions.
13 changes: 8 additions & 5 deletions datalad/downloaders/s3.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

import re
from urllib.parse import urlsplit, unquote as urlunquote
from threading import Lock

import boto3
import botocore
Expand Down Expand Up @@ -159,15 +160,17 @@ def __init__(self, size=None, filename=None, url=None, headers=None,
self.bucket = bucket
self.key = key
self.version_kwargs = version_kwargs
self.pbar_callback_lock = Lock()

def download(self, f=None, pbar=None, size=None):
# S3 specific (the rest is common with e.g. http)
def pbar_callback(downloaded):
if pbar:
try:
pbar.update(downloaded, increment=True)
except: # MIH: what does it do? MemoryError?
pass # do not let pbar spoil our fun
with self.pbar_callback_lock:
if pbar:
try:
pbar.update(downloaded, increment=True)
except: # MIH: what does it do? MemoryError?
pass # do not let pbar spoil our fun

if f:
if size is None:
Expand Down

0 comments on commit 06de1a8

Please sign in to comment.