From 6218b78e9a8347c9f736c04424541c144fb36b6f Mon Sep 17 00:00:00 2001 From: Vlad Presnyak Date: Tue, 17 Jul 2018 16:13:39 -0400 Subject: [PATCH 1/6] Fixing proper handling of file sizes. Will correctly return floating point or integer as appropriate. Previous version appeared to rely on integer division as implemented in py2 and would return a float regardless of flating_point argument in py3. --- S3/Utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/S3/Utils.py b/S3/Utils.py index 6bcfeb292..0518168ec 100644 --- a/S3/Utils.py +++ b/S3/Utils.py @@ -192,16 +192,16 @@ def dateRFC822toUnix(date): __all__.append("dateRFC822toUnix") def formatSize(size, human_readable = False, floating_point = False): - size = floating_point and float(size) or int(size) + size = int(size) if human_readable: coeffs = ['k', 'M', 'G', 'T'] coeff = "" while size > 2048: size /= 1024 coeff = coeffs.pop(0) - return (size, coeff) + return (float(size) if floating_point else int(size), coeff) else: - return (size, "") + return (float(size) if floating_point else int(size), "") __all__.append("formatSize") def formatDateTime(s3timestamp): From 5d280cb772d67e9f0c3eb79029f984b2b9815e1e Mon Sep 17 00:00:00 2001 From: Florent Viard Date: Thu, 19 Jul 2018 01:27:12 +0200 Subject: [PATCH 2/6] Fixes #987 - Fixes using IAM with python3. Config options should be unicode. --- S3/Config.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/S3/Config.py b/S3/Config.py index e6e76f0ba..10390cbc3 100644 --- a/S3/Config.py +++ b/S3/Config.py @@ -263,10 +263,10 @@ def role_config(self): conn.request('GET', "/latest/meta-data/iam/security-credentials/%s"%files.decode('UTF-8')) resp=conn.getresponse() if resp.status == 200: - creds=json.load(resp) - Config().update_option('access_key', creds['AccessKeyId'].encode('ascii')) - Config().update_option('secret_key', creds['SecretAccessKey'].encode('ascii')) - Config().update_option('access_token', creds['Token'].encode('ascii')) + creds=json.load(resp, encoding="utf-8") + Config().update_option('access_key', config_unicodise(creds['AccessKeyId'])) + Config().update_option('secret_key', config_unicodise(creds['SecretAccessKey'])) + Config().update_option('access_token', config_unicodise(creds['Token'])) else: raise IOError else: From 647258236d424ffb0cc8b1ace252775852bb8787 Mon Sep 17 00:00:00 2001 From: Florent Viard Date: Thu, 19 Jul 2018 01:45:03 +0200 Subject: [PATCH 3/6] Fixes #987 - Before py36, py3 json was only able to load unicode str --- S3/Config.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/S3/Config.py b/S3/Config.py index 10390cbc3..046c1f444 100644 --- a/S3/Config.py +++ b/S3/Config.py @@ -260,10 +260,11 @@ def role_config(self): resp = conn.getresponse() files = resp.read() if resp.status == 200 and len(files)>1: - conn.request('GET', "/latest/meta-data/iam/security-credentials/%s"%files.decode('UTF-8')) + conn.request('GET', "/latest/meta-data/iam/security-credentials/%s" % files.decode('utf-8')) resp=conn.getresponse() if resp.status == 200: - creds=json.load(resp, encoding="utf-8") + resp_content = config_unicodise(resp.read()) + creds=json.loads(resp_content) Config().update_option('access_key', config_unicodise(creds['AccessKeyId'])) Config().update_option('secret_key', config_unicodise(creds['SecretAccessKey'])) Config().update_option('access_token', config_unicodise(creds['Token'])) From fcf70a71a289d7754193a89821ecc697e16c200c Mon Sep 17 00:00:00 2001 From: Florent Viard Date: Sun, 22 Jul 2018 12:28:41 +0200 Subject: [PATCH 4/6] Complete the fix #986 - To satisfy py2 and p3, there should be float before and after the division --- S3/Utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/S3/Utils.py b/S3/Utils.py index 0518168ec..95026d291 100644 --- a/S3/Utils.py +++ b/S3/Utils.py @@ -192,16 +192,16 @@ def dateRFC822toUnix(date): __all__.append("dateRFC822toUnix") def formatSize(size, human_readable = False, floating_point = False): - size = int(size) + size = floating_point and float(size) or int(size) if human_readable: coeffs = ['k', 'M', 'G', 'T'] coeff = "" while size > 2048: size /= 1024 coeff = coeffs.pop(0) - return (float(size) if floating_point else int(size), coeff) + return (floating_point and float(size) or int(size), coeff) else: - return (float(size) if floating_point else int(size), "") + return (size, "") __all__.append("formatSize") def formatDateTime(s3timestamp): From 7a9aa8592bb87c8ebdaf349ce1df2f2cd38d52f2 Mon Sep 17 00:00:00 2001 From: Florent Viard Date: Sun, 22 Jul 2018 15:06:24 +0200 Subject: [PATCH 5/6] Issue #956 - Fixes size reporting using k instead of K as it a multiple of 1024 Important note fot automatic scripts: Output change: "125k" in will now be 125K --- S3/Utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/S3/Utils.py b/S3/Utils.py index 95026d291..ba814e341 100644 --- a/S3/Utils.py +++ b/S3/Utils.py @@ -194,7 +194,7 @@ def dateRFC822toUnix(date): def formatSize(size, human_readable = False, floating_point = False): size = floating_point and float(size) or int(size) if human_readable: - coeffs = ['k', 'M', 'G', 'T'] + coeffs = ['K', 'M', 'G', 'T'] coeff = "" while size > 2048: size /= 1024 From 16bec963026e92f4724b5729de5a1b16daad03f9 Mon Sep 17 00:00:00 2001 From: Florent Viard Date: Sun, 22 Jul 2018 15:09:58 +0200 Subject: [PATCH 6/6] Fixes #956 - fix size and alignement of du and ls output reporting This change take into account that now, having big datasets with big file sizes is more common. Important note for automatic scripts: Output changes for du, la and ls commands:display column sizes and alignements changed. --- s3cmd | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/s3cmd b/s3cmd index 2ef135018..b2110eeee 100755 --- a/s3cmd +++ b/s3cmd @@ -102,8 +102,8 @@ def subcmd_bucket_usage_all(s3): buckets_size += size total_size, size_coeff = formatSize(buckets_size, cfg.human_readable_sizes) total_size_str = str(total_size) + size_coeff - output(u"".rjust(8, "-")) - output(u"%s Total" % (total_size_str.ljust(8))) + output(u"".rjust(12, "-")) + output(u"%s Total" % (total_size_str.ljust(12))) return size def subcmd_bucket_usage(s3, uri): @@ -131,9 +131,14 @@ def subcmd_bucket_usage(s3, uri): except KeyboardInterrupt as e: extra_info = u' [interrupted]' - total_size, size_coeff = formatSize(bucket_size, Config().human_readable_sizes) - total_size_str = str(total_size) + size_coeff - output(u"%s %s objects %s%s" % (total_size_str.ljust(8), object_count, uri, extra_info)) + total_size_str = u"%d%s" % formatSize(bucket_size, + Config().human_readable_sizes) + if Config().human_readable_sizes: + total_size_str = total_size_str.rjust(5) + else: + total_size_str = total_size_str.rjust(12) + output(u"%s %7s objects %s%s" % (total_size_str, object_count, uri, + extra_info)) return bucket_size def cmd_ls(args): @@ -184,18 +189,25 @@ def subcmd_bucket_list(s3, uri, limit): error(S3.codes[e.info["Code"]] % bucket) raise + # md5 are 32 char long, but for multipart there could be a suffix + if Config().human_readable_sizes: + # %(size)5s%(coeff)1s + format_size = u"%5d%1s" + dir_str = u"DIR".rjust(6) + else: + format_size = u"%12d%s" + dir_str = u"DIR".rjust(12) if cfg.long_listing: - format_string = u"%(timestamp)16s %(size)9s%(coeff)1s %(md5)32s %(storageclass)s %(uri)s" + format_string = u"%(timestamp)16s %(size)s %(md5)-35s %(storageclass)-11s %(uri)s" elif cfg.list_md5: - format_string = u"%(timestamp)16s %(size)9s%(coeff)1s %(md5)32s %(uri)s" + format_string = u"%(timestamp)16s %(size)s %(md5)-35s %(uri)s" else: - format_string = u"%(timestamp)16s %(size)9s%(coeff)1s %(uri)s" + format_string = u"%(timestamp)16s %(size)s %(uri)s" for prefix in response['common_prefixes']: output(format_string % { "timestamp": "", - "size": "DIR", - "coeff": "", + "size": dir_str, "md5": "", "storageclass": "", "uri": uri.compose_uri(bucket, prefix["Prefix"])}) @@ -213,11 +225,11 @@ def subcmd_bucket_list(s3, uri, limit): except KeyError: pass - size, size_coeff = formatSize(object["Size"], Config().human_readable_sizes) + size_and_coeff = formatSize(object["Size"], + Config().human_readable_sizes) output(format_string % { "timestamp": formatDateTime(object["LastModified"]), - "size" : str(size), - "coeff": size_coeff, + "size" : format_size % size_and_coeff, "md5" : md5, "storageclass" : storageclass, "uri": uri.compose_uri(bucket, object["Key"]),