From 42a8203e40d0302c705fbea9300c68ca218e28de Mon Sep 17 00:00:00 2001 From: Kyle Walker Date: Tue, 22 Dec 2015 10:59:10 -0800 Subject: [PATCH 1/2] Support S3 with Python3 --- fs/s3fs.py | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/fs/s3fs.py b/fs/s3fs.py index 4f51a50..ca86add 100644 --- a/fs/s3fs.py +++ b/fs/s3fs.py @@ -1,9 +1,8 @@ +from __future__ import unicode_literals """ fs.s3fs ======= -**Currently only avaiable on Python2 due to boto not being available for Python3** - FS subclass accessing files in Amazon S3 This module provides the class 'S3FS', which implements the FS filesystem @@ -106,8 +105,6 @@ def __init__(self, bucket, prefix="", aws_access_key=None, aws_secret_key=None, prefix = prefix[1:] if not prefix.endswith(separator) and prefix != "": prefix = prefix + separator - if isinstance(prefix,unicode): - prefix = prefix.encode("utf8") if aws_access_key is None: if "AWS_ACCESS_KEY_ID" not in os.environ: raise CreateFailedError("AWS_ACCESS_KEY_ID not set") @@ -179,8 +176,6 @@ def _s3path(self,path): s3path = self._prefix + path if s3path and s3path[-1] == self._separator: s3path = s3path[:-1] - if isinstance(s3path,unicode): - s3path = s3path.encode("utf8") return s3path def _uns3path(self,s3path,roots3path=None): @@ -398,8 +393,6 @@ def _iter_keys(self,path): # Skip over the entry for the directory itself, if it exists name = self._uns3path(k.name,s3path) if name != "": - if not isinstance(name,unicode): - name = name.decode("utf8") if name.endswith(self._separator): name = name[:-1] yield (name,k) @@ -579,8 +572,6 @@ def _get_key_info(self,key,name=None): info['size'] = int(key.size) etag = getattr(key,"etag",None) if etag is not None: - if isinstance(etag,unicode): - etag = etag.encode("utf8") info['etag'] = etag.strip('"').strip("'") if hasattr(key,"last_modified"): # TODO: does S3 use any other formats? @@ -663,8 +654,6 @@ def walkfiles(self, for k in self._s3bukt.list(prefix=prefix): name = relpath(self._uns3path(k.name,prefix)) if name != "": - if not isinstance(name,unicode): - name = name.decode("utf8") if not k.name.endswith(self._separator): if wildcard is not None: if callable(wildcard): @@ -691,8 +680,6 @@ def walkinfo(self, for k in self._s3bukt.list(prefix=prefix): name = relpath(self._uns3path(k.name,prefix)) if name != "": - if not isinstance(name,unicode): - name = name.decode("utf8") if wildcard is not None: if callable(wildcard): if not wildcard(basename(name)): @@ -718,8 +705,6 @@ def walkfilesinfo(self, for k in self._s3bukt.list(prefix=prefix): name = relpath(self._uns3path(k.name,prefix)) if name != "": - if not isinstance(name,unicode): - name = name.decode("utf8") if not k.name.endswith(self._separator): if wildcard is not None: if callable(wildcard): @@ -745,4 +730,3 @@ def _startswith_utf8(name1,name2): if isinstance(name2,unicode): name2 = name2.encode("utf8") return name1.startswith(name2) - From 9d2bff770a5141e80f819fa331fa951e4e1f2de5 Mon Sep 17 00:00:00 2001 From: Kyle Walker Date: Tue, 28 Aug 2018 14:50:15 -0700 Subject: [PATCH 2/2] Fix bad merge conflict. --- fs/s3fs.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/fs/s3fs.py b/fs/s3fs.py index 354659e..70d677d 100644 --- a/fs/s3fs.py +++ b/fs/s3fs.py @@ -112,12 +112,6 @@ def __init__(self, bucket, prefix="", aws_access_key=None, aws_secret_key=None, prefix = prefix[1:] if not prefix.endswith(separator) and prefix != "": prefix = prefix + separator - if aws_access_key is None: - if "AWS_ACCESS_KEY_ID" not in os.environ: - raise CreateFailedError("AWS_ACCESS_KEY_ID not set") - if aws_secret_key is None: - if "AWS_SECRET_ACCESS_KEY" not in os.environ: - raise CreateFailedError("AWS_SECRET_ACCESS_KEY not set") self._prefix = prefix self._tlocal = thread_local() super(S3FS, self).__init__(thread_synchronize=thread_synchronize)