diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 1e3de47bda..fea3896a58 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -23,7 +23,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7, 3.8] + python-version: [3.7, 3.8, 3.12] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} diff --git a/deploy-agent/deployd/download/download_helper_factory.py b/deploy-agent/deployd/download/download_helper_factory.py index f992f77c8c..36bf3dfe0e 100644 --- a/deploy-agent/deployd/download/download_helper_factory.py +++ b/deploy-agent/deployd/download/download_helper_factory.py @@ -16,9 +16,9 @@ from typing import Optional from future.moves.urllib.parse import urlparse -from boto.s3.connection import S3Connection from deployd.download.download_helper import DownloadHelper +from deployd.download.s3_client import S3Client from deployd.download.s3_download_helper import S3DownloadHelper from deployd.download.http_download_helper import HTTPDownloadHelper from deployd.download.local_download_helper import LocalDownloadHelper @@ -38,7 +38,7 @@ def gen_downloader(url, config) -> Optional[DownloadHelper]: if aws_access_key_id is None or aws_secret_access_key is None: log.error("aws access key id and secret access key not found") return None - aws_conn = S3Connection(aws_access_key_id, aws_secret_access_key, True) + aws_conn = S3Client(aws_access_key_id, aws_secret_access_key) return S3DownloadHelper(local_full_fn=url, aws_connection=aws_conn, url=None, config=config) elif url_parse.scheme == 'file': return LocalDownloadHelper(url=url) diff --git a/deploy-agent/deployd/download/s3_client.py b/deploy-agent/deployd/download/s3_client.py new file mode 100644 index 0000000000..71b54dc5af --- /dev/null +++ b/deploy-agent/deployd/download/s3_client.py @@ -0,0 +1,87 @@ +USE_BOTO3 = False +try: + from boto.s3.connection import S3Connection +except ImportError: + import botocore + import boto3 + + USE_BOTO3 = True + + +class Boto2Client: + """ + Client to handle boto2 operations. This can be removed + once boto2 is no longer used + """ + def __init__(self, aws_access_key_id, aws_secret_access_key): + self.client = S3Connection(aws_access_key_id, aws_secret_access_key, True) + + def get_key(self, bucket_name, key_name): + """ + Return the object at the specified key + """ + return self.client.get_bucket(bucket_name).get_key(key_name) + + def download_object_to_file(self, obj, file_name): + """ + Download the object to the specified file name + + :param obj: the object returned from `self.get_key` + :param file_name str: the file_name to download to + """ + obj.get_contents_to_filename(file_name) + + def get_etag(self, obj): + """ + Get the etag of the specified object + + :param obj: the object returned from `self.get_key` + """ + return obj.etag + + +class Boto3Client: + """ + Client to handle boto3 operations. This can be renamed to + `S3Client` once boto2 is no longer used. + """ + def __init__(self, aws_access_key_id, aws_secret_access_key): + session = boto3.Session( + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + ) + self.client = session.resource('s3') + + def get_key(self, bucket_name, key_name): + """ + Return the object at the specified key + """ + obj = self.client.Bucket(bucket_name).Object(key_name) + try: + # To be compatible with boto2, return None if key does not exist + obj.load() + return obj + except botocore.exceptions.ClientError as e: + if e.response['Error']['Code'] == "404": + return None + else: + raise + + def download_object_to_file(self, obj, file_name): + """ + Download the object to the specified file name + + :param obj: the object returned from `self.get_key` + :param file_name str: the file_name to download to + """ + obj.download_file(file_name) + + def get_etag(self, obj): + """ + Get the etag of the specified object + + :param obj: the object returned from `self.get_key` + """ + return obj.e_tag + +S3Client = Boto3Client if USE_BOTO3 else Boto2Client diff --git a/deploy-agent/deployd/download/s3_download_helper.py b/deploy-agent/deployd/download/s3_download_helper.py index 3540791a58..593e3c57f9 100644 --- a/deploy-agent/deployd/download/s3_download_helper.py +++ b/deploy-agent/deployd/download/s3_download_helper.py @@ -16,10 +16,10 @@ import logging import re -from boto.s3.connection import S3Connection from deployd.common.config import Config from deployd.common.status_code import Status from deployd.download.download_helper import DownloadHelper, DOWNLOAD_VALIDATE_METRICS +from deployd.download.s3_client import S3Client from deployd.common.stats import create_sc_increment log = logging.getLogger(__name__) @@ -36,7 +36,7 @@ def __init__(self, local_full_fn, aws_connection=None, url=None, config=None) -> else: aws_access_key_id = self._config.get_aws_access_key() aws_secret_access_key = self._config.get_aws_access_secret() - self._aws_connection = S3Connection(aws_access_key_id, aws_secret_access_key, True) + self._aws_connection = S3Client(aws_access_key_id, aws_secret_access_key) if url: self._url = url @@ -52,13 +52,13 @@ def download(self, local_full_fn) -> int: return Status.FAILED try: - filekey = self._aws_connection.get_bucket(self._bucket_name).get_key(self._key) + filekey = self._aws_connection.get_key(self._bucket_name, self._key) if filekey is None: log.error("s3 key {} not found".format(self._key)) return Status.FAILED - filekey.get_contents_to_filename(local_full_fn) - etag = filekey.etag + self._aws_connection.download_object_to_file(filekey, local_full_fn) + etag = self._aws_connection.get_etag(filekey) if "-" not in etag: if etag.startswith('"') and etag.endswith('"'): etag = etag[1:-1] diff --git a/deploy-agent/requirements.txt b/deploy-agent/requirements.txt index 3828e62ed7..e0c24ab7c6 100755 --- a/deploy-agent/requirements.txt +++ b/deploy-agent/requirements.txt @@ -1,8 +1,8 @@ -e . -PyYAML==5.3.1 +PyYAML==5.3.1; python_version < '3.12' +PyYAML==6.0.1; python_version >= '3.12' zipp==1.2.0 configparser==4.0.2 python-daemon==2.0.6 -setuptools==44.1.1; python_version < '3' -setuptools==54.2.0; python_version >= '3' +setuptools==54.2.0 diff --git a/deploy-agent/requirements_test.txt b/deploy-agent/requirements_test.txt index 2c381a9ea1..f66d06921b 100755 --- a/deploy-agent/requirements_test.txt +++ b/deploy-agent/requirements_test.txt @@ -3,8 +3,8 @@ # tests tox==1.6.1 pytest-cov -pytest==6.2.2; python_version >= '3' -pytest==4.6.11; python_version < '3' +pytest==6.2.2; python_version < '3.12' +pytest==8.2.1; python_version >= '3.12' mock==1.0.1 flake8==2.5.4 pep8 diff --git a/deploy-agent/setup.py b/deploy-agent/setup.py index b6fb9f4446..4eb3b6ee75 100644 --- a/deploy-agent/setup.py +++ b/deploy-agent/setup.py @@ -24,12 +24,14 @@ 'deploy-stager = deployd.staging.stager:main'] install_requires = [ - "requests==2.20.0", - "gevent>=1.0.2,<=1.2.2; python_version < '3'", + "requests==2.20.0; python_version < '3.12'", + "requests==2.32.3; python_version >= '3.12'", "gevent>=1.0.2,<=1.5.0; python_version < '3.8'", - "gevent==20.12.0; python_version >= '3.8'", + "gevent==20.12.0; python_version >= '3.8' and python_version < '3.12'", + "gevent==24.2.1; python_version >= '3.12'", "lockfile==0.10.2", - "boto>=2.39.0", + "boto>=2.39.0; python_version < '3.12'", + "boto3==1.34.114; python_version >= '3.12'", "python-daemon==2.0.6", "future==0.18.2" ] diff --git a/deploy-agent/tests/unit/deploy/download/test_download_helper.py b/deploy-agent/tests/unit/deploy/download/test_download_helper.py index 58cb221119..3361ef1017 100644 --- a/deploy-agent/tests/unit/deploy/download/test_download_helper.py +++ b/deploy-agent/tests/unit/deploy/download/test_download_helper.py @@ -12,7 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +USE_BOTO3 = False +try: + from boto.s3.connection import S3Connection +except ImportError: + import boto3 + USE_BOTO3 = True + from deployd.download.s3_download_helper import S3DownloadHelper +from deployd.download.s3_client import S3Client import os import mock import shutil @@ -35,23 +43,33 @@ def setUpClass(cls): target = os.path.join(builds_dir, 'mock.txt') cls.target = target - cls.aws_conn = mock.Mock() - aws_filekey = cls.aws_conn.get_bucket.return_value.get_key.return_value + cls.aws_conn = mock.Mock(wraps=S3Client('test_access_key_id', 'test_secret_access_key')) + aws_filekey = cls.aws_conn.get_key.return_value def get_contents_to_filename(fn): with open(fn, 'w') as file: file.write("hello mock\n") - aws_filekey.get_contents_to_filename = mock.Mock(side_effect=get_contents_to_filename) - aws_filekey.etag = "f7673f4693aab49e3f8e643bc54cb70a" + + if USE_BOTO3: + aws_filekey.download_file = mock.Mock(side_effect=get_contents_to_filename) + aws_filekey.e_tag = "f7673f4693aab49e3f8e643bc54cb70a" + else: + aws_filekey.get_contents_to_filename = mock.Mock(side_effect=get_contents_to_filename) + aws_filekey.etag = "f7673f4693aab49e3f8e643bc54cb70a" def test_download_s3(self): downloader = S3DownloadHelper(self.target, self.aws_conn, self.url) downloader.download(self.target) - self.aws_conn.get_bucket.assert_called_once_with("pinterest-builds") - self.aws_conn.get_bucket.return_value.get_key.assert_called_once_with("teletraan/mock.txt") - self.aws_conn.get_bucket.return_value.get_key.return_value\ - .get_contents_to_filename\ - .assert_called_once_with(self.target) + self.aws_conn.get_key.assert_called_once_with("pinterest-builds", "teletraan/mock.txt") + + if USE_BOTO3: + self.aws_conn.get_key.return_value\ + .download_file\ + .assert_called_once_with(self.target) + else: + self.aws_conn.get_key.return_value\ + .get_contents_to_filename\ + .assert_called_once_with(self.target) @classmethod def tearDownClass(cls): diff --git a/deploy-agent/tox.ini b/deploy-agent/tox.ini index 4b3576faa8..26c18a6319 100644 --- a/deploy-agent/tox.ini +++ b/deploy-agent/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist=py36,py38 +envlist=py36,py38,py312 recreate=True [testenv] @@ -10,3 +10,4 @@ commands=py.test --cov=deployd {posargs} python = 3.6: py36 3.8: py38 + 3.12: py312