Py312 upgrade step 2: Update dependencies, integ tests and unit tests

.githooks/pre-push

@@ -12,5 +12,5 @@ start_time=`date +%s`
 tox -e sphinx,doc8 --parallel all
 ./ci-scripts/displaytime.sh 'sphinx,doc8' $start_time
 start_time=`date +%s`
-tox -e py38,py39,py310 --parallel all -- tests/unit
-./ci-scripts/displaytime.sh 'py38,py39,py310 unit' $start_time
+tox -e py39,py310,py311,py312 --parallel all -- tests/unit
+./ci-scripts/displaytime.sh 'py39,py310,py311,py312 unit' $start_time

.pylintrc

@@ -94,7 +94,24 @@ disable=
     useless-object-inheritance, # TODO: Enable this check and fix code once Python 2 is no longer supported.
     super-with-arguments,
     raise-missing-from,
-    E1136,
+    C0116,  # Missing function or method docstring
+    C0209,  # Use f-string instead of format
+    E0015,  # Unrecognized option found in config
+    E0702,  # Raising a string instead of an exception
+    E1101,  # Module has no member (likely dynamic attr)
+    E1136,  # Value assigned to something inferred as None
+    R0022,  # Useless option value in config
+    R1710,  # Inconsistent return statements
+    R1714,  # Consider using `in` with comparisons
+    R1729,  # Use a generator
+    R1732,
+    R1735,  # Consider using a dict or list literal
+    W0237,  # Argument renamed in override
+    W0613,  # Unused argument
+    W0621,  # Redefining name from outer scope
+    W0719
+    W1404,  # Implicit string concatenation
+    W1514,  # `open()` used without encoding
 
 [REPORTS]
 # Set the output format. Available formats are text, parseable, colorized, msvs
@@ -436,4 +453,4 @@ analyse-fallback-blocks=no
 
 # Exceptions that will emit a warning when being caught. Defaults to
 # "Exception"
-overgeneral-exceptions=Exception
+overgeneral-exceptions=builtins.Exception

.readthedocs.yaml

@@ -5,9 +5,9 @@
 version: 2
 
 build:
-  os: ubuntu-20.04
+  os: ubuntu-22.04
   tools:
-    python: "3.9"
+    python: "3.12"
 
 
 python:

doc/api/inference/model_builder.rst

@@ -3,14 +3,14 @@ Model Builder
 
 This module contains classes related to Amazon Sagemaker Model Builder
 
-.. autoclass:: sagemaker.serve.builder.model_builder.ModelBuilder
+.. autoclass:: sagemaker.serve.ModelBuilder
 
-.. automethod:: sagemaker.serve.builder.model_builder.ModelBuilder.build
+.. automethod:: sagemaker.serve.ModelBuilder.build
 
-.. automethod:: sagemaker.serve.builder.model_builder.ModelBuilder.save
+.. automethod:: sagemaker.serve.ModelBuilder.save
 
-.. autoclass:: sagemaker.serve.spec.inference_spec.InferenceSpec
+.. autoclass:: sagemaker.serve.InferenceSpec
 
-.. autoclass:: sagemaker.serve.builder.schema_builder.SchemaBuilder
+.. autoclass:: sagemaker.serve.SchemaBuilder
 
-.. autoclass:: sagemaker.serve.marshalling.custom_payload_translator.CustomPayloadTranslator
+.. autoclass:: sagemaker.serve.CustomPayloadTranslator

doc/api/training/index.rst

@@ -3,7 +3,7 @@ Training APIs
 #############
 
 .. toctree::
-   :maxdepth: 4
+   :maxdepth: 1
 
    model_trainer
    algorithm

doc/conf.py

@@ -83,16 +83,11 @@
 
 html_css_files = [
     "https://cdn.datatables.net/1.10.23/css/jquery.dataTables.min.css",
+    "theme_overrides.css",
+    "pagination.css",
+    "search_accessories.css",
 ]
 
-html_context = {
-    "css_files": [
-        "_static/theme_overrides.css",
-        "_static/pagination.css",
-        "_static/search_accessories.css",
-    ]
-}
-
 # Example configuration for intersphinx: refer to the Python standard library.
 intersphinx_mapping = {"python": ("http://docs.python.org/", None)}
 

doc/requirements.txt

@@ -1,7 +1,7 @@
-sphinx==5.1.1
-sphinx-rtd-theme==0.5.0
-docutils==0.15.2
-packaging==20.9
+sphinx==7.2.6
+sphinx-rtd-theme==3.0.0
+docutils>=0.18.1,<0.21
+packaging>=23.0,<25
 jinja2==3.1.6
 schema==0.7.5
 accelerate>=0.24.1,<=0.27.0

pyproject.toml

@@ -7,7 +7,7 @@ name = "sagemaker"
 dynamic = ["version", "optional-dependencies"]
 description = "Open source library for training and deploying models on Amazon SageMaker."
 readme = "README.rst"
-requires-python = ">=3.8"
+requires-python = ">=3.9"
 authors = [
   { name = "Amazon Web Services" },
 ]
@@ -25,10 +25,10 @@ classifiers = [
   "License :: OSI Approved :: Apache Software License",
   "Natural Language :: English",
   "Programming Language :: Python",
-  "Programming Language :: Python :: 3.8",
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
 ]
 dependencies = [
   "attrs>=23.1.0,<24",
@@ -39,15 +39,15 @@ dependencies = [
   "google-pasta",
   "importlib-metadata>=1.4.0,<7.0",
   "jsonschema",
-  "numpy>=1.9.0,<2.0",
+  "numpy==1.26.4",
   "omegaconf>=2.2,<=2.3",
-  "packaging>=20.0",
+  "packaging>=23.0,<25",
   "pandas",
   "pathos",
   "platformdirs",
   "protobuf>=3.12,<6.0",
   "psutil",
-  "PyYAML~=6.0",
+  "PyYAML>=6.0.1",
   "requests",
   "sagemaker-core>=1.0.17,<2.0.0",
   "schema",

requirements/extras/local_requirements.txt

@@ -1,3 +1,3 @@
 urllib3>=1.26.8,<3.0.0
 docker>=5.0.2,<8.0.0
-PyYAML>=5.4.1,<7
+PyYAML>=6.0.1,<7

requirements/extras/scipy_requirements.txt

@@ -1 +1 @@
-scipy==1.10.1
+scipy==1.11.3

requirements/extras/test_requirements.txt

@@ -1,7 +1,7 @@
 tox==3.24.5
-numpy>=1.24.0
+numpy==1.26.4
 build[virtualenv]==1.2.1
-flake8==4.0.1
+flake8==7.1.2
 pytest==6.2.5
 pytest-cov==3.0.0
 pytest-rerunfailures==10.2
@@ -14,10 +14,10 @@ awslogs==0.14.0
 black==24.3.0
 stopit==1.1.2
 # Update tox.ini to have correct version of airflow constraints file
-apache-airflow==2.9.3
+apache-airflow==2.10.4
 apache-airflow-providers-amazon==7.2.1
 attrs>=23.1.0,<24
-fabric==2.6.0
+fabric==3.2.2
 requests==2.32.2
 sagemaker-experiments==0.1.35
 Jinja2==3.1.6
@@ -26,13 +26,13 @@ pandas==1.4.4
 scikit-learn==1.3.0
 cloudpickle==2.2.1
 jsonpickle<4.0.0
-PyYAML==6.0
+PyYAML>=6.0.1
 # TODO find workaround
 xgboost>=1.6.2,<=1.7.6
 pillow>=10.0.1,<=11
 opentelemetry-proto==1.27.0
 protobuf==4.25.5
-tensorboard>=2.9.0,<=2.15.2
+tensorboard>=2.16.2,<=2.18.0
 transformers==4.48.0
 sentencepiece==0.1.99
 # https://github.com/triton-inference-server/server/issues/6246
@@ -42,7 +42,7 @@ onnx==1.17.0
 nbformat>=5.9,<6
 accelerate>=0.24.1,<=0.27.0
 schema==0.7.5
-tensorflow>=2.9.0,<=2.15.1
+tensorflow>=2.16.2,<=2.18.0
 mlflow>=2.12.2,<2.13
 huggingface_hub==0.26.2
 uvicorn>=0.30.1
@@ -51,3 +51,4 @@ nest-asyncio
 sagemaker-mlflow>=0.1.0
 deepdiff>=8.0.0
 orderly-set<5.4.0
+lexicon

requirements/tox/doc8_requirements.txt

@@ -1,2 +1,2 @@
-doc8==0.10.1
-Pygments==2.15.0
+doc8==1.1.2
+Pygments==2.18.0

requirements/tox/flake8_requirements.txt

@@ -1,2 +1,2 @@
-flake8==4.0.1
-flake8-future-import==0.4.6
+flake8==7.1.2
+flake8-future-import==0.4.7

requirements/tox/pylint_requirements.txt

@@ -1,2 +1,2 @@
-pylint==2.6.2
-astroid==2.4.2
+pylint==3.0.3
+astroid==3.0.2

requirements/tox/spelling_requirements.txt

@@ -1,2 +1,2 @@
 pyenchant==3.2.2
-pylint==2.6.2
+pylint==3.0.3

src/sagemaker/config/config_schema.py

@@ -540,7 +540,8 @@ def _simple_path(*args: str):
             "minItems": 0,
             "maxItems": 50,
         },
-        # Regex is taken from https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_CreateTrainingJob.html#sagemaker-CreateTrainingJob-request-Environment
+        # Regex is taken from https://docs.aws.amazon.com/sagemaker/latest/APIReference/
+        #   API_CreateTrainingJob.html#sagemaker-CreateTrainingJob-request-Environment
         "environmentVariables": {
             TYPE: OBJECT,
             ADDITIONAL_PROPERTIES: False,
@@ -553,13 +554,15 @@ def _simple_path(*args: str):
             },
             "maxProperties": 48,
         },
-        # Regex is taken from https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_S3DataSource.html#sagemaker-Type-S3DataSource-S3Uri
+        # Regex is taken from https://docs.aws.amazon.com/sagemaker/latest/APIReference/
+        #   API_S3DataSource.html#sagemaker-Type-S3DataSource-S3Uri
         "s3Uri": {
             TYPE: "string",
             "pattern": "^(https|s3)://([^/]+)/?(.*)$",
             "maxLength": 1024,
         },
-        # Regex is taken from https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_AlgorithmSpecification.html#sagemaker-Type-AlgorithmSpecification-ContainerEntrypoint
+        # Regex is taken from https://docs.aws.amazon.com/sagemaker/latest/APIReference/
+        #   API_AlgorithmSpecification.html#sagemaker-Type-AlgorithmSpecification-ContainerEntrypoint
         "preExecutionCommand": {TYPE: "string", "pattern": r".*"},
         # Regex based on https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_PipelineDefinitionS3Location.html
         # except with an additional ^ and $ for the beginning and the end to closer align to
@@ -570,7 +573,8 @@ def _simple_path(*args: str):
             "minLength": 3,
             "maxLength": 63,
         },
-        # Regex is taken from https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_MonitoringJobDefinition.html#sagemaker-Type-MonitoringJobDefinition-Environment
+        # Regex is taken from https://docs.aws.amazon.com/sagemaker/latest/APIReference/
+        #   API_MonitoringJobDefinition.html#sagemaker-Type-MonitoringJobDefinition-Environment
         "environment-Length256-Properties50": {
             TYPE: OBJECT,
             ADDITIONAL_PROPERTIES: False,
@@ -583,7 +587,8 @@ def _simple_path(*args: str):
             },
             "maxProperties": 50,
         },
-        # Regex is taken from https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_CreateTransformJob.html#sagemaker-CreateTransformJob-request-Environment
+        # Regex is taken from https://docs.aws.amazon.com/sagemaker/latest/APIReference/
+        #   API_CreateTransformJob.html#sagemaker-CreateTransformJob-request-Environment
         "environment-Length10240-Properties16": {
             TYPE: OBJECT,
             ADDITIONAL_PROPERTIES: False,
@@ -596,7 +601,8 @@ def _simple_path(*args: str):
             },
             "maxProperties": 16,
         },
-        # Regex is taken from https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_ContainerDefinition.html#sagemaker-Type-ContainerDefinition-Environment
+        # Regex is taken from https://docs.aws.amazon.com/sagemaker/latest/APIReference/
+        #   API_ContainerDefinition.html#sagemaker-Type-ContainerDefinition-Environment
         "environment-Length1024-Properties16": {
             TYPE: OBJECT,
             ADDITIONAL_PROPERTIES: False,
@@ -609,7 +615,8 @@ def _simple_path(*args: str):
             },
             "maxProperties": 16,
         },
-        # Regex is taken from https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_CreateProcessingJob.html#sagemaker-CreateProcessingJob-request-Environment
+        # Regex is taken from https://docs.aws.amazon.com/sagemaker/latest/APIReference/
+        #   API_CreateProcessingJob.html#sagemaker-CreateProcessingJob-request-Environment
         "environment-Length256-Properties100": {
             TYPE: OBJECT,
             ADDITIONAL_PROPERTIES: False,
@@ -622,7 +629,8 @@ def _simple_path(*args: str):
             },
             "maxProperties": 100,
         },
-        # Regex is taken from https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_CreateTrainingJob.html#sagemaker-CreateTrainingJob-request-Environment
+        # Regex is taken from https://docs.aws.amazon.com/sagemaker/latest/APIReference/
+        # API_CreateTrainingJob.html#sagemaker-CreateTrainingJob-request-Environment
         "environment-Length512-Properties48": {
             TYPE: OBJECT,
             ADDITIONAL_PROPERTIES: False,

src/sagemaker/feature_store/dataset_builder.py

@@ -929,7 +929,7 @@ def _construct_query_string(self, base: FeatureGroupToBeMerged) -> str:
                 selected_features += ", "
                 selected_features += ", ".join(
                     [
-                        f'fg_{i}."{feature_name}" as "{feature_name}.{(i+1)}"'
+                        f'fg_{i}."{feature_name}" as "{feature_name}.{(i + 1)}"'
                         for feature_name in feature_group.projected_feature_names
                     ]
                 )

src/sagemaker/jumpstart/factory/model.py

@@ -104,7 +104,7 @@ def get_default_predictor(
     """
 
     # if there's a non-default predictor, do not mutate -- return as is
-    if type(predictor) != Predictor:  # pylint: disable=C0123
+    if not isinstance(predictor, Predictor):
         raise RuntimeError(
             "Can only get default predictor from base Predictor class. "
             f"Using Predictor class '{type(predictor).__name__}'."

src/sagemaker/local/entities.py

@@ -845,10 +845,10 @@ def _initialize_and_validate_parameters(self, overridden_parameters):
                     )
                     raise ClientError(error_msg, "start_pipeline_execution")
                 parameter_type = default_parameters[param_name].parameter_type
-                if type(param_value) != parameter_type.python_type:  # pylint: disable=C0123
+                if not isinstance(param_value, parameter_type.python_type):
                     error_msg = self._construct_validation_exception_message(
-                        "Unexpected type for parameter '{}'. Expected {} but found "
-                        "{}.".format(param_name, parameter_type.python_type, type(param_value))
+                        f"Unexpected type for parameter '{param_name}'. Expected \
+                            {parameter_type.python_type} but found {type(param_value)}."
                     )
                     raise ClientError(error_msg, "start_pipeline_execution")
                 if param_value == "":

src/sagemaker/model_monitor/clarify_model_monitoring.py

@@ -86,11 +86,9 @@ def __init__(
                 object that configures network isolation, encryption of
                 inter-container traffic, security group IDs, and subnets.
         """
-        if type(self) == __class__:  # pylint: disable=unidiomatic-typecheck
+        if self.__class__ is __class__:
             raise TypeError(
-                "{} is abstract, please instantiate its subclasses instead.".format(
-                    __class__.__name__
-                )
+                f"{__class__.__name__} is abstract, please instantiate its subclasses instead."
             )
 
         session = sagemaker_session or Session()

src/sagemaker/serve/model_server/multi_model_server/prepare.py

@@ -84,7 +84,8 @@ def prepare_for_mms(
     image_uri: str,
     inference_spec: InferenceSpec = None,
 ) -> str:
-    """Prepares for InferenceSpec using model_path, writes inference.py, and captures dependencies to generate secret_key.
+    """Prepares for InferenceSpec using model_path, writes inference.py, \
+        and captures dependencies to generate secret_key.
 
     Args:to
         model_path (str) : Argument