|
| 1 | +import argparse |
| 2 | +import os |
| 3 | +import random |
| 4 | +import sys |
| 5 | +import logging |
| 6 | +import re |
| 7 | +import uuid |
| 8 | + |
| 9 | + |
| 10 | +import boto3 |
| 11 | +import pytest |
| 12 | + |
| 13 | +from invoke import run |
| 14 | +from invoke.context import Context |
| 15 | + |
| 16 | + |
| 17 | +from tests.utils.report import Report |
| 18 | +from tests.utils import ( |
| 19 | + S3_BUCKET_BENCHMARK_ARTIFACTS, |
| 20 | + DEFAULT_REGION, |
| 21 | + DEFAULT_DOCKER_DEV_ECR_REPO, |
| 22 | + YamlHandler, |
| 23 | + DockerImageHandler, |
| 24 | +) |
| 25 | + |
| 26 | +LOGGER = logging.getLogger(__name__) |
| 27 | +LOGGER.setLevel(logging.DEBUG) |
| 28 | +LOGGER.addHandler(logging.StreamHandler(sys.stdout)) |
| 29 | + |
| 30 | + |
| 31 | +def build_docker_container(torchserve_branch="master", push_image=True, use_local_serve_folder=False): |
| 32 | + LOGGER.info(f"Setting up docker image to be used") |
| 33 | + |
| 34 | + docker_dev_image_config_path = os.path.join( |
| 35 | + os.getcwd(), "benchmarks", "automated", "tests", "suite", "docker", "docker.yaml" |
| 36 | + ) |
| 37 | + |
| 38 | + |
| 39 | + if use_local_serve_folder: |
| 40 | + LOGGER.info(f"*** Using the local 'serve' folder closure when creating the container image.") |
| 41 | + |
| 42 | + |
| 43 | + local_serve_folder = os.getcwd() |
| 44 | + tmp_local_serve_folder = os.path.join("/tmp", "serve") |
| 45 | + serve_folder_in_docker_context = os.path.join(os.getcwd(), "docker", "serve") |
| 46 | + |
| 47 | + run(f"mkdir -p {tmp_local_serve_folder}") |
| 48 | + run(f"mkdir -p {serve_folder_in_docker_context}") |
| 49 | + |
| 50 | + run(f"rsync -av --progress {local_serve_folder}/ {tmp_local_serve_folder}/") |
| 51 | + run(f"rsync -av --progress {tmp_local_serve_folder}/ {serve_folder_in_docker_context}/") |
| 52 | + |
| 53 | + run(f"rm -rf {tmp_local_serve_folder}") |
| 54 | + |
| 55 | + docker_config = YamlHandler.load_yaml(docker_dev_image_config_path) |
| 56 | + YamlHandler.validate_docker_yaml(docker_config) |
| 57 | + |
| 58 | + account_id = run("aws sts get-caller-identity --query Account --output text").stdout.strip() |
| 59 | + |
| 60 | + for processor, config in docker_config.items(): |
| 61 | + docker_tag = None |
| 62 | + cuda_version = None |
| 63 | + dockerhub_image = None |
| 64 | + for config_key, config_value in config.items(): |
| 65 | + if processor == "gpu" and config_key == "cuda_version": |
| 66 | + cuda_version = config_value |
| 67 | + if config_key == "docker_tag": |
| 68 | + docker_tag = config_value |
| 69 | + |
| 70 | + if config_key == "dockerhub_image": |
| 71 | + dockerhub_image = config_value |
| 72 | + |
| 73 | + dockerImageHandler = DockerImageHandler(docker_tag, cuda_version, torchserve_branch) |
| 74 | + |
| 75 | + if not dockerhub_image: |
| 76 | + dockerImageHandler.build_image(use_local_serve_folder=use_local_serve_folder) |
| 77 | + else: |
| 78 | + # Image is pulled by process_docker_config in __init__.py |
| 79 | + LOGGER.info(f"*** Note: dockerhub_image specified in docker.yaml. This container image will be used for benchmark.") |
| 80 | + dockerImageHandler.pull_docker_image(dockerhub_image, docker_tag=docker_tag) |
| 81 | + |
| 82 | + if push_image: |
| 83 | + dockerImageHandler.push_docker_image_to_ecr( |
| 84 | + account_id, DEFAULT_REGION, f"{DEFAULT_DOCKER_DEV_ECR_REPO}:{docker_tag}" |
| 85 | + ) |
| 86 | + else: |
| 87 | + LOGGER.warn(f"Docker image will not be pushed to ECR repo in local execution.") |
| 88 | + |
| 89 | + |
| 90 | +def main(): |
| 91 | + |
| 92 | + parser = argparse.ArgumentParser() |
| 93 | + |
| 94 | + parser.add_argument( |
| 95 | + "--use-instances", |
| 96 | + action="store", |
| 97 | + help="Supply a .yaml file with test_name, instance_id, and key_filename to re-use already-running instances", |
| 98 | + ) |
| 99 | + parser.add_argument( |
| 100 | + "--do-not-terminate", |
| 101 | + action="store_true", |
| 102 | + default=False, |
| 103 | + help="Use with caution: does not terminate instances, instead saves the list to a file in order to re-use", |
| 104 | + ) |
| 105 | + |
| 106 | + parser.add_argument( |
| 107 | + "--run-only", nargs="+", default=None, help="Runs the tests that contain the supplied keyword as a substring" |
| 108 | + ) |
| 109 | + |
| 110 | + parser.add_argument( |
| 111 | + "--use-torchserve-branch", |
| 112 | + default="master", |
| 113 | + help="Specify a specific torchserve branch to build a container to benchmark on, else uses 'master' by default", |
| 114 | + ) |
| 115 | + |
| 116 | + parser.add_argument( |
| 117 | + "--use-local-serve-folder", |
| 118 | + action="store_true", |
| 119 | + default=False, |
| 120 | + help="Specify this option if you'd like to build a container image out of your current 'serve' folder." |
| 121 | + ) |
| 122 | + |
| 123 | + parser.add_argument( |
| 124 | + "--skip-docker-build", |
| 125 | + action="store_true", |
| 126 | + default=False, |
| 127 | + help="Use if you already have a docker image built and available locally and have specified it in docker.yaml", |
| 128 | + ) |
| 129 | + |
| 130 | + parser.add_argument( |
| 131 | + "--local-execution", |
| 132 | + action="store_true", |
| 133 | + default=False, |
| 134 | + help="Specify when you want to execute benchmarks on the current instance. Note: this will execute the model benchmarks sequentially, and will ignore instances specified in the model config *.yaml files.", |
| 135 | + ) |
| 136 | + |
| 137 | + parser.add_argument( |
| 138 | + "--local-instance-type", |
| 139 | + default=None, |
| 140 | + help="Specify the current ec2 instance on which the benchmark executes. May not specify any other value than a valid ec2 instance type." |
| 141 | + ) |
| 142 | + |
| 143 | + |
| 144 | + arguments = parser.parse_args() |
| 145 | + |
| 146 | + if arguments.local_instance_type and not arguments.local_execution: |
| 147 | + LOGGER.error(f"--local-instance-type may only be used with --local-execution") |
| 148 | + sys.exit(1) |
| 149 | + |
| 150 | + if arguments.local_execution and not arguments.local_instance_type: |
| 151 | + LOGGER.error(f"--local-instance-type must be specified when using --local-execution") |
| 152 | + sys.exit(1) |
| 153 | + |
| 154 | + do_not_terminate_string = "" if not arguments.do_not_terminate else "--do-not-terminate" |
| 155 | + local_execution_string = "" if not arguments.local_execution else "--local-execution" |
| 156 | + use_instances_arg_list = ["--use-instances", f"{arguments.use_instances}"] if arguments.use_instances else [] |
| 157 | + run_only_test = arguments.run_only |
| 158 | + |
| 159 | + if run_only_test: |
| 160 | + LOGGER.info(f"run_only_test:{run_only_test}") |
| 161 | + LOGGER.info(f"run_only_test type:{type(run_only_test)}") |
| 162 | + run_only_string_list = " or ".join([model for model in run_only_test]) |
| 163 | + run_only_string = f"-k {run_only_string_list}" |
| 164 | + LOGGER.info(f"Note: running only the tests that have the name '{run_only_string_list}'.") |
| 165 | + else: |
| 166 | + run_only_string = "" |
| 167 | + |
| 168 | + if arguments.local_execution: |
| 169 | + number_of_threads_string = "" |
| 170 | + local_instance_type_list = ["--local-instance-type", arguments.local_instance_type] |
| 171 | + else: |
| 172 | + number_of_threads_string = "-n=4" |
| 173 | + local_instance_type_list = [] |
| 174 | + |
| 175 | + torchserve_branch = arguments.use_torchserve_branch |
| 176 | + use_local_serve_folder = arguments.use_local_serve_folder |
| 177 | + |
| 178 | + # Build docker containers as specified in docker.yaml |
| 179 | + if not arguments.skip_docker_build: |
| 180 | + push_image = False if arguments.local_execution else True |
| 181 | + build_docker_container(torchserve_branch=torchserve_branch, push_image=push_image, use_local_serve_folder=use_local_serve_folder) |
| 182 | + else: |
| 183 | + LOGGER.warn(f"Skipping docker build.") |
| 184 | + |
| 185 | + # Run this script from the root directory 'serve', it changes directory below as required |
| 186 | + os.chdir(os.path.join(os.getcwd(), "benchmarks", "automated")) |
| 187 | + |
| 188 | + execution_id = f"ts-benchmark-run-{str(uuid.uuid4())}" |
| 189 | + |
| 190 | + test_path = os.path.join(os.getcwd(), "tests") |
| 191 | + LOGGER.info(f"Running tests from directory: {test_path}") |
| 192 | + |
| 193 | + pytest_args = [ |
| 194 | + "-s", |
| 195 | + run_only_string, |
| 196 | + "-rA", |
| 197 | + test_path, |
| 198 | + number_of_threads_string, |
| 199 | + "--disable-warnings", |
| 200 | + "-v", |
| 201 | + "--execution-id", |
| 202 | + execution_id, |
| 203 | + do_not_terminate_string, |
| 204 | + local_execution_string, |
| 205 | + ] + local_instance_type_list + use_instances_arg_list |
| 206 | + |
| 207 | + LOGGER.info(f"Running pytest") |
| 208 | + |
| 209 | + pytest.main(pytest_args) |
| 210 | + |
| 211 | + # Generate report |
| 212 | + s3_results_uri = f"{S3_BUCKET_BENCHMARK_ARTIFACTS}/{execution_id}" |
| 213 | + |
| 214 | + report = Report() |
| 215 | + report.download_benchmark_results_from_s3(s3_results_uri) |
| 216 | + report.generate_comprehensive_report() |
| 217 | + |
| 218 | + |
| 219 | +if __name__ == "__main__": |
| 220 | + main() |
0 commit comments