From e799682e74f38437400993cf87ac4276e32ba961 Mon Sep 17 00:00:00 2001 From: Samuel Monson Date: Thu, 1 May 2025 14:45:09 -0400 Subject: [PATCH 01/16] Second Scenario implmentation attempt --- src/guidellm/__main__.py | 118 ++++++++++++++++++-------- src/guidellm/benchmark/entrypoints.py | 14 +++ src/guidellm/benchmark/scenario.py | 62 ++++++++++++++ 3 files changed, 160 insertions(+), 34 deletions(-) create mode 100644 src/guidellm/benchmark/scenario.py diff --git a/src/guidellm/__main__.py b/src/guidellm/__main__.py index d81b7ddf..730915d9 100644 --- a/src/guidellm/__main__.py +++ b/src/guidellm/__main__.py @@ -6,7 +6,9 @@ import click from guidellm.backend import BackendType -from guidellm.benchmark import ProfileType, benchmark_generative_text +from guidellm.benchmark import ProfileType +from guidellm.benchmark.entrypoints import benchmark_with_scenario +from guidellm.benchmark.scenario import GenerativeTextScenario from guidellm.config import print_config from guidellm.scheduler import StrategyType @@ -38,6 +40,19 @@ def parse_number_str(ctx, param, value): # noqa: ARG001 ) from err +def set_if_not_default(ctx: click.Context, **kwargs): + """ + Set the value of a click option if it is not the default value. + This is useful for setting options that are not None by default. + """ + values = {} + for k, v in kwargs.items(): + if ctx.get_parameter_source(k) != click.core.ParameterSource.DEFAULT: + values[k] = v + + return values + + @click.group() def cli(): pass @@ -46,6 +61,14 @@ def cli(): @cli.command( help="Run a benchmark against a generative model using the specified arguments." ) +@click.option( + "--scenario", + type=str, + default=None, + help=( + "TODO: A scenario or path to config" + ), +) @click.option( "--target", required=True, @@ -59,12 +82,12 @@ def cli(): "The type of backend to use to run requests against. Defaults to 'openai_http'." f" Supported types: {', '.join(get_args(BackendType))}" ), - default="openai_http", + default=GenerativeTextScenario.backend_type, ) @click.option( "--backend-args", callback=parse_json, - default=None, + default=GenerativeTextScenario.backend_args, help=( "A JSON string containing any arguments to pass to the backend as a " "dict with **kwargs." @@ -72,7 +95,7 @@ def cli(): ) @click.option( "--model", - default=None, + default=GenerativeTextScenario.model, type=str, help=( "The ID of the model to benchmark within the backend. " @@ -81,7 +104,7 @@ def cli(): ) @click.option( "--processor", - default=None, + default=GenerativeTextScenario.processor, type=str, help=( "The processor or tokenizer to use to calculate token counts for statistics " @@ -91,7 +114,7 @@ def cli(): ) @click.option( "--processor-args", - default=None, + default=GenerativeTextScenario.processor_args, callback=parse_json, help=( "A JSON string containing any arguments to pass to the processor constructor " @@ -110,6 +133,7 @@ def cli(): ) @click.option( "--data-args", + default=GenerativeTextScenario.data_args, callback=parse_json, help=( "A JSON string containing any arguments to pass to the dataset creation " @@ -118,7 +142,7 @@ def cli(): ) @click.option( "--data-sampler", - default=None, + default=GenerativeTextScenario.data_sampler, type=click.Choice(["random"]), help=( "The data sampler type to use. 'random' will add a random shuffle on the data. " @@ -136,7 +160,7 @@ def cli(): ) @click.option( "--rate", - default=None, + default=GenerativeTextScenario.rate, callback=parse_number_str, help=( "The rates to run the benchmark at. " @@ -150,6 +174,7 @@ def cli(): @click.option( "--max-seconds", type=float, + default=GenerativeTextScenario.max_seconds, help=( "The maximum number of seconds each benchmark can run for. " "If None, will run until max_requests or the data is exhausted." @@ -158,6 +183,7 @@ def cli(): @click.option( "--max-requests", type=int, + default=GenerativeTextScenario.max_requests, help=( "The maximum number of requests each benchmark can run for. " "If None, will run until max_seconds or the data is exhausted." @@ -166,7 +192,7 @@ def cli(): @click.option( "--warmup-percent", type=float, - default=None, + default=GenerativeTextScenario.warmup_percent, help=( "The percent of the benchmark (based on max-seconds, max-requets, " "or lenth of dataset) to run as a warmup and not include in the final results. " @@ -176,6 +202,7 @@ def cli(): @click.option( "--cooldown-percent", type=float, + default=GenerativeTextScenario.cooldown_percent, help=( "The percent of the benchmark (based on max-seconds, max-requets, or lenth " "of dataset) to run as a cooldown and not include in the final results. " @@ -185,16 +212,19 @@ def cli(): @click.option( "--disable-progress", is_flag=True, + default=not GenerativeTextScenario.show_progress, help="Set this flag to disable progress updates to the console", ) @click.option( "--display-scheduler-stats", is_flag=True, + default=GenerativeTextScenario.show_progress_scheduler_stats, help="Set this flag to display stats for the processes running the benchmarks", ) @click.option( "--disable-console-outputs", is_flag=True, + default=not GenerativeTextScenario.output_console, help="Set this flag to disable console output", ) @click.option( @@ -211,6 +241,7 @@ def cli(): @click.option( "--output-extras", callback=parse_json, + default=GenerativeTextScenario.output_extras, help="A JSON string of extra data to save with the output benchmarks", ) @click.option( @@ -220,15 +251,16 @@ def cli(): "The number of samples to save in the output file. " "If None (default), will save all samples." ), - default=None, + default=GenerativeTextScenario.output_sampling, ) @click.option( "--random-seed", - default=42, + default=GenerativeTextScenario.random_seed, type=int, help="The random seed to use for benchmarking to ensure reproducibility.", ) def benchmark( + scenario, target, backend_type, backend_args, @@ -252,30 +284,48 @@ def benchmark( output_sampling, random_seed, ): + click_ctx = click.get_current_context() + + # If a scenario file was specified read from it + # TODO: This should probably be a factory method + if scenario is None: + _scenario = {} + else: + # TODO: Support pre-defined scenarios + # TODO: Support other formats + with Path(scenario).open() as f: + _scenario = json.load(f) + + # If any command line arguments are specified, override the scenario + _scenario.update(set_if_not_default( + click_ctx, + target=target, + backend_type=backend_type, + backend_args=backend_args, + model=model, + processor=processor, + processor_args=processor_args, + data=data, + data_args=data_args, + data_sampler=data_sampler, + rate_type=rate_type, + rate=rate, + max_seconds=max_seconds, + max_requests=max_requests, + warmup_percent=warmup_percent, + cooldown_percent=cooldown_percent, + show_progress=not disable_progress, + show_progress_scheduler_stats=display_scheduler_stats, + output_console=not disable_console_outputs, + output_path=output_path, + output_extras=output_extras, + output_sampling=output_sampling, + random_seed=random_seed, + )) + asyncio.run( - benchmark_generative_text( - target=target, - backend_type=backend_type, - backend_args=backend_args, - model=model, - processor=processor, - processor_args=processor_args, - data=data, - data_args=data_args, - data_sampler=data_sampler, - rate_type=rate_type, - rate=rate, - max_seconds=max_seconds, - max_requests=max_requests, - warmup_percent=warmup_percent, - cooldown_percent=cooldown_percent, - show_progress=not disable_progress, - show_progress_scheduler_stats=display_scheduler_stats, - output_console=not disable_console_outputs, - output_path=output_path, - output_extras=output_extras, - output_sampling=output_sampling, - random_seed=random_seed, + benchmark_with_scenario( + scenario=GenerativeTextScenario(**_scenario) ) ) diff --git a/src/guidellm/benchmark/entrypoints.py b/src/guidellm/benchmark/entrypoints.py index 2f6c7182..421904a2 100644 --- a/src/guidellm/benchmark/entrypoints.py +++ b/src/guidellm/benchmark/entrypoints.py @@ -15,9 +15,23 @@ ) from guidellm.benchmark.profile import ProfileType, create_profile from guidellm.benchmark.progress import GenerativeTextBenchmarkerProgressDisplay +from guidellm.benchmark.scenario import GenerativeTextScenario, Scenario from guidellm.request import GenerativeRequestLoader from guidellm.scheduler import StrategyType +type benchmark_type = Literal["generative_text"] + + +async def benchmark_with_scenario(scenario: Scenario, **kwargs): + """ + Run a benchmark using a scenario and specify any extra arguments + """ + + if isinstance(scenario, GenerativeTextScenario): + return await benchmark_generative_text(**vars(scenario), **kwargs) + else: + raise ValueError(f"Unsupported Scenario type {type(scenario)}") + async def benchmark_generative_text( target: str, diff --git a/src/guidellm/benchmark/scenario.py b/src/guidellm/benchmark/scenario.py new file mode 100644 index 00000000..1bd2dc06 --- /dev/null +++ b/src/guidellm/benchmark/scenario.py @@ -0,0 +1,62 @@ +from collections.abc import Iterable +from pathlib import Path +from typing import Any, Literal, Optional, Self, Union + +from datasets import Dataset, DatasetDict, IterableDataset, IterableDatasetDict +from transformers.tokenization_utils_base import ( # type: ignore[import] + PreTrainedTokenizerBase, +) + +from guidellm.backend.backend import BackendType +from guidellm.benchmark.profile import ProfileType +from guidellm.objects.pydantic import StandardBaseModel +from guidellm.scheduler.strategy import StrategyType + +__ALL__ = ["Scenario", "GenerativeTextScenario"] + + +class Scenario(StandardBaseModel): + target: str + + def _update(self, **fields: Any) -> Self: + for k, v in fields.items(): + if not hasattr(self, k): + raise ValueError(f"Invalid field {k}") + setattr(self, k, v) + + return self + + def update(self, **fields: Any) -> Self: + return self._update(**{k: v for k, v in fields.items() if v is not None}) + + +class GenerativeTextScenario(Scenario): + backend_type: BackendType = "openai_http" + backend_args: Optional[dict[str, Any]] = None + model: Optional[str] = None + processor: Optional[Union[str, Path, PreTrainedTokenizerBase]] = None + processor_args: Optional[dict[str, Any]] = None + data: Union[ + str, + Path, + Iterable[Union[str, dict[str, Any]]], + Dataset, + DatasetDict, + IterableDataset, + IterableDatasetDict, + ] + data_args: Optional[dict[str, Any]] = None + data_sampler: Optional[Literal["random"]] = None + rate_type: Union[StrategyType, ProfileType] + rate: Optional[Union[int, float, list[Union[int, float]]]] = None + max_seconds: Optional[float] = None + max_requests: Optional[int] = None + warmup_percent: Optional[float] = None + cooldown_percent: Optional[float] = None + show_progress: bool = True + show_progress_scheduler_stats: bool = True + output_console: bool = True + output_path: Optional[Union[str, Path]] = None + output_extras: Optional[dict[str, Any]] = None + output_sampling: Optional[int] = None + random_seed: int = 42 From 7e3be63f42329d54b34486e1bc7941737826b721 Mon Sep 17 00:00:00 2001 From: Samuel Monson Date: Thu, 1 May 2025 15:07:45 -0400 Subject: [PATCH 02/16] Fix pydantic model parsing issues --- src/guidellm/__main__.py | 36 +++++++++++++++--------------- src/guidellm/benchmark/scenario.py | 4 ++++ 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/src/guidellm/__main__.py b/src/guidellm/__main__.py index 730915d9..a1aa1aa9 100644 --- a/src/guidellm/__main__.py +++ b/src/guidellm/__main__.py @@ -82,12 +82,12 @@ def cli(): "The type of backend to use to run requests against. Defaults to 'openai_http'." f" Supported types: {', '.join(get_args(BackendType))}" ), - default=GenerativeTextScenario.backend_type, + default=GenerativeTextScenario.model_fields["backend_type"].default, ) @click.option( "--backend-args", callback=parse_json, - default=GenerativeTextScenario.backend_args, + default=GenerativeTextScenario.model_fields["backend_args"].default, help=( "A JSON string containing any arguments to pass to the backend as a " "dict with **kwargs." @@ -95,7 +95,7 @@ def cli(): ) @click.option( "--model", - default=GenerativeTextScenario.model, + default=GenerativeTextScenario.model_fields["model"].default, type=str, help=( "The ID of the model to benchmark within the backend. " @@ -104,7 +104,7 @@ def cli(): ) @click.option( "--processor", - default=GenerativeTextScenario.processor, + default=GenerativeTextScenario.model_fields["processor"].default, type=str, help=( "The processor or tokenizer to use to calculate token counts for statistics " @@ -114,7 +114,7 @@ def cli(): ) @click.option( "--processor-args", - default=GenerativeTextScenario.processor_args, + default=GenerativeTextScenario.model_fields["processor_args"].default, callback=parse_json, help=( "A JSON string containing any arguments to pass to the processor constructor " @@ -133,7 +133,7 @@ def cli(): ) @click.option( "--data-args", - default=GenerativeTextScenario.data_args, + default=GenerativeTextScenario.model_fields["data_args"].default, callback=parse_json, help=( "A JSON string containing any arguments to pass to the dataset creation " @@ -142,7 +142,7 @@ def cli(): ) @click.option( "--data-sampler", - default=GenerativeTextScenario.data_sampler, + default=GenerativeTextScenario.model_fields["data_sampler"].default, type=click.Choice(["random"]), help=( "The data sampler type to use. 'random' will add a random shuffle on the data. " @@ -160,7 +160,7 @@ def cli(): ) @click.option( "--rate", - default=GenerativeTextScenario.rate, + default=GenerativeTextScenario.model_fields["rate"].default, callback=parse_number_str, help=( "The rates to run the benchmark at. " @@ -174,7 +174,7 @@ def cli(): @click.option( "--max-seconds", type=float, - default=GenerativeTextScenario.max_seconds, + default=GenerativeTextScenario.model_fields["max_seconds"].default, help=( "The maximum number of seconds each benchmark can run for. " "If None, will run until max_requests or the data is exhausted." @@ -183,7 +183,7 @@ def cli(): @click.option( "--max-requests", type=int, - default=GenerativeTextScenario.max_requests, + default=GenerativeTextScenario.model_fields["max_requests"].default, help=( "The maximum number of requests each benchmark can run for. " "If None, will run until max_seconds or the data is exhausted." @@ -192,7 +192,7 @@ def cli(): @click.option( "--warmup-percent", type=float, - default=GenerativeTextScenario.warmup_percent, + default=GenerativeTextScenario.model_fields["warmup_percent"].default, help=( "The percent of the benchmark (based on max-seconds, max-requets, " "or lenth of dataset) to run as a warmup and not include in the final results. " @@ -202,7 +202,7 @@ def cli(): @click.option( "--cooldown-percent", type=float, - default=GenerativeTextScenario.cooldown_percent, + default=GenerativeTextScenario.model_fields["cooldown_percent"].default, help=( "The percent of the benchmark (based on max-seconds, max-requets, or lenth " "of dataset) to run as a cooldown and not include in the final results. " @@ -212,19 +212,19 @@ def cli(): @click.option( "--disable-progress", is_flag=True, - default=not GenerativeTextScenario.show_progress, + default=not GenerativeTextScenario.model_fields["show_progress"].default, help="Set this flag to disable progress updates to the console", ) @click.option( "--display-scheduler-stats", is_flag=True, - default=GenerativeTextScenario.show_progress_scheduler_stats, + default=GenerativeTextScenario.model_fields["show_progress_scheduler_stats"].default, help="Set this flag to display stats for the processes running the benchmarks", ) @click.option( "--disable-console-outputs", is_flag=True, - default=not GenerativeTextScenario.output_console, + default=not GenerativeTextScenario.model_fields["output_console"].default, help="Set this flag to disable console output", ) @click.option( @@ -241,7 +241,7 @@ def cli(): @click.option( "--output-extras", callback=parse_json, - default=GenerativeTextScenario.output_extras, + default=GenerativeTextScenario.model_fields["output_extras"].default, help="A JSON string of extra data to save with the output benchmarks", ) @click.option( @@ -251,11 +251,11 @@ def cli(): "The number of samples to save in the output file. " "If None (default), will save all samples." ), - default=GenerativeTextScenario.output_sampling, + default=GenerativeTextScenario.model_fields["output_sampling"].default, ) @click.option( "--random-seed", - default=GenerativeTextScenario.random_seed, + default=GenerativeTextScenario.model_fields["random_seed"].default, type=int, help="The random seed to use for benchmarking to ensure reproducibility.", ) diff --git a/src/guidellm/benchmark/scenario.py b/src/guidellm/benchmark/scenario.py index 1bd2dc06..6bea9f7d 100644 --- a/src/guidellm/benchmark/scenario.py +++ b/src/guidellm/benchmark/scenario.py @@ -31,6 +31,10 @@ def update(self, **fields: Any) -> Self: class GenerativeTextScenario(Scenario): + # FIXME: This solves an issue with Pydantic and class types + class Config: + arbitrary_types_allowed = True + backend_type: BackendType = "openai_http" backend_args: Optional[dict[str, Any]] = None model: Optional[str] = None From 9791edaa4162e578139b22b3f7a9231ed7b0bc8d Mon Sep 17 00:00:00 2001 From: Samuel Monson Date: Thu, 1 May 2025 15:16:38 -0400 Subject: [PATCH 03/16] Move CLI and output options out of scenario --- src/guidellm/__main__.py | 16 ++++++---------- src/guidellm/benchmark/scenario.py | 5 ----- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/src/guidellm/__main__.py b/src/guidellm/__main__.py index a1aa1aa9..0dbf9e1e 100644 --- a/src/guidellm/__main__.py +++ b/src/guidellm/__main__.py @@ -212,19 +212,16 @@ def cli(): @click.option( "--disable-progress", is_flag=True, - default=not GenerativeTextScenario.model_fields["show_progress"].default, help="Set this flag to disable progress updates to the console", ) @click.option( "--display-scheduler-stats", is_flag=True, - default=GenerativeTextScenario.model_fields["show_progress_scheduler_stats"].default, help="Set this flag to display stats for the processes running the benchmarks", ) @click.option( "--disable-console-outputs", is_flag=True, - default=not GenerativeTextScenario.model_fields["output_console"].default, help="Set this flag to disable console output", ) @click.option( @@ -241,7 +238,6 @@ def cli(): @click.option( "--output-extras", callback=parse_json, - default=GenerativeTextScenario.model_fields["output_extras"].default, help="A JSON string of extra data to save with the output benchmarks", ) @click.option( @@ -314,18 +310,18 @@ def benchmark( max_requests=max_requests, warmup_percent=warmup_percent, cooldown_percent=cooldown_percent, - show_progress=not disable_progress, - show_progress_scheduler_stats=display_scheduler_stats, - output_console=not disable_console_outputs, - output_path=output_path, - output_extras=output_extras, output_sampling=output_sampling, random_seed=random_seed, )) asyncio.run( benchmark_with_scenario( - scenario=GenerativeTextScenario(**_scenario) + scenario=GenerativeTextScenario(**_scenario), + show_progress=not disable_progress, + show_progress_scheduler_stats=display_scheduler_stats, + output_console=not disable_console_outputs, + output_path=output_path, + output_extras=output_extras, ) ) diff --git a/src/guidellm/benchmark/scenario.py b/src/guidellm/benchmark/scenario.py index 6bea9f7d..2d8748f7 100644 --- a/src/guidellm/benchmark/scenario.py +++ b/src/guidellm/benchmark/scenario.py @@ -57,10 +57,5 @@ class Config: max_requests: Optional[int] = None warmup_percent: Optional[float] = None cooldown_percent: Optional[float] = None - show_progress: bool = True - show_progress_scheduler_stats: bool = True - output_console: bool = True - output_path: Optional[Union[str, Path]] = None - output_extras: Optional[dict[str, Any]] = None output_sampling: Optional[int] = None random_seed: int = 42 From 0811fc58e63f8e7b4c4db09c9498f06415daf349 Mon Sep 17 00:00:00 2001 From: Samuel Monson Date: Wed, 21 May 2025 13:51:29 -0400 Subject: [PATCH 04/16] Drop int type from rate --- src/guidellm/benchmark/scenario.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/guidellm/benchmark/scenario.py b/src/guidellm/benchmark/scenario.py index 2d8748f7..8c5106a4 100644 --- a/src/guidellm/benchmark/scenario.py +++ b/src/guidellm/benchmark/scenario.py @@ -52,7 +52,7 @@ class Config: data_args: Optional[dict[str, Any]] = None data_sampler: Optional[Literal["random"]] = None rate_type: Union[StrategyType, ProfileType] - rate: Optional[Union[int, float, list[Union[int, float]]]] = None + rate: Optional[Union[float, list[float]]] = None max_seconds: Optional[float] = None max_requests: Optional[int] = None warmup_percent: Optional[float] = None From 0baf0d6e849986a86813469f975a25cd04d132f4 Mon Sep 17 00:00:00 2001 From: Samuel Monson Date: Wed, 21 May 2025 15:41:19 -0400 Subject: [PATCH 05/16] Handle reading scenario from file in factory method --- src/guidellm/__main__.py | 24 ++++++++++------------- src/guidellm/benchmark/scenario.py | 31 ++++++++++++++++++++---------- 2 files changed, 31 insertions(+), 24 deletions(-) diff --git a/src/guidellm/__main__.py b/src/guidellm/__main__.py index 0dbf9e1e..2086325b 100644 --- a/src/guidellm/__main__.py +++ b/src/guidellm/__main__.py @@ -282,18 +282,7 @@ def benchmark( ): click_ctx = click.get_current_context() - # If a scenario file was specified read from it - # TODO: This should probably be a factory method - if scenario is None: - _scenario = {} - else: - # TODO: Support pre-defined scenarios - # TODO: Support other formats - with Path(scenario).open() as f: - _scenario = json.load(f) - - # If any command line arguments are specified, override the scenario - _scenario.update(set_if_not_default( + overrides = set_if_not_default( click_ctx, target=target, backend_type=backend_type, @@ -312,11 +301,18 @@ def benchmark( cooldown_percent=cooldown_percent, output_sampling=output_sampling, random_seed=random_seed, - )) + ) + + # If a scenario file was specified read from it + if scenario is None: + _scenario = GenerativeTextScenario.model_validate(overrides) + else: + # TODO: Support pre-defined scenarios + _scenario = GenerativeTextScenario.from_file(scenario, overrides) asyncio.run( benchmark_with_scenario( - scenario=GenerativeTextScenario(**_scenario), + scenario=_scenario, show_progress=not disable_progress, show_progress_scheduler_stats=display_scheduler_stats, output_console=not disable_console_outputs, diff --git a/src/guidellm/benchmark/scenario.py b/src/guidellm/benchmark/scenario.py index 8c5106a4..b8e9f504 100644 --- a/src/guidellm/benchmark/scenario.py +++ b/src/guidellm/benchmark/scenario.py @@ -1,8 +1,11 @@ +import json from collections.abc import Iterable from pathlib import Path -from typing import Any, Literal, Optional, Self, Union +from typing import Any, Literal, Optional, TypeVar, Union +import yaml from datasets import Dataset, DatasetDict, IterableDataset, IterableDatasetDict +from loguru import logger from transformers.tokenization_utils_base import ( # type: ignore[import] PreTrainedTokenizerBase, ) @@ -14,20 +17,28 @@ __ALL__ = ["Scenario", "GenerativeTextScenario"] +T = TypeVar("T", bound="Scenario") + class Scenario(StandardBaseModel): target: str - def _update(self, **fields: Any) -> Self: - for k, v in fields.items(): - if not hasattr(self, k): - raise ValueError(f"Invalid field {k}") - setattr(self, k, v) - - return self + @classmethod + def from_file( + cls: type[T], filename: Union[str, Path], overrides: Optional[dict] = None + ) -> T: + try: + with open(filename) as f: + if str(filename).endswith(".yaml") or str(filename).endswith(".yml"): + data = yaml.safe_load(f) + else: # Assume everything else is json + data = json.load(f) + except (json.JSONDecodeError, yaml.YAMLError) as e: + logger.error("Failed to parse scenario") + raise e - def update(self, **fields: Any) -> Self: - return self._update(**{k: v for k, v in fields.items() if v is not None}) + data.update(overrides) + return cls.model_validate(data) class GenerativeTextScenario(Scenario): From 0e226d12b1620a2737135432d1903f6744c19526 Mon Sep 17 00:00:00 2001 From: Samuel Monson Date: Wed, 21 May 2025 15:56:35 -0400 Subject: [PATCH 06/16] Handle required arg parsing with pydantic --- src/guidellm/__main__.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/src/guidellm/__main__.py b/src/guidellm/__main__.py index 2086325b..4b48da0c 100644 --- a/src/guidellm/__main__.py +++ b/src/guidellm/__main__.py @@ -4,6 +4,7 @@ from typing import get_args import click +from pydantic import ValidationError from guidellm.backend import BackendType from guidellm.benchmark import ProfileType @@ -65,13 +66,10 @@ def cli(): "--scenario", type=str, default=None, - help=( - "TODO: A scenario or path to config" - ), + help=("TODO: A scenario or path to config"), ) @click.option( "--target", - required=True, type=str, help="The target path for the backend to run benchmarks against. For example, http://localhost:8000", ) @@ -123,7 +121,6 @@ def cli(): ) @click.option( "--data", - required=True, type=str, help=( "The HuggingFace dataset ID, a path to a HuggingFace dataset, " @@ -151,7 +148,6 @@ def cli(): ) @click.option( "--rate-type", - required=True, type=click.Choice(STRATEGY_PROFILE_CHOICES), help=( "The type of benchmark to run. " @@ -303,12 +299,19 @@ def benchmark( random_seed=random_seed, ) - # If a scenario file was specified read from it - if scenario is None: - _scenario = GenerativeTextScenario.model_validate(overrides) - else: - # TODO: Support pre-defined scenarios - _scenario = GenerativeTextScenario.from_file(scenario, overrides) + try: + # If a scenario file was specified read from it + if scenario is None: + _scenario = GenerativeTextScenario.model_validate(overrides) + else: + # TODO: Support pre-defined scenarios + _scenario = GenerativeTextScenario.from_file(scenario, overrides) + except ValidationError as e: + errs = e.errors(include_url=False, include_context=True, include_input=True) + param_name = "--" + str(errs[0]["loc"][0]).replace("_", "-") + raise click.BadParameter( + errs[0]["msg"], ctx=click_ctx, param_hint=param_name + ) from e asyncio.run( benchmark_with_scenario( From a54695746c8ef02a6fd1085e603e30788711de74 Mon Sep 17 00:00:00 2001 From: Samuel Monson Date: Wed, 21 May 2025 16:20:28 -0400 Subject: [PATCH 07/16] Move rate string parsing into scenario --- src/guidellm/__main__.py | 15 --------------- src/guidellm/benchmark/scenario.py | 22 ++++++++++++++++++++-- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/src/guidellm/__main__.py b/src/guidellm/__main__.py index 4b48da0c..db744534 100644 --- a/src/guidellm/__main__.py +++ b/src/guidellm/__main__.py @@ -27,20 +27,6 @@ def parse_json(ctx, param, value): # noqa: ARG001 raise click.BadParameter(f"{param.name} must be a valid JSON string.") from err -def parse_number_str(ctx, param, value): # noqa: ARG001 - if value is None: - return None - - values = value.split(",") if "," in value else [value] - - try: - return [float(val) for val in values] - except ValueError as err: - raise click.BadParameter( - f"{param.name} must be a number or comma-separated list of numbers." - ) from err - - def set_if_not_default(ctx: click.Context, **kwargs): """ Set the value of a click option if it is not the default value. @@ -157,7 +143,6 @@ def cli(): @click.option( "--rate", default=GenerativeTextScenario.model_fields["rate"].default, - callback=parse_number_str, help=( "The rates to run the benchmark at. " "Can be a single number or a comma-separated list of numbers. " diff --git a/src/guidellm/benchmark/scenario.py b/src/guidellm/benchmark/scenario.py index b8e9f504..64872298 100644 --- a/src/guidellm/benchmark/scenario.py +++ b/src/guidellm/benchmark/scenario.py @@ -1,11 +1,12 @@ import json from collections.abc import Iterable from pathlib import Path -from typing import Any, Literal, Optional, TypeVar, Union +from typing import Annotated, Any, Literal, Optional, TypeVar, Union import yaml from datasets import Dataset, DatasetDict, IterableDataset, IterableDatasetDict from loguru import logger +from pydantic import BeforeValidator from transformers.tokenization_utils_base import ( # type: ignore[import] PreTrainedTokenizerBase, ) @@ -17,6 +18,23 @@ __ALL__ = ["Scenario", "GenerativeTextScenario"] + +def parse_float_list(value: Union[str, float, list[float]]) -> list[float]: + if isinstance(value, (int, float)): + return [value] + elif isinstance(value, list): + return value + + values = value.split(",") if "," in value else [value] + + try: + return [float(val) for val in values] + except ValueError as err: + raise ValueError( + "must be a number or comma-separated list of numbers." + ) from err + + T = TypeVar("T", bound="Scenario") @@ -63,7 +81,7 @@ class Config: data_args: Optional[dict[str, Any]] = None data_sampler: Optional[Literal["random"]] = None rate_type: Union[StrategyType, ProfileType] - rate: Optional[Union[float, list[float]]] = None + rate: Annotated[Optional[list[float]], BeforeValidator(parse_float_list)] = None max_seconds: Optional[float] = None max_requests: Optional[int] = None warmup_percent: Optional[float] = None From 9f0850023b6d2246e4c3045c6b449f419c0f7827 Mon Sep 17 00:00:00 2001 From: Samuel Monson Date: Wed, 21 May 2025 16:54:38 -0400 Subject: [PATCH 08/16] Annotate scenario number fields with bounds checks --- src/guidellm/benchmark/scenario.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/guidellm/benchmark/scenario.py b/src/guidellm/benchmark/scenario.py index 64872298..36f2fbd4 100644 --- a/src/guidellm/benchmark/scenario.py +++ b/src/guidellm/benchmark/scenario.py @@ -6,7 +6,7 @@ import yaml from datasets import Dataset, DatasetDict, IterableDataset, IterableDatasetDict from loguru import logger -from pydantic import BeforeValidator +from pydantic import BeforeValidator, Field, NonNegativeInt, PositiveFloat, PositiveInt from transformers.tokenization_utils_base import ( # type: ignore[import] PreTrainedTokenizerBase, ) @@ -81,10 +81,12 @@ class Config: data_args: Optional[dict[str, Any]] = None data_sampler: Optional[Literal["random"]] = None rate_type: Union[StrategyType, ProfileType] - rate: Annotated[Optional[list[float]], BeforeValidator(parse_float_list)] = None - max_seconds: Optional[float] = None - max_requests: Optional[int] = None - warmup_percent: Optional[float] = None - cooldown_percent: Optional[float] = None - output_sampling: Optional[int] = None + rate: Annotated[ + Optional[list[PositiveFloat]], BeforeValidator(parse_float_list) + ] = None + max_seconds: Optional[PositiveFloat] = None + max_requests: Optional[PositiveInt] = None + warmup_percent: Annotated[Optional[float], Field(gt=0, le=1)] = None + cooldown_percent: Annotated[Optional[float], Field(gt=0, le=1)] = None + output_sampling: Optional[NonNegativeInt] = None random_seed: int = 42 From f0fe82c5153fbd2d8ff65248f425b1f66cb2ad6f Mon Sep 17 00:00:00 2001 From: Samuel Monson Date: Thu, 22 May 2025 12:48:49 -0400 Subject: [PATCH 09/16] Add a helper method to get scenario defaults --- src/guidellm/__main__.py | 32 +++++++++++++-------------- src/guidellm/benchmark/entrypoints.py | 2 -- src/guidellm/benchmark/scenario.py | 5 +++++ 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/src/guidellm/__main__.py b/src/guidellm/__main__.py index db744534..ecd3f392 100644 --- a/src/guidellm/__main__.py +++ b/src/guidellm/__main__.py @@ -1,7 +1,7 @@ import asyncio import json from pathlib import Path -from typing import get_args +from typing import Any, get_args import click from pydantic import ValidationError @@ -27,7 +27,7 @@ def parse_json(ctx, param, value): # noqa: ARG001 raise click.BadParameter(f"{param.name} must be a valid JSON string.") from err -def set_if_not_default(ctx: click.Context, **kwargs): +def set_if_not_default(ctx: click.Context, **kwargs) -> dict[str, Any]: """ Set the value of a click option if it is not the default value. This is useful for setting options that are not None by default. @@ -66,12 +66,12 @@ def cli(): "The type of backend to use to run requests against. Defaults to 'openai_http'." f" Supported types: {', '.join(get_args(BackendType))}" ), - default=GenerativeTextScenario.model_fields["backend_type"].default, + default=GenerativeTextScenario.get_default("backend_type"), ) @click.option( "--backend-args", callback=parse_json, - default=GenerativeTextScenario.model_fields["backend_args"].default, + default=GenerativeTextScenario.get_default("backend_args"), help=( "A JSON string containing any arguments to pass to the backend as a " "dict with **kwargs." @@ -79,7 +79,7 @@ def cli(): ) @click.option( "--model", - default=GenerativeTextScenario.model_fields["model"].default, + default=GenerativeTextScenario.get_default("model"), type=str, help=( "The ID of the model to benchmark within the backend. " @@ -88,7 +88,7 @@ def cli(): ) @click.option( "--processor", - default=GenerativeTextScenario.model_fields["processor"].default, + default=GenerativeTextScenario.get_default("processor"), type=str, help=( "The processor or tokenizer to use to calculate token counts for statistics " @@ -98,7 +98,7 @@ def cli(): ) @click.option( "--processor-args", - default=GenerativeTextScenario.model_fields["processor_args"].default, + default=GenerativeTextScenario.get_default("processor_args"), callback=parse_json, help=( "A JSON string containing any arguments to pass to the processor constructor " @@ -116,7 +116,7 @@ def cli(): ) @click.option( "--data-args", - default=GenerativeTextScenario.model_fields["data_args"].default, + default=GenerativeTextScenario.get_default("data_args"), callback=parse_json, help=( "A JSON string containing any arguments to pass to the dataset creation " @@ -125,7 +125,7 @@ def cli(): ) @click.option( "--data-sampler", - default=GenerativeTextScenario.model_fields["data_sampler"].default, + default=GenerativeTextScenario.get_default("data_sampler"), type=click.Choice(["random"]), help=( "The data sampler type to use. 'random' will add a random shuffle on the data. " @@ -142,7 +142,7 @@ def cli(): ) @click.option( "--rate", - default=GenerativeTextScenario.model_fields["rate"].default, + default=GenerativeTextScenario.get_default("rate"), help=( "The rates to run the benchmark at. " "Can be a single number or a comma-separated list of numbers. " @@ -155,7 +155,7 @@ def cli(): @click.option( "--max-seconds", type=float, - default=GenerativeTextScenario.model_fields["max_seconds"].default, + default=GenerativeTextScenario.get_default("max_seconds"), help=( "The maximum number of seconds each benchmark can run for. " "If None, will run until max_requests or the data is exhausted." @@ -164,7 +164,7 @@ def cli(): @click.option( "--max-requests", type=int, - default=GenerativeTextScenario.model_fields["max_requests"].default, + default=GenerativeTextScenario.get_default("max_requests"), help=( "The maximum number of requests each benchmark can run for. " "If None, will run until max_seconds or the data is exhausted." @@ -173,7 +173,7 @@ def cli(): @click.option( "--warmup-percent", type=float, - default=GenerativeTextScenario.model_fields["warmup_percent"].default, + default=GenerativeTextScenario.get_default("warmup_percent"), help=( "The percent of the benchmark (based on max-seconds, max-requets, " "or lenth of dataset) to run as a warmup and not include in the final results. " @@ -183,7 +183,7 @@ def cli(): @click.option( "--cooldown-percent", type=float, - default=GenerativeTextScenario.model_fields["cooldown_percent"].default, + default=GenerativeTextScenario.get_default("cooldown_percent"), help=( "The percent of the benchmark (based on max-seconds, max-requets, or lenth " "of dataset) to run as a cooldown and not include in the final results. " @@ -228,11 +228,11 @@ def cli(): "The number of samples to save in the output file. " "If None (default), will save all samples." ), - default=GenerativeTextScenario.model_fields["output_sampling"].default, + default=GenerativeTextScenario.get_default("output_sampling"), ) @click.option( "--random-seed", - default=GenerativeTextScenario.model_fields["random_seed"].default, + default=GenerativeTextScenario.get_default("random_seed"), type=int, help="The random seed to use for benchmarking to ensure reproducibility.", ) diff --git a/src/guidellm/benchmark/entrypoints.py b/src/guidellm/benchmark/entrypoints.py index 421904a2..fcb69ce9 100644 --- a/src/guidellm/benchmark/entrypoints.py +++ b/src/guidellm/benchmark/entrypoints.py @@ -19,8 +19,6 @@ from guidellm.request import GenerativeRequestLoader from guidellm.scheduler import StrategyType -type benchmark_type = Literal["generative_text"] - async def benchmark_with_scenario(scenario: Scenario, **kwargs): """ diff --git a/src/guidellm/benchmark/scenario.py b/src/guidellm/benchmark/scenario.py index 36f2fbd4..88a58a96 100644 --- a/src/guidellm/benchmark/scenario.py +++ b/src/guidellm/benchmark/scenario.py @@ -41,6 +41,11 @@ def parse_float_list(value: Union[str, float, list[float]]) -> list[float]: class Scenario(StandardBaseModel): target: str + @classmethod + def get_default(cls: type[T], field: str) -> Any: + """Get default values for model fields""" + return cls.model_fields[field].default + @classmethod def from_file( cls: type[T], filename: Union[str, Path], overrides: Optional[dict] = None From c79b60e91faccc1ab01b019e8bd737f8c520e70c Mon Sep 17 00:00:00 2001 From: Samuel Monson Date: Thu, 22 May 2025 13:34:15 -0400 Subject: [PATCH 10/16] Move scenario helper methods to base pydantic class --- src/guidellm/__main__.py | 2 +- src/guidellm/benchmark/scenario.py | 30 +----------------------------- src/guidellm/objects/pydantic.py | 28 +++++++++++++++++++++++++++- 3 files changed, 29 insertions(+), 31 deletions(-) diff --git a/src/guidellm/__main__.py b/src/guidellm/__main__.py index ecd3f392..c77ff979 100644 --- a/src/guidellm/__main__.py +++ b/src/guidellm/__main__.py @@ -290,7 +290,7 @@ def benchmark( _scenario = GenerativeTextScenario.model_validate(overrides) else: # TODO: Support pre-defined scenarios - _scenario = GenerativeTextScenario.from_file(scenario, overrides) + _scenario = GenerativeTextScenario.from_file(Path(scenario), overrides) except ValidationError as e: errs = e.errors(include_url=False, include_context=True, include_input=True) param_name = "--" + str(errs[0]["loc"][0]).replace("_", "-") diff --git a/src/guidellm/benchmark/scenario.py b/src/guidellm/benchmark/scenario.py index 88a58a96..8de73af0 100644 --- a/src/guidellm/benchmark/scenario.py +++ b/src/guidellm/benchmark/scenario.py @@ -1,11 +1,8 @@ -import json from collections.abc import Iterable from pathlib import Path -from typing import Annotated, Any, Literal, Optional, TypeVar, Union +from typing import Annotated, Any, Literal, Optional, Union -import yaml from datasets import Dataset, DatasetDict, IterableDataset, IterableDatasetDict -from loguru import logger from pydantic import BeforeValidator, Field, NonNegativeInt, PositiveFloat, PositiveInt from transformers.tokenization_utils_base import ( # type: ignore[import] PreTrainedTokenizerBase, @@ -35,34 +32,9 @@ def parse_float_list(value: Union[str, float, list[float]]) -> list[float]: ) from err -T = TypeVar("T", bound="Scenario") - - class Scenario(StandardBaseModel): target: str - @classmethod - def get_default(cls: type[T], field: str) -> Any: - """Get default values for model fields""" - return cls.model_fields[field].default - - @classmethod - def from_file( - cls: type[T], filename: Union[str, Path], overrides: Optional[dict] = None - ) -> T: - try: - with open(filename) as f: - if str(filename).endswith(".yaml") or str(filename).endswith(".yml"): - data = yaml.safe_load(f) - else: # Assume everything else is json - data = json.load(f) - except (json.JSONDecodeError, yaml.YAMLError) as e: - logger.error("Failed to parse scenario") - raise e - - data.update(overrides) - return cls.model_validate(data) - class GenerativeTextScenario(Scenario): # FIXME: This solves an issue with Pydantic and class types diff --git a/src/guidellm/objects/pydantic.py b/src/guidellm/objects/pydantic.py index 8365be33..27be1fc1 100644 --- a/src/guidellm/objects/pydantic.py +++ b/src/guidellm/objects/pydantic.py @@ -1,10 +1,14 @@ -from typing import Any, Generic, TypeVar +import json +from pathlib import Path +from typing import Any, Generic, Optional, TypeVar +import yaml from loguru import logger from pydantic import BaseModel, ConfigDict, Field __all__ = ["StandardBaseModel", "StatusBreakdown"] +T = TypeVar("T", bound="StandardBaseModel") class StandardBaseModel(BaseModel): """ @@ -27,6 +31,28 @@ def __init__(self, /, **data: Any) -> None: data, ) + @classmethod + def get_default(cls: type[T], field: str) -> Any: + """Get default values for model fields""" + return cls.model_fields[field].default + + @classmethod + def from_file( + cls: type[T], filename: Path, overrides: Optional[dict] = None + ) -> T: + try: + with filename.open() as f: + if str(filename).endswith((".yaml", ".yml")): + data = yaml.safe_load(f) + else: # Assume everything else is json + data = json.load(f) + except (json.JSONDecodeError, yaml.YAMLError) as e: + logger.error(f"Failed to parse {filename} as type {cls.__name__}") + raise e + + data.update(overrides) + return cls.model_validate(data) + SuccessfulT = TypeVar("SuccessfulT") ErroredT = TypeVar("ErroredT") From 099f4e55566f0535f72f7efa720a77cf6101ae0a Mon Sep 17 00:00:00 2001 From: Samuel Monson Date: Thu, 22 May 2025 13:43:34 -0400 Subject: [PATCH 11/16] Move cli helpers to separate file and add click Union type back --- src/guidellm/__main__.py | 41 +++++++------------------ src/guidellm/utils/cli.py | 63 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 30 deletions(-) create mode 100644 src/guidellm/utils/cli.py diff --git a/src/guidellm/__main__.py b/src/guidellm/__main__.py index c77ff979..7157ff89 100644 --- a/src/guidellm/__main__.py +++ b/src/guidellm/__main__.py @@ -1,7 +1,6 @@ import asyncio -import json from pathlib import Path -from typing import Any, get_args +from typing import get_args import click from pydantic import ValidationError @@ -12,34 +11,13 @@ from guidellm.benchmark.scenario import GenerativeTextScenario from guidellm.config import print_config from guidellm.scheduler import StrategyType +from guidellm.utils import cli as cli_tools STRATEGY_PROFILE_CHOICES = set( list(get_args(ProfileType)) + list(get_args(StrategyType)) ) -def parse_json(ctx, param, value): # noqa: ARG001 - if value is None: - return None - try: - return json.loads(value) - except json.JSONDecodeError as err: - raise click.BadParameter(f"{param.name} must be a valid JSON string.") from err - - -def set_if_not_default(ctx: click.Context, **kwargs) -> dict[str, Any]: - """ - Set the value of a click option if it is not the default value. - This is useful for setting options that are not None by default. - """ - values = {} - for k, v in kwargs.items(): - if ctx.get_parameter_source(k) != click.core.ParameterSource.DEFAULT: - values[k] = v - - return values - - @click.group() def cli(): pass @@ -50,7 +28,10 @@ def cli(): ) @click.option( "--scenario", - type=str, + type=cli_tools.Union( + click.Path(exists=True, readable=True, file_okay=True, dir_okay=False), + click.STRING + ), default=None, help=("TODO: A scenario or path to config"), ) @@ -70,7 +51,7 @@ def cli(): ) @click.option( "--backend-args", - callback=parse_json, + callback=cli_tools.parse_json, default=GenerativeTextScenario.get_default("backend_args"), help=( "A JSON string containing any arguments to pass to the backend as a " @@ -99,7 +80,7 @@ def cli(): @click.option( "--processor-args", default=GenerativeTextScenario.get_default("processor_args"), - callback=parse_json, + callback=cli_tools.parse_json, help=( "A JSON string containing any arguments to pass to the processor constructor " "as a dict with **kwargs." @@ -117,7 +98,7 @@ def cli(): @click.option( "--data-args", default=GenerativeTextScenario.get_default("data_args"), - callback=parse_json, + callback=cli_tools.parse_json, help=( "A JSON string containing any arguments to pass to the dataset creation " "as a dict with **kwargs." @@ -218,7 +199,7 @@ def cli(): ) @click.option( "--output-extras", - callback=parse_json, + callback=cli_tools.parse_json, help="A JSON string of extra data to save with the output benchmarks", ) @click.option( @@ -263,7 +244,7 @@ def benchmark( ): click_ctx = click.get_current_context() - overrides = set_if_not_default( + overrides = cli_tools.set_if_not_default( click_ctx, target=target, backend_type=backend_type, diff --git a/src/guidellm/utils/cli.py b/src/guidellm/utils/cli.py new file mode 100644 index 00000000..507a384b --- /dev/null +++ b/src/guidellm/utils/cli.py @@ -0,0 +1,63 @@ +import json +from typing import Any + +import click + + +def parse_json(ctx, param, value): # noqa: ARG001 + if value is None: + return None + try: + return json.loads(value) + except json.JSONDecodeError as err: + raise click.BadParameter(f"{param.name} must be a valid JSON string.") from err + + +def set_if_not_default(ctx: click.Context, **kwargs) -> dict[str, Any]: + """ + Set the value of a click option if it is not the default value. + This is useful for setting options that are not None by default. + """ + values = {} + for k, v in kwargs.items(): + if ctx.get_parameter_source(k) != click.core.ParameterSource.DEFAULT: + values[k] = v + + return values + + +class Union(click.ParamType): + """ + A custom click parameter type that allows for multiple types to be accepted. + """ + + def __init__(self, *types: click.ParamType): + self.types = types + self.name = "".join(t.name for t in types) + + def convert(self, value, param, ctx): + fails = [] + for t in self.types: + try: + return t.convert(value, param, ctx) + except click.BadParameter as e: + fails.append(str(e)) + continue + + self.fail("; ".join(fails) or f"Invalid value: {value}") # noqa: RET503 + + + def get_metavar(self, param: click.Parameter) -> str: + def get_choices(t: click.ParamType) -> str: + meta = t.get_metavar(param) + return meta if meta is not None else t.name + + # Get the choices for each type in the union. + choices_str = "|".join(map(get_choices, self.types)) + + # Use curly braces to indicate a required argument. + if param.required and param.param_type_name == "argument": + return f"{{{choices_str}}}" + + # Use square braces to indicate an option or optional argument. + return f"[{choices_str}]" From d81e6f1ed05ea4cd9a77512922599aa2db3fd78c Mon Sep 17 00:00:00 2001 From: Samuel Monson Date: Thu, 22 May 2025 14:19:52 -0400 Subject: [PATCH 12/16] Properly detect if scenario is a Path --- src/guidellm/__main__.py | 14 +++++++++++--- src/guidellm/utils/cli.py | 2 +- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/guidellm/__main__.py b/src/guidellm/__main__.py index 7157ff89..0f2d7133 100644 --- a/src/guidellm/__main__.py +++ b/src/guidellm/__main__.py @@ -29,7 +29,13 @@ def cli(): @click.option( "--scenario", type=cli_tools.Union( - click.Path(exists=True, readable=True, file_okay=True, dir_okay=False), + click.Path( + exists=True, + readable=True, + file_okay=True, + dir_okay=False, + path_type=Path, # type: ignore[type-var] + ), click.STRING ), default=None, @@ -269,9 +275,11 @@ def benchmark( # If a scenario file was specified read from it if scenario is None: _scenario = GenerativeTextScenario.model_validate(overrides) + elif isinstance(scenario, Path): + _scenario = GenerativeTextScenario.from_file(scenario, overrides) else: - # TODO: Support pre-defined scenarios - _scenario = GenerativeTextScenario.from_file(Path(scenario), overrides) + # TODO: Add support for builtin scenarios + raise NotImplementedError except ValidationError as e: errs = e.errors(include_url=False, include_context=True, include_input=True) param_name = "--" + str(errs[0]["loc"][0]).replace("_", "-") diff --git a/src/guidellm/utils/cli.py b/src/guidellm/utils/cli.py index 507a384b..9af6359b 100644 --- a/src/guidellm/utils/cli.py +++ b/src/guidellm/utils/cli.py @@ -20,7 +20,7 @@ def set_if_not_default(ctx: click.Context, **kwargs) -> dict[str, Any]: """ values = {} for k, v in kwargs.items(): - if ctx.get_parameter_source(k) != click.core.ParameterSource.DEFAULT: + if ctx.get_parameter_source(k) != click.core.ParameterSource.DEFAULT: # type: ignore[attr-defined] values[k] = v return values From bcbaca9a521ecfa9134a26e97b942f32ff794cc1 Mon Sep 17 00:00:00 2001 From: Samuel Monson Date: Thu, 22 May 2025 14:32:45 -0400 Subject: [PATCH 13/16] Set defaults for console output --- src/guidellm/benchmark/entrypoints.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/guidellm/benchmark/entrypoints.py b/src/guidellm/benchmark/entrypoints.py index fcb69ce9..ce43fca3 100644 --- a/src/guidellm/benchmark/entrypoints.py +++ b/src/guidellm/benchmark/entrypoints.py @@ -55,13 +55,13 @@ async def benchmark_generative_text( max_requests: Optional[int], warmup_percent: Optional[float], cooldown_percent: Optional[float], - show_progress: bool, - show_progress_scheduler_stats: bool, - output_console: bool, output_path: Optional[Union[str, Path]], output_extras: Optional[dict[str, Any]], output_sampling: Optional[int], random_seed: int, + show_progress: bool = True, + show_progress_scheduler_stats: bool = False, + output_console: bool = True, ) -> tuple[GenerativeBenchmarksReport, Optional[Path]]: console = GenerativeBenchmarksConsole(enabled=show_progress) console.print_line("Creating backend...") From 2ce5d8c075242fdddadc26828d6dbe0ec0c9521b Mon Sep 17 00:00:00 2001 From: Samuel Monson Date: Fri, 23 May 2025 15:38:48 -0400 Subject: [PATCH 14/16] Add builtin scenarios --- src/guidellm/__main__.py | 7 +++--- src/guidellm/benchmark/scenario.py | 24 ++++++++++++++++++-- src/guidellm/benchmark/scenarios/__init__.py | 0 3 files changed, 25 insertions(+), 6 deletions(-) create mode 100644 src/guidellm/benchmark/scenarios/__init__.py diff --git a/src/guidellm/__main__.py b/src/guidellm/__main__.py index 0f2d7133..42b94801 100644 --- a/src/guidellm/__main__.py +++ b/src/guidellm/__main__.py @@ -8,7 +8,7 @@ from guidellm.backend import BackendType from guidellm.benchmark import ProfileType from guidellm.benchmark.entrypoints import benchmark_with_scenario -from guidellm.benchmark.scenario import GenerativeTextScenario +from guidellm.benchmark.scenario import GenerativeTextScenario, get_builtin_scenarios from guidellm.config import print_config from guidellm.scheduler import StrategyType from guidellm.utils import cli as cli_tools @@ -36,7 +36,7 @@ def cli(): dir_okay=False, path_type=Path, # type: ignore[type-var] ), - click.STRING + click.Choice(get_builtin_scenarios()), ), default=None, help=("TODO: A scenario or path to config"), @@ -278,8 +278,7 @@ def benchmark( elif isinstance(scenario, Path): _scenario = GenerativeTextScenario.from_file(scenario, overrides) else: - # TODO: Add support for builtin scenarios - raise NotImplementedError + _scenario = GenerativeTextScenario.from_builtin(scenario, overrides) except ValidationError as e: errs = e.errors(include_url=False, include_context=True, include_input=True) param_name = "--" + str(errs[0]["loc"][0]).replace("_", "-") diff --git a/src/guidellm/benchmark/scenario.py b/src/guidellm/benchmark/scenario.py index 8de73af0..8198ed6f 100644 --- a/src/guidellm/benchmark/scenario.py +++ b/src/guidellm/benchmark/scenario.py @@ -1,6 +1,7 @@ from collections.abc import Iterable +from functools import cache from pathlib import Path -from typing import Annotated, Any, Literal, Optional, Union +from typing import Annotated, Any, Literal, Optional, TypeVar, Union from datasets import Dataset, DatasetDict, IterableDataset, IterableDatasetDict from pydantic import BeforeValidator, Field, NonNegativeInt, PositiveFloat, PositiveInt @@ -13,7 +14,14 @@ from guidellm.objects.pydantic import StandardBaseModel from guidellm.scheduler.strategy import StrategyType -__ALL__ = ["Scenario", "GenerativeTextScenario"] +__ALL__ = ["Scenario", "GenerativeTextScenario", "get_builtin_scenarios"] + +SCENARIO_DIR = Path(__file__).parent / "scenarios/" + + +@cache +def get_builtin_scenarios() -> list[str]: + return [p.stem for p in SCENARIO_DIR.glob("*.json")] def parse_float_list(value: Union[str, float, list[float]]) -> list[float]: @@ -32,9 +40,21 @@ def parse_float_list(value: Union[str, float, list[float]]) -> list[float]: ) from err +T = TypeVar("T", bound="Scenario") + + class Scenario(StandardBaseModel): target: str + @classmethod + def from_builtin(cls: type[T], name: str, overrides: Optional[dict] = None) -> T: + filename = SCENARIO_DIR / f"{name}.json" + + if not filename.is_file(): + raise ValueError(f"{name} is not a vaild builtin scenario") + + return cls.from_file(filename, overrides) + class GenerativeTextScenario(Scenario): # FIXME: This solves an issue with Pydantic and class types diff --git a/src/guidellm/benchmark/scenarios/__init__.py b/src/guidellm/benchmark/scenarios/__init__.py new file mode 100644 index 00000000..e69de29b From a5cb05d273937ec80df5df5166634a979bae1cae Mon Sep 17 00:00:00 2001 From: Samuel Monson Date: Fri, 23 May 2025 16:04:26 -0400 Subject: [PATCH 15/16] Documentation pass --- src/guidellm/__main__.py | 9 +++++++-- src/guidellm/benchmark/scenario.py | 19 +++++++++++++++++-- src/guidellm/objects/pydantic.py | 11 +++++++---- 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/src/guidellm/__main__.py b/src/guidellm/__main__.py index 42b94801..4b059655 100644 --- a/src/guidellm/__main__.py +++ b/src/guidellm/__main__.py @@ -39,7 +39,11 @@ def cli(): click.Choice(get_builtin_scenarios()), ), default=None, - help=("TODO: A scenario or path to config"), + help=( + "The name of a builtin scenario or path to a config file. " + "Missing values from the config will use defaults. " + "Options specified on the commandline will override the scenario." + ), ) @click.option( "--target", @@ -277,9 +281,10 @@ def benchmark( _scenario = GenerativeTextScenario.model_validate(overrides) elif isinstance(scenario, Path): _scenario = GenerativeTextScenario.from_file(scenario, overrides) - else: + else: # Only builtins can make it here; click will catch anything else _scenario = GenerativeTextScenario.from_builtin(scenario, overrides) except ValidationError as e: + # Translate pydantic valdation error to click argument error errs = e.errors(include_url=False, include_context=True, include_input=True) param_name = "--" + str(errs[0]["loc"][0]).replace("_", "-") raise click.BadParameter( diff --git a/src/guidellm/benchmark/scenario.py b/src/guidellm/benchmark/scenario.py index 8198ed6f..af43e426 100644 --- a/src/guidellm/benchmark/scenario.py +++ b/src/guidellm/benchmark/scenario.py @@ -21,10 +21,16 @@ @cache def get_builtin_scenarios() -> list[str]: + """Returns list of builtin scenario names.""" return [p.stem for p in SCENARIO_DIR.glob("*.json")] def parse_float_list(value: Union[str, float, list[float]]) -> list[float]: + """ + Parse a comma separated string to a list of float + or convert single float list of one or pass float + list through. + """ if isinstance(value, (int, float)): return [value] elif isinstance(value, list): @@ -44,6 +50,10 @@ def parse_float_list(value: Union[str, float, list[float]]) -> list[float]: class Scenario(StandardBaseModel): + """ + Parent Scenario class with common options for all benchmarking types. + """ + target: str @classmethod @@ -51,14 +61,19 @@ def from_builtin(cls: type[T], name: str, overrides: Optional[dict] = None) -> T filename = SCENARIO_DIR / f"{name}.json" if not filename.is_file(): - raise ValueError(f"{name} is not a vaild builtin scenario") + raise ValueError(f"{name} is not a valid builtin scenario") return cls.from_file(filename, overrides) class GenerativeTextScenario(Scenario): - # FIXME: This solves an issue with Pydantic and class types + """ + Scenario class for generative text benchmarks. + """ + class Config: + # NOTE: This prevents errors due to unvalidatable + # types like PreTrainedTokenizerBase arbitrary_types_allowed = True backend_type: BackendType = "openai_http" diff --git a/src/guidellm/objects/pydantic.py b/src/guidellm/objects/pydantic.py index 27be1fc1..f4b1e2da 100644 --- a/src/guidellm/objects/pydantic.py +++ b/src/guidellm/objects/pydantic.py @@ -10,6 +10,7 @@ T = TypeVar("T", bound="StandardBaseModel") + class StandardBaseModel(BaseModel): """ A base class for Pydantic models throughout GuideLLM enabling standard @@ -37,9 +38,11 @@ def get_default(cls: type[T], field: str) -> Any: return cls.model_fields[field].default @classmethod - def from_file( - cls: type[T], filename: Path, overrides: Optional[dict] = None - ) -> T: + def from_file(cls: type[T], filename: Path, overrides: Optional[dict] = None) -> T: + """ + Attempt to create a new instance of the model using + data loaded from json or yaml file. + """ try: with filename.open() as f: if str(filename).endswith((".yaml", ".yml")): @@ -48,7 +51,7 @@ def from_file( data = json.load(f) except (json.JSONDecodeError, yaml.YAMLError) as e: logger.error(f"Failed to parse {filename} as type {cls.__name__}") - raise e + raise ValueError(f"Error when parsing file: {filename}") from e data.update(overrides) return cls.model_validate(data) From 112d987474a7241c878b30616ddcc85ed9491e34 Mon Sep 17 00:00:00 2001 From: Samuel Monson Date: Fri, 23 May 2025 16:13:40 -0400 Subject: [PATCH 16/16] Add default scenarios --- .gitignore | 4 ++++ src/guidellm/benchmark/scenarios/chat.json | 13 +++++++++++++ src/guidellm/benchmark/scenarios/rag.json | 13 +++++++++++++ 3 files changed, 30 insertions(+) create mode 100644 src/guidellm/benchmark/scenarios/chat.json create mode 100644 src/guidellm/benchmark/scenarios/rag.json diff --git a/.gitignore b/.gitignore index d4186ed6..2010f626 100644 --- a/.gitignore +++ b/.gitignore @@ -178,3 +178,7 @@ cython_debug/ # Project specific files *.json *.yaml + +# But not scenarios +!src/guidellm/benchmark/scenarios/*.json +!src/guidellm/benchmark/scenarios/*.yaml diff --git a/src/guidellm/benchmark/scenarios/chat.json b/src/guidellm/benchmark/scenarios/chat.json new file mode 100644 index 00000000..024438c5 --- /dev/null +++ b/src/guidellm/benchmark/scenarios/chat.json @@ -0,0 +1,13 @@ +{ + "rate_type": "sweep", + "data": { + "prompt_tokens": 512, + "prompt_tokens_stdev": 128, + "prompt_tokens_min": 1, + "prompt_tokens_max": 1024, + "output_tokens": 256, + "output_tokens_stdev": 64, + "output_tokens_min": 1, + "output_tokens_max": 1024 + } +} diff --git a/src/guidellm/benchmark/scenarios/rag.json b/src/guidellm/benchmark/scenarios/rag.json new file mode 100644 index 00000000..c7ee2f27 --- /dev/null +++ b/src/guidellm/benchmark/scenarios/rag.json @@ -0,0 +1,13 @@ +{ + "rate_type": "sweep", + "data": { + "prompt_tokens": 4096, + "prompt_tokens_stdev": 512, + "prompt_tokens_min": 2048, + "prompt_tokens_max": 6144, + "output_tokens": 512, + "output_tokens_stdev": 128, + "output_tokens_min": 1, + "output_tokens_max": 1024 + } +}