optimized-baseline.yaml

# OPTIMIZED BASELINE WELL LIT PATH for LLM-D Deployment
# Migrated from bash environment variables to YAML format
# Based on https://github.com/llm-d/llm-d/tree/main/guides/optimized-baseline

scenario:
  # ============================================================================
  # Optimized Baseline - Qwen3-32B with 2 decode pods
  # ============================================================================
  - name: "optimized-baseline"

    # =========================================================================
    # COMMON -- Applies to all deployment methods
    # =========================================================================
    common:
      # Detect the device-plugin resource and select the matching machine
      # profile. Explicit accelerator settings supplied by the user still win.
      accelerator:
        profile: auto
        resource: auto

      # -------------------------------------------------------------------------
      # Affinity / Node Selection
      # By default, affinity is disabled and pods schedule on any available node.
      # Options:
      #   1. Leave commented out -- no node selection constraint
      #   2. Set nodeSelector to "auto" -- auto-detects GPU labels from the cluster
      #   3. Set explicit labels -- pods only schedule on matching nodes
      # -------------------------------------------------------------------------
      # affinity:
      #   enabled: true
      #   nodeSelector:
      #     nvidia.com/gpu.product: NVIDIA-H100-80GB-HBM3      # OpenShift
      #     # gpu.nvidia.com/model: H200                        # Kubernetes
      #     # cloud.google.com/gke-accelerator: nvidia-h100-80gb  # GKE

      model:
        name: Qwen/Qwen3-32B
        shortName: qwen-qwen3-32b
        path: models/Qwen/Qwen3-32B
        huggingfaceId: Qwen/Qwen3-32B
        size: 1Ti
        maxModelLen: 16000
        blockSize: 64
        gpuMemoryUtilization: 0.95

      storage:
        modelPvc:
          size: 1Ti

      vllmCommon:
  #      priorityClassName: nightly-gpu-critical
        volumes:
          - name: shared-config
            type: emptyDir
            emptyDir: {}
          - name: dshm
            type: emptyDir
            emptyDir:
              medium: Memory
              sizeLimit: 16Gi

        volumeMounts:
          - name: dshm
            mountPath: /dev/shm
          - name: shared-config
            mountPath: /shared-config

    # =========================================================================
    # KUSTOMIZE DEPLOY METHOD   (scope: the ENTIRE stack)
    # -------------------------------------------------------------------------
    # When kustomize.enabled is true (or `-t kustomize` is passed) this
    # scenario is deployed by applying the UPSTREAM llm-d guide directly
    # (guides/<guideName> in the llm-d repo). In that mode the deployment is
    # defined ENTIRELY by the guide's own manifests:
    #
    #   * EVERYTHING ELSE in this scenario is IGNORED -- model, modelservice,
    #     standalone, decode/prefill, vllmCommon, storage, harness model, etc.
    #   * `-m/--models` and all other CLI/scenario tuning are ignored.
    #   * DoE `experiment` SETUP sweeps do NOT alter a kustomize deploy
    #     (only run/workload treatments still apply).
    #
    # The ONLY way to modify a kustomize deployment is the kustomize.* keys
    # below:
    #   patches / overlayPath           -> modelserver (the workload pods)
    #   extraHelmValues / extraHelmSets -> router / GAIE helm release
    #   guideVariableOverrides          -> override the guide README's ${VAR}
    # See docs/kustomize.md for the full reference and worked examples.
    #
    # NOT SUPPORTED with kustomize: multi-model / multi-stack scenarios
    # (the deploy is keyed on guideName with no per-stack uniquification --
    # stacks would collide). Use the modelservice method for multi-model.
    #
    # When kustomize.enabled is false (the DEFAULT) this whole block is
    # inert and the scenario deploys via modelservice/standalone as
    # configured further below.
    # =========================================================================
    kustomize:
      enabled: false

      guideName: "optimized-baseline"
      repoPath: ""
      repoRef: "main"
      gaieVersion: ""
      acceleratorBackend: "gpu/vllm"
# Alternative: use --set kustomize.acceleratorBackend=gpu/sglang on the CLI to switch sglang
      monitoring: false
      overlayPath: ""
      extraHelmValues: []
      extraHelmSets: {}
#      guideVariableOverrides: {}   # override/fill the guide README's ${VAR}
      guideVariableOverrides:
        ROUTER_CHART_VERSION: v0.10.0
        ROUTER_GATEWAY_CHART: oci://ghcr.io/llm-d/charts/llm-d-router-gateway
      # Inline kustomize strategic merge patches applied on top of the
      # guide's modelserver base. Default is empty -- the upstream guide
      # already injects HF_TOKEN from the `llm-d-hf-token` Secret (which
      # the benchmark's standup auto-creates from the HF_TOKEN env var),
      # so no patch is needed for gated-model access.
      #
      # Use this list to override anything else upstream sets -- e.g.
      # bump replicas or pin a priorityClassName:
      #
      #   patches:
      #     - patch: |
      #         apiVersion: apps/v1
      #         kind: Deployment
      #         metadata:
      #           name: decode
      #         spec:
      #           replicas: 2
      #           template:
      #             spec:
      #               priorityClassName: nightly-gpu-critical
      patches: []

    # =========================================================================
    # MODELSERVICE -- Only applies when modelservice.enabled: true
    # The following sections configure the llm-d modelservice Helm chart
    # deployment including decode pods, prefill pods, router, and gateway.
    # =========================================================================
    modelservice:
      enabled: true

      gateway:
        className: epponly

      # -------------------------------------------------------------------------
      # Routing / GAIE Configuration
      # -------------------------------------------------------------------------
      router:
        extraServicePorts:
          - name: http
            port: 80
            protocol: TCP
            targetPort: 8081
        epp:
          replicas: 1
          flags:
            v: 2
          pluginsConfigFile: "optimized-baseline-plugins.yaml"
          pluginsCustomConfig:
            optimized-baseline-plugins.yaml: |
              apiVersion: llm-d.ai/v1alpha1
              kind: EndpointPickerConfig
              plugins:
              - type: queue-scorer
              - type: kv-cache-utilization-scorer
              - type: prefix-cache-scorer
              - type: no-hit-lru-scorer
              schedulingProfiles:
              - name: default
                plugins:
                - pluginRef: queue-scorer
                  weight: 2
                - pluginRef: kv-cache-utilization-scorer
                  weight: 2
                - pluginRef: prefix-cache-scorer
                  weight: 3
                - pluginRef: no-hit-lru-scorer
                  weight: 2
          resources:
            requests:
              cpu: "4"
              memory: 8Gi
            limits:
              memory: 16Gi
        proxy:
          # CLI args for the Envoy sidecar. Override the chart's defaults of
          # `--log-level trace` (very expensive under load) and no `--concurrency`
          # (Envoy spawns ~hardware_concurrency() worker threads, oversubscribing
          # the cgroup's CPU slice). `--concurrency 8` matches the resources.limits
          # equivalent below; trace-level logging would burn a meaningful share of
          # the sidecar's CPU at high QPS.
          args:
            - "--service-node"
            - "envoy-sidecar"
            - "--log-level"
            - "warn"
            - "--concurrency"
            - "8"
            - "--drain-strategy"
            - "immediate"
            - "--drain-time-s"
            - "60"
            - "-c"
            - "/etc/envoy/envoy.yaml"
          # Resource requests and limits for the Envoy sidecar.
          resources:
            requests:
              cpu: "4"
              memory: 8Gi
            limits:
              memory: 16Gi
        inferencePool:
          failureMode: "FailOpen"
          providerConfig:
            destinationRule:
              trafficPolicy:
                connectionPool:
                  http:
                    http1MaxPendingRequests: 256000
                    maxRequestsPerConnection: 256000
                    http2MaxRequests: 256000
                    idleTimeout: "900s"
                  tcp:
                    maxConnections: 256000
                    maxConnectionDuration: "1800s"
                    connectTimeout: "900s"
        modelServers:
          protocol: http
          matchLabels:
            llm-d.ai/guide: "optimized-baseline"
        tracing:
          otelExporterEndpoint: "http://localhost:4317"
          sampling:
            sampler: "parentbased_traceidratio"
            samplerArg: "0.1"

      # -------------------------------------------------------------------------
      # HTTPRoute Configuration
      # requestTimeout: end-to-end Gateway-API HTTPRoute timeout the istio
      # gateway enforces per request. Long agentic turns that exceed it are
      # aborted by the gateway with a 502 (surfaces as "connection reset by
      # peer" on the decode routing-proxy loopback hop). Set "0s" to disable.
      # -------------------------------------------------------------------------
      httpRoute:
        requestTimeout: "300s"
        backendRequestTimeout: "0s"

      # -------------------------------------------------------------------------
      # Prefill Configuration
      # -------------------------------------------------------------------------
      prefill:
        enabled: false
        replicas: 0

      # -------------------------------------------------------------------------
      # Decode Configuration
      # -------------------------------------------------------------------------
      decode:
        extraContainerConfig:
          ports:
            - containerPort: 5557  # NIXL side channel port
              protocol: TCP
            - containerPort: 8200  # Metrics port
              name: metrics
              protocol: TCP
          securityContext:
            capabilities:
              add:
                - "IPC_LOCK"
                - "SYS_RAWIO"
            runAsGroup: 0
            runAsUser: 0
          imagePullPolicy: Always
        replicas: 2
  #      replicas: 8
        vllm:
          customCommand: |
            ${accelerator.runtimePreamble}
            vllm serve $MODEL_SERVE_REF \
            --host 0.0.0.0 \
            --served-model-name $MODEL_NAME \
            --port $VLLM_METRICS_PORT \
            --block-size $VLLM_BLOCK_SIZE \
            --max-model-len $VLLM_MAX_MODEL_LEN \
            --max-num-seq $VLLM_MAX_NUM_SEQ \
            --max-num-batched-tokens $VLLM_MAX_NUM_BATCHED_TOKENS \
            --tensor-parallel-size $TP_SIZE \
            --gpu-memory-utilization $VLLM_ACCELERATOR_MEM_UTIL \
            ${accelerator.dtypeArgs} \
            ${accelerator.executionArgs} \
            --no-enable-log-requests \
            --disable-uvicorn-access-log \
            --disable-access-log-for-endpoints=/health,/metrics,/v1/models \
            --no-enable-prefix-caching

        initContainers:
          - name: preprocess
            imageKey: benchmark
            imagePullPolicy: Always
            command: ["set_llmdbench_environment.py", "-e", "/shared-config/llmdbench_env.sh", "-i"]
            volumeMounts:
              - name: shared-config
                mountPath: /shared-config

        parallelism:
          tensor: 1
          data: 1
          dataLocal: 1
          workers: 1

        # Resource configuration
        resources:
          limits:
            memory: 64Gi
            cpu: "16"
          requests:
            memory: 64Gi
            cpu: "16"

        # No customCommand -- the vLLM serve command is auto-generated
        # from model.*, vllmCommon.flags.*, and decode.vllm.additionalFlags.
        # -----------------------------------------------------------------------
        # Extra Environment Variables
        # No custom env vars for this scenario -- uses default ENVVARS_TO_YAML
        # mechanism. Auto-populated vars (UCX_TLS, VLLM_NIXL_*, etc.) are set
        # by the preprocess script at pod runtime.
        # -----------------------------------------------------------------------
        extraEnvVars: []

        # -----------------------------------------------------------------------
        # Additional Volume Mounts
        # -----------------------------------------------------------------------
        additionalVolumeMounts: []

        # -----------------------------------------------------------------------
        # Additional Volumes
        # -----------------------------------------------------------------------
        additionalVolumes: []

    harness:
      name: inference-perf
      experimentProfile: shared_prefix_synthetic.yaml

    workDir: "~/data/optimized-baseline"