Pin FMA+KEDA workloads to a target node for benchmarking

Adds explicit node targeting for the FMA launcher/requester workloads so a
benchmark can be confined to one node, including a tainted one:

Signed-off-by: dumb0002 <Braulio.Dumba@ibm.com>

Author: dumb0002

config/scenarios/cicd/ocp-keda-fma-hotstart.yaml

@@ -108,9 +108,12 @@ scenario:
       # type and all free, labels it, and pins the LauncherPopulationPolicy +
       # requester Deployment to it; launcherCount/replicas are sized to that
       # node's GPU count. Standup fails if no fully-free single-type GPU node.
+      # Node/label/taint come from the shared top-level `nodePinning` block
+      # (config/templates/values/defaults.yaml) so they are set in ONE place.
+      # Uncomment nodeLabel/nodeLabelValue/nodeName here only to override it
+      # for this scenario alone.
       launcherNodeSelection:
         enabled: true
-        nodeLabel: fma-hotstart
 
       # HOT-START CHANGE #1: Deploy all replicas at standup for parallel model loading
       # Initially set to maxReplicas so all launcher pods load models simultaneously

config/templates/jinja/24_fma-deployment.yaml.j2

@@ -4,7 +4,17 @@
    fma-deployment.yaml.j2
 
    Fast Model Actuation Deployment.
+
+   Node pinning resolves from the shared `nodePinning` block (defaults.yaml)
+   unless the scenario sets the fma.* equivalent, which wins.
    ============================================================================ #}
+{% set _np = nodePinning | default({}) %}
+{% set _np_on = _np.enabled | default(false) %}
+{% set _lns = fma.launcherNodeSelection | default({}) %}
+{% set _pin_on = _lns.enabled | default(false) or _np_on %}
+{% set _pin_label = _lns.nodeLabel | default('', true) or (_np.nodeLabel if _np_on else '') or 'fma-hotstart' %}
+{% set _pin_value = _lns.nodeLabelValue | default('', true) or (_np.nodeLabelValue if _np_on else '') or 'true' %}
+{% set _pin_tolerations = fma.tolerations | default([], true) or (_np.tolerations | default([], true) if _np_on else []) %}
 
 apiVersion: fma.llm-d.ai/v1alpha1
 kind: InferenceServerConfig
@@ -84,6 +94,12 @@ spec:
       serviceAccountName: fma-launcher
 {% if fma.launcher.runtimeClassName is defined and fma.launcher.runtimeClassName %}
       runtimeClassName: {{ fma.launcher.runtimeClassName }}
+{% endif %}
+{% if _pin_tolerations %}
+      # Launcher pods are created by the FMA controller from this podTemplate,
+      # so a taint on the target node has to be tolerated here.
+      tolerations:
+{{ _pin_tolerations | toyaml | indent(8, first=True) }}
 {% endif %}
       containers:
         - name: inference-server
@@ -173,9 +189,9 @@ spec:
     labelSelector:
       matchLabels:
         nvidia.com/gpu.present: "true"
-{% if fma.launcherNodeSelection is defined and fma.launcherNodeSelection.enabled | default(false) %}
+{% if _pin_on %}
         # Pin launchers to the single GPU node step_06 selected + labeled.
-        {{ fma.launcherNodeSelection.nodeLabel | default('fma-hotstart') }}: "true"
+        {{ _pin_label }}: "{{ _pin_value }}"
 {% endif %}
       # Skip unhealthy nodes
       matchExpressions:
@@ -216,11 +232,15 @@ spec:
         dual-pods.llm-d.ai/admin-port: "8081"
         dual-pods.llm-d.ai/inference-server-config: fma-{{ model_id_label }}
     spec:
-{% if fma.launcherNodeSelection is defined and fma.launcherNodeSelection.enabled | default(false) %}
+{% if _pin_on %}
       # Pin the requester to the single GPU node step_06 selected + labeled.
       # ANDed with the nodeAffinity below (which stays load-bearing for WVA).
       nodeSelector:
-        {{ fma.launcherNodeSelection.nodeLabel | default('fma-hotstart') }}: "true"
+        {{ _pin_label }}: "{{ _pin_value }}"
+{% endif %}
+{% if _pin_tolerations %}
+      tolerations:
+{{ _pin_tolerations | toyaml | indent(8, first=True) }}
 {% endif %}
       {# Pick a resolved accelerator type from decode/prefill/standalone
          (cluster_resource_resolver.py rewrites `labelValue: auto` into the

config/templates/values/defaults.yaml

@@ -1525,6 +1525,43 @@ kustomize:
 
 # ============================================================================
 # FMA DEPLOYMENT CONFIGURATION
+# ============================================================================
+# Single-node pinning (benchmark isolation)
+#
+# ONE place to name the reserved node and its taint. Set these three values
+# and every workload that reads them lands on that node -- no per-scenario
+# edits. Consumed by:
+#   - FMA launchers + requester (24_fma-deployment.yaml.j2)
+#   - EPP                       (12_router-values.yaml.j2)
+#
+# `enabled: false` leaves every consumer untouched, so this is inert until
+# you fill it in. A scenario can still override any consumer individually
+# (fma.tolerations, router.epp.affinity) -- an explicit per-scenario value
+# always wins over these defaults.
+#
+# The harness and data-access pods are deliberately NOT pinned: with a taint
+# on the node they cannot land there, which is what keeps load generation off
+# the measurement node.
+# ============================================================================
+nodePinning:
+  enabled: false
+  # Node to pin to. Also used as fma.launcherNodeSelection.nodeName, which
+  # skips FMA's free-GPU/CPU auto-scan (that scan ignores taints and could
+  # otherwise pick a node whose pods never schedule).
+  nodeName: ""
+  # Label the node already carries, used for pod placement. EPP has no
+  # nodeSelector in the router chart, so this becomes a nodeAffinity term
+  # there and a nodeSelector for the FMA requester/launchers.
+  nodeLabel: ""
+  nodeLabelValue: ""
+  # Taints to tolerate. A label places a pod; only a matching toleration lets
+  # it schedule on a tainted node. Standard corev1.Toleration entries, e.g.
+  #   - key: reserved
+  #     operator: Equal
+  #     value: my-workload
+  #     effect: NoSchedule
+  tolerations: []
+
 # For Fast Model Actuation
 # ============================================================================
 fma:
@@ -1538,7 +1575,19 @@ fma:
   # (used by cicd/ocp-wva-fma-hotstart).
   launcherNodeSelection:
     enabled: false
-    nodeLabel: fma-hotstart
+    # Empty so an inherited nodePinning value is not masked by a default; the
+    # effective fallback when neither is set is still fma-hotstart=true.
+    nodeLabel: ""
+    nodeLabelValue: ""
+    # Pin to this node instead of auto-selecting one. Required when the target
+    # node is tainted: the auto-scan reads allocatable/committed resources and
+    # does not consider taints, so it can pick a node whose pods never schedule.
+    nodeName: ""
+
+  # Tolerations applied to the FMA launcher pods and the requester Deployment.
+  # Needed when the target node carries a taint -- a nodeSelector alone places
+  # a pod, but only a matching toleration lets it schedule there.
+  tolerations: []
 
   # Deployment naming
   deployment: