Multi-node serving on Dynamo

# EKS g6.2xlarge, one NVIDIA L4 per node. Like the getting started tour's node,
# but a size up: its 32 GiB of memory holds the larger model's weights as they
# load, and the 100 GB disk holds the vLLM image. This guide runs two so a gang
# can span both.
apiVersion: modelplane.ai/v1alpha1
kind: InferenceClass
metadata:
  name: l4-1x-g6
spec:
  description: "EKS g6.2xlarge, 1x NVIDIA L4"
  provisioning:
    provider: EKS
    eks:
      instanceType: g6.2xlarge
      diskSizeGb: 100
      accelerator:
        type: nvidia-l4
        count: 1
  devices:
  - name: gpu
    claim: DRA
    driver: gpu.nvidia.com
    deviceClassName: gpu.nvidia.com
    count: 1
    attributes:
      architecture: { string: Ada Lovelace }
    capacity:
      memory: { value: "23034Mi" }   # L4's real reported VRAM (not the nominal 24GB)

# An EKS cluster running the Dynamo serving stack, with a two-node L4 pool so a
# gang can span both nodes. spec.stack: Dynamo installs Grove and the KAI
# Scheduler, which gang-schedule the leader and worker together and compose them
# as a Grove PodCliqueSet.
#
# g6.2xlarge has no EFA, so the gang's cross-node traffic goes over TCP. The
# ModelDeployment is pipeline-parallel to keep that traffic light; tensor
# parallelism across nodes would need a fast fabric.
apiVersion: modelplane.ai/v1alpha1
kind: InferenceCluster
metadata:
  name: eks-us-east
  labels:
    modelplane.ai/region: us-east
spec:
  stack: Dynamo
  cluster:
    source: EKS
    eks:
      region: us-east-1
  nodePools:
  - name: gpu-l4
    className: l4-1x-g6
    nodeCount: 2
    minNodeCount: 2
    maxNodeCount: 2
    zones:
    - us-east-1b

# The shared read-write-many cache the gang serves from, hydrated once from
# Hugging Face. Both gang pods mount it and read weights from it over EFS,
# instead of each pulling its own copy. Qwen2.5-14B is open, so it needs no
# token. Its FP16 weights are about 29 GB, so sizeGiB leaves headroom.
apiVersion: modelplane.ai/v1alpha1
kind: ModelCache
metadata:
  name: qwen2-5-14b
  namespace: ml-team
spec:
  source: HuggingFace
  huggingFace:
    repo: Qwen/Qwen2.5-14B-Instruct
    sizeGiB: 40

# Qwen2.5-14B served across two L4 nodes as a gang on a Dynamo cluster. The FP16
# weights (~29 GB) don't fit one L4's 23 GB, so the engine is a Leader + Worker
# gang, pipeline-parallel across two g6.2xlarge nodes with one L4 each. Both pods
# mount the shared ModelCache and read weights from it.
#
# The cluster runs the Dynamo stack, so Grove and the KAI Scheduler
# gang-schedule the two pods, and Modelplane composes them as a Grove
# PodCliqueSet. $(MODELPLANE_LEADER_ADDRESS) resolves to the leader on Dynamo,
# but $(MODELPLANE_RANK) isn't injected there yet (modelplaneai/modelplane#418),
# so each command sets its own --node-rank: 0 on the leader, and
# $$((GROVE_PCLQ_POD_INDEX + 1)) on the worker ($$ escapes past Kubernetes,
# leaving $((...)) for the shell to evaluate).
#
# Notes on the engine flags:
#   --pipeline-parallel-size=2 splits the model across the two nodes;
#     --tensor-parallel-size=1 keeps one GPU per node. Pipeline parallelism sends
#     only activations between nodes, so it tolerates g6.2xlarge's plain-Ethernet
#     interconnect (no EFA) far better than tensor parallelism would.
#   --distributed-executor-backend=mp is vLLM's native multiprocessing multi-node
#     path; vllm/vllm-openai:v0.23.0 no longer ships Ray.
#   --load-format=runai_streamer reads the EFS-backed cache quickly.
#   --max-model-len=8192 caps context so the KV cache fits alongside the weights.
apiVersion: modelplane.ai/v1alpha1
kind: ModelDeployment
metadata:
  name: qwen2-5-14b
  namespace: ml-team
spec:
  replicas: 1
  template:
    spec:
      modelCacheRef:
        name: qwen2-5-14b
      engines:
      - name: qwen
        members:
        - role: Leader
          nodeSelector:
            devices:
            - name: gpu
              count: 1
              selectors:
              - cel: |
                  device.capacity["gpu.nvidia.com"].memory.compareTo(quantity("20Gi")) >= 0
          template:
            spec:
              containers:
              - name: engine
                image: vllm/vllm-openai:v0.23.0
                command:
                - /bin/sh
                - -c
                - >-
                  exec vllm serve Qwen/Qwen2.5-14B-Instruct
                  --served-model-name=qwen2.5-14b
                  --tensor-parallel-size=1
                  --pipeline-parallel-size=2
                  --distributed-executor-backend=mp
                  --nnodes=2 --node-rank=0
                  --master-addr=$(MODELPLANE_LEADER_ADDRESS)
                  --load-format=runai_streamer
                  --max-model-len=8192
                  --gpu-memory-utilization=0.90
                  --port=8000
        - role: Worker
          worker:
            nodes: 1
          nodeSelector:
            devices:
            - name: gpu
              count: 1
              selectors:
              - cel: |
                  device.capacity["gpu.nvidia.com"].memory.compareTo(quantity("20Gi")) >= 0
          template:
            spec:
              containers:
              - name: engine
                image: vllm/vllm-openai:v0.23.0
                command:
                - /bin/sh
                - -c
                - >-
                  exec vllm serve Qwen/Qwen2.5-14B-Instruct
                  --served-model-name=qwen2.5-14b
                  --tensor-parallel-size=1
                  --pipeline-parallel-size=2
                  --distributed-executor-backend=mp
                  --nnodes=2 --node-rank=$$((GROVE_PCLQ_POD_INDEX + 1))
                  --master-addr=$(MODELPLANE_LEADER_ADDRESS)
                  --headless
                  --load-format=runai_streamer
                  --max-model-len=8192
                  --gpu-memory-utilization=0.90
                  --port=8000

# Exposes the gang as one OpenAI-compatible URL. Modelplane composes one
# ModelEndpoint per replica, labeled modelplane.ai/deployment: qwen2-5-14b, so
# this selector reaches it. Read the public address from status.address:
#   kubectl get ms qwen2-5-14b -n ml-team -o jsonpath='{.status.address}'
apiVersion: modelplane.ai/v1alpha1
kind: ModelService
metadata:
  name: qwen2-5-14b
  namespace: ml-team
spec:
  endpoints:
  - selector:
      matchLabels:
        modelplane.ai/deployment: qwen2-5-14b