LLM 스터디 7주차 - kind에 llm-d 올리기 — 배포부터 게이트웨이 오버헤드 실측까지

$ sudo nvidia-ctk runtime configure --runtime=docker --set-as-default
$ sudo systemctl restart docker

accept-nvidia-visible-devices-as-volume-mounts = true

apiVersion: kind.x-k8s.io/v1alpha4
kind: Cluster
nodes:
  - role: control-plane
    image: kindest/node:v1.36.1@sha256:3489c7674813ba5d8b1a9977baea8a6e553784dab7b84759d1014dbd78f7ebd5
    extraMounts:
      - hostPath: /dev/null
        containerPath: /var/run/nvidia-container-devices/all

$ kind create cluster --name gpu-lab --config kind-gpu.yaml
$ kubectl get node -o wide
NAME                    STATUS   ROLES           VERSION   CONTAINER-RUNTIME
gpu-lab-control-plane   Ready    control-plane   v1.36.1   containerd://2.3.1

$ kubectl apply -f https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/v0.17.1/deployments/static/nvidia-device-plugin.yml

$ kubectl get node gpu-lab-control-plane -o jsonpath='{.status.capacity}' | tr ',' '\n'
{"cpu":"12"
"ephemeral-storage":"478512880Ki"
"hugepages-1Gi":"0"
"hugepages-2Mi":"0"
"memory":"32683004Ki"
"nvidia.com/gpu":"1"
"pods":"110"}

export REPO_ROOT=~/llm-d
export GUIDE_NAME=optimized-baseline
export NAMESPACE=llm-d-lab
export ACCELERATOR_TYPE=gpu
export MODEL_SERVER=vllm

export ROUTER_STANDALONE_CHART=oci://ghcr.io/llm-d/charts/llm-d-router-standalone
export ROUTER_CHART_VERSION=v0
export ROUTER_BASE_VALUES="${REPO_ROOT}/guides/recipes/router/base.values.yaml"
export ROUTER_VALUES="${REPO_ROOT}/guides/${GUIDE_NAME}/router/${GUIDE_NAME}.values.yaml"
export MONITORING_VALUES=""

$ helm install ${GUIDE_NAME} \
    ${ROUTER_STANDALONE_CHART} \
    -f ${ROUTER_BASE_VALUES} \
    ${MONITORING_VALUES} \
    -f ${ROUTER_VALUES} \
    -n ${NAMESPACE} --version ${ROUTER_CHART_VERSION}

router:
  proxy:
    proxyType: envoy          # 같은 파드에 사이드카로
    args: ["--service-node", "envoy-sidecar", "--concurrency", "8", ...]
  extraServicePorts:
    - name: http
      port: 80
      targetPort: 8081        # 서비스 80 → Envoy 8081
  inferencePool:
    failureMode: FailOpen     # EPP 장애 시 요청을 막지 않는다
  modelServers:
    matchLabels:
      llm-d.ai/guide: optimized-baseline   # 이 라벨이 붙은 파드가 후보

apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
  - ../../../../../recipes/modelserver/base/single-host/default
namePrefix: optimized-baseline-nvidia-gpu-vllm-
components:
  - ../../../../../recipes/modelserver/components/images/gpu-vllm/release
labels:
  - pairs:
      llm-d.ai/model: Qwen3-0.6B
      llm-d.ai/guide: optimized-baseline      # Router의 selector와 맞춘다
      llm-d.ai/accelerator-variant: gpu
      llm-d.ai/accelerator-vendor: nvidia
    includeSelectors: true
    includeTemplates: true
patches:
  - path: patch-vllm.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: decode
spec:
  replicas: 1
  template:
    spec:
      containers:
        - name: modelserver
          command: ["vllm", "serve"]
          args:
            - "Qwen/Qwen3-0.6B"
            - "--disable-access-log-for-endpoints=/health,/metrics,/v1/models"
            - "--max-model-len=2048"
            - "--gpu-memory-utilization=0.6"
            - "--enforce-eager"
          env:
            - name: HF_TOKEN
              valueFrom:
                secretKeyRef: {name: llm-d-hf-token, key: HF_TOKEN}
          resources:
            limits: {cpu: '4', memory: 8Gi, nvidia.com/gpu: 1}
            requests: {cpu: '1', memory: 4Gi, nvidia.com/gpu: 1}
          startupProbe:
            initialDelaySeconds: 10
            periodSeconds: 10
            failureThreshold: 60          # 최대 10분까지 기동을 기다린다
      volumes:
        - name: shm
          emptyDir: {medium: Memory, sizeLimit: 2Gi}

$ kubectl create secret generic llm-d-hf-token \
    --from-literal=HF_TOKEN=<your-token> -n ${NAMESPACE}

$ kubectl apply -n ${NAMESPACE} \
    -k ${REPO_ROOT}/guides/${GUIDE_NAME}/modelserver/gpu/vllm/lab-3050/

[cuda.py:482] Using FLASH_ATTN attention backend
[flash_attn.py:776] Using FlashAttention version 2
[gpu_worker.py:560] Available KV cache memory: 2.14 GiB
[kv_cache_utils.py:2177] GPU KV cache size: 20,032 tokens
[kv_cache_utils.py:2178] Maximum concurrency for 2,048 tokens per request: 9.78x
[core.py:347] init engine (profile, create kv cache, warmup model) took 8.08 s

$ kubectl get pod -n llm-d-lab
NAME                                                         READY   STATUS
optimized-baseline-epp-5fbd7457f9-9xcr7                      2/2     Running
optimized-baseline-nvidia-gpu-vllm-decode-7978d9d5c9-h59gz   1/1     Running

$ kubectl get pod -n llm-d-lab optimized-baseline-epp-5fbd7457f9-9xcr7 \
    -o jsonpath='{range .spec.containers[*]}{.name}: {.image}{"\n"}{end}'
envoy-proxy: docker.io/envoyproxy/envoy:distroless-v1.33.2
epp: ghcr.io/llm-d/llm-d-router-endpoint-picker:main

$ kubectl get svc -n llm-d-lab optimized-baseline-epp \
    -o jsonpath='{range .spec.ports[*]}{.name} {.port}->{.targetPort}{"\n"}{end}'
grpc-ext-proc 9002->9002
http-metrics 9090->9090
http 80->8081

apiVersion: inference.networking.k8s.io/v1
kind: InferencePool
metadata:
  name: optimized-baseline
  namespace: llm-d-lab
spec:
  appProtocol: http
  selector:
    matchLabels:
      llm-d.ai/guide: optimized-baseline
  targetPorts:
    - number: 8000
  endpointPickerRef:
    kind: Service
    name: optimized-baseline-epp
    port:
      number: 9002
    failureMode: FailOpen

apiVersion: llm-d.ai/v1alpha1
kind: EndpointPickerConfig
plugins:
  - type: approx-prefix-cache-producer
  - type: inflight-load-producer
  - type: prefix-cache-affinity-filter
  - type: token-load-scorer
schedulingProfiles:
  - name: default
    plugins:
      - pluginRef: prefix-cache-affinity-filter
      - pluginRef: token-load-scorer

http_filters:
  - name: envoy.filters.http.ext_proc
    typed_config:
      failure_mode_allow: true
      grpc_service:
        envoy_grpc:
          cluster_name: ext_proc
          authority: localhost:9002
        timeout: 10s
      processing_mode:
        request_body_mode: FULL_DUPLEX_STREAMED
        response_body_mode: FULL_DUPLEX_STREAMED
      message_timeout: 1000s

clusters:
  - name: original_destination_cluster
    type: ORIGINAL_DST
    lb_policy: CLUSTER_PROVIDED
    original_dst_lb_config:
      use_http_header: true
      http_header_name: x-gateway-destination-endpoint

def req(base, prompt, max_tokens=64):
    body = json.dumps({"model": MODEL, "prompt": prompt, "max_tokens": max_tokens,
                       "temperature": 0, "stream": True}).encode()
    r = urllib.request.Request(base + "/v1/completions", data=body,
                               headers={"Content-Type": "application/json"})
    t0 = time.perf_counter(); ttft = None
    with urllib.request.urlopen(r, timeout=180) as resp:
        for raw in resp:
            if raw.startswith(b"data: ") and b"[DONE]" not in raw:
                if ttft is None:                 # 첫 토큰 도착 = TTFT
                    ttft = time.perf_counter() - t0
    return ttft * 1000, (time.perf_counter() - t0) * 1000

for i in range(20):                              # 교차 실행
    p = f"Question {i}: name one benefit of Kubernetes in a single sentence."
    d_ttft.append(req(VLLM, p)[0])
    e_ttft.append(req(EPP, p + " ")[0])

$ # 8회 요청 전후 vllm:prefix_cache_* 차이
queries delta  4504.0    # 캐시를 조회한 토큰 수
hits    delta  3808.0    # 그중 실제로 재사용한 토큰 수
→ 적중률 84.5%

{"body":"EPP received request",
 "x-request-id":"3f721c05-b714-49c8-ab60-4dc568dd3dff"}
{"body":"EPP sent request body response(s) to proxy",
 "x-request-id":"3f721c05-b714-49c8-ab60-4dc568dd3dff",
 "modelName":"Qwen/Qwen3-0.6B","targetModelName":"Qwen/Qwen3-0.6B"}
{"body":"EPP sent response body back to proxy",
 "x-request-id":"3f721c05-b714-49c8-ab60-4dc568dd3dff"}