kind + RTX 3050 6GB에서 Ray Serve LLM으로 vLLM 서빙하기 — RayService 실습 매뉴얼

@serve.deployment
class MyFirstDeployment:
    def __init__(self, msg):
        self.msg = msg
    def __call__(self):
        return self.msg

nvidia-smi --query-compute-apps=pid,process_name,used_memory --format=csv

docker info --format '{{.DefaultRuntime}}'                    # nvidia
grep accept-nvidia /etc/nvidia-container-runtime/config.toml  # = true

# kind-gpu.yaml
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
name: gpu
nodes:
  - role: control-plane
    extraMounts:
      # GPU 전체를 노드에 주입
      - hostPath: /dev/null
        containerPath: /var/run/nvidia-container-devices/all
      # NVIDIA Container Toolkit 을 노드 안으로
      - {hostPath: /usr/bin/nvidia-container-runtime,      containerPath: /usr/bin/nvidia-container-runtime}
      - {hostPath: /usr/bin/nvidia-container-runtime-hook, containerPath: /usr/bin/nvidia-container-runtime-hook}
      - {hostPath: /usr/bin/nvidia-container-cli,          containerPath: /usr/bin/nvidia-container-cli}
      - {hostPath: /usr/bin/nvidia-container-toolkit,      containerPath: /usr/bin/nvidia-container-toolkit}
      - {hostPath: /usr/bin/nvidia-ctk,                    containerPath: /usr/bin/nvidia-ctk}
      - {hostPath: /usr/lib/x86_64-linux-gnu/libnvidia-container.so.1.19.1,    containerPath: /usr/lib/x86_64-linux-gnu/libnvidia-container.so.1}
      - {hostPath: /usr/lib/x86_64-linux-gnu/libnvidia-container-go.so.1.19.1, containerPath: /usr/lib/x86_64-linux-gnu/libnvidia-container-go.so.1}
      - {hostPath: /etc/nvidia-container-runtime/config.toml, containerPath: /etc/nvidia-container-runtime/config.toml}
    extraPortMappings:
      - {containerPort: 30005, hostPort: 30005, listenAddress: "0.0.0.0"}   # Ray Serve
      - {containerPort: 30006, hostPort: 30006, listenAddress: "0.0.0.0"}   # Ray Dashboard

kind create cluster --config kind-gpu.yaml
docker exec gpu-control-plane nvidia-smi -L

docker exec gpu-control-plane \
  nvidia-ctk runtime configure --runtime=containerd \
             --config=/etc/containerd/config.toml --set-as-default
docker exec gpu-control-plane systemctl restart containerd

kubectl apply -f - <<'EOF'
apiVersion: node.k8s.io/v1
kind: RuntimeClass
metadata:
  name: nvidia
handler: nvidia
EOF

kubectl label node gpu-control-plane nvidia.com/gpu.present=true --overwrite

helm repo add nvdp https://nvidia.github.io/k8s-device-plugin
helm install nvdp nvdp/nvidia-device-plugin \
  -n nvidia-device-plugin --create-namespace

kubectl get node gpu-control-plane -o jsonpath='{.status.allocatable}'

{
  "cpu": "12",
  "memory": "32683004Ki",
  "nvidia.com/gpu": "1"
}

apiVersion: v1
kind: Pod
metadata:
  name: gpu-test
spec:
  restartPolicy: Never
  runtimeClassName: nvidia
  containers:
    - name: cuda
      image: nvidia/cuda:12.4.1-base-ubuntu22.04
      command: ["nvidia-smi"]
      resources:
        limits:
          nvidia.com/gpu: 1

helm repo add kuberay https://ray-project.github.io/kuberay-helm/
helm repo update kuberay
helm search repo kuberay/kuberay-operator --versions | head -5

kubectl create namespace kuberay-system
helm install kuberay-operator kuberay/kuberay-operator \
  --version 1.6.0 -n kuberay-system

kubectl explain rayservices.ray.io.spec
kubectl logs -n kuberay-system -l app.kubernetes.io/name=kuberay-operator --tail=30

docker exec gpu-control-plane crictl pull docker.io/rayproject/ray-llm:2.52.0-py311-cu128

apiVersion: ray.io/v1
kind: RayService
metadata:
  name: vllm-service
  namespace: kuberay
spec:
  serveConfigV2: |
    applications:
      - name: llms
        import_path: ray.serve.llm:build_openai_app
        route_prefix: "/"
        args:
          llm_configs:
            - model_loading_config:
                model_id: qwen2.5-1.5b-instruct-awq
                model_source: Qwen/Qwen2.5-1.5B-Instruct-AWQ
              engine_kwargs:
                dtype: auto
                quantization: awq
                max_model_len: 2048
                gpu_memory_utilization: 0.70
              deployment_config:
                autoscaling_config:
                  min_replicas: 1
                  max_replicas: 1
                  target_ongoing_requests: 4
                max_ongoing_requests: 8
  rayClusterConfig:
    headGroupSpec:
      rayStartParams:
        num-gpus: "0"
      template:
        spec:
          containers:
            - name: ray-head
              image: rayproject/ray-llm:2.52.0-py311-cu128
              imagePullPolicy: IfNotPresent
              resources:
                limits:   {cpu: "2", memory: "5Gi"}
                requests: {cpu: "2", memory: "4Gi"}
              ports:
                - {containerPort: 8000, name: serve}
                - {containerPort: 8080, name: metrics}
                - {containerPort: 6379, name: gcs}
                - {containerPort: 8265, name: dashboard}
                - {containerPort: 10001, name: client}
    workerGroupSpecs:
      - groupName: gpu-group
        replicas: 1
        minReplicas: 1
        maxReplicas: 1
        rayStartParams:
          num-gpus: "1"
        template:
          spec:
            runtimeClassName: nvidia
            containers:
              - name: ray-worker
                image: rayproject/ray-llm:2.52.0-py311-cu128
                imagePullPolicy: IfNotPresent
                resources:
                  limits:   {cpu: "4", memory: "12Gi", nvidia.com/gpu: 1}
                  requests: {cpu: "4", memory: "12Gi", nvidia.com/gpu: 1}

kubectl create namespace kuberay
kubectl apply -f vllm-service-6gb.yaml
kubectl get pods -n kuberay -w

kubectl describe rayservices.ray.io vllm-service -n kuberay

kubectl get rayservice vllm-service -n kuberay \
  -o jsonpath='{.status.activeServiceStatus.rayClusterName}'

apiVersion: v1
kind: Service
metadata:
  name: vllm-service-nodeport
  namespace: kuberay
spec:
  type: NodePort
  selector:
    ray.io/node-type: head
    ray.io/cluster: vllm-service-bslgp    # 배포 후 확인한 실제 이름
  ports:
    - {name: serve, port: 8000, targetPort: 8000, nodePort: 30005}

curl -s http://localhost:30005/v1/chat/completions -H 'Content-Type: application/json' \
  -d '{"model":"qwen2.5-1.5b-instruct-awq",
       "messages":[{"role":"user","content":"안녕, 너는 어떤 모델이야?"}],
       "max_tokens":80}' | jq

{
  "id": "chatcmpl-d84b8adf-7ec8-44fa-96df-909b471cd59e",
  "model": "qwen2.5-1.5b-instruct-awq",
  "choices": [{
    "index": 0,
    "message": {
      "role": "assistant",
      "content": "나는 Qwen이라고 불릴 수 있습니다. 이것은 대규모의 언어 모델입니다."
    },
    "finish_reason": "stop"
  }],
  "usage": {"prompt_tokens": 40, "total_tokens": 62, "completion_tokens": 22}
}

apiVersion: v1
kind: Service
metadata:
  name: vllm-service-dashboard-nodeport
  namespace: kuberay
spec:
  type: NodePort
  selector:
    ray.io/node-type: head
    ray.io/cluster: vllm-service-bslgp
  ports:
    - {name: dashboard, port: 8265, targetPort: 8265, nodePort: 30006}

curl -s http://localhost:30006/api/serve/applications/ | jq

{
  "controller_info": {
    "node_id": "6b647d452cfe4f4e135c68b4695518c0ab8645111ec5361cf069fee0",
    "node_ip": "10.244.0.9",
    "node_instance_id": "vllm-service-bslgp-head-ks8hd",
    "actor_id": "8cc20adab0daf4293ad5396701000000",
    "actor_name": "SERVE_CONTROLLER_ACTOR"
  }
}

# kps-values.yaml
# kind 환경 조정: 호스트에서 접근 불가한 컨트롤플레인 컴포넌트 스크레이프 비활성
kubeControllerManager: {enabled: false}
kubeScheduler:         {enabled: false}
kubeEtcd:              {enabled: false}
kubeProxy:             {enabled: false}

prometheus:
  service:
    type: NodePort
    nodePort: 30008
  prometheusSpec:
    # 차트 기본값은 자기 릴리스 라벨이 붙은 것만 수집한다. 전부 수집하도록 해제
    podMonitorSelectorNilUsesHelmValues: false
    serviceMonitorSelectorNilUsesHelmValues: false
    retention: 6h

grafana:
  service:
    type: NodePort
    nodePort: 30007
  adminPassword: admin

helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm install kube-prometheus-stack prometheus-community/kube-prometheus-stack \
  -n monitoring --create-namespace -f kps-values.yaml --timeout 15m

kubectl get node -o wide      # INTERNAL-IP 확인 → 172.18.0.2
curl -s -o /dev/null -w "grafana:    HTTP %{http_code}\n" http://172.18.0.2:30007/login
curl -s -o /dev/null -w "prometheus: HTTP %{http_code}\n" http://172.18.0.2:30008/-/ready

apiVersion: monitoring.coreos.com/v1
kind: PodMonitor
metadata:
  name: ray-head-monitor
  namespace: monitoring
spec:
  jobLabel: ray-head
  namespaceSelector: {matchNames: [kuberay]}
  selector: {matchLabels: {ray.io/node-type: head}}
  podMetricsEndpoints:
    - port: metrics
      relabelings:
        - {action: replace, sourceLabels: [__meta_kubernetes_pod_label_ray_io_cluster], targetLabel: ray_io_cluster}
---
apiVersion: monitoring.coreos.com/v1
kind: PodMonitor
metadata:
  name: ray-workers-monitor
  namespace: monitoring
spec:
  jobLabel: ray-workers
  namespaceSelector: {matchNames: [kuberay]}
  selector: {matchLabels: {ray.io/node-type: worker}}
  podMetricsEndpoints:
    - port: metrics
      relabelings:
        - {action: replace, sourceLabels: [__meta_kubernetes_pod_label_ray_io_cluster], targetLabel: ray_io_cluster}

curl -s "http://172.18.0.2:30008/api/v1/targets?state=active" | jq -r \
  '.data.activeTargets[] | select(.scrapePool|test("ray")) | "\(.scrapePool) \(.health) \(.labels.pod)"'

BASE=https://raw.githubusercontent.com/ray-project/kuberay/master/config/grafana
for f in default_grafana_dashboard.json serve_grafana_dashboard.json \
         serve_deployment_grafana_dashboard.json serve_llm_grafana_dashboard.json; do
  curl -sfLO $BASE/$f
  n=$(echo $f | sed 's/_grafana_dashboard.json//' | tr '_' '-')
  kubectl create configmap ray-dash-$n -n monitoring --from-file=$f \
    --dry-run=client -o yaml \
  | kubectl label -f - --local -o yaml grafana_dashboard=1 \
  | kubectl apply -f -
done

curl -s "http://172.18.0.2:30008/api/v1/query?query=sum(ray_serve_num_http_requests_total)"

    headGroupSpec:
      template:
        spec:
          containers:
            - name: ray-head
              env:
                - {name: RAY_GRAFANA_HOST,        value: "http://kube-prometheus-stack-grafana.monitoring.svc:80"}
                - {name: RAY_PROMETHEUS_HOST,     value: "http://kube-prometheus-stack-prometheus.monitoring.svc:9090"}
                - {name: RAY_GRAFANA_IFRAME_HOST, value: "http://172.16.0.44:30007"}
                - {name: RAY_PROMETHEUS_NAME,     value: "Prometheus"}

  grafana.ini:
    security:
      allow_embedding: true       # iframe 삽입 허용
    auth.anonymous:
      enabled: true
      org_role: Viewer            # 로그인 없이 패널 조회

docker run -d --name grafana-fwd --network host alpine/socat \
  tcp-listen:30007,fork,reuseaddr tcp-connect:172.18.0.2:30007

kubectl delete raycluster vllm-service-bslgp -n kuberay

kubectl patch svc vllm-service-nodeport -n kuberay \
  -p '{"spec":{"selector":{"ray.io/node-type":"head","ray.io/cluster":"vllm-service-xrcnp"}}}'

curl -s http://localhost:30006/api/grafana_health | jq

{
  "result": true,
  "msg": "Grafana running",
  "data": {
    "grafanaHost": "http://172.16.0.44:30007",
    "grafanaOrgId": "1",
    "dashboardUids": {
      "default": "rayDefaultDashboard",
      "serve": "rayServeDashboard",
      "serveDeployment": "rayServeDeploymentDashboard",
      "serveLlm": "rayServeLlmDashboard"
    }
  }
}

import concurrent.futures, json, time, urllib.request

URL = "http://localhost:30005/v1/chat/completions"
PAYLOAD = json.dumps({
    "model": "qwen2.5-1.5b-instruct-awq",
    "messages": [{"role": "user", "content": "Explain Kubernetes in one sentence."}],
    "max_tokens": 64,
}).encode()

def call():
    t0 = time.time()
    req = urllib.request.Request(URL, data=PAYLOAD, headers={"Content-Type": "application/json"})
    try:
        with urllib.request.urlopen(req, timeout=60) as r:
            r.read()
        return time.time() - t0, True
    except Exception:
        return time.time() - t0, False

def worker(stop_at):
    out = []
    while time.time() < stop_at:
        out.append(call())
    return out

DUR, CONC = 180, 4
stop_at = time.time() + DUR
with concurrent.futures.ThreadPoolExecutor(max_workers=CONC) as ex:
    fs = [ex.submit(worker, stop_at) for _ in range(CONC)]
    res = [r for f in fs for r in f.result()]

lat = sorted(r[0] for r in res)
ok = sum(1 for r in res if r[1])
p = lambda q: lat[min(int(len(lat)*q), len(lat)-1)]
print(f"total={len(res)} success={ok} fail={len(res)-ok}")
print(f"avg={sum(lat)/len(lat):.2f}s p50={p(0.50):.2f}s p95={p(0.95):.2f}s p99={p(0.99):.2f}s")
print(f"throughput={len(res)/DUR:.2f} req/s  (concurrency={CONC})")

kubectl delete -f vllm-service-6gb.yaml
kubectl delete svc -n kuberay vllm-service-nodeport vllm-service-dashboard-nodeport
docker rm -f grafana-fwd                       # socat 중계 제거
helm uninstall kube-prometheus-stack -n monitoring
helm uninstall kuberay-operator -n kuberay-system
helm uninstall nvdp -n nvidia-device-plugin
kubectl delete namespace kuberay kuberay-system nvidia-device-plugin monitoring

nvidia-smi                                      # GPU 반환 확인
kind delete cluster --name gpu                  # 클러스터째 삭제
docker exec gpu-control-plane crictl rmi \
  docker.io/rayproject/ray-llm:2.52.0-py311-cu128   # 클러스터를 남긴다면 이미지만 회수