$ aws iam list-open-id-connect-providers
{
    "OpenIDConnectProviderList": []
}

"tp_degree": 2,
"pp_degree": 1,
"cp_degree": 1,
"ep_degree": 1

"speculation_length": 0,
"enable_fused_speculation": false,
"enable_eagle_speculation": false,
"num_medusa_heads": 0

# workshop 계정에서 발급된 EC2에 접속한다
MYEC2IP=<EC2_PUBLIC_IP>
ssh -i ws-default-keypair.pem ubuntu@$MYEC2IP

#
whoami
hostnamectl
...
  Hardware Model: t3.2xlarge

htop
df -hT
ip -c addr
docker info

#
sudo apt update && sudo apt install tree jq -y
tree -a workshop/
workshop/
└── .env

cat workshop/.env
HF_TOKEN="hf_****************************"

cd workshop
pwd

# Update package list and install tools
sudo apt update
sudo apt install -y python3-pip jq unzip

# Install AWS CLI v2
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
unzip awscliv2.zip
sudo ./aws/install --update

# Install Helm
curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash

# Enable kubectl autocompletion for current session and add to bashrc
source <(kubectl completion bash) && echo "source <(kubectl completion bash)" >> ~/.bashrc

# Verify installations
aws --version
helm version --short
jq --version

$ aws --version
aws-cli/2.36.43 Python/3.14.6 Linux/6.8.0-1035-aws exe/x86_64.ubuntu.22

$ helm version --short
v3.22.0+g144ca65

$ eksctl version
0.230.0

$ kubectl version --client
Client Version: v1.37.0
Kustomize Version: v5.8.1

export AWS_REGION=us-west-2
export CLUSTER_NAME=ai-infra-summit-test-cluster
export EKS_VERSION=1.33
export INSTANCE_TYPE=trn1.2xlarge
export DESIRED_NODES=1
export WORKER_AMI=$(aws ssm get-parameter \
    --name /aws/service/eks/optimized-ami/1.33/amazon-linux-2023/x86_64/neuron/recommended/image_id \
    --region $AWS_REGION \
    --query "Parameter.Value" \
    --output text)
export AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
export BUCKET_NAME=ai-infra-summit-vllm-models-cache-${AWS_ACCOUNT_ID}

echo "$CLUSTER_NAME $WORKER_AMI $AWS_ACCOUNT_ID $BUCKET_NAME"

ai-infra-summit-test-cluster ami-0e08c07b0376ba3f8 <ACCOUNT_ID> ai-infra-summit-vllm-models-cache-<ACCOUNT_ID>

$ aws sts get-caller-identity
{
    "UserId": "<ROLE_ID>:<EC2_INSTANCE_ID>",
    "Account": "<ACCOUNT_ID>",
    "Arn": "arn:aws:sts::<ACCOUNT_ID>:assumed-role/ec2-workshop-WorkshopInstanceRole-MXbUd3DpMGVq/<EC2_INSTANCE_ID>"
}

aws eks update-kubeconfig --region $AWS_REGION --name $CLUSTER_NAME

Updated context arn:aws:eks:us-west-2:<ACCOUNT_ID>:cluster/ai-infra-summit-test-cluster in /home/ubuntu/.kube/config

$ aws eks describe-cluster --name $CLUSTER_NAME --region $AWS_REGION \
    --query "cluster.{status:status,version:version,createdAt:createdAt,vpc:resourcesVpcConfig.vpcId}"
{
    "status": "ACTIVE",
    "version": "1.33",
    "createdAt": "2026-09-11T03:55:40.647000+00:00",
    "vpc": "<VPC_ID>"
}

aws eks describe-cluster --name $CLUSTER_NAME --region $AWS_REGION \
    --query "cluster.resourcesVpcConfig"

{
    "subnetIds": [
        "<SUBNET-2A-PUBLIC>",
        "<SUBNET-2B-PUBLIC>",
        "<SUBNET-2C-PUBLIC>",
        "<SUBNET-2D-PUBLIC>",
        "<SUBNET-2A-PRIVATE>",
        "<SUBNET-2B-PRIVATE>",
        "<SUBNET-2C-PRIVATE>",
        "<SUBNET-2D-PRIVATE>"
    ],
    "securityGroupIds": [],
    "clusterSecurityGroupId": "<CLUSTER-SG>",
    "vpcId": "<VPC_ID>",
    "endpointPublicAccess": false,
    "endpointPrivateAccess": true,
    "publicAccessCidrs": [],
    "controlPlaneEgressMode": "AWS_MANAGED"
}

aws ec2 describe-subnets --region $AWS_REGION \
    --filters "Name=vpc-id,Values=<VPC_ID>" \
    --query "Subnets[].{id:SubnetId,az:AvailabilityZone,cidr:CidrBlock,public:MapPublicIpOnLaunch}" \
    --output table

----------------------------------------------------------------
|                       DescribeSubnets                        |
+------------+--------------+-----------------------+----------+
|     az     |     cidr     |           id          |  public  |
+------------+--------------+-----------------------+----------+
|  us-west-2b|  10.0.7.0/24 |  <SUBNET-2B-PRIVATE>  |  False   |
|  us-west-2b|  10.0.5.0/24 |  <SUBNET-2B-PUBLIC>   |  True    |
|  us-west-2c|  10.0.8.0/24 |  <SUBNET-2C-PRIVATE>  |  False   |
|  us-west-2a|  10.0.3.0/24 |  <SUBNET-2A-PRIVATE>  |  False   |
|  us-west-2a|  10.0.1.0/24 |  <SUBNET-2A-PUBLIC>   |  True    |
|  us-west-2d|  10.0.2.0/24 |  <SUBNET-2D-PUBLIC>   |  True    |
|  us-west-2d|  10.0.4.0/24 |  <SUBNET-2D-PRIVATE>  |  False   |
|  us-west-2c|  10.0.6.0/24 |  <SUBNET-2C-PUBLIC>   |  True    |
+------------+--------------+-----------------------+----------+

ssh-keygen -t rsa -b 3072 -f ~/.ssh/id_rsa -N ""

apiVersion: eksctl.io/v1alpha5
kind: ClusterConfig
metadata:
  name: ai-infra-summit-test-cluster
  region: us-west-2
  version: "1.33"
vpc:
  id: <VPC_ID>
  subnets:
    public:
      us-west-2b: { id: <SUBNET-2B-PUBLIC> }
      us-west-2d: { id: <SUBNET-2D-PUBLIC> }
  securityGroup: <CLUSTER-SG>
managedNodeGroups:
- name: neuron-trn1-2x
  ami: ami-0e08c07b0376ba3f8
  amiFamily: AmazonLinux2023
  subnets: ["<SUBNET-2B-PUBLIC>", "<SUBNET-2D-PUBLIC>"]
  iam:
    attachPolicyARNs:
    - arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy
    - arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly
    - arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore
    - arn:aws:iam::aws:policy/AmazonS3FullAccess
    - arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy
  instanceType: trn1.2xlarge
  desiredCapacity: 1
  volumeSize: 100
  volumeType: gp2
  ssh:
     allow: true
     publicKeyPath: ~/.ssh/id_rsa.pub

eksctl create nodegroup -f eks_nodegroup.yaml

$ eksctl get nodegroup --cluster $CLUSTER_NAME --region $AWS_REGION
CLUSTER                       NODEGROUP       STATUS  CREATED               MIN SIZE  MAX SIZE  DESIRED CAPACITY  INSTANCE TYPE  IMAGE ID               TYPE
ai-infra-summit-test-cluster  neuron-trn1-2x  ACTIVE  2026-09-11T13:47:03Z  1         1         1                 trn1.2xlarge   ami-0e08c07b0376ba3f8  managed

$ kubectl get nodes -o wide
NAME                                       STATUS   ROLES    AGE   VERSION                INTERNAL-IP   EXTERNAL-IP     OS-IMAGE                        KERNEL-VERSION                     CONTAINER-RUNTIME
ip-10-0-5-142.us-west-2.compute.internal   Ready    <none>   11m   v1.33.13-eks-cb19647   10.0.5.142    34.222.123.83   Amazon Linux 2023.12.20260831   6.12.103-127.188.amzn2023.x86_64   containerd://2.2.5+unknown

helm install neuron-helm-chart oci://public.ecr.aws/neuron/neuron-helm-chart \
  --set "npd.enabled=false"

helm upgrade neuron-helm-chart oci://public.ecr.aws/neuron/neuron-helm-chart \
  --set "npd.enabled=false" \
  --set "scheduler.enabled=true"

$ helm history neuron-helm-chart -n default
REVISION  UPDATED                   STATUS      CHART                     APP VERSION  DESCRIPTION
1         Fri Sep 11 13:56:32 2026  superseded  neuron-helm-chart-1.10.0  1.10.0       Install complete
2         Fri Sep 11 13:56:53 2026  deployed    neuron-helm-chart-1.10.0  1.10.0       Upgrade complete

$ helm get values neuron-helm-chart -n default
USER-SUPPLIED VALUES:
npd:
  enabled: false
scheduler:
  enabled: true

$ kubectl get pods -n kube-system | grep -E "neuron|scheduler"
k8s-neuron-scheduler-785c8d99f8-zbrt6   1/1     Running   0   2m23s
my-scheduler-55f56bc9f8-25m4c           1/1     Running   0   2m23s
neuron-device-plugin-vl2jb              1/1     Running   0   2m45s

$ kubectl get node -o json | jq -r '.items[] | .status.allocatable'
{
  "aws.amazon.com/neuron": "1",
  "aws.amazon.com/neuroncore": "2",
  "cpu": "7910m",
  "ephemeral-storage": "95491281146",
  "hugepages-1Gi": "0",
  "hugepages-2Mi": "0",
  "memory": "31315408Ki",
  "pods": "58"
}

aws s3 mb s3://${BUCKET_NAME} --region $AWS_REGION
aws s3 ls

2026-09-11 13:56:09 ai-infra-summit-vllm-models-cache-<ACCOUNT_ID>

helm repo add aws-mountpoint-s3-csi-driver https://awslabs.github.io/mountpoint-s3-csi-driver
helm repo update
helm upgrade --install aws-mountpoint-s3-csi-driver \
  aws-mountpoint-s3-csi-driver/aws-mountpoint-s3-csi-driver \
  --namespace kube-system

$ kubectl get pods -n kube-system | grep s3-csi
s3-csi-controller-5df587766f-dqb2z   1/1     Running   0   2m
s3-csi-node-5b6lt                    3/3     Running   0   2m

$ helm list -A
NAME                          NAMESPACE    REVISION  STATUS    CHART                               APP VERSION
aws-mountpoint-s3-csi-driver  kube-system  1         deployed  aws-mountpoint-s3-csi-driver-2.8.0
neuron-helm-chart             default      2         deployed  neuron-helm-chart-1.10.0            1.10.0

$ kubectl get pods -A
NAMESPACE     NAME                                    READY   STATUS    RESTARTS   AGE
kube-system   aws-node-hkdj6                          2/2     Running   0          11m
kube-system   coredns-75cb89d95b-lf4lz                1/1     Running   0          9h
kube-system   coredns-75cb89d95b-v5rzs                1/1     Running   0          9h
kube-system   k8s-neuron-scheduler-785c8d99f8-zbrt6   1/1     Running   0          2m23s
kube-system   kube-proxy-jtwxj                        1/1     Running   0          11m
kube-system   my-scheduler-55f56bc9f8-25m4c           1/1     Running   0          2m23s
kube-system   neuron-device-plugin-vl2jb              1/1     Running   0          2m45s
kube-system   s3-csi-controller-5df587766f-dqb2z      1/1     Running   0          2m
kube-system   s3-csi-node-5b6lt                       3/3     Running   0          2m

$ aws iam list-open-id-connect-providers
{
    "OpenIDConnectProviderList": []
}

$ eksctl get cluster --region us-west-2
Error: failed to list clusters in region "us-west-2": ... AccessDeniedException:
User: arn:aws:sts::<ACCOUNT_ID>:assumed-role/ec2-workshop-WorkshopInstanceRole-... is not
authorized to perform: eks:ListClusters ...

rm -rf /home/ubuntu/workshop/.env
echo 'HF_TOKEN="hf_****"' > /home/ubuntu/workshop/.env

source /home/ubuntu/workshop/.env
kubectl create secret generic hf-token-secret \
    --from-literal=HF_TOKEN="$HF_TOKEN" \
    --dry-run=client -o yaml | kubectl apply -f -

$ kubectl get secret hf-token-secret
NAME              TYPE     DATA   AGE
hf-token-secret   Opaque   1      40m

cat > vllm-configmap.yaml <<EOF
apiVersion: v1
kind: ConfigMap
metadata:
  name: vllm-shared-config
data:
  HF_TOKEN: "$HF_TOKEN"
  MODEL_NAME: "tinyLlama/TinyLlama-1.1B-Chat-v1.0"
  S3_BUCKET: "ai-infra-summit-vllm-models-cache-$(aws sts get-caller-identity | jq -r .Account)"
  S3_PREFIX: "compiled-models"
  MAX_NUM_SEQS: "4"
  PORT: "8080"
  NEURON_COMPILED_ARTIFACTS: "/shared/model/cache"
  NEURON_COMPILE_CACHE_URL: "/shared/model/cache"
  TENSOR_PARALLEL_SIZE: "2"
  MAX_MODEL_LEN: "1024"
  NEURON_RT_VISIBLE_CORES: "0-1"
  NEURON_RT_LOG_LEVEL: "ERROR"
  NEURON_RT_ASYNC_EXEC_MAX_INFLIGHT_REQUESTS: "4"
  VLLM_NEURON_FRAMEWORK: "neuronx-distributed-inference"
EOF
kubectl apply -f vllm-configmap.yaml

{
  "MAX_MODEL_LEN": "1024",
  "MAX_NUM_SEQS": "4",
  "MODEL_NAME": "tinyLlama/TinyLlama-1.1B-Chat-v1.0",
  "NEURON_COMPILED_ARTIFACTS": "/shared/model/cache",
  "NEURON_COMPILE_CACHE_URL": "/shared/model/cache",
  "NEURON_RT_ASYNC_EXEC_MAX_INFLIGHT_REQUESTS": "4",
  "NEURON_RT_LOG_LEVEL": "ERROR",
  "NEURON_RT_VISIBLE_CORES": "0-1",
  "PORT": "8080",
  "S3_BUCKET": "ai-infra-summit-vllm-models-cache-<ACCOUNT_ID>",
  "S3_PREFIX": "compiled-models",
  "TENSOR_PARALLEL_SIZE": "2",
  "VLLM_NEURON_FRAMEWORK": "neuronx-distributed-inference"
}

cat > vllm-storage.yaml <<EOF
apiVersion: v1
kind: PersistentVolume
metadata:
  name: s3-model-cache-pv
spec:
  capacity:
    storage: 100Gi
  accessModes:
    - ReadWriteMany
  persistentVolumeReclaimPolicy: Retain
  csi:
    driver: s3.csi.aws.com
    volumeHandle: ai-infra-summit-vllm-models-cache-$(aws sts get-caller-identity | jq -r .Account)
    volumeAttributes:
      bucketName: ai-infra-summit-vllm-models-cache-$(aws sts get-caller-identity | jq -r .Account)
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: s3-model-cache-pvc
spec:
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 100Gi
  volumeName: s3-model-cache-pv
EOF
kubectl apply -f vllm-storage.yaml

$ kubectl get pvc,pv
NAME                                       STATUS   VOLUME              CAPACITY   ACCESS MODES   RECLAIM POLICY   AGE
persistentvolumeclaim/s3-model-cache-pvc   Bound    s3-model-cache-pv   100Gi      RWX                             39m

NAME                                 CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM                        AGE
persistentvolume/s3-model-cache-pv   100Gi      RWX            Retain           Bound    default/s3-model-cache-pvc   39m

$ kubectl describe node | grep nodegroup-name
                    alpha.eksctl.io/nodegroup-name=neuron-trn1-2x

spec:
  schedulerName: my-scheduler
  nodeSelector:
    alpha.eksctl.io/nodegroup-name: neuron-trn1-2x
  volumes:
    - name: model-storage
      persistentVolumeClaim:
        claimName: s3-model-cache-pvc
  initContainers:
    - name: model-prep
      image: public.ecr.aws/neuron/pytorch-inference-vllm-neuronx:0.9.1-neuronx-py310-sdk2.25.0-ubuntu22.04
      envFrom:
        - configMapRef: { name: vllm-shared-config }
        - secretRef:    { name: hf-token-secret }
      resources:
        limits:   { aws.amazon.com/neuron: 1, ephemeral-storage: 50Gi }
        requests: { aws.amazon.com/neuron: 1, ephemeral-storage: 50Gi }
      volumeMounts:
        - name: model-storage
          mountPath: /shared/model
  containers:
    - name: vllm-server
      image: public.ecr.aws/neuron/pytorch-inference-vllm-neuronx:0.9.1-neuronx-py310-sdk2.25.0-ubuntu22.04
      ports:
        - containerPort: 8080
          name: http-vllm
      volumeMounts:
        - name: model-storage
          mountPath: /shared/model
          readOnly: true
      resources:
        limits:   { aws.amazon.com/neuron: 1, ephemeral-storage: 50Gi, cpu: "8000m" }
        requests: { aws.amazon.com/neuron: 1, ephemeral-storage: 50Gi, cpu: "4000m" }

kubectl apply -f vllm-deployment.yaml
kubectl get pod -l app.kubernetes.io/name=vllm-server -w

$ kubectl get deployment vllm-deployment -o wide
NAME              READY   UP-TO-DATE   AVAILABLE   AGE   CONTAINERS    IMAGES
vllm-deployment   1/1     1            1           38m   vllm-server   public.ecr.aws/neuron/pytorch-inference-vllm-neuronx:0.9.1-...

$ kubectl get pods -l app.kubernetes.io/name=vllm-server -o wide
NAME                               READY   STATUS    RESTARTS   AGE   IP          NODE
vllm-deployment-64597fb8cc-cfjqk   1/1     Running   0          38m   10.0.5.40   ip-10-0-5-142.us-west-2.compute.internal

$ kubectl get pod -l app.kubernetes.io/name=vllm-server \
    -o jsonpath='{.items[0].spec.schedulerName}{"\n"}{.items[0].spec.nodeSelector}'
my-scheduler
{"alpha.eksctl.io/nodegroup-name":"neuron-trn1-2x"}

kubectl get pod -l app.kubernetes.io/name=vllm-server -o json | jq -r '.items[] |
{podCreated: .metadata.creationTimestamp,
 initStart:  .status.initContainerStatuses[0].state.terminated.startedAt,
 initFinish: .status.initContainerStatuses[0].state.terminated.finishedAt,
 mainStart:  .status.containerStatuses[0].state.running.startedAt}'

$ kubectl logs -l app.kubernetes.io/name=vllm-server -c model-prep --tail=8
INFO:Neuron:Finished weights loading in 12.716184116000022 seconds
INFO:Neuron:Warming up the model.
INFO:Neuron:Warmup completed in 0.4222989082336426 seconds.
INFO 09-11 14:23:11 [executor_base.py:112] # neuron blocks: 4, # CPU blocks: 0
INFO 09-11 14:23:11 [executor_base.py:117] Maximum concurrency for 1024 tokens per request: 4.00x
INFO 09-11 14:23:11 [llm_engine.py:434] init engine (profile, create kv cache, warmup model) took 0.00 seconds
Model compiled successfully!

CCOM WARN NET/Plugin: Error: libnccom-net.so load failed. libfabric.so.1: cannot open shared object file

$ aws s3 ls s3://$BUCKET_NAME --recursive --human-readable
2026-09-11 14:23:13    4.6 MiB cache/model.pt
2026-09-11 14:23:14    6.1 KiB cache/neuron_config.json
2026-09-11 14:23:14  359 Bytes cache/neuronxcc-2.20.9961.0+0acef03a/MODULE_56f0d314fda2b6e1e336+617f6939/compile_flags.json
2026-09-11 14:23:15    0 Bytes cache/neuronxcc-2.20.9961.0+0acef03a/MODULE_56f0d314fda2b6e1e336+617f6939/model.done
2026-09-11 14:23:14  547.2 KiB cache/neuronxcc-2.20.9961.0+0acef03a/MODULE_56f0d314fda2b6e1e336+617f6939/model.hlo_module.pb
2026-09-11 14:23:14    1.5 MiB cache/neuronxcc-2.20.9961.0+0acef03a/MODULE_56f0d314fda2b6e1e336+617f6939/model.neff
2026-09-11 14:23:15    1.6 MiB cache/neuronxcc-2.20.9961.0+0acef03a/MODULE_56f0d314fda2b6e1e336+617f6939/wrapped_neff.hlo
2026-09-11 14:23:15  359 Bytes cache/neuronxcc-2.20.9961.0+0acef03a/MODULE_ae92d68443828ba4e463+ad9e832d/compile_flags.json
2026-09-11 14:23:16    0 Bytes cache/neuronxcc-2.20.9961.0+0acef03a/MODULE_ae92d68443828ba4e463+ad9e832d/model.done
2026-09-11 14:23:15  851.5 KiB cache/neuronxcc-2.20.9961.0+0acef03a/MODULE_ae92d68443828ba4e463+ad9e832d/model.hlo_module.pb
2026-09-11 14:23:15  741.0 KiB cache/neuronxcc-2.20.9961.0+0acef03a/MODULE_ae92d68443828ba4e463+ad9e832d/model.neff

Total Objects: 11
   Total Size: 10320638

$ kubectl exec -it deploy/vllm-deployment -c vllm-server -- ls -R -1 /shared/model
/shared/model:
cache

/shared/model/cache:
model.pt
neuron_config.json
neuronxcc-2.20.9961.0+0acef03a

/shared/model/cache/neuronxcc-2.20.9961.0+0acef03a:
MODULE_56f0d314fda2b6e1e336+617f6939
MODULE_ae92d68443828ba4e463+ad9e832d

$ kubectl exec -it deploy/vllm-deployment -c vllm-server -- env | grep -E 'NEURON|VLLM|MAX|TENSOR' | sort
MAX_MODEL_LEN=1024
MAX_NUM_SEQS=4
NEURON_COMPILED_ARTIFACTS=/shared/model/cache
NEURON_COMPILE_CACHE_URL=/shared/model/cache
NEURON_LOGICAL_NC_CONFIG=1
NEURON_RT_ASYNC_EXEC_MAX_INFLIGHT_REQUESTS=4
NEURON_RT_LOG_LEVEL=ERROR
NEURON_RT_VISIBLE_CORES=0-1
TENSOR_PARALLEL_SIZE=2
VLLM_NEURON_FRAMEWORK=neuronx-distributed-inference
VLLM_TARGET_DEVICE=neuron

kubectl debug node/ip-10-0-5-142.us-west-2.compute.internal -it \
  --image=public.ecr.aws/amazonlinux/amazonlinux:2023 --profile=sysadmin \
  -- chroot /host bash

$ ctr -n k8s.io images ls | grep -i vllm
public.ecr.aws/neuron/pytorch-inference-vllm-neuronx:0.9.1-neuronx-py310-sdk2.25.0-ubuntu22.04
public.ecr.aws/neuron/pytorch-inference-vllm-neuronx@sha256:01f0f7b1e2cf256019a80c16712e79a5f254b04a3a77dbf8ac196de4ee380928

$ ps -ef | grep -E "mount-s3|api_server" | grep -v grep
ec2-user 14990 14969 0 14:16 ? 00:00:00 /mountpoint-s3/bin/mount-s3 ai-infra-summit-vllm-models-cache-<ACCOUNT_ID> /dev/fd/3 --allow-root --foreground --user-agent-prefix=s3-csi-driver/2.8.0 credential-source#driver k8s/v1.33.13-eks-4cc7921 md/install#helm
root     17827 15119 0 14:23 ? 00:00:09 python -m vllm.entrypoints.openai.api_server --model=tinyLlama/TinyLlama-1.1B-Chat-v1.0 --max-num-seqs=4 --max-model-len=1024 --tensor-parallel-size=2 --port=8080 --device=neuron --override-neuron-config={"enable_bucketing":false}

$ mount | grep -i s3
mountpoint-s3 on /var/lib/kubelet/plugins/s3.csi.aws.com/mnt/mp-8jfx5 type fuse (rw,nosuid,nodev,noatime,user_id=0,group_id=0,default_permissions,allow_other)
mountpoint-s3 on /var/lib/kubelet/pods/ea2412ea-.../volumes/kubernetes.io~csi/s3-model-cache-pv/mount type fuse (rw,nosuid,nodev,noatime,user_id=0,group_id=0,default_permissions,allow_other)

$ /opt/aws/neuron/bin/neuron-ls
instance-type: trn1.2xlarge
instance-id: <NODE_INSTANCE_ID>
+--------+--------+----------+--------+--------------+-------+----------+------+------------------------------------------+---------+
| NEURON | NEURON |  NEURON  | NEURON |     PCI      |  PID  |   CPU    | NUMA |                 COMMAND                  | RUNTIME |
| DEVICE | CORES  | CORE IDS | MEMORY |     BDF      |       | AFFINITY | NODE |                                          | VERSION |
+--------+--------+----------+--------+--------------+-------+----------+------+------------------------------------------+---------+
| 0      | 2      | 0-1      | 32 GB  | 0000:00:1e.0 | 17954 | 0-7      | -1   | /opt/conda/bin/python -c from multipr... | 2.27.23 |
+--------+--------+----------+--------+--------------+-------+----------+------+------------------------------------------+---------+

kubectl exec -it deploy/vllm-deployment -c vllm-server -- \
  curl -s http://localhost:8080/v1/models | jq .data

cat > vllm-service.yaml <<EOF
apiVersion: v1
kind: Service
metadata:
  name: vllm-service
spec:
  selector:
    app.kubernetes.io/name: vllm-server
  ports:
    - protocol: TCP
      port: 8080
      targetPort: http-vllm
  type: LoadBalancer
EOF
kubectl apply -f vllm-service.yaml

$ kubectl get svc,ep vllm-service
NAME                   TYPE           CLUSTER-IP       EXTERNAL-IP                                                               PORT(S)          AGE
service/vllm-service   LoadBalancer   172.20.216.178   <VLLM-ELB>   8080:30732/TCP   13m

NAME                     ENDPOINTS        AGE
endpoints/vllm-service   10.0.5.40:8080   13m

$ aws ec2 describe-security-groups --group-ids <ELB-SG> \
    --query "SecurityGroups[0].{name:GroupName,desc:Description}"
{
    "name": "k8s-elb-<VLLM-ELB-ID>",
    "desc": "Security group for Kubernetes ELB <VLLM-ELB-ID> (default/vllm-service)"
}

export VLLM_ENDPOINT="http://<VLLM-ELB>:8080"

curl -s $VLLM_ENDPOINT/v1/models | jq .data

[
  {
    "id": "tinyLlama/TinyLlama-1.1B-Chat-v1.0",
    "object": "model",
    "created": 1789138204,
    "owned_by": "vllm",
    "root": "tinyLlama/TinyLlama-1.1B-Chat-v1.0",
    "parent": null,
    "max_model_len": 1024
  }
]

curl -s $VLLM_ENDPOINT/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "tinyLlama/TinyLlama-1.1B-Chat-v1.0",
    "messages": [{"role": "user", "content": "What is AWS Trainium in one sentence?"}],
    "max_tokens": 80
  }'

{
  "id": "chatcmpl-1f0fa6b77ee34cf298c48602bba5267b",
  "model": "tinyLlama/TinyLlama-1.1B-Chat-v1.0",
  "choices": [{
    "index": 0,
    "message": {
      "role": "assistant",
      "content": "AWS Trainium is an open-source, interactive website that provides demonstrations of Amazon Web Services (AWS) functionality and examples of successful deployments to help users better understand the capabilities and features of the platform."
    },
    "finish_reason": "stop"
  }],
  "usage": {"prompt_tokens": 25, "completion_tokens": 45, "total_tokens": 70}
}

kubectl port-forward svc/vllm-service 8080:8080 &
export VLLM_ENDPOINT="http://localhost:8080"
curl -s $VLLM_ENDPOINT/v1/models | jq .data

kill %1

from openai import OpenAI
import sys, os

def main():
    base_endpoint = os.getenv("VLLM_ENDPOINT")
    if not base_endpoint:
        print("Error: VLLM_ENDPOINT environment variable is not set")
        sys.exit(1)

    client = OpenAI(api_key="EMPTY", base_url=f"{base_endpoint}/v1")
    model_name = client.models.list().data[0].id
    print(f"Connected! Using model: {model_name}")

    print("Chat (type 'exit' to quit):")
    while True:
        user_input = input("\nYou: ").strip()
        if user_input.lower() in ['exit', 'quit', 'bye'] or not user_input:
            break
        response = client.chat.completions.create(
            model=model_name,
            messages=[{"role": "user", "content": user_input}],
            max_tokens=900,
            temperature=1.0,
            extra_body={'top_k': 50})
        print("AI:", response.choices[0].message.content)

if __name__ == "__main__":
    main()

pip install openai
python3 test-vllm-pod.py

# 컨트롤러 로그
kubectl logs -n kube-system -l name=neuron-device-plugin-ds
kubectl logs -n kube-system -l app.kubernetes.io/component=k8s-neuron-scheduler

# 이벤트 / 리소스 확인
kubectl get events --sort-by=.metadata.creationTimestamp
kubectl describe pods -l app.kubernetes.io/name=vllm-server | grep -A 10 "Requests"
kubectl describe nodes -l alpha.eksctl.io/nodegroup-name=neuron-trn1-2x

# 컨테이너별 로그
kubectl logs -l app.kubernetes.io/name=vllm-server -c model-prep -f
kubectl logs -l app.kubernetes.io/name=vllm-server -c vllm-server -f

echo "$AWS_REGION $CLUSTER_NAME"

us-west-2 ai-infra-summit-test-cluster

# Add NGINX Ingress Controller Helm repository
helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx

# Install NGINX Ingress Controller
helm upgrade --install ingress-nginx ingress-nginx \
  --repo https://kubernetes.github.io/ingress-nginx \
  --namespace ingress-nginx \
  --create-namespace

# Wait for the ingress controller to be ready
kubectl wait --namespace ingress-nginx \
  --for=condition=ready pod \
  --selector=app.kubernetes.io/component=controller \
  --timeout=90s

"ingress-nginx" has been added to your repositories
...
pod/ingress-nginx-controller-6797f4dc8c-7cqcq condition met

$ helm list -n ingress-nginx
NAME           NAMESPACE      REVISION  UPDATED                                 STATUS    CHART                 APP VERSION
ingress-nginx  ingress-nginx  1         2026-09-11 15:09:53.621833545 +0000 UTC deployed  ingress-nginx-4.15.1  1.15.1

$ kubectl get pod -n ingress-nginx -o wide
NAME                                        READY   STATUS    RESTARTS   AGE   IP           NODE
ingress-nginx-controller-6797f4dc8c-7cqcq   1/1     Running   0          18s   10.0.5.252   ip-10-0-5-142.us-west-2.compute.internal

$ kubectl get svc,ep -n ingress-nginx ingress-nginx-controller
NAME                               TYPE           CLUSTER-IP       EXTERNAL-IP                                                               PORT(S)                      AGE
service/ingress-nginx-controller   LoadBalancer   172.20.238.179   <INGRESS-ELB>   80:30381/TCP,443:32447/TCP   19s

NAME                                 ENDPOINTS                      AGE
endpoints/ingress-nginx-controller   10.0.5.252:443,10.0.5.252:80   19s

$ kubectl get ingressclass
NAME    CONTROLLER             PARAMETERS   AGE
nginx   k8s.io/ingress-nginx   <none>       20s

$ curl -i http://<INGRESS-ELB>/
HTTP/1.1 404 Not Found
Date: Fri, 11 Sep 2026 15:11:52 GMT
Content-Type: text/html
Content-Length: 146
Connection: keep-alive

<html>
<head><title>404 Not Found</title></head>
<body>
<center><h1>404 Not Found</h1></center>
<hr><center>nginx</center>
</body>
</html>

http://<VLLM-ELB>:8080/v1/models

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: vllm-ingress-simple
  namespace: default
  annotations:
    nginx.ingress.kubernetes.io/rewrite-target: /
spec:
  ingressClassName: nginx
  rules:
  - http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: vllm-service
            port:
              number: 8080

kubectl apply -f vllm-ingress-simple.yaml

$ kubectl get ingress -o wide
NAME                  CLASS   HOSTS   ADDRESS                                                                   PORTS   AGE
vllm-ingress-simple   nginx   *       <INGRESS-ELB>   80      2m14s

$ kubectl describe ingress vllm-ingress-simple
Name:             vllm-ingress-simple
Namespace:        default
Address:          <INGRESS-ELB>
Ingress Class:    nginx
Default backend:  <default>
Rules:
  Host        Path  Backends
  ----        ----  --------
  *
              /   vllm-service:8080 (10.0.5.40:8080)
Annotations:  nginx.ingress.kubernetes.io/rewrite-target: /
Events:
  Type    Reason  Age                   From                      Message
  ----    ------  ----                  ----                      -------
  Normal  Sync    109s (x2 over 2m15s)  nginx-ingress-controller  Scheduled for sync

export VLLM_ENDPOINT="http://$(kubectl get ingress vllm-ingress-simple -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')"
echo $VLLM_ENDPOINT

http://<INGRESS-ELB>

$ curl -s $VLLM_ENDPOINT/v1/models
{"object":"list","data":[{"id":"tinyLlama/TinyLlama-1.1B-Chat-v1.0","object":"model",
 "created":1789139702,"owned_by":"vllm","max_model_len":1024, ...}]}

$ curl -s -o /dev/null -w "%{http_code}\n" $VLLM_ENDPOINT/health
200

curl -s -X POST "$VLLM_ENDPOINT/v1/chat/completions" \
  -H "Content-Type: application/json" \
  -d '{
    "model": "tinyLlama/TinyLlama-1.1B-Chat-v1.0",
    "messages": [{"role": "user", "content": "Hello, how are you?"}],
    "max_tokens": 100,
    "temperature": 0.7
  }' | jq -r '.choices[0].message.content'

I am fine, thank you. How about you?

i am doing well, thank you.

i hope you are too.

feel free to contact me if you need anything.

i'm here to help you.

see you soon.

meanwhile, stay safe and healthy.

$ kubectl logs -n ingress-nginx deployment/ingress-nginx-controller --tail=5
10.0.5.142 - - [11/Sep/2026:15:13:45 +0000] "POST /v1/chat/completions HTTP/1.1" 200 828 "-" "curl/7.81.0" 380 0.832 [default-vllm-service-8080] [] 10.0.5.40:8080 828 0.832 200 ebb5664e...
10.0.5.142 - - [11/Sep/2026:15:15:02 +0000] "GET /v1/models HTTP/1.1" 200 518 "-" "curl/8.7.1" 143 0.001 [default-vllm-service-8080] [] 10.0.5.40:8080 518 0.001 200 6246691a...
10.0.5.142 - - [11/Sep/2026:15:15:03 +0000] "GET /health HTTP/1.1" 200 0 "-" "curl/8.7.1" 140 0.001 [default-vllm-service-8080] [] 10.0.5.40:8080 0 0.001 200 5c208dcb...
10.0.5.142 - - [11/Sep/2026:15:15:03 +0000] "POST /v1/chat/completions HTTP/1.1" 200 676 "-" "curl/8.7.1" 349 0.601 [default-vllm-service-8080] [] 10.0.5.40:8080 676 0.601 200 0415e926...

python3 test-vllm-pod.py

# Ingress 상태
kubectl get ingress vllm-ingress-simple -o wide
kubectl describe ingress vllm-ingress-simple
kubectl get ingress vllm-ingress-simple -o yaml

# controller
kubectl get pods -n ingress-nginx
kubectl get svc -n ingress-nginx
kubectl logs -n ingress-nginx deployment/ingress-nginx-controller --tail=50

# 이벤트 / endpoint
kubectl get events --sort-by=.metadata.creationTimestamp | grep -i ingress
kubectl get endpoints vllm-service

# 클러스터 내부에서 연결 테스트
kubectl run test-pod --image=curlimages/curl:latest -it --rm --restart=Never -- /bin/sh
# 파드 안에서: curl http://vllm-service:8080/health

$ kubectl get endpoints vllm-service
NAME           ENDPOINTS        AGE
vllm-service   10.0.5.40:8080   34m

kubectl patch svc vllm-service -p '{"spec":{"type":"ClusterIP"}}'

kubectl delete ingress vllm-ingress-simple
helm uninstall ingress-nginx -n ingress-nginx
kubectl delete namespace ingress-nginx

helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo update

server:
  persistentVolume:
    enabled: false
  retention: "15d"
  resources:
    requests: { cpu: 500m, memory: 1Gi }
    limits:   { cpu: 1000m, memory: 2Gi }
  global:
    scrape_interval: 15s
    evaluation_interval: 15s

alertmanager:
  enabled: false
  persistentVolume:
    enabled: false

nodeExporter:
  enabled: true
kubeStateMetrics:
  enabled: true

serverFiles:
  prometheus.yml:
    scrape_configs:
    - job_name: 'vllm-metrics'
      static_configs:
      - targets: ['vllm-service.default.svc.cluster.local:8080']
      metrics_path: '/metrics'
      scrape_interval: 10s

helm upgrade -i prometheus prometheus-community/prometheus \
  --namespace monitoring --create-namespace \
  --values prometheus-values.yaml

$ kubectl get svc -n monitoring
NAME                                  TYPE        CLUSTER-IP       PORT(S)    AGE
prometheus-kube-state-metrics         ClusterIP   172.20.158.73    8080/TCP   5m17s
prometheus-prometheus-node-exporter   ClusterIP   172.20.67.81     9100/TCP   5m17s
prometheus-prometheus-pushgateway     ClusterIP   172.20.214.63    9091/TCP   5m17s
prometheus-server                     ClusterIP   172.20.70.164    80/TCP     5m17s

$ kubectl get ingressclasses.networking.k8s.io
NAME    CONTROLLER             PARAMETERS   AGE
nginx   k8s.io/ingress-nginx   <none>       3h21m

cat <<EOF | kubectl apply -f -
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: prometheus-ingress
  namespace: monitoring
spec:
  ingressClassName: nginx
  rules:
  - http:
      paths:
      - path: /p8s
        pathType: Prefix
        backend:
          service:
            name: prometheus-server
            port:
              number: 80
EOF

$ curl -i http://<ELB>/p8s/targets
HTTP/1.1 404 Not Found
Content-Type: text/plain; charset=utf-8
X-Content-Type-Options: nosniff

404 page not found

$ curl -o /dev/null -w "%{http_code}" http://prometheus-server.monitoring.svc/targets
200
$ curl -o /dev/null -w "%{http_code}" http://prometheus-server.monitoring.svc/p8s/targets
404

helm upgrade prometheus prometheus-community/prometheus -n monitoring --reuse-values \
  --set server.prefixURL=/p8s \
  --set server.baseURL=http://<INGRESS-ELB>/p8s

$ kubectl get deploy -n monitoring prometheus-server \
    -o jsonpath='{.spec.template.spec.containers[?(@.name=="prometheus-server")].args}'
["--storage.tsdb.retention.time=15d",
 "--config.file=/etc/config/prometheus.yml",
 "--storage.tsdb.path=/data",
 "--web.console.libraries=/etc/prometheus/console_libraries",
 "--web.console.templates=/etc/prometheus/consoles",
 "--web.enable-lifecycle",
 "--web.route-prefix=/p8s",
 "--web.external-url=http://<INGRESS-ELB>/p8s"]

$ kubectl describe ingress -n monitoring prometheus-ingress
Address:          <INGRESS-ELB>
Ingress Class:    nginx
Rules:
  Host        Path  Backends
  ----        ----  --------
  *           /p8s   prometheus-server:80 (10.0.5.241:9090)

/p8s/targets                     200
/p8s/graph                       302  → Location: /p8s/query?
/p8s/-/healthy                   200
/p8s/-/ready                     200
/p8s/api/v1/query?query=up       200

curl -s "http://<ELB>/p8s/api/v1/query?query=up"

helm repo add grafana https://grafana.github.io/helm-charts
helm repo update

persistence:
  enabled: false
adminPassword: "<GRAFANA_PASSWORD>"
service:
  type: ClusterIP

datasources:
  datasources.yaml:
    apiVersion: 1
    datasources:
    - name: Prometheus
      type: prometheus
      url: http://prometheus-server.monitoring.svc.cluster.local/p8s
      access: proxy
      isDefault: true

dashboardProviders:
  dashboardproviders.yaml:
    apiVersion: 1
    providers:
    - name: 'default'
      orgId: 1
      folder: ''
      type: file
      disableDeletion: false
      editable: true
      options:
        path: /var/lib/grafana/dashboards/default

dashboards:
  default:
    kubernetes-cluster:
      gnetId: 7249
      revision: 1
      datasource: Prometheus
    kubernetes-pods:
      gnetId: 6336
      revision: 1
      datasource: Prometheus

resources:
  requests: { cpu: 250m, memory: 512Mi }
  limits:   { cpu: 500m, memory: 1Gi }

# /p8s 없이 뒀을 때
{"message":"404 Not Found - There was an error returned querying the Prometheus API.","status":"ERROR"}

helm upgrade -i grafana grafana/grafana --namespace monitoring --values grafana-values.yaml

cat <<EOF | kubectl apply -f -
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: grafana-ingress
  namespace: monitoring
spec:
  ingressClassName: nginx
  rules:
  - http:
      paths:
      - path: /grafana
        pathType: Prefix
        backend:
          service:
            name: grafana
            port:
              number: 80
EOF

grafana.ini:
  server:
    root_url: "http://<INGRESS-ELB>/grafana"
    serve_from_sub_path: true
readinessProbe:
  httpGet:
    path: /grafana/api/health
    port: grafana
livenessProbe:
  httpGet:
    path: /grafana/api/health
    port: grafana
  initialDelaySeconds: 60
  timeoutSeconds: 30
  failureThreshold: 10

helm upgrade grafana grafana/grafana -n monitoring --reuse-values -f grafana-subpath-values.yaml

$ kubectl get deploy -n monitoring grafana \
    -o jsonpath='{.spec.template.spec.containers[0].readinessProbe.httpGet.path}'
/grafana/api/health

$ kubectl get ingress -A
NAMESPACE    NAME                  CLASS   HOSTS   ADDRESS                                                                   PORTS   AGE
default      vllm-ingress-simple   nginx   *       <INGRESS-ELB>   80      12m
monitoring   grafana-ingress       nginx   *       <INGRESS-ELB>   80      67s
monitoring   prometheus-ingress    nginx   *       <INGRESS-ELB>   80      6m21s

$ kubectl describe ingress -n monitoring grafana-ingress
Rules:
  Host        Path      Backends
  ----        ----      --------
  *           /grafana   grafana:80 (10.0.5.239:3000)

http://<INGRESS-ELB>/grafana
admin / <GRAFANA_PASSWORD>

$ curl -s http://<ELB>/grafana/api/health
{
  "database": "ok",
  "version": "12.3.1",
  "commit": "3a1c80ca7ce612f309fdc99338dd3c5e486339be"
}

$ curl -i http://<ELB>/grafana | grep -iE "^HTTP|^location"
HTTP/1.1 302 Found
Location: /grafana/login?redirectTo=

$ curl -s -u admin:<GRAFANA_PASSWORD> http://<ELB>/grafana/api/datasources/uid/PBFA97CFB590B2093/health
{"details":{"application":"Prometheus","features":{"rulerApiEnabled":false}},
 "message":"Successfully queried the Prometheus API.","status":"OK"}

cat > vllm-dashboard.json <<'EOF'
{
  "id": null,
  "title": "vLLM Inference Metrics",
  "tags": ["vllm", "inference", "llm"],
  "panels": [ ... ],
  "time": {"from": "now-15m", "to": "now"},
  "refresh": "10s"
}
EOF

kubectl create configmap vllm-dashboard \
  --from-file=vllm-dashboard.json \
  -n monitoring

kubectl patch deployment grafana -n monitoring --type='json' -p='[
  {
    "op": "add",
    "path": "/spec/template/spec/volumes/-",
    "value": {
      "name": "vllm-dashboard",
      "configMap": { "name": "vllm-dashboard" }
    }
  },
  {
    "op": "add",
    "path": "/spec/template/spec/containers/0/volumeMounts/-",
    "value": {
      "name": "vllm-dashboard",
      "mountPath": "/var/lib/grafana/dashboards/default/vllm-dashboard.json",
      "subPath": "vllm-dashboard.json"
    }
  }
]'

$ kubectl get deploy grafana -n monitoring -o jsonpath='{.spec.template.spec.volumes[*].name}'
config dashboards-default storage search vllm-dashboard

kubectl annotate deployment vllm-deployment -n default \
  prometheus.io/scrape=true \
  prometheus.io/port=8080 \
  prometheus.io/path=/metrics

$ kubectl get deploy vllm-deployment -o jsonpath='{.metadata.annotations}' | jq
{
  "prometheus.io/path": "/metrics",
  "prometheus.io/port": "8080",
  "prometheus.io/scrape": "true"
}

for q in vllm:request_success_total vllm:num_requests_running vllm:num_requests_waiting \
         vllm:gpu_cache_usage_perc vllm:prompt_tokens_total vllm:generation_tokens_total; do
  curl -s "http://<ELB>/p8s/api/v1/query?query=$q"
done

# 이렇게 하면 scrape_configs 배열 전체가 교체된다
helm upgrade prometheus ... --reuse-values \
  --set 'serverFiles.prometheus\.yml.scrape_configs[0].metrics_path=/p8s/metrics'

- job_name: prometheus-pushgateway
- metrics_path: /p8s/metrics      ← job_name 없음

server:
  persistentVolume:
    enabled: false
  retention: "15d"
  resources:
    requests: { cpu: 500m, memory: 1Gi }
    limits:   { cpu: 1000m, memory: 2Gi }
  global:
    scrape_interval: 15s
    evaluation_interval: 15s
  prefixURL: /p8s
  baseURL: http://<INGRESS-ELB>/p8s

alertmanager:
  enabled: false
  persistentVolume:
    enabled: false

nodeExporter:
  enabled: true
kubeStateMetrics:
  enabled: true

serverFiles:
  prometheus.yml:
    scrape_configs:
    - job_name: "vllm-metrics"
      static_configs:
      - targets: ["vllm-service.default.svc.cluster.local:8080"]
      metrics_path: "/metrics"
      scrape_interval: 10s

helm upgrade prometheus prometheus-community/prometheus -n monitoring \
  --values prometheus-values.yaml --dry-run > /tmp/p8s-dry.yaml

grep -nE "job_name" /tmp/p8s-dry.yaml

 98:    - job_name: kubernetes-api-servers
112:    - job_name: kubernetes-nodes
123:    - job_name: kubernetes-nodes-cadvisor
138:    - job_name: kubernetes-pods
196:    - job_name: kubernetes-pods-slow
252:    - job_name: kubernetes-service-endpoints
299:    - job_name: kubernetes-service-endpoints-slow
344:    - job_name: kubernetes-services
373:    - job_name: prometheus
377:    - job_name: prometheus-pushgateway
386:    - job_name: vllm-metrics

helm upgrade prometheus prometheus-community/prometheus -n monitoring --values prometheus-values.yaml
kubectl rollout status deploy/prometheus-server -n monitoring

export AWS_REGION=us-west-2
export CLUSTER_NAME=ai-infra-summit-test-cluster
export MONITORING_NAMESPACE=monitoring

export VLLM_ENDPOINT=$(kubectl get service vllm-service -n default \
  -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')
export VLLM_NAMESPACE=default
export VLLM_URL="http://$VLLM_ENDPOINT:8080/v1"

echo "vLLM Endpoint: $VLLM_URL"

vLLM Endpoint: http://<VLLM-ELB>:8080/v1
vLLM Namespace: default

kubectl create namespace performance-testing

apiVersion: v1
kind: Pod
metadata:
  name: performance-test-runner
  namespace: performance-testing
spec:
  containers:
  - name: performance-tester
    image: python:3.10-slim
    command: ["sleep", "infinity"]
    resources:
      requests: { cpu: 500m, memory: 1Gi }
      limits:   { cpu: 2000m, memory: 4Gi }
    volumeMounts:
    - name: test-scripts
      mountPath: /scripts
  volumes:
  - name: test-scripts
    emptyDir: {}
  restartPolicy: Never

kubectl apply -f performance-test-pod.yaml
kubectl wait --for=condition=ready pod/performance-test-runner -n performance-testing --timeout=180s

$ kubectl get pod -n performance-testing
NAME                      READY   STATUS    RESTARTS   AGE
performance-test-runner   1/1     Running   0          5s

kubectl exec -n performance-testing performance-test-runner -- \
  pip install requests aiohttp numpy matplotlib pandas locust

$ kubectl exec -n performance-testing performance-test-runner -- pip list
aiohttp            3.14.3
locust             2.46.0
matplotlib         3.10.9
numpy              2.2.6
pandas             2.3.3
requests           2.34.2

kubectl cp basic_load_test.py performance-testing/performance-test-runner:/scripts/
kubectl exec -n performance-testing performance-test-runner -- \
  python /scripts/basic_load_test.py $VLLM_URL 30 5

Starting load test with 5 workers
Target: http://<VLLM-ELB>:8080/v1
Completed 10/30 requests
Completed 20/30 requests
Completed 30/30 requests

=== LOAD TEST RESULTS ===
Total Requests: 30
Successful: 30 (100.0%)
Failed: 0 (0.0%)

=== LATENCY STATISTICS ===
Average Latency: 0.96s
Median Latency: 1.06s
95th Percentile: 1.52s
99th Percentile: 1.77s
Min Latency: 0.09s
Max Latency: 1.77s

=== TOKEN STATISTICS ===
Average Tokens per Response: 50.8
Tokens per Second: 52.8

total_time = sum(latencies)          # 동시 실행된 요청들의 지연 시간을 그냥 더함
print(f"Tokens per Second: {total_tokens/total_time:.1f}")

print(f"[wall-clock] Elapsed: {self.wall_clock:.2f}s")
print(f"[wall-clock] Tokens per Second: {total_tokens/self.wall_clock:.1f}")
print(f"[wall-clock] Requests per Second: {len(self.results)/self.wall_clock:.2f}")

[wall-clock] Elapsed: 6.28s
[wall-clock] Tokens per Second: 243.0
[wall-clock] Requests per Second: 4.78

kubectl exec -n performance-testing performance-test-runner -- bash -c "
pip install --upgrade pip && \
apt-get update && apt-get install -y git && \
cd /tmp && git clone https://github.com/ray-project/llmperf.git && \
cd llmperf && pip install ray && pip install -e .
"

kubectl exec -n performance-testing performance-test-runner -- bash -c "
cd /tmp/llmperf && \
export OPENAI_API_KEY=EMPTY && \
export OPENAI_API_BASE=$VLLM_URL && \
python token_benchmark_ray.py \
    --model 'tinyLlama/TinyLlama-1.1B-Chat-v1.0' \
    --mean-input-tokens 256 --stddev-input-tokens 50 \
    --mean-output-tokens 100 --stddev-output-tokens 20 \
    --max-num-completed-requests 50 \
    --timeout 600 \
    --num-concurrent-requests 5 \
    --results-dir 'result_outputs' \
    --llm-api openai \
    --additional-sampling-params '{\"temperature\": 0.7}'
"

WARNING: The object store is using /tmp/ray instead of /dev/shm because /dev/shm has only
67108864 bytes available. This will harm performance!

inter_token_latency_s
    p25 = 0.010533197675579501
    p50 = 0.011335195329905044
    p75 = 0.013357803658334447
    p90 = 0.01458673198562109
    p95 = 0.014918960312485338
    p99 = 0.016072675808620207
    mean = 0.01195091227547865
    min = 0.009657615346942311
    max = 0.0170578906052972
    stddev = 0.0017345265009493843
ttft_s
    p25 = 0.11523566200003188
    p50 = 0.20684104649944857
    p75 = 0.3595698380004251
    p90 = 0.4603797805995783
    p95 = 0.5340842209003311
    p99 = 0.5588042465605576
    mean = 0.25021471850001037
    min = 0.04793875699942873
    max = 0.5594372560008196
    stddev = 0.15323682653572493
end_to_end_latency_s
    p25 = 1.058154924749715
    p50 = 1.1764374354997926
    p75 = 1.3464375240000663
    p90 = 1.4255717264004488
    p95 = 1.508258455750456
    p99 = 1.5572798763800983
    mean = 1.182102578119975
    min = 0.6697167339998487
    max = 1.5696425460000683
    stddev = 0.2160172698857819
request_output_throughput_token_per_s
    p25 = 74.72318697474945
    p50 = 88.05565188536687
    p75 = 94.92815383462235
    p90 = 96.77919871768835
    p95 = 99.36473786914019
    p99 = 102.16478482190938
    mean = 85.0578411708009
    min = 58.619424018192035
    max = 103.53285630406408
    stddev = 11.227464153934132
number_input_tokens
    mean = 256.94   min = 131   max = 418   stddev = 50.23999951254387
number_output_tokens
    mean = 99.06    min = 62    max = 128   stddev = 15.261542971716556

Number Of Errored Requests: 0
Overall Output Throughput: 339.01856647830186
Number Of Completed Requests: 50
Completed Requests Per Minute: 205.3413485634778

$ kubectl exec -n performance-testing performance-test-runner -- ls /tmp/llmperf/result_outputs/
tinyLlama-TinyLlama-1-1B-Chat-v1-0_256_100_individual_responses.json
tinyLlama-TinyLlama-1-1B-Chat-v1-0_256_100_summary.json

{
    "error_code": null,
    "inter_token_latency_s": 0.009924852558099722,
    "ttft_s": 0.052236619000723294,
    "end_to_end_latency_s": 0.853675344000294,
    "request_output_throughput_token_per_s": 100.74087368742184,
    "number_total_tokens": 352,
    "number_output_tokens": 86,
    "number_input_tokens": 266
}

curl -s "http://<ELB>/p8s/api/v1/query?query=vllm:request_success_total"

stop     14
length   75

$ curl -s --data-urlencode 'query=sum(vllm:request_success_total)' "http://<ELB>/p8s/api/v1/query"
89

kubectl delete namespace performance-testing

kubectl cp performance-testing/performance-test-runner:/tmp/llmperf/result_outputs ./result_outputs