LLM 스터디 2주차 - 멀티 모델 서빙과 Triton 위임 직접 해보기

pkill -f "python main.py"; sleep 2
nvidia-smi --query-gpu=memory.used,memory.free --format=csv   # 반환 확인

export M_SENT=550e8400-e29b-41d4-a716-446655440000
export M_SPAM=6ba7b810-9dad-11d1-80b4-00c04fd430c8
export M_IMG=7c9e6679-7425-40de-944b-e07fc1f90ae7
export M_TRITON=8ba7b810-9dad-11d1-80b4-00c04fd430c9

cd llm-model-inference/ch03/multi_model_serving
python3.12 -m venv venv
source venv/bin/activate
pip install -U pip
pip install -r requirements.txt        # 5~10분

python -c "import torch, torchvision, transformers, tritonclient.http; print(torch.__version__, torchvision.__version__, transformers.__version__)"
# 기대: 2.2.1 0.17.1 4.35.2

docker pull nvcr.io/nvidia/tritonserver:24.12-py3 &

grep -n "cuda\|\.to(\|device" app/worker.py
# 아무것도 안 나온다

# app/worker.py — TransformerWorker
    def _load_model(self):
        if self.model is None:
            self.device = "cuda" if torch.cuda.is_available() else "cpu"     # 추가
            self.model = AutoModelForSequenceClassification.from_pretrained(
                self.model_metadata.name).to(self.device).eval()             # 추가
            self.tokenizer = AutoTokenizer.from_pretrained(self.model_metadata.name)

    def predict(self, input_data: Any) -> Dict[str, Any]:
        inputs = self.tokenizer(input_data, return_tensors="pt",
                                padding=True, truncation=True).to(self.device)   # 추가
        with torch.no_grad():
            outputs = self.model(**inputs)
        predictions = torch.softmax(outputs.logits, dim=-1)
        return {"predictions": predictions.cpu().tolist()}                       # 추가

# app/manager.py — get_model_worker() 안, eviction 블록
        if len(self.model_cache) >= self.max_models:
            id, model_worker = self.model_cache.popitem(last=False)
            self.model_engine.delete_worker(id)
            del model_worker                      # 추가 — 마지막 참조 제거
            gc.collect()                          # 추가 — 순환 참조까지 정리
            if torch.cuda.is_available():
                torch.cuda.empty_cache()          # 추가 — allocator가 쥔 VRAM 반환

# 반드시 multi_model_serving/ 에서 (ModelStore가 상대경로를 쓴다)
python -m app.server 2>&1 | tee mms_run.log

watch -n 1 'nvidia-smi --query-gpu=memory.used,memory.free --format=csv; \
            nvidia-smi --query-compute-apps=pid,used_memory --format=csv'

curl -s http://localhost:8001/models | jq

{
  "available_models": {
    "550e8400-e29b-41d4-a716-446655440000": {
      "name": "distilbert-base-uncased-finetuned-sst-2-english",
      "framework": "transformers"
    },
    "6ba7b810-9dad-11d1-80b4-00c04fd430c8": {
      "name": "mrm8488/bert-tiny-finetuned-sms-spam-detection",
      "framework": "transformers"
    },
    "7c9e6679-7425-40de-944b-e07fc1f90ae7": {
      "name": "pytorch/vision:mobilenet_v2",
      "framework": "torchvision"
    },
    "8ba7b810-9dad-11d1-80b4-00c04fd430c9": {
      "name": "densenet_onnx",
      "framework": "triton"
    }
  },
  "loaded_models": {}
}

curl -s -X POST http://localhost:8001/predict \
  -H "Content-Type: application/json" \
  -d "{\"model_id\": \"$M_SENT\", \"input_data\": \"This movie was great! I really enjoyed it.\"}" | jq

{
  "predictions": [
    [
      0.00011904446728294715,
      0.9998809099197388
    ]
  ]
}

time curl -s -X POST http://localhost:8001/predict -H "Content-Type: application/json" \
  -d "{\"model_id\": \"$M_SENT\", \"input_data\": \"good\"}" > /dev/null

curl -s http://localhost:8001/models | jq '.loaded_models'

{
  "550e8400-e29b-41d4-a716-446655440000": "distilbert-base-uncased-finetuned-sst-2-english"
}

curl -s -X POST http://localhost:8001/predict \
  -H "Content-Type: application/json" \
  -d "{\"model_id\": \"$M_SPAM\", \"input_data\": \"This movie was great! I really enjoyed it.\"}" | jq

{
  "predictions": [
    [
      0.9374048113822937,
      0.06259513646364212
    ]
  ]
}

curl -s http://localhost:8001/models | jq '.loaded_models'

{
  "550e8400-e29b-41d4-a716-446655440000": "distilbert-base-uncased-finetuned-sst-2-english",
  "6ba7b810-9dad-11d1-80b4-00c04fd430c8": "mrm8488/bert-tiny-finetuned-sms-spam-detection"
}

curl -s -X POST http://localhost:8001/predict \
  -H "Content-Type: application/json" \
  -d "{\"model_id\": \"$M_IMG\", \"input_data\": \"tests/images/cat1.jpg\"}" | jq '.predictions[0] | length'
# 1000

curl -s http://localhost:8001/models | jq '.loaded_models'

{
  "6ba7b810-9dad-11d1-80b4-00c04fd430c8": "mrm8488/bert-tiny-finetuned-sms-spam-detection",
  "7c9e6679-7425-40de-944b-e07fc1f90ae7": "pytorch/vision:mobilenet_v2"
}

curl -s -X POST localhost:8001/predict -H "Content-Type: application/json" -d "{\"model_id\":\"$M_SENT\",\"input_data\":\"a\"}" >/dev/null
curl -s -X POST localhost:8001/predict -H "Content-Type: application/json" -d "{\"model_id\":\"$M_SPAM\",\"input_data\":\"a\"}" >/dev/null
curl -s -X POST localhost:8001/predict -H "Content-Type: application/json" -d "{\"model_id\":\"$M_SENT\",\"input_data\":\"a\"}" >/dev/null   # 여기서 move_to_end
curl -s -X POST localhost:8001/predict -H "Content-Type: application/json" -d "{\"model_id\":\"$M_IMG\",\"input_data\":\"tests/images/cat1.jpg\"}" >/dev/null
curl -s localhost:8001/models | jq '.loaded_models'

{
  "550e8400-e29b-41d4-a716-446655440000": "distilbert-base-uncased-finetuned-sst-2-english",
  "7c9e6679-7425-40de-944b-e07fc1f90ae7": "pytorch/vision:mobilenet_v2"
}

curl -s -w "\nHTTP %{http_code}\n" -X POST http://localhost:8001/predict \
  -H "Content-Type: application/json" -d '{"model_id":"invalid-id","input_data":"test"}'

VRAM() { nvidia-smi --query-gpu=memory.used --format=csv,noheader; }

# ② sentiment 로드 (distilbert 66M)
curl -s -X POST localhost:8001/predict -H "Content-Type: application/json" \
     -d "{\"model_id\":\"$M_SENT\",\"input_data\":\"a\"}" >/dev/null; VRAM

# ③ spam 추가 (bert-tiny 4.4M) → 캐시 2/2
curl -s -X POST localhost:8001/predict -H "Content-Type: application/json" \
     -d "{\"model_id\":\"$M_SPAM\",\"input_data\":\"a\"}" >/dev/null; VRAM

# ④ image 호출 → sentiment 축출
curl -s -X POST localhost:8001/predict -H "Content-Type: application/json" \
     -d "{\"model_id\":\"$M_IMG\",\"input_data\":\"tests/images/cat1.jpg\"}" >/dev/null; VRAM

docker run --gpus all -d --name triton-densenet \
  --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864 \
  -p8009:8000 -p8010:8001 -p8011:8002 \
  -v $(pwd)/model_dir:/models \
  nvcr.io/nvidia/tritonserver:24.12-py3 \
  tritonserver --model-repository=/models --model-control-mode=explicit

curl -s -X POST localhost:8009/v2/repository/index | jq

[
  {
    "name": "densenet_onnx"
  }
]

docker exec triton-densenet ps -ef | head -3

nvidia-smi

# ① 모델 관리 API — 로드
curl -s -X POST localhost:8009/v2/repository/models/densenet_onnx/load
curl -s localhost:8009/v2/models/densenet_onnx | jq

{
  "name": "densenet_onnx",
  "versions": [ "1" ],
  "platform": "onnxruntime_onnx",
  "inputs": [
    { "name": "data_0", "datatype": "FP32", "shape": [ 3, 224, 224 ] }
  ],
  "outputs": [
    { "name": "fc6_1", "datatype": "FP32", "shape": [ 1000 ] }
  ]
}

# ② 추론 API — 고양이 사진으로
curl -s -X POST localhost:8009/v2/models/densenet_onnx/infer \
  -H "Content-Type: application/json" -d @/tmp/triton_payload.json \
  > /tmp/triton_response.json

curl -s -X POST http://localhost:8001/predict -H "Content-Type: application/json" \
  -d @/tmp/predict_triton.json | python3 -c "..."

curl -s localhost:8001/models | jq '.loaded_models'

{
  "7c9e6679-7425-40de-944b-e07fc1f90ae7": "pytorch/vision:mobilenet_v2",
  "8ba7b810-9dad-11d1-80b4-00c04fd430c9": "densenet_onnx"
}

docker logs -f triton-densenet 2>&1 | grep --line-buffered -E "successfully (loaded|unloaded)"

# 1) triton 모델 로드
curl -s -X POST localhost:8001/predict -H "Content-Type: application/json" -d @/tmp/predict_triton.json >/dev/null
# 2) 다른 모델 2개 호출 → 캐시가 밀리면서 TritonWorker 소멸
curl -s -X POST localhost:8001/predict -H "Content-Type: application/json" \
     -d "{\"model_id\":\"$M_SENT\",\"input_data\":\"hello\"}" >/dev/null
curl -s -X POST localhost:8001/predict -H "Content-Type: application/json" \
     -d "{\"model_id\":\"$M_SPAM\",\"input_data\":\"hello\"}" >/dev/null

docker stop triton-densenet && docker rm triton-densenet
nvidia-smi                                       # GPU 반환 확인
deactivate
docker rmi nvcr.io/nvidia/tritonserver:24.12-py3   # 이미지까지 지우려면 (9.6GB 회수)