MLPE 스터디 2주차 - Ch4~5 홈랩 Kubernetes에 MLflow, Feast, Kubeflow Pipelines 설치하기

#!/usr/bin/env bash
# MLflow 실습 플랫폼 설치: RustFS(S3) → 버킷 → Postgres → MLflow(helm) → JupyterLab
# 저장소 루트 또는 mlflow/ 어디서 실행해도 된다. 멱등이라 다시 실행하면 바뀐 것만 적용된다.
# 비밀값(RustFS 계정, Postgres 비밀번호, Jupyter 토큰)은 처음 실행 때 무작위로 만들어 Secret 에만 저장하고
# 화면에 출력하지 않는다. 확인 방법은 끝에 출력하는 명령을 쓴다.
set -euo pipefail
cd "$(dirname "$0")/.."
NS=mlflow
CHART_VERSION=${CHART_VERSION:-1.11.7}

rand() { python3 -c "import secrets;print(secrets.token_hex($1))"; }
ensure_secret() {  # name key=value ...  (이미 있으면 건드리지 않는다)
  local name=$1; shift
  if kubectl -n $NS get secret "$name" >/dev/null 2>&1; then echo "secret $name: 있음 (유지)"; return; fi
  local args=(); for kv in "$@"; do args+=(--from-literal="$kv"); done
  kubectl -n $NS create secret generic "$name" "${args[@]}" >/dev/null && echo "secret $name: 생성"
}

echo "== 1. 네임스페이스와 Secret"
kubectl get ns $NS >/dev/null 2>&1 || kubectl create ns $NS
ensure_secret rustfs-credentials "AWS_ACCESS_KEY_ID=mlflow$(rand 4)" "AWS_SECRET_ACCESS_KEY=$(rand 20)"
ensure_secret mlflow-postgres "username=mlflow" "password=$(rand 16)" "database=mlflow"
ensure_secret jupyter-token "token=$(rand 24)"

echo "== 2. RustFS"
kubectl apply -f 00-rustfs/rustfs.yaml
kubectl -n $NS rollout status deploy/rustfs --timeout=5m
kubectl -n $NS delete job rustfs-make-buckets --ignore-not-found >/dev/null
kubectl apply -f 00-rustfs/buckets-job.yaml
kubectl -n $NS wait --for=condition=complete job/rustfs-make-buckets --timeout=5m
kubectl -n $NS logs job/rustfs-make-buckets

echo "== 3. Postgres"
kubectl apply -f 01-postgres/postgres.yaml
kubectl -n $NS rollout status deploy/mlflow-postgres --timeout=5m

echo "== 4. MLflow (community-charts/mlflow $CHART_VERSION)"
helm repo add community-charts https://community-charts.github.io/helm-charts >/dev/null 2>&1 || true
helm repo update community-charts >/dev/null
helm upgrade --install mlflow community-charts/mlflow --version "$CHART_VERSION" -n $NS \
  -f 02-mlflow/values.yaml --wait --timeout 10m

echo "== 5. JupyterLab"
kubectl apply -f 03-jupyter/jupyter.yaml
kubectl -n $NS rollout status deploy/jupyter --timeout=15m

echo "== 6. 확인"
MLFLOW_IP=$(kubectl -n $NS get svc mlflow -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
RUSTFS_IP=$(kubectl -n $NS get svc rustfs-lb -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
JUPYTER_IP=$(kubectl -n $NS get svc jupyter -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
printf "mlflow /health  : "; curl -s -m 10 "http://$MLFLOW_IP:5000/health"; echo
printf "rustfs /health  : "; curl -s -m 10 -o /dev/null -w '%{http_code}\n' "http://$RUSTFS_IP:9000/health"
printf "jupyter /api    : "; curl -s -m 10 -o /dev/null -w '%{http_code}\n' "http://$JUPYTER_IP:8888/api"
kubectl -n $NS get pods -o wide

cat <<EOF

접속
  MLflow UI      http://$MLFLOW_IP:5000
  RustFS 콘솔    http://$RUSTFS_IP:9001   (S3 API http://$RUSTFS_IP:9000)
  JupyterLab     http://$JUPYTER_IP:8888/lab?token=<토큰>
비밀값 확인 (화면에 출력되니 주의)
  kubectl -n $NS get secret jupyter-token -o jsonpath='{.data.token}' | base64 -d; echo
  kubectl -n $NS get secret rustfs-credentials -o jsonpath='{.data.AWS_ACCESS_KEY_ID}' | base64 -d; echo
  kubectl -n $NS get secret rustfs-credentials -o jsonpath='{.data.AWS_SECRET_ACCESS_KEY}' | base64 -d; echo
EOF

#!/usr/bin/env bash
# Feast 피처 스토어 설치. 멱등: 다시 실행해도 Secret 과 데이터는 유지된다.
# 전제: mlflow/ 가 설치되어 있어야 한다 (RustFS 와 Secret rustfs-credentials 를 그대로 쓴다).
# 순서: ns/Redis(Secret redis-auth 생성) -> rustfs-credentials 복사 -> 버킷+parquet Job -> ConfigMap feast-repo
#       -> feast apply Job -> feast materialize Job -> 피처 서버/UI -> Jupyter 에 redis-auth 복사 후 재기동
set -euo pipefail
cd "$(dirname "$0")/.."
NS=feast
SRC_NS=mlflow

kubectl create namespace "$NS" --dry-run=client -o yaml | kubectl apply -f -

# 1) Redis 암호 (없을 때만 생성)
if ! kubectl -n "$NS" get secret redis-auth >/dev/null 2>&1; then
  kubectl -n "$NS" create secret generic redis-auth \
    --from-literal=redis-password="$(python3 -c 'import secrets;print(secrets.token_hex(20))')"
  echo "[install] Secret redis-auth 생성"
fi

# 2) RustFS 계정을 mlflow 네임스페이스에서 복사 (Secret 은 네임스페이스를 넘지 못한다)
AK=$(kubectl -n "$SRC_NS" get secret rustfs-credentials -o jsonpath='{.data.AWS_ACCESS_KEY_ID}' | base64 -d)
SK=$(kubectl -n "$SRC_NS" get secret rustfs-credentials -o jsonpath='{.data.AWS_SECRET_ACCESS_KEY}' | base64 -d)
kubectl -n "$NS" create secret generic rustfs-credentials \
  --from-literal=AWS_ACCESS_KEY_ID="$AK" --from-literal=AWS_SECRET_ACCESS_KEY="$SK" \
  --dry-run=client -o yaml | kubectl apply -f -

# 3) Redis
kubectl apply -f 00-redis/redis.yaml
kubectl -n "$NS" rollout status statefulset/redis --timeout=5m

# 4) 버킷 + parquet 업로드 (Job 은 spec 이 불변이라 지우고 다시 만든다)
kubectl -n "$NS" delete job feast-prepare-buckets --ignore-not-found
kubectl apply -f 01-buckets/buckets-job.yaml
kubectl -n "$NS" wait --for=condition=complete job/feast-prepare-buckets --timeout=5m
kubectl -n "$NS" logs job/feast-prepare-buckets

# 5) 피처 저장소 정의 -> ConfigMap (feast 네임스페이스와 Jupyter 가 있는 mlflow 네임스페이스 양쪽)
for ns in "$NS" "$SRC_NS"; do
  kubectl -n "$ns" create configmap feast-repo --from-file=02-feature-repo/ --dry-run=client -o yaml | kubectl apply -f -
done

# 6) feast apply, feast materialize
kubectl -n "$NS" delete job feast-apply feast-materialize --ignore-not-found
kubectl apply -f 03-registry/apply-job.yaml
kubectl -n "$NS" wait --for=condition=complete job/feast-apply --timeout=10m
kubectl -n "$NS" logs job/feast-apply
kubectl apply -f 03-registry/materialize-job.yaml
kubectl -n "$NS" wait --for=condition=complete job/feast-materialize --timeout=15m
kubectl -n "$NS" logs job/feast-materialize | tail -n 12

# 7) 피처 서버 + UI
kubectl apply -f 04-feature-server/feature-server.yaml
kubectl -n "$NS" rollout status deployment/feast-server --timeout=5m
kubectl -n "$NS" rollout status deployment/feast-ui --timeout=5m

# 8) Jupyter(mlflow 네임스페이스)에서 SDK 를 쓰도록 Redis 암호를 복사하고 재기동 (mlflow/03-jupyter 가 REDIS_PASSWORD 를 읽는다)
PW=$(kubectl -n "$NS" get secret redis-auth -o jsonpath='{.data.redis-password}' | base64 -d)
kubectl -n "$SRC_NS" create secret generic redis-auth --from-literal=redis-password="$PW" \
  --dry-run=client -o yaml | kubectl apply -f -
if kubectl -n "$SRC_NS" get deployment jupyter >/dev/null 2>&1; then
  kubectl -n "$SRC_NS" rollout restart deployment/jupyter
  kubectl -n "$SRC_NS" rollout status deployment/jupyter --timeout=15m
fi

echo
echo "[install] 완료"
echo "  Redis keys : $(kubectl -n "$NS" exec redis-0 -- redis-cli DBSIZE 2>/dev/null)"
echo "  피처 서버  : http://192.168.0.240:6566/docs  (health: $(curl -s -o /dev/null -w '%{http_code}' http://192.168.0.240:6566/health))"
echo "  Feast UI   : http://192.168.0.241:8080"
echo "  Redis 암호 : kubectl -n $NS get secret redis-auth -o jsonpath='{.data.redis-password}' | base64 -d"

#!/usr/bin/env bash
set -euo pipefail

# ==============================================================================
# Kubeflow Pipelines 2.17.3 standalone 설치 (번들 Argo 없이 기존 Argo Workflows v4.1.4 공유)
# Components: cert-manager, KFP cluster-scoped(CRD), KFP(API·UI·MySQL·SeaweedFS·메타데이터·캐시), UI LoadBalancer
# 전제: argo-workflows/ 설치 완료 (Workflow CRD + 전체 네임스페이스 감시 컨트롤러), 기본 StorageClass (local-path)
# 사용법: ./scripts/install.sh
# ==============================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
BASE_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
NAMESPACE=kubeflow

if ! kubectl get crd workflows.argoproj.io >/dev/null 2>&1; then
    echo "❌ Argo Workflow CRD 없음 : argo-workflows/scripts/install.sh 를 먼저 실행하세요"
    exit 1
fi
if kubectl -n argo get deploy argo-workflows-workflow-controller -o jsonpath='{.spec.template.spec.containers[0].args}' | grep -q -- '--namespaced'; then
    echo "❌ 기존 Argo 컨트롤러가 --namespaced 모드 : kubeflow 네임스페이스 워크플로를 처리하지 못함"
    exit 1
fi
if ! kubectl get sc -o jsonpath='{range .items[*]}{.metadata.annotations.storageclass\.kubernetes\.io/is-default-class}{"\n"}{end}' | grep -q true; then
    echo "❌ 기본 StorageClass 없음 : observability/00-storage (local-path) 를 먼저 적용하세요"
    exit 1
fi

echo "=========================================================="
echo "🚀 Kubeflow Pipelines 2.17.3 배포 시작"
echo "=========================================================="

echo ""
echo "[Step 1/4] 🔐 cert-manager v1.21.3..."
kubectl apply -k "${BASE_DIR}/00-cert-manager/"
kubectl -n cert-manager rollout status deploy/cert-manager-webhook --timeout=5m

echo ""
echo "[Step 2/4] 📜 KFP 클러스터 범위 리소스 (Namespace, CRD : Argo CRD 제외)..."
kubectl apply -k "${BASE_DIR}/01-cluster-scoped/"
kubectl wait --for=condition=Established crd/applications.app.k8s.io crd/scheduledworkflows.kubeflow.org --timeout=60s

echo ""
echo "[Step 3/4] 🧪 KFP 본체 (번들 Argo·cache-deployer 제외)..."
kubectl apply -k "${BASE_DIR}/02-pipelines/"
echo "   (mysql·seaweedfs가 뜰 때까지 다른 파드가 몇 번 재시작할 수 있음)"
kubectl -n "${NAMESPACE}" wait --for=condition=Available deploy --all --timeout=15m

echo ""
echo "[Step 4/4] 🔌 UI LoadBalancer..."
kubectl apply -f "${BASE_DIR}/03-access/ml-pipeline-ui-lb.yaml"

echo ""
echo "=========================================================="
echo "🎉 Kubeflow Pipelines 배포 완료!"
echo "=========================================================="
kubectl -n "${NAMESPACE}" get pods,svc,pvc
LB_IP=$(kubectl -n "${NAMESPACE}" get svc ml-pipeline-ui-lb -o jsonpath='{.status.loadBalancer.ingress[0].ip}' 2>/dev/null || true)
echo ""
echo "📌 UI: http://${LB_IP:-<LB-IP>}"
echo "📌 예제: KFP_HOST=http://${LB_IP:-<LB-IP>} python 04-examples/llm_eval_pipeline.py"
echo "=========================================================="