skypilot-tmp.yaml

# sky serve up -n gum-api skypilot.yaml

service:
  # Specifying the path to the endpoint to check the readiness of the service.
  readiness_probe: /v1/models

  # How many replicas to manage.
  replicas: 1

# Fields below are the same with `serve-openai-api.yaml`.
envs:
  MODEL_NAME: Qwen/Qwen2.5-VL-32B-Instruct-AWQ
  HF_TOKEN: TOKEN_HERE
  VLLM_USE_V1: 1
  PYTHONUTF8: 1

resources:
  cloud: gcp
  accelerators: {H100:1}
  use_spot: true
  ports:
    - 8000
  any_of:
    - region: us-central1
    - region: us-east1
    - region: us-east4
    - region: us-east5
    - region: us-south1
    - region: us-west1
    - region: us-west2
    - region: us-west3
    - region: us-west4

setup: |
  conda activate vllm
  if [ $? -ne 0 ]; then
    conda create -n vllm python=3.10 -y
    conda activate vllm
  fi

  pip install accelerate
  pip install qwen-vl-utils
  pip install 'vllm>0.7.2'
  pip install flash-attn --no-build-isolation

run: |
  conda activate vllm
  echo 'Starting vllm openai api server...'
  python -m vllm.entrypoints.openai.api_server \
    --model $MODEL_NAME \
    --host 0.0.0.0 \
    --max-model-len 64000 \
    --limit-mm-per-prompt '{"image":15}'

# curl  http://SERVER_IP:8000/v1/chat/completions \
#     -H "Content-Type: application/json" \
#     -d '{
#         "model": "Qwen/Qwen2.5-VL-32B-Instruct-AWQ",
#         "messages": [
#             {"role": "system", "content": "You are a helpful assistant."},
#             {"role": "user", "content": "What is 2 + 2?"}
#         ]
#     }'