File size: 252 Bytes
d651825 | 1 2 3 4 5 6 7 8 9 10 11 12 | #!/bin/bash
set -euo pipefail
MODEL_DIR="$(cd "$(dirname "$0")" && pwd)"
PORT=8004
python -m vllm.entrypoints.openai.api_server \
--model "$MODEL_DIR" \
--tensor-parallel-size 1 \
--port $PORT \
--max-num-seqs 1 --gpu-memory-utilization 0.95
|