wsl.exe -d Ubuntu
sudo apt update
sudo apt upgrade
sudo apt install python3-pip python3-venv -y
cd ~
python3 -m venv vllm-env
source vllm-env/bin/activate
pip install --upgrade pip
pip install vllm
python -m vllm.entrypoints.openai.api_server \
--model Qwen/Qwen2.5-7B-Instruct-AWQ \
--quantization awq_marlin \
--gpu-memory-utilization 0.8 \
--max-model-len 32768 \
--enable-auto-tool-choice \
--tool-call-parser hermes