1
git clone https://github.com/vllm-project/vllm.git && cd vllm

1
uv venv venv-vllm
2
source venv-vllm/bin/activate
3
uv pip install -U pip

1
pip install torch torchvision --index-url https://download.pytorch.org/whl/cu132
2
python use_existing_torch.py
3
pip install setuptools-rust setuptools-scm
4
pip install -r requirements/cuda.txt

1
pip install --no-build-isolation --editable .

1
export MODEL_HOME=$HOME/Models/Qwen3.6-35B-A3B-NVFP4
2
export MODEL_NAME=Qwen3.6-35B-A3B-NVFP4
3

4
export CUDA_HOME=/usr/local/cuda
5
export FLASHINFER_NVCC="$CUDA_HOME/bin/nvcc"
6
export FLASHINFER_CUDA_ARCH_LIST="12.0f"
7
export NVCC_PREPEND_FLAGS="-DCCCL_DISABLE_CTK_COMPATIBILITY_CHECK"
8
export LIBRARY_PATH="$CUDA_HOME/lib:$LIBRARY_PATH"
9
export LD_LIBRARY_PATH="$CUDA_HOME/lib:$LD_LIBRARY_PATH"
10
export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
11
export VLLM_USE_FLASHINFER_MOE_FP4=0
12
export VLLM_USE_FLASHINFER_SAMPLER=0
13

14

15
vllm serve $MODEL_HOME \
16
    --served-model-name $MODEL_NAME \
17
    --host 0.0.0.0 --port 8082 \
18
    --max-model-len 196608 \
19
    --max-num-seqs 1 \
20
    --max-num-batched-tokens 384 \
21
    --gpu-memory-utilization 0.93 \
22
    --kv-cache-dtype fp8 \
23
    --quantization modelopt \
24
    --async-scheduling \
25
    --enable-chunked-prefill \
26
    --language-model-only \
27
    --skip-mm-profiling \
28
    --no-enable-prefix-caching \
29
    --no-calculate-kv-scales \
30
    --max-cudagraph-capture-size 64 \
31
    --attention-backend flashinfer \
32
    --moe-backend marlin \
33
    --reasoning-parser qwen3 \
34
    --enable-auto-tool-choice \
35
    --tool-call-parser qwen3_xml \
36
    --trust-remote-code
37

38

39
## NVIDIA defaults
40
vllm serve $MODEL_HOME \
41
    --served-model-name $MODEL_NAME \
42
    --host 0.0.0.0 --port 8082 \
43
    --tensor-parallel-size 1 \
44
    --trust-remote-code \
45
    --dtype auto \
46
    --quantization modelopt \
47
    --kv-cache-dtype fp8 \
48
    --attention-backend flashinfer \
49
    --moe-backend marlin \
50
    --gpu-memory-utilization 0.85 \
51
    --max-model-len 65536 \
52
    --max-num-seqs 4 \
53
    --max-num-batched-tokens 8192 \
54
    --enable-chunked-prefill \
55
    --async-scheduling \
56
    --enable-prefix-caching \
57
    --speculative-config '{"method":"mtp","num_speculative_tokens":3,"moe_backend":"triton"}'
58

59
# Gemini version
60
vllm serve $MODEL_HOME \
61
    --served-model-name $MODEL_NAME \
62
    --host 0.0.0.0 --port 8082 \
63
    --max-model-len 196608 \
64
    --max-num-seqs 1 \
65
    --max-num-batched-tokens 4096 \
66
    --gpu-memory-utilization 0.94 \
67
    --kv-cache-dtype fp8 \
68
    --quantization modelopt \
69
    --async-scheduling \
70
    --enable-chunked-prefill \
71
    --language-model-only \
72
    --skip-mm-profiling \
73
    --no-calculate-kv-scales \
74
    --enable-prefix-caching \
75
    --attention-backend flashinfer \
76
    --moe-backend marlin \
77
    --reasoning-parser qwen3 \
78
    --enable-auto-tool-choice \
79
    --tool-call-parser qwen3_xml \
80
    --trust-remote-code \
81
    --enforce-eager