1
# Add NVIDIA PPA
2
sudo add-apt-repository ppa:graphics-drivers/ppa
3
sudo apt update
4

5
# Install NVIDIA driver
6
sudo apt install nvidia-driver-470

1
nvidia-smi

1
# Download CUDA 11.8
2
wget https://developer.download.nvidia.com/compute/cuda/11.8.0/local_installers/cuda_11.8.0_520.61.05_linux.run
3

4
# Make executable and run installer
5
chmod +x cuda_11.8.0_520.61.05_linux.run
6
sudo ./cuda_11.8.0_520.61.05_linux.run

1
nvcc --version

1
# Install rootless Docker
2
curl -fsSL https://get.docker.com/rootless | sh
3
export PATH=$HOME/bin:$PATH
4
export DOCKER_HOST=unix://$XDG_RUNTIME_DIR/docker.sock
5

6
# Add to shell profile
7
echo 'export PATH=$HOME/bin:$PATH' >> ~/.bashrc
8
echo 'export DOCKER_HOST=unix://$XDG_RUNTIME_DIR/docker.sock' >> ~/.bashrc
9

10
# Start Docker daemon
11
systemctl --user start docker

1
# Install NVIDIA Container Toolkit
2
distribution=$(. /etc/os-release;echo $ID$VERSION_ID)
3
curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg
4
curl -s -L https://nvidia.github.io/libnvidia-container/experimental/$distribution/libnvidia-container.list | sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list
5

6
sudo apt-get update
7
sudo apt-get install -y nvidia-container-toolkit
8

9
# Configure for rootless Docker
10
sudo nvidia-ctk runtime configure --runtime=docker --no-cgroups
11
sudo nvidia-ctk runtime configure --runtime=docker --config=$HOME/.config/docker/daemon.json
12
systemctl --user restart docker

1
# Test NVIDIA Container Toolkit
2
docker run --rm --gpus all nvidia/cuda:11.8-base-ubuntu20.04 nvidia-smi

1
# Pull vLLM image
2
docker pull vllm/vllm-openai:latest
3

4
# Run vLLM server
5
docker run --gpus all --shm-size 1g \
6
  -p 8000:8000 \
7
  -v $HOME/.cache/huggingface:/root/.cache/huggingface \
8
  vllm/vllm-openai:latest \
9
  --model microsoft/DialoGPT-medium \
10
  --gpu-memory-utilization 0.9

1
# Install Python environment
2
sudo apt install python3.8 python3.8-venv
3
python3 -m venv vllm-env
4
source vllm-env/bin/activate
5

6
# Install vLLM
7
pip install vllm
8
pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118

1
# Environment variables
2
export CUDA_VISIBLE_DEVICES=1
3
export PYTORCH_CUDA_ALLOC_CONF=max_split_size_mb:512

1
# Test API endpoint
2
curl -X POST "http://localhost:8000/v1/chat/completions" \
3
  -H "Content-Type: application/json" \
4
  -d '{
5
    "model": "microsoft/DialoGPT-medium",
6
    "messages": [{"role": "user", "content": "Hello"}],
7
    "max_tokens": 100
8
  }'

1
# Monitor GPU usage
2
watch -n 1 nvidia-smi