67 lines
1.9 KiB
YAML
67 lines
1.9 KiB
YAML
version: '3.8'
|
|
|
|
# Ollama - GPU-Accelerated ML Model Serving
|
|
# Phase 1: Foundation Setup
|
|
# Ports: 11434 (API)
|
|
# GPU: YES - Requires NVIDIA Container Toolkit
|
|
# Storage: SSD or HDD for models (models are 2-15GB each)
|
|
|
|
services:
|
|
ollama:
|
|
image: ollama/ollama:latest
|
|
container_name: ollama
|
|
restart: unless-stopped
|
|
ports:
|
|
- "11434:11434" # Ollama API endpoint
|
|
volumes:
|
|
# Model storage - choose based on available space:
|
|
# SSD (faster load times): /home/jpmschweitzer/docker-data/ollama/models
|
|
# HDD (more space): /mnt/media/ollama/models
|
|
- /home/jpmschweitzer/docker-data/ollama/models:/root/.ollama
|
|
environment:
|
|
- TZ=Europe/Amsterdam
|
|
- NVIDIA_VISIBLE_DEVICES=all
|
|
- NVIDIA_DRIVER_CAPABILITIES=all
|
|
deploy:
|
|
resources:
|
|
limits:
|
|
memory: 8G
|
|
reservations:
|
|
memory: 1G
|
|
devices:
|
|
- driver: nvidia
|
|
count: 1
|
|
capabilities: [gpu]
|
|
networks:
|
|
- docker-dataplane
|
|
|
|
networks:
|
|
docker-dataplane:
|
|
external: true
|
|
name: docker-dataplane
|
|
|
|
# GPU Requirements:
|
|
# - RTX 2080 Ti (11GB VRAM)
|
|
# - Suitable for 3B-13B parameter models
|
|
# - NVIDIA Container Toolkit must be installed
|
|
#
|
|
# After Deployment:
|
|
# 1. Verify GPU access: docker exec ollama nvidia-smi
|
|
# 2. Pull a model: docker exec ollama ollama pull llama3.2:3b
|
|
# 3. List models: docker exec ollama ollama list
|
|
# 4. Test inference: docker exec ollama ollama run llama3.2:3b "Hello"
|
|
# 5. Monitor GPU during inference: watch -n 1 nvidia-smi
|
|
#
|
|
# Recommended Models for RTX 2080 Ti (11GB VRAM):
|
|
# - llama3.2:3b (2GB) - Fast, general purpose
|
|
# - mistral:7b (4GB) - High quality, coding
|
|
# - codellama:7b (4GB) - Code-specialized
|
|
# - phi3:mini (2GB) - Fast reasoning
|
|
#
|
|
# API Usage:
|
|
# curl http://localhost:11434/api/generate -d '{
|
|
# "model": "llama3.2:3b",
|
|
# "prompt": "Why is the sky blue?",
|
|
# "stream": false
|
|
# }'
|