OpenWebUI + Ollama + LiteLLM setup on Bazzite using Podman Quadlets (NVIDIA GPU)

litellm-config.yaml

general_settings:
  master_key: sk-1234 # enter your own master key, ensure it starts with 'sk-'
  alerting: [] # Setup slack alerting - get alerts on LLM exceptions, Budget Alerts, Slow LLM Responses
  proxy_batch_write_at: 60 # Batch write spend updates every 60s
  database_connection_pool_limit: 10 # limit the number of database connections to = MAX Number of DB Connections/Number of instances of litellm proxy (Around 10-20 is good number)

  # OPTIONAL Best Practices
  disable_spend_logs: True # turn off writing each transaction to the db. We recommend doing this is you don't need to see Usage on the LiteLLM UI and are tracking metrics via Prometheus
  disable_error_logs: True # turn off writing LLM Exceptions to DB
  allow_requests_on_db_unavailable: True # Only USE when running LiteLLM on your VPC. Allow requests to still be processed even if the DB is unavailable. We recommend doing this if you're running LiteLLM on VPC that cannot be accessed from the public internet.

litellm_settings:
  request_timeout: 600 # raise Timeout error if call takes longer than 600 seconds. Default value is 6000seconds if not set
  set_verbose: False # Switch off Debug Logging, ensure your logs do not have any debugging on
  json_logs: true # Get debug logs in json format

litellm.container

[Container]
ContainerName=litellm
Image=ghcr.io/berriai/litellm-database:main-latest
Pull=newer
Environment=DATABASE_URL=postgresql://litellm:secure@postgres:5432/litellm
Environment=STORE_MODEL_IN_DB='True'
Volume=litellm:/config
PublishPort=127.0.0.1:4000:4000
Exec=--config /config/config.yaml
Network=postgres
Network=ollama
AutoUpdate=registry

[Service]
TimeoutStartSec=300
TimeoutStopSec=300

[Install]
WantedBy=default.target

ollama.container

[Container]
ContainerName=ollama
Image=docker.io/ollama/ollama:latest
Volume=ollama:/root/.ollama
Pull=newer
PodmanArgs=--security-opt=label=disable
PodmanArgs=--hooks-dir=/usr/share/containers/oci/hooks.d/
PodmanArgs=--device=nvidia.com/gpu=all
PublishPort=127.0.0.1:11434:11434
SecurityLabelType=container_runtime_t
Network=ollama
AutoUpdate=registry

[Service]
TimeoutStartSec=300
TimeoutStopSec=300

[Install]
WantedBy=default.target

open-webui.container

[Container]
ContainerName=open-webui
Image=ghcr.io/open-webui/open-webui:main
Pull=newer
Volume=open-webui:/app/backend/data
Environment=ENV=dev
Environment=WEBUI_AUTH=false
Environment=USE_CUDA_DOCKER=true
Environment=TIKA_SERVER_URL=http://tika:9998
Environment=CONTENT_EXTRACTION_ENGINE=tika
Environment=OLLAMA_BASE_URL=http://ollama:11434
Environment=OPENAI_API_BASE_URL=http://litellm:4000
Environment=OPENAI_API_KEY=sk-1234
Network=ollama
PublishPort=127.0.0.1:8080:8080
PodmanArgs=--security-opt=label=disable
PodmanArgs=--hooks-dir=/usr/share/containers/oci/hooks.d/
PodmanArgs=--device=nvidia.com/gpu=all
SecurityLabelType=container_runtime_t
AutoUpdate=registry

[Service]
TimeoutStartSec=300
TimeoutStopSec=300

[Install]
WantedBy=default.target

postgres.container

[Container]
ContainerName=postgres
Image=docker.io/pgduckdb/pgduckdb:16-main
Pull=newer
Environment=POSTGRES_PASSWORD='secure'
Environment=PGDATA=/var/lib/postgresql/data/pgdata
Volume=postgres:/var/lib/postgresql/data:z
PublishPort=127.0.0.1:5432:5432
Network=postgres
AutoUpdate=registry

[Service]
TimeoutStartSec=300
TimeoutStopSec=300

[Install]
WantedBy=default.target