Add LLM compose stack for phy-srv-gpu01 (not deployed yet)
Follows the homelab pattern: ironicbadger.docker_compose_generator v2 renders services/<host>/NN-<stack>/compose.yml templates into ~/docker/compose.yaml on the host. - 01-vllm: chat model, fixed --gpu-memory-utilization - 02-embeddings: second vLLM instance (--task embed) rather than a separate toolchain, so SM120 support only has to be solved once - 03-openwebui: Open WebUI + pgvector (not chroma — corpus size) - 99-network: shared bridge; leading comment keeps networks: top-level - pin docker_compose_generator to 2.0.1 — galaxy tags mix v1/v2 formats - group_vars: stack config incl. LDAP placeholders still to be filled The role only writes the compose file; starting the stack stays manual. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.8
@@ -0,0 +1,36 @@
|
||||
services:
|
||||
vllm:
|
||||
image: "{{ vllm_image }}"
|
||||
container_name: vllm
|
||||
networks:
|
||||
- llmnet
|
||||
ports:
|
||||
- "8000:8000"
|
||||
volumes:
|
||||
# model cache on local disk — tens of GB per model
|
||||
- "{{ appdata_path }}/models/huggingface:/root/.cache/huggingface"
|
||||
environment:
|
||||
- NVIDIA_VISIBLE_DEVICES=all
|
||||
- NVIDIA_DRIVER_CAPABILITIES=compute,utility
|
||||
- "HUGGING_FACE_HUB_TOKEN={{ hf_token | default('') }}"
|
||||
command:
|
||||
- --model
|
||||
- "{{ vllm_model }}"
|
||||
- --served-model-name
|
||||
- "{{ vllm_served_model_name }}"
|
||||
# fixed VRAM share so the embedding server keeps its slice (§0: "it just works")
|
||||
- --gpu-memory-utilization
|
||||
- "{{ vllm_gpu_memory_utilization }}"
|
||||
- --max-model-len
|
||||
- "{{ vllm_max_model_len }}"
|
||||
# vLLM needs a large shared-memory segment; without this it dies on startup
|
||||
ipc: host
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: 1
|
||||
capabilities: [gpu]
|
||||
runtime: nvidia
|
||||
restart: unless-stopped
|
||||
@@ -0,0 +1,37 @@
|
||||
services:
|
||||
# Embeddings run on a second vLLM instance rather than a separate toolchain
|
||||
# (e.g. text-embeddings-inference): whatever vLLM build works on SM120 then
|
||||
# covers embeddings too, instead of having to solve Blackwell support twice.
|
||||
embeddings:
|
||||
image: "{{ vllm_image }}"
|
||||
container_name: embeddings
|
||||
networks:
|
||||
- llmnet
|
||||
ports:
|
||||
- "8001:8000"
|
||||
volumes:
|
||||
- "{{ appdata_path }}/models/huggingface:/root/.cache/huggingface"
|
||||
environment:
|
||||
- NVIDIA_VISIBLE_DEVICES=all
|
||||
- NVIDIA_DRIVER_CAPABILITIES=compute,utility
|
||||
- "HUGGING_FACE_HUB_TOKEN={{ hf_token | default('') }}"
|
||||
command:
|
||||
- --model
|
||||
- "{{ embedding_model }}"
|
||||
- --served-model-name
|
||||
- "{{ embedding_model_name }}"
|
||||
- --task
|
||||
- embed
|
||||
# small fixed slice — the chat model gets the rest
|
||||
- --gpu-memory-utilization
|
||||
- "{{ embeddings_gpu_memory_utilization }}"
|
||||
ipc: host
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: 1
|
||||
capabilities: [gpu]
|
||||
runtime: nvidia
|
||||
restart: unless-stopped
|
||||
@@ -0,0 +1,62 @@
|
||||
services:
|
||||
openwebui:
|
||||
image: "{{ openwebui_image }}"
|
||||
container_name: openwebui
|
||||
networks:
|
||||
- llmnet
|
||||
ports:
|
||||
# plain HTTP for now — no TLS yet (projektplan §2.2). Put a reverse proxy
|
||||
# in front (new stack 04-proxy) once an internal CA certificate exists.
|
||||
- "80:8080"
|
||||
depends_on:
|
||||
- vllm
|
||||
- owui-db
|
||||
volumes:
|
||||
- "{{ appdata_path }}/apps/open-webui:/app/backend/data"
|
||||
environment:
|
||||
- "WEBUI_URL=http://{{ llm_dns_name }}"
|
||||
- "WEBUI_SECRET_KEY={{ vault_owui_secret_key }}"
|
||||
|
||||
# --- inference: vLLM speaks the OpenAI API, Ollama stays off ---
|
||||
- OPENAI_API_BASE_URL=http://vllm:8000/v1
|
||||
- OPENAI_API_KEY=dummy
|
||||
- ENABLE_OLLAMA_API=false
|
||||
|
||||
# --- vector store: pgvector rather than the default chroma, which does
|
||||
# not scale to the corpus sizes discussed in §2.6 ---
|
||||
- VECTOR_DB=pgvector
|
||||
- "PGVECTOR_DB_URL=postgresql://openwebui:{{ vault_owui_db_password }}@owui-db:5432/openwebui"
|
||||
|
||||
# --- RAG embeddings from the second vLLM instance ---
|
||||
- RAG_EMBEDDING_ENGINE=openai
|
||||
- RAG_OPENAI_API_BASE_URL=http://embeddings:8000/v1
|
||||
- RAG_OPENAI_API_KEY=dummy
|
||||
- "RAG_EMBEDDING_MODEL={{ embedding_model_name }}"
|
||||
|
||||
# --- AD/LDAP login (group llm_users) ---
|
||||
# NOTE: verify these variable names against the Open WebUI docs for the
|
||||
# tag actually deployed — they have changed between releases.
|
||||
- ENABLE_LDAP=true
|
||||
- LDAP_SERVER_LABEL=Phytron AD
|
||||
- "LDAP_SERVER_HOST={{ ldap_server_host }}"
|
||||
- "LDAP_SERVER_PORT={{ ldap_server_port }}"
|
||||
- "LDAP_APP_DN={{ ldap_bind_dn }}"
|
||||
- "LDAP_APP_PASSWORD={{ vault_ldap_bind_password }}"
|
||||
- "LDAP_SEARCH_BASE={{ ldap_search_base }}"
|
||||
- "LDAP_SEARCH_FILTER={{ ldap_search_filter }}"
|
||||
- LDAP_ATTRIBUTE_FOR_USERNAME=sAMAccountName
|
||||
- LDAP_USE_TLS=false
|
||||
restart: unless-stopped
|
||||
|
||||
owui-db:
|
||||
image: "{{ pgvector_image }}"
|
||||
container_name: owui-db
|
||||
networks:
|
||||
- llmnet
|
||||
volumes:
|
||||
- "{{ appdata_path }}/databases/openwebui-pgdata:/var/lib/postgresql/data"
|
||||
environment:
|
||||
- POSTGRES_DB=openwebui
|
||||
- POSTGRES_USER=openwebui
|
||||
- "POSTGRES_PASSWORD={{ vault_owui_db_password }}"
|
||||
restart: unless-stopped
|
||||
@@ -0,0 +1,7 @@
|
||||
# network definition (leading comment required: the compose generator
|
||||
# indents the first line of each fragment; networks: must stay top-level)
|
||||
networks:
|
||||
llmnet:
|
||||
driver: bridge
|
||||
driver_opts:
|
||||
com.docker.network.bridge.name: br-llm
|
||||
@@ -0,0 +1,39 @@
|
||||
# Compose stacks for phy-srv-gpu01
|
||||
|
||||
Rendered by `ironicbadger.docker_compose_generator` (pinned to 2.0.1) into
|
||||
`{{ docker_compose_generator_output_path }}/compose.yaml` on the host.
|
||||
The fragments here are **Jinja2 templates**, not plain compose files — Ansible
|
||||
variables are substituted at render time.
|
||||
|
||||
| Stack | Contains |
|
||||
| --- | --- |
|
||||
| `01-vllm` | vLLM, serves the chat model on port 8000 (OpenAI API) |
|
||||
| `02-embeddings` | second vLLM instance, embedding model on port 8001 |
|
||||
| `03-openwebui` | Open WebUI (port 80) + pgvector Postgres |
|
||||
| `99-network` | the shared `llmnet` bridge — must sort last |
|
||||
|
||||
Ordering comes from the `NN-` prefix; `99-network` keeps `networks:` at the
|
||||
bottom of the generated file. The leading comment in `99-network/compose.yml`
|
||||
is load-bearing: the role indents only the first line of each fragment, and the
|
||||
comment absorbs that indentation so `networks:` stays top-level.
|
||||
|
||||
## Deploy
|
||||
|
||||
```sh
|
||||
just compose phy_srv_gpu01 # renders ~/docker/compose.yaml on the host
|
||||
```
|
||||
|
||||
The role only **writes** the file. Bringing the stack up is deliberate:
|
||||
|
||||
```sh
|
||||
ssh sbxadmin@192.168.66.69 'cd ~/docker && docker compose up -d'
|
||||
```
|
||||
|
||||
## Before the first run
|
||||
|
||||
- Secrets must exist in `group_vars/secrets.yml` (`just vault edit`):
|
||||
`vault_owui_secret_key`, `vault_owui_db_password`, `vault_ldap_bind_password`
|
||||
- LDAP parameters in `group_vars/phy_srv_gpu01.yml` need the real bind DN and
|
||||
base DN from the customer
|
||||
- `vllm_image` must point at a build that works on SM120 (Blackwell) — verify
|
||||
before deploying, the official image is not guaranteed to work
|
||||
Reference in New Issue
Block a user