fancy multi-stage Dockerfile, remove llama.cpp from repo, add zefie's local nvfp4_quant docker launcher helper

Dockerfile

@@ -1,74 +1,125 @@
-FROM nvidia/cuda:13.2.1-cudnn-devel-ubuntu24.04
+# ---------- llama.cpp Build Stage
+FROM nvidia/cuda:13.2.1-cudnn-devel-ubuntu24.04 AS builder
+
 ENV DEBIAN_FRONTEND=noninteractive
 
-# "Unminify"
+# Install build tools, CMake, Git, and documentation tools
-RUN rm -f /etc/dpkg/dpkg.cfg.d/excludes && \
-    rm -f /etc/apt/apt.conf.d/docker-clean && \
-    rm -f /etc/apt/apt.conf.d/01-nodoc && \
-    rm -f /usr/bin/man && \
-    dpkg-divert --quiet --remove /usr/bin/man
-
-# Install packages
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
     --mount=type=cache,target=/var/lib/apt,sharing=locked \
-    apt update && apt-get dist-upgrade --autoremove -y && apt-get install -y \
+    apt-get update && apt-get install -y --no-install-recommends \
-    python3-pip \
-    python3-dev \
-    python3-venv \
     build-essential \
-    man \
+    cmake \
-    man-db \
-    wget \
-    bzip2 \
-    curl \
     git \
-    cmake
+    ninja-build \
+    ca-certificates \
+    libgomp1 \
+    pandoc \
+    && rm -rf /var/lib/apt/lists/*
 
-# Set up nvfp4 python virtual environment
+WORKDIR /workspace
-WORKDIR /opt/nvfp4
-RUN python3 -m venv /opt/nvfp4/py-venv
-ENV PATH="/opt/nvfp4/py-venv/bin:${PATH}"
-RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked pip3 install torch torchvision torchaudio --upgrade --index-url \
-    https://download.pytorch.org/whl/cu132/
 
-
+# Clone llama.cpp repository
-# Fetch convert_to_quant, and install it into our python venv.
+ARG LLAMA_CPP_TAG=master
-WORKDIR /opt/nvfp4
+RUN git clone --depth 1 --branch ${LLAMA_CPP_TAG} https://github.com/ggerganov/llama.cpp.git .
-RUN git clone --depth=1 https://github.com/silveroxides/convert_to_quant && cd convert_to_quant && rm -rf .git && pip3 install -r requirements.txt && pip3 install .
-
-# Clone and compile llama.cpp (save some space by removing unneeded .git data)
-RUN git clone --depth=1 https://github.com/ggml-org/llama.cpp && rm -rf llama.cpp/.git
-WORKDIR /opt/nvfp4/llama.cpp
 
 # Build llama.cpp, the ldconfig is important
 RUN echo "/usr/local/cuda-13.2/compat" > /etc/ld.so.conf.d/988-cuda-compat.conf && ldconfig
 RUN cmake -B build -S . \
      -DCMAKE_BUILD_TYPE=Release \
      -DGGML_CUDA=ON \
+     -DCMAKE_INSTALL_PREFIX=/install \
      -DCMAKE_CUDA_ARCHITECTURES=120 \
      -DLLAMA_BUILD_TESTS=OFF \
      -DLLAMA_BUILD_EXAMPLES=ON \
-    && cmake --build build -j$(nproc) \
+    && cmake --build build --config Release -j $(nproc) \
-    && cmake --build build --target install \
+    && cmake --install build \
     && rm -rf build
+# Generate or stage manpages into /install/share/man
+# If manpages are prebuilt or markdown docs exist, compile/place them in standard sections:
+RUN mkdir -p /install/share/man/man1 && \
+    for doc in docs/*.md; do \
+        [ -f "$doc" ] || continue; \
+        base=$(basename "$doc" .md); \
+        pandoc "$doc" -s -t man -o "/install/share/man/man1/${base}.1" 2>/dev/null || true; \
+    done
+
+# ---------- Python virtual environment builder
+
+FROM nvidia/cuda:13.2.1-cudnn-devel-ubuntu24.04 AS venv-builder
+
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    apt update && apt-get install -y --no-install-recommends \
+    python3-pip \
+    python3-dev \
+    python3-venv \
+    git
+
+WORKDIR /workspace
+RUN python3 -m venv /workspace/py-venv
+ENV PATH="/workspace/py-venv/bin:${PATH}"
+RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked pip3 install torch torchvision torchaudio --upgrade --index-url \
+    https://download.pytorch.org/whl/cu132/
+
+# Fetch convert_to_quant, and install it into our python venv.
+RUN git clone --depth=1 https://github.com/silveroxides/convert_to_quant && \
+    cd convert_to_quant && pip3 install -r requirements.txt && pip3 install .
+
+# Install heretic-llm
+RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked pip3 install heretic-llm
+
+# Install zefie's nvfp4 quant requirements
+COPY --chmod=0644 requirements.txt /workspace/requirements.txt
+RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked pip3 install -r /workspace/requirements.txt
+
+# ---------- Main image
+
+FROM nvidia/cuda:13.2.1-cudnn-devel-ubuntu24.04 AS runner
+ENV DEBIAN_FRONTEND=noninteractive
+
+# "Unminify" nvidia container to enable man pages
+RUN rm -f /etc/dpkg/dpkg.cfg.d/excludes && \
+    rm -f /etc/apt/apt.conf.d/docker-clean && \
+    rm -f /etc/apt/apt.conf.d/01-nodoc && \
+    rm -f /usr/bin/man && \
+    dpkg-divert --quiet --remove /usr/bin/man
+
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    apt update && apt-get install -y \
+    python3 \
+    man \
+    man-db \
+    wget \
+    bzip2 \
+    curl
+
+WORKDIR /opt/nvfp4
 
-# Copy and install zefie's nvfp4 quant scripts
 RUN mkdir nvfp4_gguf_quant
-WORKDIR /opt/nvfp4/nvfp4_gguf_quant
+COPY --exclude=__pycache__ nvfp4gguf ./nvfp4_gguf_quant/nvfp4gguf
-COPY --exclude=__pycache__ nvfp4gguf ./nvfp4gguf
+COPY --chmod=0644 gguf_to_nvfp4_gguf.py hf_to_nvfp4_gguf.py hf_to_nvfp4_safetensors.py safetensors_to_nvfp4_gguf.py hf_info.py gguf_info.py ./nvfp4_gguf_quant/
-COPY --chmod=0644 gguf_to_nvfp4_gguf.py hf_to_nvfp4_gguf.py hf_to_nvfp4_safetensors.py requirements.txt safetensors_to_nvfp4_gguf.py hf_info.py gguf_info.py .
 
-# Install requirements for zefie's nvfp4 quant scripts
-RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked pip3 install -r requirements.txt
 COPY --chmod=0644 docker/README.txt /workspace/README.txt
-COPY --chmod=0755 docker/hf_info docker/gguf_info docker/hf_to_nvfp4_gguf docker/hf_to_nvfp4_safetensors docker/gguf_to_nvfp4_gguf docker/safetensors_to_nvfp4_gguf /usr/local/bin/
+COPY --chmod=0755 --exclude=README.txt docker/* /usr/local/bin/
 
-# Install manpages for zefie's nvfp4 quant scripts, ctq, and heretic
+# Install manpages for zefie's nvfp4 quant scripts, ctq and heretic
 RUN mkdir -p /usr/local/share/man/man1
 COPY --chmod=0644 man/man1/* /usr/local/share/man/man1/
-RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked pip3 install heretic-llm
+
+# Copy llama.cpp compiled binaries, manpages and shared libraries from builder
+COPY --from=builder /install/bin/ /usr/local/bin/
+COPY --from=builder /install/lib/ /usr/local/lib/
+COPY --from=builder /install/share/man/ /usr/share/man/
+
+# Set up a minimal llama.cpp source for conversions
+RUN mkdir /opt/nvfp4/llama.cpp
+COPY --from=builder /workspace/conversion /opt/nvfp4/llama.cpp/conversion/
+COPY --from=builder --chmod=0644 /workspace/*.py /opt/nvfp4/llama.cpp/
+
+COPY --from=venv-builder /workspace/py-venv /opt/nvfp4/py-venv
+ENV PATH="/opt/nvfp4/py-venv/bin:${PATH}"
 
 # Finalize and settle at /workspace
 ENV DEBIAN_FRONTEND=
 WORKDIR /workspace
-ENTRYPOINT []

docker.sh

@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+CURDIR="$(realpath "$(dirname "${0}")")"
+
 get_latest() {
 # Find all tags for zefie/nvfp4_quant, sort by creation timestamp (descending), return the newest
 	latest_image=$(docker images --format '{{.CreatedAt}}\t{{.Repository}}:{{.Tag}}' zefie/nvfp4_quant \
@@ -33,7 +35,7 @@ elif [ "${1}" == "run" ]; then
 		exit 1;
 	fi
 	export EMBEDDED=1
-	source ${HOME}/bin/nvfp4_quant
+	source ${CURDIR}/nvfp4_quant
 	CMD=("${COMMON[@]}" "${@}")
 	echo "Executing ${CMD[@]}"
 	"${CMD[@]}"

nvfp4_quant

@@ -0,0 +1,17 @@
+#!/bin/bash
+TAG="${TAG:-latest}"
+CMD="$(basename "${0}")"
+if [ -z "${USER}" ]; then
+	USER="$(whoami)"
+fi
+
+IMAGE="${IMAGE:-zefie/nvfp4_quant:${TAG}}"
+COMMON=(docker run --user 1000:1000 --rm --gpus all -it -v ./:/workspace "${IMAGE}")
+
+if [ -z "${EMBEDDED}" ]; then
+	if [ "${CMD}" == "nvfp4_quant" ]; then
+		"${COMMON[@]}" "${@}"
+	else
+		"${COMMON[@]}" "${CMD}" "${@}"
+	fi
+fi