diff --git a/Dockerfile b/Dockerfile index e7c6094..bade85a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,74 +1,125 @@ -FROM nvidia/cuda:13.2.1-cudnn-devel-ubuntu24.04 +# ---------- llama.cpp Build Stage +FROM nvidia/cuda:13.2.1-cudnn-devel-ubuntu24.04 AS builder + ENV DEBIAN_FRONTEND=noninteractive -# "Unminify" -RUN rm -f /etc/dpkg/dpkg.cfg.d/excludes && \ - rm -f /etc/apt/apt.conf.d/docker-clean && \ - rm -f /etc/apt/apt.conf.d/01-nodoc && \ - rm -f /usr/bin/man && \ - dpkg-divert --quiet --remove /usr/bin/man - -# Install packages +# Install build tools, CMake, Git, and documentation tools RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \ - apt update && apt-get dist-upgrade --autoremove -y && apt-get install -y \ - python3-pip \ - python3-dev \ - python3-venv \ + apt-get update && apt-get install -y --no-install-recommends \ build-essential \ - man \ - man-db \ - wget \ - bzip2 \ - curl \ + cmake \ git \ - cmake + ninja-build \ + ca-certificates \ + libgomp1 \ + pandoc \ + && rm -rf /var/lib/apt/lists/* -# Set up nvfp4 python virtual environment -WORKDIR /opt/nvfp4 -RUN python3 -m venv /opt/nvfp4/py-venv -ENV PATH="/opt/nvfp4/py-venv/bin:${PATH}" -RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked pip3 install torch torchvision torchaudio --upgrade --index-url \ - https://download.pytorch.org/whl/cu132/ +WORKDIR /workspace - -# Fetch convert_to_quant, and install it into our python venv. -WORKDIR /opt/nvfp4 -RUN git clone --depth=1 https://github.com/silveroxides/convert_to_quant && cd convert_to_quant && rm -rf .git && pip3 install -r requirements.txt && pip3 install . - -# Clone and compile llama.cpp (save some space by removing unneeded .git data) -RUN git clone --depth=1 https://github.com/ggml-org/llama.cpp && rm -rf llama.cpp/.git -WORKDIR /opt/nvfp4/llama.cpp +# Clone llama.cpp repository +ARG LLAMA_CPP_TAG=master +RUN git clone --depth 1 --branch ${LLAMA_CPP_TAG} https://github.com/ggerganov/llama.cpp.git . # Build llama.cpp, the ldconfig is important RUN echo "/usr/local/cuda-13.2/compat" > /etc/ld.so.conf.d/988-cuda-compat.conf && ldconfig RUN cmake -B build -S . \ -DCMAKE_BUILD_TYPE=Release \ -DGGML_CUDA=ON \ + -DCMAKE_INSTALL_PREFIX=/install \ -DCMAKE_CUDA_ARCHITECTURES=120 \ -DLLAMA_BUILD_TESTS=OFF \ -DLLAMA_BUILD_EXAMPLES=ON \ - && cmake --build build -j$(nproc) \ - && cmake --build build --target install \ + && cmake --build build --config Release -j $(nproc) \ + && cmake --install build \ && rm -rf build +# Generate or stage manpages into /install/share/man +# If manpages are prebuilt or markdown docs exist, compile/place them in standard sections: +RUN mkdir -p /install/share/man/man1 && \ + for doc in docs/*.md; do \ + [ -f "$doc" ] || continue; \ + base=$(basename "$doc" .md); \ + pandoc "$doc" -s -t man -o "/install/share/man/man1/${base}.1" 2>/dev/null || true; \ + done + +# ---------- Python virtual environment builder + +FROM nvidia/cuda:13.2.1-cudnn-devel-ubuntu24.04 AS venv-builder + +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt update && apt-get install -y --no-install-recommends \ + python3-pip \ + python3-dev \ + python3-venv \ + git + +WORKDIR /workspace +RUN python3 -m venv /workspace/py-venv +ENV PATH="/workspace/py-venv/bin:${PATH}" +RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked pip3 install torch torchvision torchaudio --upgrade --index-url \ + https://download.pytorch.org/whl/cu132/ + +# Fetch convert_to_quant, and install it into our python venv. +RUN git clone --depth=1 https://github.com/silveroxides/convert_to_quant && \ + cd convert_to_quant && pip3 install -r requirements.txt && pip3 install . + +# Install heretic-llm +RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked pip3 install heretic-llm + +# Install zefie's nvfp4 quant requirements +COPY --chmod=0644 requirements.txt /workspace/requirements.txt +RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked pip3 install -r /workspace/requirements.txt + +# ---------- Main image + +FROM nvidia/cuda:13.2.1-cudnn-devel-ubuntu24.04 AS runner +ENV DEBIAN_FRONTEND=noninteractive + +# "Unminify" nvidia container to enable man pages +RUN rm -f /etc/dpkg/dpkg.cfg.d/excludes && \ + rm -f /etc/apt/apt.conf.d/docker-clean && \ + rm -f /etc/apt/apt.conf.d/01-nodoc && \ + rm -f /usr/bin/man && \ + dpkg-divert --quiet --remove /usr/bin/man + +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt update && apt-get install -y \ + python3 \ + man \ + man-db \ + wget \ + bzip2 \ + curl + +WORKDIR /opt/nvfp4 -# Copy and install zefie's nvfp4 quant scripts RUN mkdir nvfp4_gguf_quant -WORKDIR /opt/nvfp4/nvfp4_gguf_quant -COPY --exclude=__pycache__ nvfp4gguf ./nvfp4gguf -COPY --chmod=0644 gguf_to_nvfp4_gguf.py hf_to_nvfp4_gguf.py hf_to_nvfp4_safetensors.py requirements.txt safetensors_to_nvfp4_gguf.py hf_info.py gguf_info.py . +COPY --exclude=__pycache__ nvfp4gguf ./nvfp4_gguf_quant/nvfp4gguf +COPY --chmod=0644 gguf_to_nvfp4_gguf.py hf_to_nvfp4_gguf.py hf_to_nvfp4_safetensors.py safetensors_to_nvfp4_gguf.py hf_info.py gguf_info.py ./nvfp4_gguf_quant/ -# Install requirements for zefie's nvfp4 quant scripts -RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked pip3 install -r requirements.txt COPY --chmod=0644 docker/README.txt /workspace/README.txt -COPY --chmod=0755 docker/hf_info docker/gguf_info docker/hf_to_nvfp4_gguf docker/hf_to_nvfp4_safetensors docker/gguf_to_nvfp4_gguf docker/safetensors_to_nvfp4_gguf /usr/local/bin/ +COPY --chmod=0755 --exclude=README.txt docker/* /usr/local/bin/ -# Install manpages for zefie's nvfp4 quant scripts, ctq, and heretic +# Install manpages for zefie's nvfp4 quant scripts, ctq and heretic RUN mkdir -p /usr/local/share/man/man1 COPY --chmod=0644 man/man1/* /usr/local/share/man/man1/ -RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked pip3 install heretic-llm + +# Copy llama.cpp compiled binaries, manpages and shared libraries from builder +COPY --from=builder /install/bin/ /usr/local/bin/ +COPY --from=builder /install/lib/ /usr/local/lib/ +COPY --from=builder /install/share/man/ /usr/share/man/ + +# Set up a minimal llama.cpp source for conversions +RUN mkdir /opt/nvfp4/llama.cpp +COPY --from=builder /workspace/conversion /opt/nvfp4/llama.cpp/conversion/ +COPY --from=builder --chmod=0644 /workspace/*.py /opt/nvfp4/llama.cpp/ + +COPY --from=venv-builder /workspace/py-venv /opt/nvfp4/py-venv +ENV PATH="/opt/nvfp4/py-venv/bin:${PATH}" # Finalize and settle at /workspace ENV DEBIAN_FRONTEND= -WORKDIR /workspace -ENTRYPOINT [] +WORKDIR /workspace \ No newline at end of file diff --git a/docker.sh b/docker.sh index 4961e12..f3becca 100755 --- a/docker.sh +++ b/docker.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash set -euo pipefail +CURDIR="$(realpath "$(dirname "${0}")")" + get_latest() { # Find all tags for zefie/nvfp4_quant, sort by creation timestamp (descending), return the newest latest_image=$(docker images --format '{{.CreatedAt}}\t{{.Repository}}:{{.Tag}}' zefie/nvfp4_quant \ @@ -33,7 +35,7 @@ elif [ "${1}" == "run" ]; then exit 1; fi export EMBEDDED=1 - source ${HOME}/bin/nvfp4_quant + source ${CURDIR}/nvfp4_quant CMD=("${COMMON[@]}" "${@}") echo "Executing ${CMD[@]}" "${CMD[@]}" diff --git a/llama.cpp b/llama.cpp deleted file mode 160000 index 9e2a04a..0000000 --- a/llama.cpp +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 9e2a04a439edc49f08c5acdafefae19dfa25a658 diff --git a/nvfp4_quant b/nvfp4_quant new file mode 100755 index 0000000..1763fd3 --- /dev/null +++ b/nvfp4_quant @@ -0,0 +1,17 @@ +#!/bin/bash +TAG="${TAG:-latest}" +CMD="$(basename "${0}")" +if [ -z "${USER}" ]; then + USER="$(whoami)" +fi + +IMAGE="${IMAGE:-zefie/nvfp4_quant:${TAG}}" +COMMON=(docker run --user 1000:1000 --rm --gpus all -it -v ./:/workspace "${IMAGE}") + +if [ -z "${EMBEDDED}" ]; then + if [ "${CMD}" == "nvfp4_quant" ]; then + "${COMMON[@]}" "${@}" + else + "${COMMON[@]}" "${CMD}" "${@}" + fi +fi