---

license: apache-2.0

---

• Model Q2_0.gguf is completely degraded by the quantization.
• Model Q4_1.gguf shows no quantization problems.

MTP active GPU RTX 3090 | 24 GB VRAM

• Q4_1 use: 18.8 GB VRAM
• Q5_K_S use: 20 GB VRAM

MTP active GPU RTX 6000 | 48 GB VRAM

 max_tok | compl_tok |  time(s) |  tok/s medido |  tok/s nativo | draft acc% |   finish
---------------------------------------------------------------------------------------
     512 |       512 |    14.07 |         36.38 | 38.03026971780351 |      83.6% |   length
    1024 |      1024 |    27.29 |         37.53 | 38.620231872251516 |      83.0% |   length
    1536 |      1536 |     40.4 |         38.02 | 38.91217863284138 |      85.2% |   length
    2048 |      2048 |    57.74 |         35.47 | 36.07018250085741 |      85.5% |   length
    2560 |      2560 |    69.86 |         36.64 | 37.41751994689986 |      83.4% |   length
    3072 |      3072 |    83.79 |         36.66 | 37.40154927882399 |      84.5% |   length
    3584 |      3584 |    98.34 |         36.44 | 37.083063983609044 |      84.5% |   length
    4096 |      4096 |   109.02 |         37.57 | 38.18485281655008 |      85.8% |   length
    4608 |      4608 |   122.13 |         37.73 | 38.337046416603904 |      85.9% |   length

Q4_1

max_tok | compl_tok |  time(s) |  tok/s medido |  tok/s nativo | draft acc% |   finish
---------------------------------------------------------------------------------------
     512 |       512 |     9.24 |         55.39 | 62.4539752626147 |      44.1% |   length
    1024 |      1024 |    16.02 |         63.93 | 68.03560414785971 |      50.7% |   length
    1536 |      1536 |    23.58 |         65.15 | 68.60468040246114 |      52.7% |   length
    2048 |      2048 |    31.26 |         65.52 | 68.2785767264003 |      51.9% |   length
    2560 |      2560 |    40.58 |         63.09 | 65.22341018001381 |      49.0% |   length
    3072 |      3072 |    46.15 |         66.57 | 68.65996160719114 |      52.5% |   length
    3584 |      3584 |    57.67 |         62.15 | 63.758617842674866 |      46.5% |   length
    4096 |      4096 |    61.97 |          66.1 | 67.72380170206316 |      51.4% |   length
    4608 |      4608 |    69.65 |         66.16 | 67.63730802868203 |      51.4% |   length

Q5_K_S

 max_tok | compl_tok |  time(s) |  tok/s medido |  tok/s nativo | draft acc% |   finish
---------------------------------------------------------------------------------------
     512 |       512 |    10.54 |         48.57 | 53.69416370658363 |      58.0% |   length
    1024 |      1024 |    21.48 |         47.67 | 49.9450896660683 |      51.6% |   length
    1536 |      1536 |    31.48 |         48.79 | 50.980782634670916 |      54.3% |   length
    2048 |      2048 |    44.64 |         45.88 | 47.13598921616946 |      47.8% |   length
    2560 |      2560 |    54.88 |         46.65 | 47.733756915731036 |      50.1% |   length
    3072 |      3072 |    70.81 |         43.39 | 44.329723967380424 |      43.6% |   length
    3584 |      3584 |    76.61 |         46.78 | 47.751962450436196 |      49.6% |   length
    4096 |      4096 |    85.52 |          47.9 | 48.840519974586236 |      51.8% |   length
    4608 |      4608 |    97.05 |         47.48 | 48.30432023651003 |      51.7% |   length

auto detect GPU - build_llamacpp_auto.sh

#!/usr/bin/env bash
# build_llamacpp_auto.sh
# Compila llama.cpp (llama-cli + llama-quantize) detectando automáticamente
# la arquitectura CUDA de las GPUs presentes (A40=86, RTX 4090=89, A100=80,
# H100=90, RTX PRO 6000 Blackwell=120, etc.)
#
# Uso:
#   ./build_llamacpp_auto.sh                 # desde la raíz de llama.cpp
#   ./build_llamacpp_auto.sh /ruta/llama.cpp # o pasando la ruta
#   CUDA_ARCHS=86 ./build_llamacpp_auto.sh   # forzar arquitectura manualmente

set -euo pipefail

LLAMA_DIR="${1:-.}"
BUILD_DIR="build-cuda"
TARGETS="llama-cli llama-quantize llama-server"

cd "$LLAMA_DIR"

if [[ ! -f CMakeLists.txt ]]; then
  echo "ERROR: no se encontró CMakeLists.txt en $(pwd). ¿Estás en la raíz de llama.cpp?" >&2
  exit 1
fi

# ---------------------------------------------------------------
# 1. Detectar arquitecturas CUDA
# ---------------------------------------------------------------
detect_archs() {
  # Permitir override manual: CUDA_ARCHS=86 o CUDA_ARCHS="80;86"
  if [[ -n "${CUDA_ARCHS:-}" ]]; then
    echo "$CUDA_ARCHS"
    return 0
  fi

  # Método principal: nvidia-smi (drivers >= 470 soportan compute_cap)
  if command -v nvidia-smi &>/dev/null; then
    local caps
    caps=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null \
           | tr -d ' ' | grep -E '^[0-9]+\.[0-9]+$' || true)
    if [[ -n "$caps" ]]; then
      # "8.6" -> "86", "12.0" -> "120"; deduplicar y unir con ';'
      echo "$caps" | tr -d '.' | sort -un | paste -sd ';' -
      return 0
    fi
  fi

  # Fallback 1: CMake >= 3.24 acepta "native" (nvcc detecta la GPU local)
  local cmake_ver
  cmake_ver=$(cmake --version | head -n1 | grep -oE '[0-9]+\.[0-9]+' | head -n1)
  if [[ -n "$cmake_ver" ]] && awk -v v="$cmake_ver" 'BEGIN{split(v,a,"."); exit !(a[1]>3 || (a[1]==3 && a[2]>=24))}'; then
    echo "native"
    return 0
  fi

  return 1
}

ARCHS=$(detect_archs) || {
  echo "ERROR: no pude detectar la arquitectura de la GPU." >&2
  echo "  - Verifica que nvidia-smi funcione, o" >&2
  echo "  - Fuerza el valor: CUDA_ARCHS=86 $0" >&2
  exit 1
}

# ---------------------------------------------------------------
# 2. Info y validaciones
# ---------------------------------------------------------------
echo "=== GPUs detectadas ==="
nvidia-smi --query-gpu=name,compute_cap,memory.total --format=csv,noheader 2>/dev/null || echo "(nvidia-smi no disponible)"
echo "CMAKE_CUDA_ARCHITECTURES=$ARCHS"

# Advertencia: sm_120 (Blackwell) requiere CUDA 12.8+
if [[ "$ARCHS" == *"120"* ]] && command -v nvcc &>/dev/null; then
  CUDA_VER=$(nvcc --version | grep -oE 'release [0-9]+\.[0-9]+' | grep -oE '[0-9]+\.[0-9]+')
  if awk -v v="$CUDA_VER" 'BEGIN{split(v,a,"."); exit !(a[1]<12 || (a[1]==12 && a[2]<8))}'; then
    echo "ADVERTENCIA: sm_120 (Blackwell) requiere CUDA >= 12.8, tienes $CUDA_VER" >&2
  fi
fi

# ---------------------------------------------------------------
# 3. Compilar
# ---------------------------------------------------------------
rm -rf "$BUILD_DIR"
cmake -B "$BUILD_DIR" \
  -DGGML_CUDA=ON \
  -DCMAKE_CUDA_ARCHITECTURES="$ARCHS" \
  -DBUILD_SHARED_LIBS=OFF \
  -DLLAMA_CURL=OFF \
  > /dev/null

cmake --build "$BUILD_DIR" --config Release -j "$(nproc)" --target $TARGETS > /dev/null

echo ""
echo "✅ Compilado para arquitectura(s): $ARCHS"
echo "Binarios en: $(pwd)/$BUILD_DIR/bin/"
ls -lh "$BUILD_DIR/bin/" | grep -E 'llama-(cli|quantize)' || true

Download models

hf download tepirale/Qwen3.5-27B-ThinkSonic-Coder-dare_ties-MTP-GGUF \
  --include "ThinkSonic-Coder-Q5_K_M.gguf" \
  --include "ThinkSonic-Coder-Q5_K_S.gguf" \
  --local-dir /workspace/models/ThinkSonic

run server

./build-cuda/bin/llama-server \
  -m /workspace/models/ThinkSonic/ThinkSonic-Coder-Q5_K_S.gguf \
  -ngl 999 \
  -c 21504 \
  --slots \
  --parallel 2 \
  --spec-type draft-mtp \
  --spec-draft-n-max 3 \
  --spec-draft-n-min 1 \
  --host 0.0.0.0 \
  --port 8080 \
  --flash-attn on \
  --batch-size 1024 \
  --ubatch-size 1024 \
  --api-key tu-clave-secreta \
  --dry-multiplier 0.8 \
  --alias "ThinkSonic-Custom" \
  --metrics