In this tutorial, we work with Dory 1.0 4B, a small reasoning model from NVIDIA’s Nemotron Labs whose middle layers form a shared block that can be applied once, twice, three, or four times for every token. Each extra pass, or loop, spends more computation per token without adding a single parameter, so one checkpoint can run at four different compute budgets with no retraining. We build the llama.cpp branch that NVIDIA’s model card pins, download the 3.7 GB GGUF, and read the architecture straight from the file to predict what each loop should cost in parameters and cache memory. We then check those predictions against the running server, measure decode speed at every loop count, test whether the cheaper last-loop cache changes the model’s predictions, and run a paired GSM8K evaluation to see how accuracy, answer length, and time per solved question move as the loops increase.
import os
import re
import sys
import json
import time
import math
import random
import signal
import shutil
import hashlib
import platform
import statistics
import traceback
import subprocess
import collections
import urllib.request
from pathlib import Path
RESULTS = {} # step -> one-line result, for the summary table
CLAIMS = [] # (step, claim, held)
CURRENT = ["1"]
def banner(title):
print("\n" + "=" * 78)
print(title)
print("=" * 78)
def section(name):
def wrap(fn):
def run(*a, **kw):
CURRENT[0] = name.split(".")[0]
banner(name)
t0 = time.time()
try:
out = fn(*a, **kw)
RESULTS[name] = out if isinstance(out, str) else "ok"
return out
except Exception as e:
RESULTS[name] = f"SKIPPED / FAILED -> {type(e).__name__}: {e}"
print(f"\n[!] {name} did not complete: {type(e).__name__}: {e}")
traceback.print_exc(limit=3)
return None
finally:
print(f"\n[{time.time() - t0:.1f} s]")
return run
return wrap
def claim(text, held):
CLAIMS.append((CURRENT[0], text, bool(held)))
return held
banner("1. Build the pinned Dory llama.cpp fork and record the environment")
FORK = "https://github.com/brianchmielNV/llama.cpp.git"
FORK_COMMIT = "98a74c97ea09ac72f6ceda1559808fc0a2f447ce" # the commit NVIDIA's model card pins; ggml-org PR #30306 is still a draft
GGUF_REPO = "nvidia/NVIDIA-Nemotron-Labs-Dory1.0-4B-GGUF"
GGUF_REV = "df53755cb8741843af8e019e088819fbef17e29b"
GGUF_FILE = "NVIDIA-Nemotron-Labs-Dory1.0-4B-Q5_K_M.gguf" # the card's commands call it dory-q5_k_m.gguf
GGUF_SIZE = 3_673_288_864
GGUF_SHA256 = "96080be49ffd67b916018b7bdef8c484762576fb52780e10ab98ac4978a660e0"
GSM8K_REV = "740312add88f781978c0658806c59bc2815b9866"
WORK = Path(os.environ.get("DORY_WORK") or ("/content/dory" if os.path.isdir("/content") else "dory_work")).resolve()
WORK.mkdir(parents=True, exist_ok=True)
(WORK / "logs").mkdir(exist_ok=True)
subprocess.run([sys.executable, "-m", "pip", "install", "-q", "gguf==0.19.0", "huggingface_hub", "pandas", "pyarrow"], check=True)
SYSTEM = platform.system()
GPU = ""
if shutil.which("nvidia-smi"):
q = subprocess.run(["nvidia-smi", "--query-gpu=name,memory.total,compute_cap", "--format=csv,noheader"],
capture_output=True, text=True)
GPU = q.stdout.strip().splitlines()[0] if q.returncode == 0 and q.stdout.strip() else ""
BACKEND = "metal" if SYSTEM == "Darwin" else ("cuda" if GPU else "cpu")
SRC = WORK / "llama.cpp-dory"
SERVER_BIN = SRC / "build" / "bin" / "llama-server"
free_gb = shutil.disk_usage(WORK).free / 1e9
print(f"{SYSTEM} {platform.machine()}, Python {platform.python_version()}, {os.cpu_count()} CPU cores, "
f"backend {BACKEND}{' (' + GPU + ')' if GPU else ''}, {free_gb:.1f} GB free disk")
if not SERVER_BIN.exists():
t0 = time.time()
if not (SRC / ".git").exists():
subprocess.run(["git", "init", "-q", str(SRC)], check=True)
subprocess.run(["git", "-C", str(SRC), "remote", "add", "origin", FORK], check=True)
subprocess.run(["git", "-C", str(SRC), "fetch", "-q", "--depth", "1", "origin", FORK_COMMIT], check=True)
subprocess.run(["git", "-C", str(SRC), "checkout", "-q", "--detach", "FETCH_HEAD"], check=True)
flags = ["-DCMAKE_BUILD_TYPE=Release", "-DLLAMA_BUILD_SERVER=ON", "-DLLAMA_BUILD_TESTS=OFF",
"-DLLAMA_BUILD_EXAMPLES=OFF", "-DLLAMA_BUILD_APP=OFF", "-DLLAMA_BUILD_UI=OFF",
"-DLLAMA_USE_PREBUILT_UI=ON", "-DLLAMA_OPENSSL=OFF"]
if BACKEND == "metal": # the card's Apple-silicon flags
flags += ["-DCMAKE_OSX_ARCHITECTURES=arm64", "-DGGML_METAL=ON", "-DGGML_METAL_EMBED_LIBRARY=ON",
"-DGGML_BLAS=ON", "-DGGML_BLAS_VENDOR=Apple"]
elif BACKEND == "cuda": # kernels for this GPU only (a T4 is 7.5); flash-attention kernels are unused with --flash-attn off
cap = GPU.split(",")[-1].strip().replace(".", "")
flags += ["-DGGML_CUDA=ON", f"-DCMAKE_CUDA_ARCHITECTURES={cap}", "-DGGML_CUDA_FA=OFF", "-DGGML_NATIVE=OFF"]
env = dict(os.environ, HF_UI_VERSION="b11551")
with open(WORK / "build.log", "w") as log:
subprocess.run(["cmake", "-S", str(SRC), "-B", str(SRC / "build")] + flags, check=True, env=env, stdout=log, stderr=log)
subprocess.run(["cmake", "--build", str(SRC / "build"), "--parallel", str(os.cpu_count()), "--target", "llama-server"],
check=True, env=env, stdout=log, stderr=log)
print(f"built llama-server in {time.time() - t0:.0f} s")
head = subprocess.run(["git", "-C", str(SRC), "rev-parse", "HEAD"], capture_output=True, text=True).stdout.strip()
assert head == FORK_COMMIT, f"checked out {head}, expected {FORK_COMMIT}"
version = subprocess.run([str(SERVER_BIN), "--version"], capture_output=True, text=True)
print(f"fork commit {head[:12]}; {(version.stdout + version.stderr).strip().splitlines()[-1]}")
RESULTS["1. Build the pinned Dory llama.cpp fork and record the environment"] = f"fork {head[:12]}, backend {BACKEND}"
We build llama-server from the exact commit of the Dory branch that NVIDIA’s model card pins, because Dory support in llama.cpp is still an open draft pull request and no released binary can load the model. The cell detects the platform and passes the card’s Metal flags on a Mac, or compiles CUDA kernels for the attached GPU alone on Colab, skipping the flash-attention kernels that the card’s –flash-attn off never uses, which keeps a T4 build manageable. It then checks that the checked-out commit is the pinned one, so every later number traces to one known version of the code.
@section("2. Download the Q5_K_M GGUF at a pinned revision and check its SHA-256")
def download_step():
global GGUF
from huggingface_hub import hf_hub_download
t0 = time.time()
GGUF = Path(hf_hub_download(GGUF_REPO, GGUF_FILE, revision=GGUF_REV, local_dir=str(WORK)))
h = hashlib.sha256()
with open(GGUF, "rb") as f:
for block in iter(lambda: f.read(1 << 24), b""):
h.update(block)
size, digest = GGUF.stat().st_size, h.hexdigest()
print(f"{GGUF.name}: {size:,} bytes, sha256 {digest[:16]}... ({time.time() - t0:.0f} s)")
assert size == GGUF_SIZE and digest == GGUF_SHA256, "the file does not match the card's size and checksum"
claim("the GGUF matches the size and SHA-256 published on the card", True)
return f"{size / 1e9:.2f} GB, checksum matches the card"
download_step()
The Q5_K_M GGUF comes from Hugging Face at a fixed repository revision, and we hash it before anything else touches it. The card’s commands refer to the file as dory-q5_k_m.gguf, but the repository names it NVIDIA-Nemotron-Labs-Dory1.0-4B-Q5_K_M.gguf, so copying the card’s command verbatim fails; pinning the real name, the size, and the SHA-256 together catches both a wrong file and a corrupted download. The 3.67 GB file matches the card’s published checksum exactly, so every measurement that follows uses the same bytes NVIDIA published.
@section("3. Read the architecture from the GGUF: three layer groups and a default of four loops")
def architecture_step():
global META, GROUPS, PATTERN
import gguf
reader = gguf.GGUFReader(str(GGUF))
META = {k: f.contents() for k, f in reader.fields.items() if k.startswith("dory.")}
for k in ("dory.block_count", "dory.input_layer_count", "dory.recurrent_layer_count", "dory.output_layer_count",
"dory.num_loops", "dory.recurrent_kv_cache_mode", "dory.attention.sliding_window", "dory.context_length"):
print(f"{k:34s} {META[k]}")
PATTERN = META["dory.layer_pattern"].split("-")[:-1] # one symbol per attention layer: S sliding window, * full
n_in, n_rec = META["dory.input_layer_count"], META["dory.recurrent_layer_count"]
print(f"\nlayer entries alternate attention and MLP: {len(PATTERN)} attention layers "
f"({PATTERN.count('S')} sliding-window, {PATTERN.count('*')} full attention)")
GROUPS = collections.defaultdict(lambda: {"params": 0, "bytes": 0})
dtypes = collections.Counter()
for t in reader.tensors:
m = re.match(r"blk\.(\d+)\.", t.name)
if m:
i = int(m.group(1))
g = "input block" if i < n_in else "recurrent block" if i < n_in + n_rec else "output block"
else:
g = "embedding" if t.name.startswith("token_embd") else "LM head"
GROUPS[g]["params"] += int(math.prod(int(x) for x in t.shape))
GROUPS[g]["bytes"] += int(t.n_bytes)
dtypes[t.tensor_type.name] += 1
print(f"\n{'group':16s} {'params':>9s} {'bytes':>9s}")
for g in ("embedding", "input block", "recurrent block", "output block", "LM head"):
print(f"{g:16s} {GROUPS[g]['params'] / 1e9:7.3f} B {GROUPS[g]['bytes'] / 1e9:6.3f} GB")
print(f"tensor types: {dict(dtypes)}")
claim("the GGUF stores 16 input, 40 recurrent and 16 output layer entries (8, 20 and 8 blocks)",
(n_in, n_rec, META["dory.output_layer_count"]) == (16, 40, 16))
claim("the GGUF's built-in loop count is 4, not the 2 the card recommends", META["dory.num_loops"] == 4)
total = sum(v["params"] for v in GROUPS.values())
return (f"{total / 1e9:.2f} B parameters, {GROUPS['recurrent block']['params'] / 1e9:.2f} B of them in the shared "
f"recurrent block; built-in num_loops = {META['dory.num_loops']}")
architecture_step()
The GGUF header tells us how the model is wired before we load it. Dory’s 72 layer entries split into an input group of 16, a shared recurrent group of 40 and an output group of 16, alternating attention and MLP, with one full-attention layer for every three sliding-window layers. Summing tensor sizes by group shows the recurrent block accounts for 1.84 of the model’s 3.98 billion parameters. The header also carries a trap: its built-in loop count is 4, while the card recommends 2, so a server started without an override silently runs the most expensive setting.
LOOPS = (1, 2, 3, 4)
CTX, UBATCH = 4096, 512
def predicted_kv(loops, mode, ctx=CTX, ubatch=UBATCH):
"""KV cache under the fork's allocation rule: per_loop keeps one cache per loop for every recurrent attention layer,
last_loop keeps one in all. Full-attention layers hold ctx cells; sliding-window layers hold window + ubatch, padded to 256."""
n_in, n_rec = META["dory.input_layer_count"] // 2, META["dory.recurrent_layer_count"] // 2
rec, fixed = PATTERN[n_in:n_in + n_rec], PATTERN[:n_in] + PATTERN[n_in + n_rec:]
copies = loops if mode == "per_loop" else 1
full = fixed.count("*") + copies * rec.count("*")
swa = fixed.count("S") + copies * rec.count("S")
swa_cells = min(ctx, -(-(META["dory.attention.sliding_window"] + ubatch) // 256) * 256)
per_cell = 2 * META["dory.attention.head_count_kv"] * META["dory.attention.key_length"] * 2 # K and V, 2 bytes each
return full, swa, (full * ctx + swa * swa_cells) * per_cell / 2**20
@section("4. Predict what each extra loop costs: parameters per token and KV-cache size")
def cost_model_step():
global PER_TOKEN
once = ("input block", "output block", "LM head") # the embedding is a one-row lookup, so it costs no matrix work
PER_TOKEN = {}
print(f"{'loops':>5s} {'params per token':>17s} {'vs 1 loop':>10s} {'weight bytes read':>18s} {'vs 1 loop':>10s}")
for n in LOOPS:
p = sum(GROUPS[g]["params"] for g in once) + n * GROUPS["recurrent block"]["params"]
b = sum(GROUPS[g]["bytes"] for g in once) + n * GROUPS["recurrent block"]["bytes"]
PER_TOKEN[n] = (p, b)
print(f"{n:5d} {p / 1e9:15.2f} B {p / PER_TOKEN[1][0]:9.2f}x {b / 1e9:16.2f} GB {b / PER_TOKEN[1][1]:9.2f}x")
print(f"\npredicted KV cache at --ctx-size {CTX}, f16 keys and values, micro-batch {UBATCH}:")
print(f"{'loops':>5s} {'per_loop: full + SWA layers':>28s} {'MiB':>7s} {'last_loop: full + SWA layers':>29s} {'MiB':>7s}")
for n in LOOPS:
a, b = predicted_kv(n, "per_loop"), predicted_kv(n, "last_loop")
print(f"{n:5d} {a[0]:>21d} + {a[1]:<4d} {a[2]:7.1f} {b[0]:>22d} + {b[1]:<4d} {b[2]:7.1f}")
full_ctx = META["dory.context_length"]
print(f"\nat the full {full_ctx:,}-token context: per_loop {predicted_kv(2, 'per_loop', full_ctx)[2] / 1024:.1f} GiB at 2 loops "
f"and {predicted_kv(4, 'per_loop', full_ctx)[2] / 1024:.1f} GiB at 4; last_loop "
f"{predicted_kv(4, 'last_loop', full_ctx)[2] / 1024:.1f} GiB at any loop count")
return (f"4 loops apply {PER_TOKEN[4][0] / PER_TOKEN[1][0]:.2f}x the parameters per token of 1 loop; predicted KV at 4 loops "
f"{predicted_kv(4, 'per_loop')[2]:.0f} MiB per_loop vs {predicted_kv(4, 'last_loop')[2]:.0f} MiB last_loop")
cost_model_step()
From those group sizes, we can predict what a loop costs before running anything. A token passes through the input block, the recurrent block once per loop, the output block, and the LM head, so 4 loops apply 2.51 times the parameters of 1 loop and read 2.28 times the weight bytes, the better guide for a memory-bound decoder. The KV cache follows the fork’s allocation rule: per_loop keeps a separate cache for every loop of every recurrent attention layer, growing from 306 MiB at 1 loop to 816 MiB at 4 for a 4,096-token context, while last_loop stays at 306 MiB.
PORT = 8090
KV_LINE = re.compile(r"llama_kv_cache: size =\s*([\d.]+) MiB \(\s*(\d+) cells,\s*(\d+) layers")
class DoryServer:
"""llama-server with one loop count and KV mode. Both are read when the model loads, so each setting is a fresh start."""
def __init__(self, loops=None, mode="per_loop", ngl=999, ubatch=UBATCH):
self.loops, self.mode, self.ngl, self.ubatch = loops, mode, ngl, ubatch
self.name = f"{'default' if loops is None else loops}-{mode}-ngl{ngl}-ub{ubatch}"
def __enter__(self):
cmd = [str(SERVER_BIN), "-m", str(GGUF), "--host", "127.0.0.1", "--port", str(PORT), "-ngl", str(self.ngl),
"--ctx-size", str(CTX), "--parallel", "1", "--batch-size", str(self.ubatch), "--ubatch-size", str(self.ubatch),
"--cache-type-k", "f16", "--cache-type-v", "f16", "--flash-attn", "off", "--no-context-shift",
"--jinja", "--reasoning", "on", "--temp", "1.0", "--top-p", "0.95", "--top-k", "0", "--min-p", "0",
"--repeat-penalty", "1"]
if self.loops is not None:
cmd += ["--override-kv", f"dory.num_loops=int:{self.loops},dory.recurrent_kv_cache_mode=str:{self.mode}"]
self.log = WORK / "logs" / f"server-{self.name}.log"
with open(self.log, "w") as f:
self.proc = subprocess.Popen(cmd, stdout=f, stderr=subprocess.STDOUT, start_new_session=True)
t0 = time.time()
while True:
try:
with urllib.request.urlopen(f"http://127.0.0.1:{PORT}/health", timeout=2) as r:
if r.status == 200: