Ollama Architecture Analysis / Real-World Q&A

╔══════════════════════════════════════════════════════════════════╗
║                        Ollama System                             ║
║                                                                  ║
║  ┌──────────────────────────────────────────────────────────┐   ║
║  │                    CLI Layer (Cobra)                      │   ║
║  │  ollama run / pull / push / create / list / show / serve  │   ║
║  └──────────────────────┬───────────────────────────────────┘   ║
║                         │                                         ║
║  ┌──────────────────────▼───────────────────────────────────┐   ║
║  │              HTTP Server (Gin + CORS)                     │   ║
║  │         127.0.0.1:11434  (configurable via OLLAMA_HOST)   │   ║
║  │                                                           │   ║
║  │   OpenAI Compatibility   Anthropic Compatibility          │   ║
║  │   Middleware             Middleware                       │   ║
║  └──────────────────────┬───────────────────────────────────┘   ║
║                         │                                         ║
║  ┌──────────────────────▼───────────────────────────────────┐   ║
║  │                   Route Handlers                          │   ║
║  │  ChatHandler / GenerateHandler / EmbedHandler             │   ║
║  │  PullHandler / PushHandler / CreateHandler                │   ║
║  └──────────────────────┬───────────────────────────────────┘   ║
║                         │ scheduleRunner()                        ║
║  ┌──────────────────────▼───────────────────────────────────┐   ║
║  │                   Scheduler                               │   ║
║  │  - Model load/unload management                           │   ║
║  │  - VRAM-based Eviction                                    │   ║
║  │  - Reference Counting                                     │   ║
║  │  - Keep-Alive timer                                       │   ║
║  └──────────────────────┬───────────────────────────────────┘   ║
║                         │ spawn process                           ║
║  ┌──────────────────────▼───────────────────────────────────┐   ║
║  │               Runner Process (separate process)           │   ║
║  │                                                           │   ║
║  │  ┌─────────────┐  ┌─────────────┐  ┌─────────────────┐  │   ║
║  │  │LlamaRunner  │  │OllamaRunner │  │ImageGen Runner  │  │   ║
║  │  │(llama.cpp)  │  │(Go native)  │  │(diffusion)      │  │   ║
║  │  └──────┬──────┘  └──────┬──────┘  └────────┬────────┘  │   ║
║  │         └────────────────┴──────────────────┘           │   ║
║  │                          │                                │   ║
║  │               GGML Backend (GPU/CPU)                      │   ║
║  │           CUDA  /  ROCm  /  Metal  /  CPU                │   ║
║  └──────────────────────────────────────────────────────────┘   ║
║                                                                  ║
║  ┌──────────────────────────────────────────────────────────┐   ║
║  │              Model Storage (Content-Addressable)          │   ║
║  │   ~/.ollama/models/manifests/ + blobs/sha256-[digest]    │   ║
║  └──────────────────────────────────────────────────────────┘   ║
╚══════════════════════════════════════════════════════════════════╝

type Server struct {
    addr          net.Addr    // listening address
    sched         *Scheduler  // model scheduler
    defaultNumCtx int         // default context length
}

type LlamaServer interface {
    Load(ctx context.Context, opts api.Options) error
    Ping(ctx context.Context) error
    Completion(ctx context.Context, req CompletionRequest, fn func(CompletionResponse)) error
    Embedding(ctx context.Context, input string) ([]float64, error)
    Tokenize(ctx context.Context, content string) ([]int, error)
    Detokenize(ctx context.Context, tokens []int) (string, error)
    MemorySize(ctx context.Context) (uint64, error)
    Close() error
}

Manifest (JSON)
  └── Layers []Layer
        ├── Digest: "sha256-abc123..."
        ├── MediaType: "application/vnd.ollama.image.model"
        └── Size: 4_000_000_000

Client
    │
    │ POST /api/chat
    ▼
ChatHandler()
    │ JSON parsing + validation
    │
    ▼
scheduleRunner(modelName, options)
    │
    ▼
Scheduler.GetRunner()
    ├─ [already loaded] → return Runner reference
    └─ [not loaded]     → enqueue as LlmRequest
                            │
                            ▼
                 processPending() goroutine
                            │
                            ├─ detect GPU devices
                            ├─ calculate available VRAM
                            ├─ evict existing model if needed
                            └─ spawn Runner process
                                     │
                                     ▼
                          Runner process (separate port)
                                     │
                            HTTP RPC communication
                                     │
                                     ▼
Runner.Completion()
    │
    ├─ messages → tokenize
    ├─ initialize / reuse KV cache
    ├─ execute forward pass (GPU)
    ├─ sample next token
    └─ repeat (until EOS or max_tokens)
    │
    ▼
ChatResponse chunks streamed (NDJSON)
    │
    ▼
Client (done: true + metrics)

"llama3.2:3b"
    │
    ▼
GetModel(name)
    │
    ├─ check for local manifest
    ├─ if absent, Pull from registry
    ▼
parse manifest.json
    │
    ▼
parse GGUF header → extract model metadata
    │ (parameter count, context length, quantization type, etc.)
    ▼
auto-detect or explicitly load chat template
    │
    ▼
return Model struct

// Unit of work queued for scheduling
type LlmRequest struct {
    ctx             context.Context
    model           *Model
    opts            api.Options
    sessionDuration *api.Duration
    successCh       chan *runnerRef   // delivers Runner once loaded
    errCh           chan error
}

// Reference-counted handle to a running Runner
type runnerRef struct {
    refMu    sync.Mutex
    refCount uint          // number of in-flight requests
    llama    llm.LlamaServer
    model    *Model
    pid      int           // Runner process PID
    gpus     []ml.DeviceID // GPUs in use
    expireTimer *time.Timer
}

Request A → load llama3.2 → refCount: 1
Request B → llama3.2 already loaded → refCount: 2
Request A done → refCount: 1
Request B done → refCount: 0 → Keep-Alive timer starts
5 min later → model unloaded (reloaded on next request)

~/.ollama/models/
├── manifests/
│   └── registry.ollama.com/
│       └── library/
│           ├── llama3.2/
│           │   ├── latest
│           │   └── 3b
│           └── mistral/
│               └── latest
└── blobs/
    ├── sha256-abc123...  (model weights GGUF file)
    ├── sha256-def456...  (template)
    └── sha256-ghi789...  (parameters)

1. Parse model name (registry/namespace/name:tag)
2. Fetch Manifest from registry API
3. Parallel download of required layer blobs (16-part chunks)
4. Resumable download support (Range header)
5. SHA256 digest verification
6. Save Manifest + blobs locally

FROM llama3.2

SYSTEM """
You are a helpful Korean assistant.
항상 한국어로 답변해주세요.
"""

PARAMETER temperature 0.7
PARAMETER top_p 0.9
PARAMETER num_ctx 8192

TEMPLATE """
{{- if .System }}<|system|>
{{ .System }}<|end|>
{{- end }}
{{- range .Messages }}<|{{ .Role }}|>
{{ .Content }}<|end|>
{{- end }}<|assistant|>
"""

[Ollama Server Process]
        │
        │ spawn Runner via exec.Command()
        │ pass port/config via environment variables
        ▼
[Runner Process (llama.cpp / ollama-engine)]
        │
        │ HTTP server on localhost:<random port>
        │ JSON RPC communication
        ▼
[GGML Backend]
        │
        ├── CUDA (libcuda.so)
        ├── ROCm (librocblas.so)
        ├── Metal (Apple Framework)
        └── CPU (AVX2/AVX512)

// CGO binding example (conceptual)
/*
#include "llama.h"
*/
import "C"

func loadModel(modelPath string) *C.llama_model {
    params := C.llama_model_default_params()
    return C.llama_load_model_from_file(C.CString(modelPath), params)
}

{
  "model": "llama3.2",
  "messages": [...],
  "options": {
    "temperature": 0.8,
    "top_k": 40,
    "top_p": 0.9,
    "min_p": 0.05,
    "repeat_penalty": 1.1,
    "num_ctx": 4096,
    "num_predict": 512,
    "seed": 42,
    "stop": ["\n\n", "<|end|>"],
    "num_gpu": 35
  },
  "keep_alive": "10m"
}

POST /v1/chat/completions     → ChatHandler (OpenAI format)
POST /v1/completions          → GenerateHandler (OpenAI format)
POST /v1/embeddings           → EmbedHandler (OpenAI format)
GET  /v1/models               → ListHandler (OpenAI format)

{"model":"llama3.2","message":{"role":"assistant","content":"안"},"done":false}
{"model":"llama3.2","message":{"role":"assistant","content":"녕"},"done":false}
{"model":"llama3.2","message":{"role":"assistant","content":"하세요"},"done":false}
{"model":"llama3.2","message":{"role":"assistant","content":"!"},"done":false}
{
  "model":"llama3.2",
  "done":true,
  "total_duration": 2340000000,
  "load_duration": 100000000,
  "prompt_eval_count": 12,
  "prompt_eval_duration": 340000000,
  "eval_count": 48,
  "eval_duration": 1900000000
}

type ChatRequest struct {
    Model     string          `json:"model"`
    Messages  []Message       `json:"messages"`
    Stream    *bool           `json:"stream"`
    Tools     []Tool          `json:"tools"`         // function calling
    Think     *ThinkValue     `json:"think"`         // reasoning mode (DeepSeek, etc.)
    KeepAlive *Duration       `json:"keep_alive"`
    Options   map[string]any  `json:"options"`
}

type Message struct {
    Role      string      `json:"role"`    // system/user/assistant/tool
    Content   string      `json:"content"`
    Images    []ImageData `json:"images"`  // Base64 images (multimodal)
    ToolCalls []ToolCall  `json:"tool_calls"`
}

type Options struct {
    // Sampling parameters (can be changed per request)
    Temperature    float32   // creativity (0.0 – 2.0)
    TopK           int       // Top-K sampling
    TopP           float32   // Top-P (nucleus) sampling
    MinP           float32   // Min-P sampling
    RepeatPenalty  float32   // repetition penalty
    Seed           int       // seed for reproducibility

    // Load-time parameters (applied when the model is reloaded)
    NumCtx         int       // context window size
    NumBatch       int       // batch size
    NumGPU         int       // GPU offload layers (-1 = all)
    NumThread      int       // CPU thread count
    UseMMap        *bool     // memory mapping
    UseMLock       bool      // memory lock (prevent swapping)
}

type Model struct {
    Name           string
    Config         model.ConfigV2
    ModelPath      string
    AdapterPaths   []string     // LoRA adapters
    ProjectorPaths []string     // Vision projectors
    System         string       // system prompt
    Template       *template.Template
    Digest         string
    Options        map[string]any
    Messages       []api.Message // few-shot examples
}

ollama/
├── api/              # Go client library + type definitions
├── auth/             # Ed25519-based authentication
├── cmd/              # CLI commands (cobra)
│   └── cmd.go        # run, pull, push, list, show, create...
├── convert/          # Model format conversion (HuggingFace → GGUF, etc.)
├── discover/         # GPU detection (CUDA, ROCm, Metal)
├── envconfig/        # Environment variable config parsing
├── format/           # Formatting utilities (file size, duration, etc.)
├── integration/      # Integration tests
├── llama/            # llama.cpp CGO bindings + C/C++ sources
├── llm/              # LlamaServer interface + implementations
├── middleware/       # OpenAI/Anthropic compatibility middleware
├── ml/               # ML backend abstraction layer
├── model/            # Go-native model architecture implementations
│   ├── llama/
│   ├── mistral/
│   ├── qwen/
│   └── ...
├── manifest/         # Model manifests + blob management
├── runner/           # Model execution engines
│   ├── llamarunner/  # llama.cpp-based
│   ├── ollamarunner/ # Go-native
│   └── x/
│       ├── imagegen/ # image generation
│       └── mlxrunner/# Apple MLX
├── server/           # HTTP server + handlers + scheduler
│   ├── routes.go     # route definitions
│   ├── sched.go      # scheduler
│   └── ...
├── template/         # chat template system
└── tokenizer/        # tokenizer utilities

# Check GPU status
ollama ps

# Inspect the number of layers loaded onto VRAM (debug logs)
OLLAMA_DEBUG=1 ollama run llama3.2

# If VRAM is insufficient, use a smaller quantized model
ollama pull llama3.2:3b-instruct-q4_K_M  # 4-bit quantization, ~2GB

# Or increase the CPU thread count
OLLAMA_NUM_THREAD=8 ollama serve

# Increase num_ctx in the request
curl http://localhost:11434/api/chat -d '{
  "model": "llama3.2",
  "messages": [{"role": "user", "content": "긴 문서..."}],
  "options": {
    "num_ctx": 8192
  }
}'

FROM llama3.2
PARAMETER num_ctx 8192

ollama create my-llama --file ./Modelfile

curl http://localhost:11434/api/chat -d '{
  "model": "llama3.2",
  "messages": [{"role": "user", "content": "2+2=?"}],
  "options": {
    "temperature": 0,
    "seed": 42
  }
}'

# Listen on all interfaces
OLLAMA_HOST=0.0.0.0 ollama serve

# Or bind to a specific IP
OLLAMA_HOST=192.168.1.100:11434 ollama serve

OLLAMA_ORIGINS="http://localhost:3000,https://myapp.com" ollama serve

# Set the number of models allowed to load simultaneously (default: 1)
OLLAMA_NUM_PARALLEL=2 ollama serve

# Call the model with keep_alive=-1 to retain it indefinitely
curl http://localhost:11434/api/generate -d '{
  "model": "llama3.2",
  "keep_alive": -1,
  "prompt": ""
}'

# Leverage Unified Memory (shared CPU+GPU memory)
# MacBook Pro M3 Pro (18GB): can run llama3.2:8b Q8 fully on GPU

# Disabling memory mapping slows initial loading but can improve inference speed
OLLAMA_USE_MMAP=0 ollama serve

# Use the MLX Runner (experimental — faster Metal inference)
OLLAMA_LLM_LIBRARY=mlx ollama serve

import ollama

# Streaming chat
for chunk in ollama.chat(
    model='llama3.2',
    messages=[{'role': 'user', 'content': '한국의 수도는?'}],
    stream=True
):
    print(chunk['message']['content'], end='', flush=True)

# Embeddings
response = ollama.embed(model='nomic-embed-text', input='텍스트')
print(response['embeddings'])

import ollama from 'ollama'

const response = await ollama.chat({
  model: 'llama3.2',
  messages: [{ role: 'user', content: '안녕하세요!' }],
  stream: true,
})

for await (const part of response) {
  process.stdout.write(part.message.content)
}

from openai import OpenAI

client = OpenAI(
    base_url='http://localhost:11434/v1',
    api_key='ollama',  # any value works
)

response = client.chat.completions.create(
    model='llama3.2',
    messages=[{'role': 'user', 'content': 'Hello!'}],
)

# LangChain
from langchain_ollama import ChatOllama

llm = ChatOllama(model="llama3.2", temperature=0)
response = llm.invoke("파이썬으로 피보나치 수열을 구현해줘")

# LlamaIndex
from llama_index.llms.ollama import Ollama

llm = Ollama(model="llama3.2", request_timeout=120.0)
response = llm.complete("안녕하세요!")

# Option 1: local GGUF file
cat > Modelfile << 'EOF'
FROM /path/to/my-model.gguf

PARAMETER temperature 0.7
SYSTEM "You are a helpful assistant."
EOF

ollama create my-custom-model --file Modelfile
ollama run my-custom-model

# Option 2: directly from a HuggingFace repo
ollama run hf.co/bartowski/Llama-3.2-3B-Instruct-GGUF:Q4_K_M

# Use the LLaVA model
ollama pull llava

# Send an image via the API (Base64-encoded)
import base64, ollama

with open('image.jpg', 'rb') as f:
    image_data = base64.b64encode(f.read()).decode()

response = ollama.chat(
    model='llava',
    messages=[{
        'role': 'user',
        'content': '이 이미지에 무엇이 보이나요?',
        'images': [image_data]
    }]
)

# NVIDIA GPU
docker run -d \
  --gpus=all \
  -v ollama:/root/.ollama \
  -p 11434:11434 \
  --name ollama \
  ollama/ollama

# CPU-only (no GPU)
docker run -d \
  -v ollama:/root/.ollama \
  -p 11434:11434 \
  --name ollama \
  ollama/ollama

# Run a model
docker exec -it ollama ollama run llama3.2

import ollama

tools = [{
    'type': 'function',
    'function': {
        'name': 'get_weather',
        'description': '특정 도시의 날씨를 조회합니다',
        'parameters': {
            'type': 'object',
            'properties': {
                'city': {'type': 'string', 'description': '도시 이름'},
            },
            'required': ['city'],
        },
    },
}]

response = ollama.chat(
    model='llama3.2',
    messages=[{'role': 'user', 'content': '서울 날씨 알려줘'}],
    tools=tools,
)

# If tool_calls are present, execute the corresponding function
if response.message.tool_calls:
    for tool_call in response.message.tool_calls:
        print(f"Call: {tool_call.function.name}({tool_call.function.arguments})")