ollama pull llama3.2:3b
ollama pull llama3.2-vision:11b

ollama pull llama3.3:70b

ollama pull deepseek-r1:8b     # Runs on 8GB RAM
ollama pull deepseek-r1:14b    # Runs on 16GB RAM
ollama pull deepseek-r1:32b    # Runs on 24GB RAM

# Check your Mac's specs
system_profiler SPHardwareDataType | grep -E "Chip|Memory"

git clone https://github.com/ggerganov/llama.cpp
cd llama.cpp

# macOS with Metal (Apple Silicon)
make LLAMA_METAL=1

# Linux with CUDA
make LLAMA_CUDA=1

# CPU only (not recommended for interactive use)
make

# Interactive chat
./llama-cli -m models/llama-3.1-8b-instruct-Q4_K_M.gguf \
  -c 8192 \
  -n 512 \
  --color \
  -i \
  --temp 0.7

# Server mode (OpenAI-compatible API)
./llama-server -m models/llama-3.1-8b-instruct-Q4_K_M.gguf \
  -c 8192 \
  --port 8080 \
  --host 0.0.0.0

# Force valid JSON output
./llama-cli -m models/llama-3.1-8b-instruct-Q4_K_M.gguf \
  --grammar-file grammars/json.gbnf \
  -p "Extract the name and age from: 'John is 32 years old'"

Can the model fit at Q8_0? → Use Q8_0
Can it fit at Q5_K_M? → Use Q5_K_M
Can it fit at Q4_K_M? → Use Q4_K_M (this is the most common case)
Can it only fit at Q3_K_M or lower? → Consider a smaller model at higher quant

# Convert a HuggingFace model to GGUF
python convert_hf_to_gguf.py \
  --outfile models/my-model-f16.gguf \
  --outtype f16 \
  path/to/hf/model/

# Quantize to Q4_K_M
./llama-quantize models/my-model-f16.gguf models/my-model-Q4_K_M.gguf Q4_K_M

# Quantize with importance matrix (imatrix) for better quality
./llama-imatrix -m models/my-model-f16.gguf \
  -f calibration-data.txt \
  -o models/my-model.imatrix

./llama-quantize --imatrix models/my-model.imatrix \
  models/my-model-f16.gguf models/my-model-IQ4_XS.gguf IQ4_XS

from unsloth import FastLanguageModel
from trl import SFTTrainer
from datasets import load_dataset

model, tokenizer = FastLanguageModel.from_pretrained(
    model_name="unsloth/llama-3.1-8b-instruct-bnb-4bit",
    max_seq_length=2048,
    load_in_4bit=True,
)

model = FastLanguageModel.get_peft_model(
    model,
    r=16,              # LoRA rank — higher = more capacity, more VRAM
    lora_alpha=16,
    lora_dropout=0,
    target_modules=["q_proj", "k_proj", "v_proj", "o_proj",
                     "gate_proj", "up_proj", "down_proj"],
)

dataset = load_dataset("json", data_files="training_data.jsonl")

trainer = SFTTrainer(
    model=model,
    tokenizer=tokenizer,
    train_dataset=dataset["train"],
    dataset_text_field="text",
    max_seq_length=2048,
    per_device_train_batch_size=4,
    gradient_accumulation_steps=4,
    warmup_steps=10,
    num_train_epochs=3,
    learning_rate=2e-4,
    fp16=True,
    logging_steps=10,
    output_dir="./outputs",
)

trainer.train()
model.save_pretrained_merged("./merged_model", tokenizer)

{"text": "<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\nYou are a code review assistant for a TypeScript project.<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nReview this function:\n```typescript\nfunction getUser(id) { return db.query('SELECT * FROM users WHERE id = ' + id) }\n```<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n🔴 **SQL Injection vulnerability.** The `id` parameter is concatenated directly into the SQL query. Use parameterized queries:\n```typescript\nfunction getUser(id: string): Promise<User> {\n  return db.query('SELECT * FROM users WHERE id = $1', [id])\n}\n```\nAlso: add TypeScript types for the parameter and return value.<|eot_id|>"}

# Export to GGUF
python llama.cpp/convert_hf_to_gguf.py \
  --outfile my-model.gguf \
  --outtype q4_k_m \
  ./merged_model/

# Create Ollama Modelfile
cat > Modelfile << 'EOF'
FROM ./my-model.gguf
PARAMETER temperature 0.1
PARAMETER num_ctx 4096
SYSTEM "You are a TypeScript code review assistant."
EOF

# Import into Ollama
ollama create my-code-reviewer -f Modelfile
ollama run my-code-reviewer

import chromadb
from ollama import Client

ollama = Client()
chroma = chromadb.PersistentClient(path="./chroma_db")

collection = chroma.get_or_create_collection(
    name="codebase",
    metadata={"hnsw:space": "cosine"}
)

def index_file(filepath: str, content: str):
    chunks = chunk_by_function(content) if filepath.endswith(('.ts', '.py')) \
        else chunk_by_paragraph(content, max_tokens=512)

    for i, chunk in enumerate(chunks):
        embedding = ollama.embeddings(
            model="nomic-embed-text",
            prompt=chunk
        )["embedding"]

        collection.add(
            ids=[f"{filepath}:{i}"],
            embeddings=[embedding],
            documents=[chunk],
            metadatas=[{"filepath": filepath, "chunk_index": i}]
        )

def query_rag(question: str, n_results: int = 5) -> str:
    query_embedding = ollama.embeddings(
        model="nomic-embed-text",
        prompt=question
    )["embedding"]

    results = collection.query(
        query_embeddings=[query_embedding],
        n_results=n_results,
        include=["documents", "metadatas", "distances"]
    )

    context_parts = []
    for doc, meta, dist in zip(
        results["documents"][0],
        results["metadatas"][0],
        results["distances"][0]
    ):
        if dist < 0.7:
            context_parts.append(f"[{meta['filepath']}]\n{doc}")

    context = "\n\n---\n\n".join(context_parts)

    response = ollama.chat(
        model="llama3.1:8b",
        messages=[
            {"role": "system", "content": f"Answer based on this codebase context. "
             f"Cite file paths when referencing code.\n\n{context}"},
            {"role": "user", "content": question}
        ]
    )

    return response["message"]["content"]

FROM ollama/ollama:latest

COPY Modelfile /tmp/Modelfile
COPY my-model.gguf /tmp/my-model.gguf

RUN ollama serve & sleep 5 && \
    ollama create my-model -f /tmp/Modelfile && \
    kill %1

EXPOSE 11434

CMD ["ollama", "serve"]

# docker-compose.yml
services:
  ollama:
    image: ollama/ollama
    ports:
      - "11434:11434"
    volumes:
      - ollama_data:/root/.ollama
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: 1
              capabilities: [gpu]

  api:
    build: ./api
    environment:
      - LLM_BASE_URL=http://ollama:11434/v1
      - LLM_MODEL=llama3.1:8b
    depends_on:
      - ollama

volumes:
  ollama_data:

import { Hono } from "hono";
import { OpenAI } from "openai";
import { z } from "zod";
import { zValidator } from "@hono/zod-validator";

const app = new Hono();

const client = new OpenAI({
  baseURL: process.env.LLM_BASE_URL || "http://localhost:11434/v1",
  apiKey: "ollama",
});

const classifySchema = z.object({
  text: z.string().min(1).max(10000),
  categories: z.array(z.string()).min(2).max(20),
});

app.post("/classify", zValidator("json", classifySchema), async (c) => {
  const { text, categories } = c.req.valid("json");

  const response = await client.chat.completions.create({
    model: process.env.LLM_MODEL || "llama3.1:8b",
    messages: [
      {
        role: "system",
        content: `Classify the following text into exactly one of these categories: ${categories.join(", ")}. Respond with only the category name, nothing else.`,
      },
      { role: "user", content: text },
    ],
    temperature: 0,
    max_tokens: 50,
  });

  const category = response.choices[0].message.content?.trim() || "unknown";

  if (!categories.includes(category)) {
    return c.json({ category: "unknown", confidence: "low" }, 200);
  }

  return c.json({ category, confidence: "high" }, 200);
});

export default app;