Counting Tokens Intermediate

$ pip install tiktoken

import tiktoken

# Method 1: By encoding name
enc = tiktoken.get_encoding("cl100k_base")

# Method 2: By model name (recommended)
enc = tiktoken.encoding_for_model("gpt-4o")

# Count tokens in a string
text = "How many tokens is this sentence?"
token_count = len(enc.encode(text))
print(f"Token count: {token_count}")  # ~7 tokens

# Count tokens for a full chat message
def count_chat_tokens(messages, model="gpt-4o"):
    enc = tiktoken.encoding_for_model(model)
    total = 0
    for msg in messages:
        total += 4  # message overhead
        for key, value in msg.items():
            total += len(enc.encode(value))
    total += 2  # reply priming
    return total

messages = [
    {"role": "system", "content": "You are a helpful assistant."},
    {"role": "user", "content": "What is the capital of France?"}
]
print(f"Chat tokens: {count_chat_tokens(messages)}")

from anthropic import Anthropic

client = Anthropic()

# Count tokens before sending a request
token_count = client.messages.count_tokens(
    model="claude-sonnet-4-20250514",
    messages=[{
        "role": "user",
        "content": "Explain quantum computing in simple terms."
    }],
    system="You are a physics teacher."
)
print(f"Input tokens: {token_count.input_tokens}")

# After a response, check usage
response = client.messages.create(
    model="claude-sonnet-4-20250514",
    max_tokens=1024,
    messages=[{"role": "user", "content": "Hello!"}]
)
print(f"Input: {response.usage.input_tokens}")
print(f"Output: {response.usage.output_tokens}")

from transformers import AutoTokenizer

# Load tokenizer for any Hugging Face model
tokenizer = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3-8B")

text = "Count my tokens please!"
tokens = tokenizer.encode(text)
print(f"Token count: {len(tokens)}")

# See the actual token strings
token_strings = tokenizer.tokenize(text)
print(f"Tokens: {token_strings}")

# Works for any model on Hugging Face
mistral_tok = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-v0.1")
print(f"Mistral tokens: {len(mistral_tok.encode(text))}")

import tiktoken
enc = tiktoken.encoding_for_model("gpt-4o")

texts = {
    "English": "The quick brown fox jumps over the lazy dog.",
    "Spanish": "El rápido zorro marrón salta sobre el perro perezoso.",
    "Chinese": "敏捷的棕色狐狸跳过了懒狗。",
    "Japanese": "素早い茶色のキツネが怠け者の犬を飛び越える。",
    "Arabic": "الثعلب البني السريع يقفز فوق الكلب الكسول.",
}

for lang, text in texts.items():
    tokens = enc.encode(text)
    print(f"{lang:10}: {len(tokens):3} tokens | {text}")

# OpenAI GPT-4o image token formula:
# Low detail: 85 tokens (fixed)
# High detail: 170 * number_of_tiles + 85
# where tiles = ceil(width/512) * ceil(height/512)

# Example: 1024x768 image at high detail
# tiles = ceil(1024/512) * ceil(768/512) = 2 * 2 = 4
# tokens = 170 * 4 + 85 = 765 tokens

# Claude image tokens (approximate):
# tokens ≈ (width * height) / 750
# Example: 1024x768 = 786,432 pixels / 750 ≈ 1,049 tokens