Streaming Datasets

from datasets import load_dataset

# Enable streaming with streaming=True
dataset = load_dataset("c4", "en", split="train", streaming=True)

# This returns an IterableDataset (no download!)
print(type(dataset))  # <class 'datasets.IterableDataset'>

# Iterate through examples one at a time
for i, example in enumerate(dataset):
    print(example["text"][:100])
    if i >= 4:
        break

# Map (applied lazily during iteration)
tokenized = dataset.map(tokenize_fn)

# Filter
english = dataset.filter(lambda x: x["language"] == "en")

# Take first N examples
small = dataset.take(1000)

# Skip first N examples
rest = dataset.skip(1000)

# Shuffle with a buffer
shuffled = dataset.shuffle(seed=42, buffer_size=10000)

# Chain operations (all lazy)
processed = (
    dataset
    .filter(lambda x: len(x["text"]) > 100)
    .map(tokenize_fn)
    .shuffle(buffer_size=5000)
)

from datasets import interleave_datasets

# Mix multiple streaming datasets
en = load_dataset("mc4", "en", split="train", streaming=True)
fr = load_dataset("mc4", "fr", split="train", streaming=True)
de = load_dataset("mc4", "de", split="train", streaming=True)

# Interleave with custom probabilities
mixed = interleave_datasets(
    [en, fr, de],
    probabilities=[0.5, 0.3, 0.2],
    seed=42
)