Loading Datasets

from datasets import load_dataset

# Load entire dataset
dataset = load_dataset("squad")

# Load specific split
train = load_dataset("squad", split="train")

# Load specific configuration
dataset = load_dataset("glue", "mrpc")

# Load a subset of rows (great for prototyping)
small = load_dataset("imdb", split="train[:1000]")

# Load specific percentage
ten_pct = load_dataset("imdb", split="train[:10%]")

# Load from a specific revision/branch
dataset = load_dataset("squad", revision="main")

# CSV files
dataset = load_dataset("csv", data_files="data/train.csv")

# Multiple files with splits
dataset = load_dataset("csv", data_files={
    "train": "data/train.csv",
    "test": "data/test.csv"
})

# JSON / JSON Lines
dataset = load_dataset("json", data_files="data/*.jsonl")

# Parquet files
dataset = load_dataset("parquet", data_files="data/train.parquet")

# Text files (one example per line)
dataset = load_dataset("text", data_files="data/corpus.txt")

# Image folder (folder name = label)
dataset = load_dataset("imagefolder", data_dir="images/")

# Audio folder
dataset = load_dataset("audiofolder", data_dir="audio/")

from datasets import Dataset
import pandas as pd

# From a dictionary
data = {"text": ["hello", "world"], "label": [0, 1]}
dataset = Dataset.from_dict(data)

# From a pandas DataFrame
df = pd.read_csv("data.csv")
dataset = Dataset.from_pandas(df)

# From a list of dictionaries
records = [{"text": "hi", "label": 1}, {"text": "bye", "label": 0}]
dataset = Dataset.from_list(records)

# Convert back to pandas
df = dataset.to_pandas()

# Dataset info
print(dataset.features)    # Column types and names
print(dataset.num_rows)    # Number of examples
print(dataset.column_names) # List of column names
print(dataset.shape)       # (num_rows, num_columns)

# Access examples
example = dataset[0]       # First example (dict)
batch = dataset[:5]       # First 5 examples (dict of lists)
column = dataset["text"]  # Entire column (list)

# Unique values
print(dataset.unique("label"))  # Unique label values