Creating Datasets

from datasets import Dataset, Features, Value, ClassLabel

# Define features schema
features = Features({
    "text": Value("string"),
    "label": ClassLabel(names=["negative", "positive"]),
    "score": Value("float32"),
    "id": Value("int64")
})

# Create dataset from dictionary
data = {
    "text": ["Great product!", "Terrible quality.", "Amazing service."],
    "label": [1, 0, 1],
    "score": [4.5, 1.0, 5.0],
    "id": [1, 2, 3]
}

dataset = Dataset.from_dict(data, features=features)

from datasets import Dataset

def generate_examples():
    # Read from files, databases, APIs, etc.
    for file in Path("data/").glob("*.json"):
        with open(file) as f:
            for line in f:
                example = json.loads(line)
                yield {
                    "text": example["content"],
                    "label": example["sentiment"],
                }

dataset = Dataset.from_generator(generate_examples)

# Save to disk (Arrow format - fastest loading)
dataset.save_to_disk("./my_dataset")
loaded = load_from_disk("./my_dataset")

# Save as Parquet (compact, portable)
dataset.to_parquet("my_dataset.parquet")

# Save as CSV
dataset.to_csv("my_dataset.csv")

# Save as JSON
dataset.to_json("my_dataset.jsonl")

from huggingface_hub import login

# Authenticate
login(token="hf_your_token")

# Push dataset to the Hub
dataset.push_to_hub("username/my-dataset")

# Push with splits
from datasets import DatasetDict
ds_dict = DatasetDict({
    "train": train_dataset,
    "test": test_dataset
})
ds_dict.push_to_hub("username/my-dataset")

# Make it private
dataset.push_to_hub("username/my-dataset", private=True)