DVC Experiments

# Run an experiment with modified parameters
dvc exp run --set-param train.learning_rate=0.01

# Run with multiple parameter changes
dvc exp run \
  --set-param train.learning_rate=0.001 \
  --set-param train.epochs=100 \
  --set-param train.batch_size=64

# Run with a custom name
dvc exp run --name "high-lr-experiment" \
  --set-param train.learning_rate=0.1

# Queue multiple experiments
dvc exp run --queue --set-param train.learning_rate=0.001
dvc exp run --queue --set-param train.learning_rate=0.01
dvc exp run --queue --set-param train.learning_rate=0.1

# Run all queued experiments
dvc queue start

# Show all experiments with metrics
dvc exp show

# Output:
# ┏━━━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━┓
# ┃ Experiment     ┃ accuracy ┃ f1    ┃ lr      ┃
# ┡━━━━━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━┩
# │ workspace      │ 0.892    │ 0.885 │ 0.001   │
# │ high-lr        │ 0.856    │ 0.843 │ 0.1     │
# │ low-lr         │ 0.901    │ 0.894 │ 0.0001  │
# └────────────────┴──────────┴───────┴─────────┘

# Compare specific metrics
dvc metrics show

# Diff metrics between experiments
dvc metrics diff

# View plots
dvc plots show

# Apply the best experiment to your workspace
dvc exp apply low-lr

# Create a Git branch from an experiment
dvc exp branch low-lr feature/best-model

# Remove experiments
dvc exp remove high-lr

# Remove all experiments
dvc exp gc --workspace

# Push experiments to remote (share with team)
dvc exp push origin

# Pull experiments from remote
dvc exp pull origin

import json
import csv
import pickle
import pandas as pd
from sklearn.metrics import accuracy_score, f1_score, confusion_matrix

# Load model and test data
with open('models/model.pkl', 'rb') as f:
    model = pickle.load(f)
test_df = pd.read_csv('data/processed/test.csv')
X_test = test_df.drop('target', axis=1)
y_test = test_df['target']

# Predict
y_pred = model.predict(X_test)

# Save metrics (DVC tracks these)
metrics = {
    'accuracy': accuracy_score(y_test, y_pred),
    'f1_score': f1_score(y_test, y_pred, average='weighted')
}
with open('metrics/eval_metrics.json', 'w') as f:
    json.dump(metrics, f, indent=2)

# Save confusion matrix as CSV plot
cm = confusion_matrix(y_test, y_pred)
with open('plots/confusion_matrix.csv', 'w', newline='') as f:
    writer = csv.writer(f)
    writer.writerow(['actual', 'predicted', 'count'])
    for i in range(len(cm)):
        for j in range(len(cm[0])):
            writer.writerow([i, j, int(cm[i][j])])