Gradient Boosting Deep Dive

Bagging (Random Forest):
  Tree 1 ──┐
  Tree 2 ──┤── Average/Vote ──→ Final Prediction
  Tree 3 ──┤     (parallel)
  Tree N ──┘

Boosting (Gradient Boosting):
  Tree 1 → errors → Tree 2 → errors → Tree 3 → ... → Tree N
           (sequential: each tree learns from previous errors)

  Final = Tree 1 + lr*Tree 2 + lr*Tree 3 + ... + lr*Tree N

Gradient Boosting Algorithm:

1. Initialize with a constant prediction:
   F_0(x) = mean(y)  (for regression)

2. For m = 1 to M (number of trees):
   a. Compute residuals (negative gradient):
      r_i = y_i - F_{m-1}(x_i)
      (How much each prediction is "off")

   b. Fit a decision tree h_m(x) to the residuals r_i
      (The tree learns to predict the ERRORS)

   c. Update the model:
      F_m(x) = F_{m-1}(x) + learning_rate * h_m(x)
      (Add a fraction of the error correction)

3. Final prediction:
   F_M(x) = F_0(x) + lr * h_1(x) + lr * h_2(x) + ... + lr * h_M(x)

Key insight: Each tree is a small step toward reducing the loss.
The learning rate controls how conservative each step is.

# XGBoost early stopping
import xgboost as xgb

model = xgb.XGBClassifier(
    n_estimators=10000,       # set high, early stopping will find optimal
    learning_rate=0.05,
    max_depth=6,
    early_stopping_rounds=50  # stop if no improvement for 50 rounds
)

model.fit(
    X_train, y_train,
    eval_set=[(X_val, y_val)],
    verbose=100               # print every 100 rounds
)

print(f"Best iteration: {model.best_iteration}")
print(f"Best score: {model.best_score}")

import numpy as np
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, classification_report
from sklearn.datasets import load_breast_cancer

# Load data
data = load_breast_cancer()
X, y = data.data, data.target

X_train, X_temp, y_train, y_temp = train_test_split(
    X, y, test_size=0.3, random_state=42, stratify=y
)
X_val, X_test, y_val, y_test = train_test_split(
    X_temp, y_temp, test_size=0.5, random_state=42, stratify=y_temp
)

# ==================== XGBoost ====================
import xgboost as xgb

xgb_model = xgb.XGBClassifier(
    n_estimators=1000,
    learning_rate=0.05,
    max_depth=6,
    subsample=0.8,
    colsample_bytree=0.8,
    min_child_weight=3,
    reg_lambda=1.0,
    early_stopping_rounds=50,
    random_state=42,
    use_label_encoder=False,
    eval_metric='logloss'
)

xgb_model.fit(
    X_train, y_train,
    eval_set=[(X_val, y_val)],
    verbose=False
)

y_pred_xgb = xgb_model.predict(X_test)
print("=== XGBoost ===")
print(f"Accuracy: {accuracy_score(y_test, y_pred_xgb):.4f}")
print(f"Best iteration: {xgb_model.best_iteration}")

# XGBoost feature importance
xgb.plot_importance(xgb_model, max_num_features=15, importance_type='gain')
plt.title("XGBoost - Feature Importance (Gain)")
plt.tight_layout()
plt.show()

# ==================== LightGBM ====================
import lightgbm as lgb

lgb_model = lgb.LGBMClassifier(
    n_estimators=1000,
    learning_rate=0.05,
    max_depth=-1,            # unlimited (leaf-wise growth)
    num_leaves=31,           # controls complexity instead of max_depth
    subsample=0.8,
    colsample_bytree=0.8,
    min_child_samples=20,
    reg_lambda=1.0,
    random_state=42,
    verbose=-1
)

lgb_model.fit(
    X_train, y_train,
    eval_set=[(X_val, y_val)],
    callbacks=[lgb.early_stopping(50), lgb.log_evaluation(0)]
)

y_pred_lgb = lgb_model.predict(X_test)
print("\n=== LightGBM ===")
print(f"Accuracy: {accuracy_score(y_test, y_pred_lgb):.4f}")
print(f"Best iteration: {lgb_model.best_iteration_}")

# LightGBM feature importance
lgb.plot_importance(lgb_model, max_num_features=15, importance_type='gain')
plt.title("LightGBM - Feature Importance (Gain)")
plt.tight_layout()
plt.show()

# ==================== Comparison ====================
from sklearn.ensemble import GradientBoostingClassifier

sklearn_gb = GradientBoostingClassifier(
    n_estimators=200, learning_rate=0.1, max_depth=3, random_state=42
)
sklearn_gb.fit(X_train, y_train)
y_pred_sklearn = sklearn_gb.predict(X_test)

print("\n=== Comparison ===")
print(f"sklearn GB:  {accuracy_score(y_test, y_pred_sklearn):.4f}")
print(f"XGBoost:     {accuracy_score(y_test, y_pred_xgb):.4f}")
print(f"LightGBM:    {accuracy_score(y_test, y_pred_lgb):.4f}")