Mutual Information Intermediate

I(X; Y) = SUM_x SUM_y P(x, y) * log2( P(x, y) / (P(x) * P(y)) )

# Equivalently:
I(X; Y) = H(X) + H(Y) - H(X, Y)
I(X; Y) = H(X) - H(X | Y)
I(X; Y) = D_KL( P(X, Y) || P(X) * P(Y) )

from sklearn.metrics import mutual_info_score
from sklearn.feature_selection import mutual_info_classif
import numpy as np

# Discrete MI using sklearn
labels_true = [0, 0, 1, 1, 2, 2]
labels_pred = [0, 0, 1, 2, 2, 2]
mi = mutual_info_score(labels_true, labels_pred)
print(f"MI = {mi:.4f} nats")

# Feature selection: MI between features and target
from sklearn.datasets import load_iris
X, y = load_iris(return_X_y=True)
mi_scores = mutual_info_classif(X, y, random_state=42)
for i, score in enumerate(mi_scores):
    print(f"Feature {i}: MI = {score:.4f}")