决策树、随机森林与集成学习

from sklearn.datasets import make_classification
from sklearn.ensemble import GradientBoostingClassifier, RandomForestClassifier
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeClassifier

X, y = make_classification(
    n_samples=1200, n_features=10, n_informative=6,
    flip_y=0.08, random_state=42,
)
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.3, stratify=y, random_state=42,
)

models = [
    DecisionTreeClassifier(random_state=42),
    DecisionTreeClassifier(max_depth=4, random_state=42),
    RandomForestClassifier(n_estimators=150, max_depth=7, random_state=42, n_jobs=-1),
    GradientBoostingClassifier(random_state=42),
]
for model in models:
    model.fit(X_train, y_train)
    train_score = accuracy_score(y_train, model.predict(X_train))
    test_score = accuracy_score(y_test, model.predict(X_test))
    print(type(model).__name__, round(train_score, 3), round(test_score, 3))

from sklearn.tree import export_text

feature_names = [f"feature_{index}" for index in range(X.shape[1])]
shallow_tree = DecisionTreeClassifier(
    max_depth=3, min_samples_leaf=20, random_state=42
)
shallow_tree.fit(X_train, y_train)
print(export_text(shallow_tree, feature_names=feature_names))

from sklearn.inspection import permutation_importance

forest = models[2]
result = permutation_importance(
    forest,
    X_test,
    y_test,
    scoring="accuracy",
    n_repeats=10,
    random_state=42,
)
ranking = sorted(
    zip(feature_names, result.importances_mean, result.importances_std),
    key=lambda item: item[1],
    reverse=True,
)
for name, mean_drop, std_drop in ranking[:5]:
    print(name, round(mean_drop, 3), "+/-", round(std_drop, 3))