模型评估：指标、阈值与类别不平衡

from sklearn.metrics import precision_recall_curve

precision, recall, thresholds = precision_recall_curve(y_true, scores)
for index in range(0, len(thresholds), max(1, len(thresholds) // 5)):
    print(
        round(float(thresholds[index]), 3),
        round(float(precision[index]), 3),
        round(float(recall[index]), 3),
    )

import numpy as np
from sklearn.metrics import (
    average_precision_score, confusion_matrix, f1_score,
    precision_score, recall_score, roc_auc_score,
)

y_true = np.array([0] * 90 + [1] * 10)
scores = np.concatenate([
    np.linspace(0.01, 0.65, 90),
    np.array([0.18, 0.31, 0.44, 0.52, 0.58, 0.66, 0.73, 0.81, 0.9, 0.96]),
])

print("ROC-AUC:", round(roc_auc_score(y_true, scores), 3))
print("PR-AUC:", round(average_precision_score(y_true, scores), 3))
for threshold in (0.3, 0.5, 0.7):
    prediction = (scores >= threshold).astype(int)
    print(threshold, confusion_matrix(y_true, prediction).tolist())
    print(
        "precision", round(precision_score(y_true, prediction), 3),
        "recall", round(recall_score(y_true, prediction), 3),
        "f1", round(f1_score(y_true, prediction), 3),
    )

rng = np.random.default_rng(42)
bootstrap_f1 = []
prediction = (scores >= 0.5).astype(int)

for _ in range(1000):
    indices = rng.integers(0, len(y_true), len(y_true))
    if np.unique(y_true[indices]).size < 2:
        continue
    bootstrap_f1.append(f1_score(y_true[indices], prediction[indices]))

lower, upper = np.quantile(bootstrap_f1, [0.025, 0.975])
print("F1 interval:", round(float(lower), 3), round(float(upper), 3))