【保存版】モデル評価：指標の選び方・交差検証・閾値最適化・ビジネス接続を“実務の型”で解説

from sklearn.model_selection import train_test_split, KFold, StratifiedKFold, TimeSeriesSplit

# ホールドアウト（最終報告用のテストは最後に1回）

X\_tr, X\_te, y\_tr, y\_te = train\_test\_split(X, y, test\_size=0.2, stratify=y, random\_state=42)

# KFold（回帰）/ StratifiedKFold（分類）

cv\_cls = StratifiedKFold(n\_splits=5, shuffle=True, random\_state=42)
cv\_reg = KFold(n\_splits=5, shuffle=True, random\_state=42)

# 時系列CV

cv\_ts = TimeSeriesSplit(n\_splits=5)

import numpy as np, pandas as pd
from sklearn.metrics import (confusion_matrix, classification_report,
                             roc_auc_score, roc_curve, precision_recall_curve,
                             average_precision_score)

proba = model.predict\_proba(X\_te)\[:,1]
pred = (proba >= 0.5).astype(int)

cm = confusion\_matrix(y\_te, pred)
print(pd.DataFrame(cm, index=\["TrueNeg","TruePos"], columns=\["PredNeg","PredPos"]))

print(classification\_report(y\_te, pred, digits=3))
print("ROC-AUC=", roc\_auc\_score(y\_te, proba))

fpr, tpr, thr = roc\_curve(y\_te, proba)
prec, rec, thr2 = precision\_recall\_curve(y\_te, proba)
print("PR-AUC=", average\_precision\_score(y\_te, proba))

from sklearn.metrics import roc_curve, confusion_matrix
import numpy as np

proba = model.predict\_proba(X\_te)\[:,1]
fpr, tpr, thr = roc\_curve(y\_te, proba)

# Youden J（tpr - fpr）最大

j = tpr - fpr
thr\_opt = thr\[j.argmax()]
print("Youden best threshold=", float(thr\_opt))

# コスト最適化（例：FPコスト=1, FNコスト=10）

cost\_fp, cost\_fn = 1.0, 10.0
best = (np.inf, 0.5)
for t in np.linspace(0,1,101):
p = (proba >= t).astype(int)
tn, fp, fn, tp = confusion\_matrix(y\_te, p).ravel()
cost = cost\_fp*fp + cost\_fn*fn
best = min(best, (cost, t))
print("Cost-min threshold=", best\[1])

from sklearn.calibration import CalibratedClassifierCV
from sklearn.metrics import brier_score_loss

# 例：確率が偏りがちなツリーモデルをPlattスケーリング

cal\_model = CalibratedClassifierCV(model, method="sigmoid", cv=5).fit(X\_tr, y\_tr)
proba\_cal = cal\_model.predict\_proba(X\_te)\[:,1]
print("Brier(before)", brier\_score\_loss(y\_te, model.predict\_proba(X\_te)\[:,1]))
print("Brier(after) ", brier\_score\_loss(y\_te, proba\_cal))

from sklearn.metrics import average_precision_score
import numpy as np

proba = model.predict\_proba(X\_te)\[:,1]
print("PR-AUC=", average\_precision\_score(y\_te, proba))

# Top-k（例：上位5%を営業対象にする）

k = int(len(proba) \* 0.05)
idx = np.argsort(-proba)\[:k]
recall\_at\_k = y\_te.iloc\[idx].mean()  # Positive率
print("Recall\@5%:", float(recall\_at\_k))

from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score

pred = reg\_model.predict(X\_te)
mae = mean\_absolute\_error(y\_te, pred)
rmse = mean\_squared\_error(y\_te, pred, squared=False)
r2 = r2\_score(y\_te, pred)
print({"MAE": mae, "RMSE": rmse, "R2": r2})

import numpy as np
from sklearn.utils import resample
from sklearn.metrics import roc_auc_score

proba = model.predict\_proba(X\_te)\[:,1]
auc = roc\_auc\_score(y\_te, proba)
aucs = \[]
for \_ in range(500):
idx = resample(np.arange(len(y\_te)), replace=True, random\_state=\_)
aucs.append(roc\_auc\_score(y\_te.iloc\[idx], proba\[idx]))
lo, hi = np.percentile(aucs, \[2.5, 97.5])
print(f"AUC={auc:.3f} (95% CI: {lo:.3f}-{hi:.3f})")

import numpy as np

def apk(actual, pred, k=10):
pred = pred\[:k]
score, hits = 0.0, 0
for i, p in enumerate(pred, start=1):
if p in actual and p not in pred\[:i-1]:
hits += 1
score += hits / i
return score / min(len(actual), k) if actual else 0.0

def mapk(actual\_list, pred\_list, k=10):
return np.mean(\[apk(a, p, k) for a, p in zip(actual\_list, pred\_list)])