| name | model-evaluation |
| description | Evaluates machine learning models for performance, fairness, and reliability using appropriate metrics and validation techniques. Trigger keywords: model evaluation, metrics, accuracy, precision, recall, F1, ROC, AUC, cross-validation, ML testing. |
Model Evaluation
Overview
This skill focuses on comprehensive evaluation of machine learning models. It covers metric selection, validation strategies, fairness assessment, and production monitoring for ensuring model quality and reliability.
Instructions
1. Define Evaluation Criteria
- Identify business objectives
- Select appropriate metrics
- Define success thresholds
- Consider fairness requirements
2. Design Evaluation Strategy
- Choose validation approach
- Plan for data splits
- Handle class imbalance
- Account for temporal aspects
3. Conduct Evaluation
- Calculate performance metrics
- Analyze error patterns
- Assess model fairness
- Test edge cases
4. Report and Monitor
- Document evaluation results
- Create monitoring dashboards
- Set up alerting thresholds
- Plan for retraining
Best Practices
- Match Metrics to Goals: Choose metrics aligned with business objectives
- Use Multiple Metrics: No single metric tells the whole story
- Proper Validation: Use appropriate cross-validation schemes
- Test Distribution Shift: Evaluate on out-of-distribution data
- Check for Bias: Assess fairness across demographic groups
- Version Everything: Track models, data, and metrics
- Monitor Production: Continuously track model performance
Examples
Example 1: Classification Model Evaluation
import numpy as np
import pandas as pd
from sklearn.metrics import (
accuracy_score, precision_score, recall_score, f1_score,
roc_auc_score, average_precision_score, confusion_matrix,
classification_report, roc_curve, precision_recall_curve
)
import matplotlib.pyplot as plt
class ClassificationEvaluator:
"""Comprehensive classification model evaluator."""
def __init__(self, y_true, y_pred, y_prob=None, class_names=None):
self.y_true = y_true
self.y_pred = y_pred
self.y_prob = y_prob
self.class_names = class_names or ['Negative', 'Positive']
def compute_metrics(self) -> dict:
"""Compute all classification metrics."""
metrics = {
'accuracy': accuracy_score(self.y_true, self.y_pred),
'precision': precision_score(self.y_true, self.y_pred, average='weighted'),
'recall': recall_score(self.y_true, self.y_pred, average='weighted'),
'f1': f1_score(self.y_true, self.y_pred, average=),
}
.y_prob :
metrics[] = roc_auc_score(.y_true, .y_prob)
metrics[] = average_precision_score(.y_true, .y_prob)
metrics
() -> :
cm = confusion_matrix(.y_true, .y_pred)
tn, fp, fn, tp = cm.ravel()
{
: cm,
: tn,
: fp,
: fn,
: tp,
: tn / (tn + fp),
: tp / (tp + fn),
: fp / (fp + tn),
: fn / (fn + tp),
}
():
.y_prob :
ValueError()
fpr, tpr, thresholds = roc_curve(.y_true, .y_prob)
auc = roc_auc_score(.y_true, .y_prob)
plt.figure(figsize=(, ))
plt.plot(fpr, tpr, label=)
plt.plot([, ], [, ], , label=)
plt.xlabel()
plt.ylabel()
plt.title()
plt.legend()
plt.grid(, alpha=)
save_path:
plt.savefig(save_path, dpi=, bbox_inches=)
plt.show()
() -> :
metrics = .compute_metrics()
cm_analysis = .confusion_matrix_analysis()
report =
report
evaluator = ClassificationEvaluator(y_true, y_pred, y_prob)
(evaluator.generate_report())
evaluator.plot_roc_curve()
Example 2: Regression Model Evaluation
from sklearn.metrics import (
mean_squared_error, mean_absolute_error, r2_score,
mean_absolute_percentage_error, explained_variance_score
)
import numpy as np
class RegressionEvaluator:
"""Comprehensive regression model evaluator."""
def __init__(self, y_true, y_pred):
self.y_true = np.array(y_true)
self.y_pred = np.array(y_pred)
self.residuals = self.y_true - self.y_pred
def compute_metrics(self) -> dict:
"""Compute all regression metrics."""
mse = mean_squared_error(self.y_true, self.y_pred)
return {
'mse': mse,
'rmse': np.sqrt(mse),
'mae': mean_absolute_error(self.y_true, self.y_pred),
'mape': mean_absolute_percentage_error(self.y_true, self.y_pred) * 100,
'r2': r2_score(self.y_true, self.y_pred),
'explained_variance': explained_variance_score(self.y_true, self.y_pred),
}
def residual_analysis(self) -> dict:
{
: np.mean(.residuals),
: np.std(.residuals),
: np.(.residuals),
: np.(.residuals),
: ._skewness(.residuals),
}
():
n = (data)
mean = np.mean(data)
std = np.std(data)
(n / ((n-) * (n-))) * np.(((data - mean) / std) ** )
():
fig, axes = plt.subplots(, , figsize=(, ))
ax1 = axes[, ]
ax1.scatter(.y_true, .y_pred, alpha=)
ax1.plot([.y_true.(), .y_true.()],
[.y_true.(), .y_true.()], )
ax1.set_xlabel()
ax1.set_ylabel()
ax1.set_title()
ax2 = axes[, ]
ax2.scatter(.y_pred, .residuals, alpha=)
ax2.axhline(y=, color=, linestyle=)
ax2.set_xlabel()
ax2.set_ylabel()
ax2.set_title()
ax3 = axes[, ]
ax3.hist(.residuals, bins=, edgecolor=)
ax3.set_xlabel()
ax3.set_ylabel()
ax3.set_title()
ax4 = axes[, ]
scipy stats
stats.probplot(.residuals, dist=, plot=ax4)
ax4.set_title()
plt.tight_layout()
save_path:
plt.savefig(save_path, dpi=, bbox_inches=)
plt.show()
Example 3: Cross-Validation Strategies
from sklearn.model_selection import (
cross_val_score, StratifiedKFold, TimeSeriesSplit,
GroupKFold, cross_validate
)
def evaluate_with_cv(model, X, y, cv_strategy='stratified', n_splits=5, groups=None):
"""
Evaluate model with appropriate cross-validation strategy.
Args:
model: Sklearn-compatible model
X: Features
y: Target
cv_strategy: 'stratified', 'timeseries', 'group', or 'kfold'
n_splits: Number of CV folds
groups: Group labels for GroupKFold
Returns:
Dictionary with CV results
"""
if cv_strategy == 'stratified':
cv = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)
elif cv_strategy == 'timeseries':
cv = TimeSeriesSplit(n_splits=n_splits)
elif cv_strategy == 'group':
cv = GroupKFold(n_splits=n_splits)
else:
cv = n_splits
scoring = {
'accuracy': 'accuracy',
'precision': 'precision_weighted',
'recall': 'recall_weighted',
'f1': 'f1_weighted',
'roc_auc': 'roc_auc'
}
cv_results = cross_validate(
model, X, y,
cv=cv,
scoring=scoring,
groups=groups,
return_train_score=True,
n_jobs=-1
)
summary = {}
for metric in scoring.keys():
test_scores = cv_results[]
train_scores = cv_results[]
summary[metric] = {
: np.mean(test_scores),
: np.std(test_scores),
: np.mean(train_scores),
: np.std(train_scores),
: np.mean(train_scores) - np.mean(test_scores)
}
summary
results = evaluate_with_cv(model, X, y, cv_strategy=, n_splits=)
metric, values results.items():
()
Example 4: Fairness Evaluation
def evaluate_fairness(y_true, y_pred, sensitive_attr, favorable_label=1):
"""
Evaluate model fairness across demographic groups.
Args:
y_true: True labels
y_pred: Predicted labels
sensitive_attr: Protected attribute values
favorable_label: The favorable outcome label
Returns:
Dictionary with fairness metrics
"""
groups = np.unique(sensitive_attr)
results = {'group_metrics': {}}
for group in groups:
mask = sensitive_attr == group
group_true = y_true[mask]
group_pred = y_pred[mask]
tp = np.sum((group_true == favorable_label) & (group_pred == favorable_label))
fp = np.sum((group_true != favorable_label) & (group_pred == favorable_label))
fn = np.sum((group_true == favorable_label) & (group_pred != favorable_label))
tn = np.sum((group_true != favorable_label) & (group_pred != favorable_label))
results['group_metrics'][group] = {
'selection_rate': np.mean(group_pred == favorable_label),
'tpr': tp / (tp + fn) if (tp + fn) > 0 else 0,
'fpr': fp / (fp + tn) if (fp + tn) > 0 else 0,
'accuracy': np.mean(group_true == group_pred),
'size': len(group_true)
}
selection_rates = [m['selection_rate'] for m in results['group_metrics'].values()]
tprs = [m['tpr'] for m in results[].values()]
fprs = [m[] m results[].values()]
results[] = {
: (selection_rates) - (selection_rates),
: (tprs) - (tprs),
: (fprs) - (fprs),
}
results
