Menu
Combining multiple models to achieve better predictive performance than individual models
Install with Codex or Claude Copy this prompt, paste it into Codex, Claude, or another assistant, and let it review the skill page and install it for you.
Based on SOC occupation classification
| name | Ensemble Methods |
| category | data-science |
| description | Combining multiple models to achieve better predictive performance than individual models |
I enable superior predictive performance by combining predictions from multiple models. By leveraging diversity among models, I can reduce variance, decrease bias, and improve generalization. My techniques are among the most effective methods for winning machine learning competitions and building production systems.
Bagging: Training models on bootstrap samples to reduce variance.
Boosting: Sequentially training models to correct previous errors.
Stacking: Using predictions as features for meta-learner.
Diversity: Ensuring models make different errors for ensemble benefit.
Voting: Combining predictions through majority or weighted voting.
Blending: Using holdout predictions for stacking.
Ensemble Diversity: Measuring disagreement between ensemble members.
Weight Optimization: Learning optimal combination weights.
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from sklearn.utils import resample
class BaggingClassifier:
def __init__(self, base_model_fn, n_estimators=10):
self.base_model_fn = base_model_fn
self.n_estimators = n_estimators
self.estimators = []
def fit(self, X, y, epochs=10):
for i in range(self.n_estimators):
X_boot, y_boot = resample(X, y, random_state=i)
model = self.base_model_fn()
self._train_model(model, X_boot, y_boot, epochs)
self.estimators.append(model)
return self
def predict_proba(self, X):
predictions = []
for model in self.estimators:
with torch.no_grad():
logits = model(X)
probs = F.softmax(logits, dim=1)
predictions.append(probs.numpy())
return np.mean(predictions, axis=0)
def predict(self, X):
proba = self.predict_proba(X)
return proba.argmax(axis=1)
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
class AdaBoost:
def __init__(self, base_model_fn, n_estimators=50):
self.base_model_fn = base_model_fn
self.n_estimators = n_estimators
self.estimators = []
self.estimator_weights = []
def fit(self, X, y, epochs=10):
n_samples = len(X)
sample_weights = np.ones(n_samples) / n_samples
for t in range(self.n_estimators):
X_t, y_t = self._bootstrap_sample(X, y, sample_weights)
model = self.base_model_fn()
self._train_model(model, X_t, y_t, epochs)
predictions = model.predict(X)
error = np.sum(sample_weights * (predictions != y)) / np.sum(sample_weights)
if error >= 0.5 or error == 0:
break
alpha = 0.5 * np.log((1 - error) / max(error, 1e-10))
sample_weights = sample_weights * np.exp(alpha * (predictions != y))
sample_weights = sample_weights / (sample_weights.sum() + 1e-10)
self.estimators.append(model)
self.estimator_weights.append(alpha)
return self
def predict_proba(self, X):
predictions = np.zeros((len(X), 2))
for model, weight in zip(self.estimators, self.estimator_weights):
preds = model.predict_proba(X)
predictions[:, 1] += weight * preds[:, 1]
predictions[:, 0] += weight * preds[:, 0]
predictions[:, 1] = predictions[:, 1] / (np.sum(self.estimator_weights) + 1e-10)
predictions[:, 0] = 1 - predictions[:, 1]
return predictions
def predict(self, X):
proba = self.predict_proba(X)
return (proba[:, 1] > 0.5).astype(int)
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
class GradientBoosting:
def __init__(self, base_model_fn, n_estimators=100, learning_rate=0.1):
self.base_model_fn = base_model_fn
self.n_estimators = n_estimators
self.learning_rate = learning_rate
self.models = []
self.initial_prediction = None
def fit(self, X, y, epochs=10):
if isinstance(X, torch.Tensor):
X = X.numpy()
if isinstance(y, torch.Tensor):
y = y.numpy()
self.initial_prediction = np.log(np.mean(y) + 1e-10)
F_m = np.full(len(y), self.initial_prediction)
for m in range(self.n_estimators):
residual = y - F_m
model = self.base_model_fn()
self._train_regression_model(model, X, residual, epochs)
predictions = model.predict(X)
gamma = self._line_search(F_m, predictions, y)
F_m = F_m + self.learning_rate * gamma * predictions
self.models.append(model)
return self
def predict_proba(self, X):
if isinstance(X, torch.Tensor):
X = X.numpy()
F_m = np.full(len(X), self.initial_prediction)
for model in self.models:
predictions = model.predict(X)
F_m = F_m + self.learning_rate * predictions
return 1 / (1 + np.exp(-F_m))
def predict(self, X):
proba = self.predict_proba(X)
return (proba > 0.5).astype(int)
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
class StackingClassifier:
def __init__(self, base_estimators, meta_learner, n_folds=5):
self.base_estimators = base_estimators
self.meta_learner = meta_learner
self.n_folds = n_folds
self.fitted_base_estimators = []
def fit(self, X, y, epochs_base=5, epochs_meta=10):
n_samples = len(X)
n_estimators = len(self.base_estimators)
oof_predictions = np.zeros((n_samples, n_estimators))
fold_indices = np.array_split(np.arange(n_samples), self.n_folds)
for fold_idx, val_indices in enumerate(fold_indices):
train_indices = np.concatenate([idx for i, idx in enumerate(fold_indices) if i != fold_idx])
for est_idx, EstClass in enumerate(self.base_estimators):
model = EstClass()
X_train_fold = torch.tensor(X[train_indices], dtype=torch.float32)
y_train_fold = torch.tensor(y[train_indices], dtype=torch.long)
self._train_model(model, X_train_fold, y_train_fold, epochs_base)
X_val_fold = torch.tensor(X[val_indices], dtype=torch.float32)
with torch.no_grad():
logits = model(X_val_fold)
probs = F.softmax(logits, dim=1)
oof_predictions[val_indices, est_idx] = probs[:, 1].numpy()
for EstClass in self.base_estimators:
model = EstClass()
X_full = torch.tensor(X, dtype=torch.float32)
y_full = torch.tensor(y, dtype=torch.long)
self._train_model(model, X_full, y_full, epochs_base)
self.fitted_base_estimators.append(model)
meta_X = oof_predictions
meta_y = torch.tensor(y, dtype=torch.long)
self._train_model(self.meta_learner, meta_X, meta_y, epochs_meta)
return self
def predict_proba(self, X):
n_estimators = len(self.fitted_base_estimators)
predictions = np.zeros((len(X), n_estimators))
X_tensor = torch.tensor(X, dtype=torch.float32)
for est_idx, model in enumerate(self.fitted_base_estimators):
with torch.no_grad():
logits = model(X_tensor)
probs = F.softmax(logits, dim=1)
predictions[:, est_idx] = probs[:, 1].numpy()
meta_X = torch.tensor(predictions, dtype=torch.float32)
with torch.no_grad():
meta_probs = F.softmax(self.meta_learner(meta_X), dim=1)
return meta_probs.numpy()
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
class VotingClassifier:
def __init__(self, estimators, voting="soft", weights=None):
self.estimators = estimators
self.voting = voting
self.weights = weights or [1.0] * len(estimators)
def fit(self, X, y, epochs=10):
for name, EstClass in self.estimators:
model = EstClass()
self._train_model(model, X, y, epochs)
self.fitted_estimators.append(model)
return self
def predict_proba(self, X):
predictions = []
for model, weight in zip(self.fitted_estimators, self.weights):
with torch.no_grad():
logits = model(X)
probs = F.softmax(logits, dim=1)
predictions.append(probs * weight)
if self.voting == "soft":
return torch.stack(predictions).sum(dim=0) / sum(self.weights)
else:
votes = torch.stack([p.argmax(dim=1) for p in predictions])
return votes.mode(dim=0)[0]
def predict(self, X):
proba = self.predict_proba(X)
return proba.argmax(dim=1)
class DiversityMeasure:
def __init__(self):
pass
def disagreement(self, predictions):
n_samples, n_models = predictions.shape
disagreements = 0
for i in range(n_samples):
for j in range(i + 1, n_models):
if predictions[i] != predictions[j]:
disagreements += 1
return 2 * disagreements / (n_models * (n_models - 1))
def q_statistic(self, predictions, true_labels):
n_models = predictions.shape[1]
agreement_matrix = np.zeros((n_models, n_models))
for i in range(n_models):
for j in range(n_models):
both_correct = np.sum((predictions[:, i] == true_labels) & (predictions[:, j] == true_labels))
both_wrong = np.sum((predictions[:, i] != true_labels) & (predictions[:, j] != true_labels))
agreement_matrix[i, j] = (both_correct + both_wrong) / len(true_labels)
return agreement_matrix
Prioritize model diversity over individual model performance in ensembles.
Use diverse algorithms (neural networks, trees, linear models) for better ensembles.
Apply bagging for high-variance models, boosting for high-bias models.
Use out-of-bag predictions for stacking to avoid overfitting.
Optimize ensemble weights on validation data.
Consider computational cost vs. accuracy trade-offs for production.
Use proper cross-validation for ensemble evaluation.
Apply early stopping to prevent overfitting in boosting.
Consider model compression for deploying ensembles.
Monitor ensemble calibration for confidence estimates.
Building autonomous AI agents capable of reasoning, planning, and executing multi-step tasks
Learning from a small number of examples per class using metric learning and meta-learning
Techniques and frameworks for generating new data instances that match the distribution of training data
Advanced techniques for training and fine-tuning transformer-based language models at scale
Foundational understanding and practical implementation of transformer-based language models
Integrating and reasoning across multiple data modalities including text, images, audio, and video