optuna

name	optuna
description	Optuna hyperparameter optimization — study creation, samplers, pruners, storage, and best practices for ML tuning

Optuna

Study creation

import optuna

# In-memory (default)
study = optuna.create_study(direction="maximize")

# Persistent SQLite storage (recommended for long runs)
study = optuna.create_study(
    study_name="xgboost_fraud",
    storage="sqlite:///optuna.db",
    direction="maximize",
    load_if_exists=True,
)

Objective function

def objective(trial: optuna.Trial) -> float:
    params = {
        "n_estimators": trial.suggest_int("n_estimators", 100, 1000),
        "max_depth": trial.suggest_int("max_depth", 3, 10),
        "learning_rate": trial.suggest_float("learning_rate", 1e-4, 0.3, log=True),
        "subsample": trial.suggest_float("subsample", 0.5, 1.0),
        "colsample_bytree": trial.suggest_float("colsample_bytree", 0.5, 1.0),
    }
    model = XGBClassifier(**params)
    model.fit(X_train, y_train, eval_set=[(X_val, y_val)], verbose=False)
    return roc_auc_score(y_val, model.predict_proba(X_val)[:, 1])

study.optimize(objective, n_trials=100, n_jobs=1)

Samplers

# Default: TPE (Tree-structured Parzen Estimator) — good general choice
sampler = optuna.samplers.TPESampler(seed=42)

# Grid search over small discrete spaces
sampler = optuna.samplers.GridSampler({"n_estimators": [100, 300, 500]})

# Random (useful as baseline or for parallelism)
sampler = optuna.samplers.RandomSampler(seed=42)

# CMA-ES for continuous params
sampler = optuna.samplers.CmaEsSampler(seed=42)

study = optuna.create_study(sampler=sampler, direction="maximize")

Pruners (early stopping for trials)

# Median pruner — stop if trial is below median of completed trials at same step
pruner = optuna.pruners.MedianPruner(n_warmup_steps=10)

# Hyperband
pruner = optuna.pruners.HyperbandPruner()

# In objective: report intermediate value and check for pruning
for epoch in range(n_epochs):
    val_score = train_one_epoch(model, ...)
    trial.report(val_score, epoch)
    if trial.should_prune():
        raise optuna.TrialPruned()

Retrieving results

best_trial = study.best_trial
print(best_trial.value)   # best objective value
print(best_trial.params)  # best hyperparameters

# All completed trials as DataFrame
df = study.trials_dataframe()

# Top N trials
top = sorted(study.trials, key=lambda t: t.value, reverse=True)[:5]

Logging & callbacks

# Suppress per-trial logs
optuna.logging.set_verbosity(optuna.logging.WARNING)

# Callback: stop when best value reaches threshold
def stop_callback(study: optuna.Study, trial: optuna.Trial) -> None:
    if study.best_value >= 0.99:
        study.stop()

study.optimize(objective, n_trials=200, callbacks=[stop_callback])

Multi-objective

study = optuna.create_study(directions=["maximize", "minimize"])  # AUC, time

def objective(trial):
    ...
    return auc, train_time

# Access Pareto front
pareto = study.best_trials

Pitfalls

• Never use trial.suggest_* outside the objective function — it breaks reproducibility.
• Use log=True for learning rates and regularization params that span orders of magnitude.
• Set n_jobs=-1 only when your objective is thread-safe; XGBoost/LightGBM with internal parallelism can deadlock.
• Always set load_if_exists=True with SQLite storage to resume interrupted studies.
• Categorical params with many values slow TPE; prefer suggest_int with ranges when possible.