import numpy as np
from scipy.stats import norm

def expected_improvement(mu, sigma, best_score, xi=0.01):
    """
    Expected Improvement acquisition function (maximization framing).
    mu, sigma : surrogate's predicted mean and std-dev at candidate points
    best_score: best observed score so far
    xi        : small exploration bonus (encourages exploring beyond best_score)
    """
    sigma = np.maximum(sigma, 1e-9)  # avoid divide-by-zero
    improvement = mu - best_score - xi
    z = improvement / sigma
    ei = improvement * norm.cdf(z) + sigma * norm.pdf(z)
    ei[sigma == 0.0] = 0.0
    return ei

# Toy illustration: 3 candidate hyperparameter points
candidate_points = np.array(["lr=0.01", "lr=0.05", "lr=0.20"])
mu    = np.array([0.91, 0.93, 0.89])   # surrogate's predicted validation accuracy
sigma = np.array([0.01, 0.04, 0.02])   # surrogate's uncertainty at each point
best_score_so_far = 0.92

ei_scores = expected_improvement(mu, sigma, best_score_so_far)
for point, ei in zip(candidate_points, ei_scores):
    print(f"  {point:10s}  μ={mu[candidate_points.tolist().index(point)]:.2f}  EI={ei:.4f}")

best_next = candidate_points[ei_scores.argmax()]
print(f"\nNext trial to run: {best_next}  (highest expected improvement)")

import optuna
from sklearn.datasets import load_breast_cancer
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_val_score, StratifiedKFold

X, y = load_breast_cancer(return_X_y=True)
cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)

def objective(trial: optuna.Trial) -> float:
    """
    Optuna calls this function once per trial. `trial` is used to
    *sample* hyperparameters — Optuna decides the actual values based
    on the configured sampler (TPE by default) and the trial history.
    """
    params = {
        "n_estimators":      trial.suggest_int("n_estimators", 50, 500),
        "max_depth":         trial.suggest_int("max_depth", 2, 32),
        "min_samples_split": trial.suggest_int("min_samples_split", 2, 20),
        "min_samples_leaf":  trial.suggest_int("min_samples_leaf", 1, 10),
        "max_features":      trial.suggest_categorical("max_features", ["sqrt", "log2", None]),
        # log=True is essential for parameters that span orders of magnitude
        "ccp_alpha":         trial.suggest_float("ccp_alpha", 1e-6, 1e-1, log=True),
    }

    model = RandomForestClassifier(**params, n_jobs=-1, random_state=42)
    scores = cross_val_score(model, X, y, cv=cv, scoring="roc_auc", n_jobs=-1)
    return scores.mean()   # Optuna maximises or minimises this return value


# Create a study and run the optimization
study = optuna.create_study(
    direction="maximize",            # we're maximizing ROC-AUC
    sampler=optuna.samplers.TPESampler(seed=42),
    study_name="rf_breast_cancer",
)
study.optimize(objective, n_trials=60, show_progress_bar=True)

print(f"\nBest trial: #{study.best_trial.number}")
print(f"Best ROC-AUC: {study.best_value:.4f}")
print("Best hyperparameters:")
for k, v in study.best_params.items():
    print(f"  {k}: {v}")

# Inspect the full trial history as a DataFrame
df = study.trials_dataframe()
print(df[["number", "value", "params_n_estimators", "params_max_depth",
          "params_max_features", "state"]].sort_values("value", ascending=False).head(5))

import optuna
from sklearn.svm import SVC
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import cross_val_score, StratifiedKFold
import time

X, y = load_breast_cancer(return_X_y=True)
cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)

def svm_objective(trial):
    # Purely continuous space (good fit for CMA-ES) — C and gamma on log scale
    C     = trial.suggest_float("C", 1e-3, 1e3, log=True)
    gamma = trial.suggest_float("gamma", 1e-5, 1e1, log=True)
    model = SVC(C=C, gamma=gamma, kernel="rbf")
    return cross_val_score(model, X, y, cv=cv, scoring="roc_auc").mean()

results = {}
for sampler_name, sampler in [
    ("TPE",    optuna.samplers.TPESampler(seed=42)),
    ("CMA-ES", optuna.samplers.CmaEsSampler(seed=42)),
    ("Random", optuna.samplers.RandomSampler(seed=42)),
]:
    optuna.logging.set_verbosity(optuna.logging.WARNING)
    study = optuna.create_study(direction="maximize", sampler=sampler)
    t0 = time.time()
    study.optimize(svm_objective, n_trials=40)
    results[sampler_name] = (study.best_value, time.time() - t0)

print(f"{'Sampler':10s} {'Best ROC-AUC':>14s} {'Wall time':>12s}")
for name, (best, elapsed) in results.items():
    print(f"{name:10s} {best:14.4f} {elapsed:11.1f}s")

import optuna
import xgboost as xgb
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score

X, y = load_breast_cancer(return_X_y=True)
X_train, X_val, y_train, y_val = train_test_split(
    X, y, test_size=0.2, stratify=y, random_state=42)

def objective(trial: optuna.Trial) -> float:
    params = {
        "max_depth":        trial.suggest_int("max_depth", 2, 10),
        "learning_rate":    trial.suggest_float("learning_rate", 1e-3, 0.3, log=True),
        "subsample":        trial.suggest_float("subsample", 0.5, 1.0),
        "colsample_bytree": trial.suggest_float("colsample_bytree", 0.5, 1.0),
        "objective":        "binary:logistic",
        "eval_metric":      "auc",
    }

    dtrain = xgb.DMatrix(X_train, label=y_train)
    dval   = xgb.DMatrix(X_val, label=y_val)

    n_rounds = 200
    booster = None
    for round_num in range(n_rounds):
        booster = xgb.train(
            params, dtrain, num_boost_round=1,
            xgb_model=booster, verbose_eval=False,
        )
        val_pred = booster.predict(dval)
        val_auc  = roc_auc_score(y_val, val_pred)

        # Report the intermediate value at this step
        trial.report(val_auc, step=round_num)

        # Ask the pruner: should this trial be stopped now?
        if trial.should_prune():
            raise optuna.TrialPruned()

    return val_auc


study = optuna.create_study(
    direction="maximize",
    sampler=optuna.samplers.TPESampler(seed=42),
    pruner=optuna.pruners.MedianPruner(
        n_startup_trials=5,   # don't prune until 5 trials have completed (baseline)
        n_warmup_steps=20,    # don't prune before round 20 within any trial
    ),
)
study.optimize(objective, n_trials=50)

pruned    = [t for t in study.trials if t.state == optuna.trial.TrialState.PRUNED]
completed = [t for t in study.trials if t.state == optuna.trial.TrialState.COMPLETE]
print(f"Completed: {len(completed)}   Pruned: {len(pruned)}")
print(f"Best AUC: {study.best_value:.4f}")
print(f"Compute saved: ~{len(pruned) * 0.6:.0f} fewer boosting-round-batches on average")

import optuna

# HyperbandPruner needs to know the budget range it's allocating across
pruner = optuna.pruners.HyperbandPruner(
    min_resource=1,        # minimum number of epochs/steps before any pruning decision
    max_resource=50,       # maximum epochs a trial could run
    reduction_factor=3,    # keep top 1/3 of trials at each rung
)

study = optuna.create_study(direction="maximize", pruner=pruner)
print("HyperbandPruner allocates a successive-halving 'bracket' schedule:")
print("  Rung 0: many trials, 1-2 epochs each -- cheap screening")
print("  Rung 1: top 1/3 survive, run to ~6 epochs")
print("  Rung 2: top 1/3 of those survive, run to ~17 epochs")
print("  Rung 3: top 1/3 of those survive, run to the full 50 epochs")
print("Net effect: full budget is spent almost entirely on the most promising trials.")

import optuna
import optuna.visualization as vis

# (continuing the study from Section 3 / 5)
# study = ... already populated with trials ...

# 1. Optimization history: best score so far at each trial
fig1 = vis.plot_optimization_history(study)
fig1.write_html("optimization_history.html")

# 2. Hyperparameter importance — which params actually drove the score?
#    Computed via fANOVA (functional analysis of variance) over the trial history
fig2 = vis.plot_param_importances(study)
fig2.write_html("param_importances.html")

# 3. Slice plot: score vs each individual hyperparameter
fig3 = vis.plot_slice(study, params=["max_depth", "learning_rate", "subsample"])
fig3.write_html("slice_plot.html")

# 4. Parallel coordinate plot: see how parameters co-vary in good vs bad trials
fig4 = vis.plot_parallel_coordinate(study)
fig4.write_html("parallel_coordinate.html")

# 5. Contour plot: interaction between two hyperparameters
fig5 = vis.plot_contour(study, params=["max_depth", "learning_rate"])
fig5.write_html("contour_plot.html")

# Print importance scores as plain numbers too (useful in notebooks/CI logs)
importances = optuna.importance.get_param_importances(study)
print("Hyperparameter importance (fraction of variance explained):")
for name, score in importances.items():
    bar = "█" * int(score * 40)
    print(f"  {name:18s} {score:.3f}  {bar}")

import optuna
import xgboost as xgb
import numpy as np
import time
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score

X, y = load_breast_cancer(return_X_y=True)
X_train, X_val, y_train, y_val = train_test_split(
    X, y, test_size=0.2, stratify=y, random_state=42)

def multi_objective(trial: optuna.Trial):
    params = {
        "n_estimators":  trial.suggest_int("n_estimators", 20, 500),
        "max_depth":     trial.suggest_int("max_depth", 2, 12),
        "learning_rate": trial.suggest_float("learning_rate", 1e-3, 0.3, log=True),
        "objective":     "binary:logistic",
        "eval_metric":   "auc",
        "n_jobs":        1,   # single-threaded so latency measurement is meaningful
    }
    model = xgb.XGBClassifier(**params)
    model.fit(X_train, y_train)

    val_pred = model.predict_proba(X_val)[:, 1]
    auc = roc_auc_score(y_val, val_pred)

    # Measure realistic single-row inference latency (what matters at serving time)
    single_row = X_val[:1]
    n_repeats = 200
    t0 = time.perf_counter()
    for _ in range(n_repeats):
        model.predict_proba(single_row)
    latency_ms = (time.perf_counter() - t0) / n_repeats * 1000

    # Return (objective_1, objective_2) -- Optuna tracks both
    return auc, latency_ms


study = optuna.create_study(
    directions=["maximize", "minimize"],     # maximize AUC, minimize latency
    sampler=optuna.samplers.NSGAIISampler(seed=42),
)
study.optimize(multi_objective, n_trials=80)

print(f"Number of trials on the Pareto front: {len(study.best_trials)}")
print(f"\n{'AUC':>8s} {'Latency (ms)':>14s}   n_estimators  max_depth  learning_rate")
for t in sorted(study.best_trials, key=lambda t: -t.values[0])[:8]:
    auc, latency = t.values
    p = t.params
    print(f"{auc:8.4f} {latency:14.3f}   {p['n_estimators']:12d}  {p['max_depth']:9d}  {p['learning_rate']:.4f}")

import optuna
import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import DataLoader, TensorDataset

device = "cuda" if torch.cuda.is_available() else "cpu"

# Assume train_ds / val_ds are already-prepared TensorDatasets (e.g. MNIST-like)
# train_ds = TensorDataset(X_train, y_train)
# val_ds   = TensorDataset(X_val, y_val)

def build_model(trial, input_dim, n_classes):
    n_layers = trial.suggest_int("n_layers", 1, 3)
    layers, in_dim = [], input_dim
    for i in range(n_layers):
        out_dim = trial.suggest_int(f"n_units_l{i}", 32, 256, step=32)
        layers += [nn.Linear(in_dim, out_dim), nn.ReLU()]
        dropout = trial.suggest_float(f"dropout_l{i}", 0.0, 0.5)
        layers.append(nn.Dropout(dropout))
        in_dim = out_dim
    layers.append(nn.Linear(in_dim, n_classes))
    return nn.Sequential(*layers).to(device)


def objective(trial: optuna.Trial) -> float:
    model = build_model(trial, input_dim=784, n_classes=10)

    optimizer_name = trial.suggest_categorical("optimizer", ["Adam", "RMSprop", "SGD"])
    lr = trial.suggest_float("lr", 1e-5, 1e-1, log=True)
    optimizer = getattr(optim, optimizer_name)(model.parameters(), lr=lr)

    batch_size = trial.suggest_categorical("batch_size", [32, 64, 128, 256])
    train_loader = DataLoader(train_ds, batch_size=batch_size, shuffle=True)
    val_loader   = DataLoader(val_ds, batch_size=256)

    criterion = nn.CrossEntropyLoss()
    n_epochs = 20

    for epoch in range(n_epochs):
        model.train()
        for xb, yb in train_loader:
            xb, yb = xb.to(device), yb.to(device)
            optimizer.zero_grad()
            loss = criterion(model(xb), yb)
            loss.backward()
            optimizer.step()

        # Validation pass
        model.eval()
        correct, total = 0, 0
        with torch.no_grad():
            for xb, yb in val_loader:
                xb, yb = xb.to(device), yb.to(device)
                preds = model(xb).argmax(dim=1)
                correct += (preds == yb).sum().item()
                total += yb.size(0)
        val_acc = correct / total

        # Report to Optuna at the end of each epoch, then check for pruning
        trial.report(val_acc, step=epoch)
        if trial.should_prune():
            raise optuna.TrialPruned()

    return val_acc


study = optuna.create_study(
    direction="maximize",
    sampler=optuna.samplers.TPESampler(seed=42),
    pruner=optuna.pruners.HyperbandPruner(min_resource=1, max_resource=20, reduction_factor=3),
)
study.optimize(objective, n_trials=40, timeout=3600)  # stop after 1 hour regardless

print(f"Best validation accuracy: {study.best_value:.4f}")
print(f"Best config: {study.best_params}")

import optuna

# SQLite for local persistence (single machine, survives crashes/restarts)
study = optuna.create_study(
    study_name="xgb_fraud_tuning",
    storage="sqlite:///optuna_studies.db",
    direction="maximize",
    load_if_exists=True,   # resume the study if it already exists, instead of erroring
)
study.optimize(objective, n_trials=20)   # run a further batch of trials
print(f"Study now has {len(study.trials)} total trials")
print(f"Best so far: {study.best_value:.4f}")

# For multi-machine parallel search, use a networked database, e.g. PostgreSQL/MySQL:
#   storage="postgresql://user:password@db-host:5432/optuna_db"
# Then launch the same script on N machines -- they all read/write the same study
# and each worker independently calls study.optimize(objective, n_trials=...).

# Inspect or resume a study later without re-running anything:
loaded_study = optuna.load_study(study_name="xgb_fraud_tuning",
                                  storage="sqlite:///optuna_studies.db")
print(f"\nResumed study '{loaded_study.study_name}' has "
      f"{len(loaded_study.trials)} trials on record.")