Neural Network Hyperparameter Optimization using Optuna

import numpy as np
import pandas as pd
from sklearn.preprocessing import StandardScaler

# 学習データ
x = train.iloc[:,:-1]
y = train.iloc[:,-1]

# データの標準化
scaler = StandardScaler()
scaler.fit(x)
sc_x = scaler.transform(x)

import optuna
import tensorflow as tf
from tensorflow.keras.callbacks import EarlyStopping
from sklearn.model_selection import StratifiedKFold

def objective(trial):
    # ハイパーパラメータの探索範囲を定義
    num_hidden_layers = trial.suggest_int('num_hidden_layers', 1, 5)  # 隠れ層の数を指定

    num_units_per_layer = []
    for i in range(num_hidden_layers):
        num_units = trial.suggest_int(f'num_units_layer{i}', 4, 256)
        num_units_per_layer.append(num_units)

    learning_rate = trial.suggest_float('learning_rate', 1e-5, 1e-1, log=True)

    # モデルの構築
    model = tf.keras.Sequential()
    model.add(tf.keras.layers.Input(shape=(sc_x.shape[1],)))
    for i in range(num_hidden_layers):
        model.add(tf.keras.layers.Dense(num_units_per_layer[i], activation='relu'))
    model.add(tf.keras.layers.Dense(1, activation='sigmoid'))

    # 学習率の設定
    optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)

    # モデルのコンパイル
    model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy'])

    # アーリーストッピングの設定
    early_stopping = EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)

    # 交差検証の設定
    kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)

    # 交差検証の各分割でのバリデーション損失を記録するリスト
    val_losses = []

    # 交差検証の各分割での学習と評価
    for train_idx, val_idx in kfold.split(sc_x, y):
        X_train_fold, X_val_fold = sc_x[train_idx], sc_x[val_idx]
        y_train_fold, y_val_fold = y.iloc[train_idx], y.iloc[val_idx]

        history = model.fit(
            X_train_fold, y_train_fold,
            epochs=50, batch_size=1000, validation_data=(X_val_fold, y_val_fold),
            callbacks=[early_stopping], verbose=0
        )

        val_loss = history.history['val_loss'][-1]
        val_losses.append(val_loss)

    # 交差検証の平均バリデーション損失を返す
    return sum(val_losses) / len(val_losses)

# Optunaの最適化実行
study = optuna.create_study(direction='minimize')
study.optimize(objective, n_trials=100)

# 最適なハイパーパラメータを表示
best_params = study.best_params
print(f"Best hyperparameters: {best_params}")

# 最適なハイパーパラメータでの評価
best_model = tf.keras.Sequential()
best_model.add(tf.keras.layers.Input(shape=(sc_x.shape[1],)))
for i in range(best_params['num_hidden_layers']):
    best_model.add(tf.keras.layers.Dense(best_params[f'num_units_layer{i}'], activation='relu'))
best_model.add(tf.keras.layers.Dense(1, activation='sigmoid'))

best_optimizer = tf.keras.optimizers.Adam(learning_rate=best_params['learning_rate'])
best_model.compile(optimizer=best_optimizer, loss='binary_crossentropy', metrics=['accuracy'])
early_stopping = EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)

# 交差検証の各分割での学習と評価
kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
for train_idx, val_idx in kfold.split(sc_x, y):
    X_train_fold, X_val_fold = sc_x[train_idx], sc_x[val_idx]
    y_train_fold, y_val_fold = y.iloc[train_idx], y.iloc[val_idx]

    history = best_model.fit(
        X_train_fold, y_train_fold, 
        epochs=50, batch_size=3000, validation_data=(X_val_fold, y_val_fold), 
        callbacks=[early_stopping], verbose=2)