diff --git "a/Analysis_code/deeplearning_model_binary.ipynb" "b/Analysis_code/deeplearning_model_binary.ipynb" deleted file mode 100644--- "a/Analysis_code/deeplearning_model_binary.ipynb" +++ /dev/null @@ -1,3952 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "##### 통일 데이터셋 변환 함수 정의" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "import numpy as np\n", - "from sklearn.preprocessing import StandardScaler, LabelEncoder\n", - "import torch\n", - "from torch.utils.data import DataLoader, TensorDataset\n", - "import random\n", - "\n", - "# 데이터셋 준비 함수\n", - "def prepare_dataset(region, target = 'binary'):\n", - "\n", - " # 데이터 경로 지정\n", - " dat_path = f\"../data/data_for_modeling/{region}_train.csv\"\n", - " train_path = f'../data/data_oversampled/ctgan10000/ctgan10000_3_{region}.csv'\n", - " test_path = f\"../data/data_for_modeling/{region}_test.csv\"\n", - " drop_col = ['binary_class','multi_class','visi','year']\n", - " target_col = f\"{target}_class\"\n", - "\n", - " # 데이터 로드\n", - " region_dat = pd.read_csv(dat_path, index_col=0)\n", - " region_train = pd.read_csv(train_path)\n", - " region_val = region_dat.loc[region_dat['year'].isin([2018]), :]\n", - " region_test = pd.read_csv(test_path)\n", - "\n", - " # 컬럼 정렬 (일관성 유지)\n", - " common_columns = region_train.columns.to_list()\n", - " train_data = region_train[common_columns]\n", - " val_data = region_val[common_columns]\n", - " test_data = region_test[common_columns]\n", - "\n", - " # 설명변수 & 타겟 분리\n", - " X_train = train_data.drop(columns=drop_col)\n", - " y_train = train_data[target_col]\n", - " X_val = val_data.drop(columns=drop_col)\n", - " y_val = val_data[target_col]\n", - " X_test = test_data.drop(columns=drop_col)\n", - " y_test = test_data[target_col]\n", - "\n", - " # 범주형 & 연속형 변수 분리\n", - " categorical_cols = X_train.select_dtypes(include=['object', 'category', 'int64']).columns\n", - " numerical_cols = X_train.select_dtypes(include=['float64']).columns\n", - "\n", - " # 범주형 변수 Label Encoding\n", - " label_encoders = {}\n", - " for col in categorical_cols:\n", - " le = LabelEncoder()\n", - " le.fit(X_train[col]) # Train 데이터 기준으로 학습\n", - " label_encoders[col] = le\n", - "\n", - " # 변환 적용\n", - " for col in categorical_cols:\n", - " X_train[col] = label_encoders[col].transform(X_train[col])\n", - " X_val[col] = label_encoders[col].transform(X_val[col])\n", - " X_test[col] = label_encoders[col].transform(X_test[col])\n", - "\n", - " # 연속형 변수 Standard Scaling\n", - " scaler = StandardScaler()\n", - " scaler.fit(X_train[numerical_cols]) # Train 데이터 기준으로 학습\n", - "\n", - " # 변환 적용\n", - " X_train[numerical_cols] = scaler.transform(X_train[numerical_cols])\n", - " X_val[numerical_cols] = scaler.transform(X_val[numerical_cols])\n", - " X_test[numerical_cols] = scaler.transform(X_test[numerical_cols])\n", - "\n", - " return X_train, X_val, X_test, y_train, y_val, y_test, categorical_cols, numerical_cols" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [], - "source": [ - "# 데이터 변환 및 dataloader 생성 함수\n", - "def prepare_dataloader(region, target = 'binary', random_state = None):\n", - "\n", - " # 데이터 경로 지정\n", - " dat_path = f\"../data/data_for_modeling/{region}_train.csv\"\n", - " train_path = f'../data/data_oversampled/ctgan10000/ctgan10000_3_{region}.csv'\n", - " test_path = f\"../data/data_for_modeling/{region}_test.csv\"\n", - " drop_col = ['binary_class','multi_class','visi','year']\n", - " target_col = f'{target}_class'\n", - " \n", - " # 데이터 로드\n", - " region_dat = pd.read_csv(dat_path, index_col=0)\n", - " region_train = pd.read_csv(train_path)\n", - " region_val = region_dat.loc[region_dat['year'].isin([2018]), :]\n", - " region_test = pd.read_csv(test_path)\n", - "\n", - " # 컬럼 정렬 (일관성 유지)\n", - " common_columns = region_train.columns.to_list()\n", - " train_data = region_train[common_columns]\n", - " val_data = region_val[common_columns]\n", - " test_data = region_test[common_columns]\n", - "\n", - " # 설명변수 & 타겟 분리\n", - " X_train = train_data.drop(columns=drop_col)\n", - " y_train = train_data[target_col]\n", - " X_val = val_data.drop(columns=drop_col)\n", - " y_val = val_data[target_col]\n", - " X_test = test_data.drop(columns=drop_col)\n", - " y_test = test_data[target_col]\n", - "\n", - " # 범주형 & 연속형 변수 분리\n", - " categorical_cols = X_train.select_dtypes(include=['object', 'category', 'int64']).columns\n", - " numerical_cols = X_train.select_dtypes(include=['float64']).columns\n", - "\n", - " # 범주형 변수 Label Encoding\n", - " label_encoders = {}\n", - " for col in categorical_cols:\n", - " le = LabelEncoder()\n", - " le.fit(X_train[col]) # Train 데이터 기준으로 학습\n", - " label_encoders[col] = le\n", - "\n", - " # 변환 적용\n", - " for col in categorical_cols:\n", - " X_train[col] = label_encoders[col].transform(X_train[col])\n", - " X_val[col] = label_encoders[col].transform(X_val[col])\n", - " X_test[col] = label_encoders[col].transform(X_test[col])\n", - "\n", - " # 연속형 변수 Standard Scaling\n", - " scaler = StandardScaler()\n", - " scaler.fit(X_train[numerical_cols]) # Train 데이터 기준으로 학습\n", - "\n", - " # 변환 적용\n", - " X_train[numerical_cols] = scaler.transform(X_train[numerical_cols])\n", - " X_val[numerical_cols] = scaler.transform(X_val[numerical_cols])\n", - " X_test[numerical_cols] = scaler.transform(X_test[numerical_cols])\n", - "\n", - " # 연속형 변수와 범주형 변수 분리\n", - " X_train_num = torch.tensor(X_train[numerical_cols].values, dtype=torch.float32)\n", - " X_train_cat = torch.tensor(X_train[categorical_cols].values, dtype=torch.long)\n", - "\n", - " X_val_num = torch.tensor(X_val[numerical_cols].values, dtype=torch.float32)\n", - " X_val_cat = torch.tensor(X_val[categorical_cols].values, dtype=torch.long)\n", - "\n", - " X_test_num = torch.tensor(X_test[numerical_cols].values, dtype=torch.float32)\n", - " X_test_cat = torch.tensor(X_test[categorical_cols].values, dtype=torch.long)\n", - "\n", - " # 레이블 변환\n", - " if target == \"binary\":\n", - " y_train_tensor = torch.tensor(y_train.values, dtype=torch.float32) # 이진 분류 → float32\n", - " y_val_tensor = torch.tensor(y_val.values, dtype=torch.float32)\n", - " y_test_tensor = torch.tensor(y_test.values, dtype=torch.float32)\n", - " elif target == \"multi\":\n", - " y_train_tensor = torch.tensor(y_train.values, dtype=torch.long) # 다중 분류 → long\n", - " y_val_tensor = torch.tensor(y_val.values, dtype=torch.long)\n", - " y_test_tensor = torch.tensor(y_test.values, dtype=torch.long)\n", - " else:\n", - " raise ValueError(\"target must be 'binary' or 'multi'\")\n", - "\n", - " # TensorDataset 생성\n", - " train_dataset = TensorDataset(X_train_num, X_train_cat, y_train_tensor)\n", - " val_dataset = TensorDataset(X_val_num, X_val_cat, y_val_tensor)\n", - " test_dataset = TensorDataset(X_test_num, X_test_cat, y_test_tensor)\n", - "\n", - " # DataLoader 생성\n", - " if random_state == None:\n", - " train_loader = DataLoader(train_dataset, batch_size=64, shuffle=True)\n", - " else:\n", - " train_loader = DataLoader(train_dataset, batch_size=64, shuffle=True, generator=torch.Generator().manual_seed(random_state))\n", - " val_loader = DataLoader(val_dataset, batch_size=64, shuffle=False)\n", - " test_loader = DataLoader(test_dataset, batch_size=64, shuffle=False)\n", - " \n", - " return X_train, categorical_cols, numerical_cols, train_loader, val_loader, test_loader" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "##### 통일 최적화 함수 정의" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import torch\n", - "import torch.nn as nn\n", - "import torch.optim as optim\n", - "import optuna\n", - "from sklearn.metrics import f1_score\n", - "\n", - "# 모델을 GPU로 전송\n", - "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n", - "\n", - "# 공통 Objective 함수\n", - "def objective(train_loader, val_loader, model_class, model_params):\n", - "\n", - " # 모델 초기화\n", - " model = model_class(**model_params).to(device)\n", - "\n", - " # 손실 함수 및 옵티마이저 설정\n", - " criterion = nn.BCELoss()\n", - " optimizer = optim.AdamW(model.parameters(), lr=model_params[\"lr\"], weight_decay=model_params[\"weight_decay\"])\n", - "\n", - " # 학습 루프\n", - " epochs = 50 # 충분한 학습 기회 제공\n", - " patience = 8 # Early stopping 기준 (8 epochs 동안 개선 없으면 종료)\n", - " best_val_f1 = -float(\"inf\") # 최대화 목표\n", - " counter = 0 # 개선되지 않은 epoch 수\n", - "\n", - " for epoch in range(epochs):\n", - " model.train()\n", - " train_loss = 0\n", - "\n", - " for x_num_batch, x_cat_batch, y_batch in train_loader:\n", - " x_num_batch, x_cat_batch, y_batch = (\n", - " x_num_batch.to(device),\n", - " x_cat_batch.to(device),\n", - " y_batch.to(device)\n", - " )\n", - " optimizer.zero_grad()\n", - " y_pred = model(x_num_batch, x_cat_batch)\n", - " loss = criterion(y_pred, y_batch.unsqueeze(1))\n", - " loss.backward()\n", - " optimizer.step()\n", - " train_loss += loss.item()\n", - "\n", - " # Validation 평가\n", - " model.eval()\n", - " val_loss = 0\n", - " y_pred_val, y_true_val = [], []\n", - "\n", - " with torch.no_grad():\n", - " for x_num_batch, x_cat_batch, y_batch in val_loader:\n", - " x_num_batch, x_cat_batch, y_batch = (\n", - " x_num_batch.to(device),\n", - " x_cat_batch.to(device),\n", - " y_batch.to(device)\n", - " )\n", - " output = model(x_num_batch, x_cat_batch)\n", - " val_loss += criterion(output, y_batch.unsqueeze(1)).item()\n", - " y_pred_val.extend((output.cpu().numpy() > 0.5).astype(int))\n", - " y_true_val.extend(y_batch.cpu().numpy())\n", - "\n", - " # F1-score 계산\n", - " val_f1 = f1_score(y_true_val, y_pred_val)\n", - "\n", - " print(f\"Epoch {epoch+1}/{epochs} | Train Loss: {train_loss:.4f} | Validation Loss: {val_loss:.4f} | Validation F1: {val_f1:.4f}\")\n", - "\n", - " # Early Stopping 체크\n", - " if val_f1 > best_val_f1:\n", - " best_val_f1 = val_f1\n", - " counter = 0\n", - " else:\n", - " counter += 1\n", - " if counter >= patience:\n", - " print(f\"Early stopping at epoch {epoch+1}\")\n", - " break\n", - "\n", - " return best_val_f1 # 최종 F1-score 반환" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## FT-Transformer" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "##### 데이터셋 준비" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [], - "source": [ - "# 시드 고정\n", - "seed = 42\n", - "random.seed(seed)\n", - "np.random.seed(seed)\n", - "torch.manual_seed(seed)\n", - "torch.cuda.manual_seed_all(seed)\n", - "\n", - "# DataLoader 생성\n", - "X_train, categorical_cols, numerical_cols, train_loader, val_loader, test_loader = prepare_dataloader(\"seoul\", target='binary', random_state = seed)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "##### 모델 구현(하이퍼파라미터 최적화)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n", - "[I 2025-02-02 15:41:08,871] A new study created in memory with name: no-name-251e54c9-5a4b-408c-a8d9-c1a826aa0507\n", - "[I 2025-02-02 15:41:24,977] Trial 0 finished with value: 0.7239199157007377 and parameters: {'d_token': 128, 'n_blocks': 4, 'attention_dropout': 0.24679835610086082, 'ffn_dropout': 0.21742508365045984, 'lr': 0.0007348118405270452, 'weight_decay': 0.00015930522616241006}. Best is trial 0 with value: 0.7239199157007377.\n", - "[I 2025-02-02 15:41:42,721] Trial 1 finished with value: 0.6823069403714564 and parameters: {'d_token': 192, 'n_blocks': 5, 'attention_dropout': 0.2545474901621302, 'ffn_dropout': 0.2550213529560302, 'lr': 0.0002014847788415866, 'weight_decay': 0.00011207606211860574}. Best is trial 0 with value: 0.7239199157007377.\n", - "[I 2025-02-02 15:42:00,849] Trial 2 finished with value: 0.7318982387475538 and parameters: {'d_token': 192, 'n_blocks': 5, 'attention_dropout': 0.3099085529881075, 'ffn_dropout': 0.3368209952651108, 'lr': 0.0006097839109531517, 'weight_decay': 2.5081156860452307e-05}. Best is trial 2 with value: 0.7318982387475538.\n", - "[I 2025-02-02 15:42:16,314] Trial 3 finished with value: 0.19511692592974142 and parameters: {'d_token': 256, 'n_blocks': 4, 'attention_dropout': 0.21951547789558387, 'ffn_dropout': 0.48466566117599996, 'lr': 0.0009239150319627245, 'weight_decay': 0.0004138040112561013}. Best is trial 2 with value: 0.7318982387475538.\n", - "[I 2025-02-02 15:42:31,589] Trial 4 finished with value: 0.19511692592974142 and parameters: {'d_token': 192, 'n_blocks': 4, 'attention_dropout': 0.34855307303338107, 'ffn_dropout': 0.21031655633456553, 'lr': 0.0008115595675970503, 'weight_decay': 3.292759134423613e-05}. Best is trial 2 with value: 0.7318982387475538.\n", - "[I 2025-02-02 15:42:46,768] Trial 5 finished with value: 0.7303424987940184 and parameters: {'d_token': 64, 'n_blocks': 4, 'attention_dropout': 0.49087538832936756, 'ffn_dropout': 0.43253984700833437, 'lr': 0.0008699593128513321, 'weight_decay': 0.0006161049539380963}. Best is trial 2 with value: 0.7318982387475538.\n", - "[I 2025-02-02 15:43:02,084] Trial 6 finished with value: 0.7247845919918905 and parameters: {'d_token': 128, 'n_blocks': 4, 'attention_dropout': 0.2975990992289793, 'ffn_dropout': 0.31660318690684464, 'lr': 0.000186788025710707, 'weight_decay': 0.00045443839603360173}. Best is trial 2 with value: 0.7318982387475538.\n", - "[I 2025-02-02 15:43:27,280] Trial 7 finished with value: 0.28465346534653463 and parameters: {'d_token': 192, 'n_blocks': 8, 'attention_dropout': 0.22236519310393127, 'ffn_dropout': 0.4960660809801552, 'lr': 0.0005918951335463647, 'weight_decay': 2.497073714505272e-05}. Best is trial 2 with value: 0.7318982387475538.\n", - "[I 2025-02-02 15:43:49,893] Trial 8 finished with value: 0.6748971193415638 and parameters: {'d_token': 128, 'n_blocks': 7, 'attention_dropout': 0.22221339552022712, 'ffn_dropout': 0.30753971856328177, 'lr': 0.00013057771348997234, 'weight_decay': 0.0005323617594751496}. Best is trial 2 with value: 0.7318982387475538.\n", - "[I 2025-02-02 15:44:07,744] Trial 9 finished with value: 0.6968566862013852 and parameters: {'d_token': 64, 'n_blocks': 5, 'attention_dropout': 0.41888185350141927, 'ffn_dropout': 0.39126724140656394, 'lr': 0.000771281194715635, 'weight_decay': 8.798929749689021e-05}. Best is trial 2 with value: 0.7318982387475538.\n", - "[I 2025-02-02 15:44:28,098] Trial 10 finished with value: 0.19511692592974142 and parameters: {'d_token': 256, 'n_blocks': 6, 'attention_dropout': 0.3554111493483545, 'ffn_dropout': 0.36590280160354616, 'lr': 0.00043660479644886074, 'weight_decay': 1.1952270129143866e-05}. Best is trial 2 with value: 0.7318982387475538.\n", - "[I 2025-02-02 15:44:45,702] Trial 11 finished with value: 0.6822519083969466 and parameters: {'d_token': 64, 'n_blocks': 5, 'attention_dropout': 0.48408856291704483, 'ffn_dropout': 0.42557494926138, 'lr': 0.00042673231508545695, 'weight_decay': 4.49074466433967e-05}. Best is trial 2 with value: 0.7318982387475538.\n", - "[I 2025-02-02 15:45:05,871] Trial 12 finished with value: 0.7150368033648791 and parameters: {'d_token': 64, 'n_blocks': 6, 'attention_dropout': 0.48739285247229314, 'ffn_dropout': 0.43103225611689927, 'lr': 0.0004896552590776571, 'weight_decay': 0.0009912861232764343}. Best is trial 2 with value: 0.7318982387475538.\n", - "[I 2025-02-02 15:45:23,603] Trial 13 finished with value: 0.6933638443935927 and parameters: {'d_token': 64, 'n_blocks': 5, 'attention_dropout': 0.4047001889260864, 'ffn_dropout': 0.32569556937329, 'lr': 0.00029920804271524077, 'weight_decay': 1.2615508659239553e-05}. Best is trial 2 with value: 0.7318982387475538.\n", - "[I 2025-02-02 15:45:41,530] Trial 14 finished with value: 0.7511987213638786 and parameters: {'d_token': 192, 'n_blocks': 5, 'attention_dropout': 0.32657703152441825, 'ffn_dropout': 0.4178174448017821, 'lr': 0.0003267555125994075, 'weight_decay': 0.00021351678009204976}. Best is trial 14 with value: 0.7511987213638786.\n", - "[I 2025-02-02 15:46:04,378] Trial 15 finished with value: 0.7074829931972789 and parameters: {'d_token': 192, 'n_blocks': 7, 'attention_dropout': 0.31041768721719115, 'ffn_dropout': 0.2739122593366717, 'lr': 0.00030899938367208267, 'weight_decay': 0.00023496826834476444}. Best is trial 14 with value: 0.7511987213638786.\n", - "[I 2025-02-02 15:46:24,868] Trial 16 finished with value: 0.6610328638497652 and parameters: {'d_token': 192, 'n_blocks': 6, 'attention_dropout': 0.32398593919255186, 'ffn_dropout': 0.3867121557282759, 'lr': 0.0002330123217423081, 'weight_decay': 8.506812879738529e-05}. Best is trial 14 with value: 0.7511987213638786.\n", - "[I 2025-02-02 15:46:42,870] Trial 17 finished with value: 0.6872202884137245 and parameters: {'d_token': 192, 'n_blocks': 5, 'attention_dropout': 0.39211245016802665, 'ffn_dropout': 0.34949985542474904, 'lr': 0.0005640304117528214, 'weight_decay': 0.0002277020852390507}. Best is trial 14 with value: 0.7511987213638786.\n", - "[I 2025-02-02 15:47:05,644] Trial 18 finished with value: 0.712608473711077 and parameters: {'d_token': 192, 'n_blocks': 7, 'attention_dropout': 0.2800711256226658, 'ffn_dropout': 0.45887819655597584, 'lr': 0.00010363593787464723, 'weight_decay': 5.3707427835369724e-05}. Best is trial 14 with value: 0.7511987213638786.\n", - "[I 2025-02-02 15:47:26,042] Trial 19 finished with value: 0.7258145363408521 and parameters: {'d_token': 192, 'n_blocks': 6, 'attention_dropout': 0.36318783257719667, 'ffn_dropout': 0.3984033562225775, 'lr': 0.0003651849381739298, 'weight_decay': 1.8782195995924806e-05}. Best is trial 14 with value: 0.7511987213638786.\n", - "[I 2025-02-02 15:47:43,815] Trial 20 finished with value: 0.6697300402067777 and parameters: {'d_token': 256, 'n_blocks': 5, 'attention_dropout': 0.43274746581488754, 'ffn_dropout': 0.2824558846985788, 'lr': 0.0002556751337418656, 'weight_decay': 0.0002494683273647461}. Best is trial 14 with value: 0.7511987213638786.\n", - "[I 2025-02-02 15:47:58,938] Trial 21 finished with value: 0.7178354500276091 and parameters: {'d_token': 64, 'n_blocks': 4, 'attention_dropout': 0.44984547464488966, 'ffn_dropout': 0.43243862220781226, 'lr': 0.0009982289684266161, 'weight_decay': 0.0009101904796088088}. Best is trial 14 with value: 0.7511987213638786.\n", - "[I 2025-02-02 15:48:16,546] Trial 22 finished with value: 0.7169082125603865 and parameters: {'d_token': 64, 'n_blocks': 5, 'attention_dropout': 0.3266483061476531, 'ffn_dropout': 0.3566508287751997, 'lr': 0.0006592761670945311, 'weight_decay': 0.0006359195874655299}. Best is trial 14 with value: 0.7511987213638786.\n", - "[I 2025-02-02 15:48:31,855] Trial 23 finished with value: 0.737927292457949 and parameters: {'d_token': 192, 'n_blocks': 4, 'attention_dropout': 0.27515293275689734, 'ffn_dropout': 0.4524271854297583, 'lr': 0.0005489202273938838, 'weight_decay': 0.00033027787368051153}. Best is trial 14 with value: 0.7511987213638786.\n", - "[I 2025-02-02 15:48:49,727] Trial 24 finished with value: 0.6891252955082743 and parameters: {'d_token': 192, 'n_blocks': 5, 'attention_dropout': 0.27553431938378914, 'ffn_dropout': 0.45873360240139666, 'lr': 0.0005086245601961827, 'weight_decay': 0.000351934968810717}. Best is trial 14 with value: 0.7511987213638786.\n", - "[I 2025-02-02 15:49:05,170] Trial 25 finished with value: 0.7389215162840364 and parameters: {'d_token': 192, 'n_blocks': 4, 'attention_dropout': 0.2837638019921456, 'ffn_dropout': 0.4641514065134212, 'lr': 0.0004026800729283218, 'weight_decay': 0.00011701373306653755}. Best is trial 14 with value: 0.7511987213638786.\n", - "[I 2025-02-02 15:49:20,536] Trial 26 finished with value: 0.6642599277978339 and parameters: {'d_token': 192, 'n_blocks': 4, 'attention_dropout': 0.27251203104160765, 'ffn_dropout': 0.4607058172591405, 'lr': 0.0003601626181990028, 'weight_decay': 0.00014136037952166197}. Best is trial 14 with value: 0.7511987213638786.\n", - "[I 2025-02-02 15:49:35,930] Trial 27 finished with value: 0.7280248190279214 and parameters: {'d_token': 192, 'n_blocks': 4, 'attention_dropout': 0.33730153869002266, 'ffn_dropout': 0.472794506350429, 'lr': 0.0003797203538970235, 'weight_decay': 0.0001573156600007119}. Best is trial 14 with value: 0.7511987213638786.\n", - "[I 2025-02-02 15:49:51,312] Trial 28 finished with value: 0.6848891181021145 and parameters: {'d_token': 192, 'n_blocks': 4, 'attention_dropout': 0.24862055884209416, 'ffn_dropout': 0.4073050694157798, 'lr': 0.0002731917735846195, 'weight_decay': 0.0003171519637892753}. Best is trial 14 with value: 0.7511987213638786.\n", - "[I 2025-02-02 15:50:06,623] Trial 29 finished with value: 0.6995412844036697 and parameters: {'d_token': 128, 'n_blocks': 4, 'attention_dropout': 0.29664078129288607, 'ffn_dropout': 0.4183212661013632, 'lr': 0.00044184677057511584, 'weight_decay': 6.641491095324641e-05}. Best is trial 14 with value: 0.7511987213638786.\n", - "[I 2025-02-02 15:50:21,934] Trial 30 finished with value: 0.7068676716917923 and parameters: {'d_token': 192, 'n_blocks': 4, 'attention_dropout': 0.3782977122856468, 'ffn_dropout': 0.4963551001157477, 'lr': 0.00020313715887341092, 'weight_decay': 0.0001853642529114672}. Best is trial 14 with value: 0.7511987213638786.\n", - "[I 2025-02-02 15:50:39,805] Trial 31 finished with value: 0.1179245283018868 and parameters: {'d_token': 192, 'n_blocks': 5, 'attention_dropout': 0.3039279279944892, 'ffn_dropout': 0.44734530574119136, 'lr': 0.000664310381639209, 'weight_decay': 0.00012348127087389468}. Best is trial 14 with value: 0.7511987213638786.\n", - "[I 2025-02-02 15:50:57,852] Trial 32 finished with value: 0.7144396551724138 and parameters: {'d_token': 192, 'n_blocks': 5, 'attention_dropout': 0.2612697301073756, 'ffn_dropout': 0.3787483984065484, 'lr': 0.0005416552465077905, 'weight_decay': 0.00017487733354466299}. Best is trial 14 with value: 0.7511987213638786.\n", - "[I 2025-02-02 15:51:13,276] Trial 33 finished with value: 0.6918238993710691 and parameters: {'d_token': 192, 'n_blocks': 4, 'attention_dropout': 0.20051318674084254, 'ffn_dropout': 0.3370003290336566, 'lr': 0.0006598525178596913, 'weight_decay': 0.00030290392419648945}. Best is trial 14 with value: 0.7511987213638786.\n", - "[I 2025-02-02 15:51:33,645] Trial 34 finished with value: 0.6428571428571429 and parameters: {'d_token': 256, 'n_blocks': 6, 'attention_dropout': 0.2855644129946276, 'ffn_dropout': 0.4817800237565608, 'lr': 0.00038807742512600733, 'weight_decay': 0.00010344315875246088}. Best is trial 14 with value: 0.7511987213638786.\n", - "[I 2025-02-02 15:51:49,249] Trial 35 finished with value: 0.7339805825242719 and parameters: {'d_token': 192, 'n_blocks': 4, 'attention_dropout': 0.2399976537028991, 'ffn_dropout': 0.4445370590364987, 'lr': 0.00033171689577321723, 'weight_decay': 4.028888739382443e-05}. Best is trial 14 with value: 0.7511987213638786.\n", - "[I 2025-02-02 15:52:04,545] Trial 36 finished with value: 0.7280788177339902 and parameters: {'d_token': 192, 'n_blocks': 4, 'attention_dropout': 0.23554167653467295, 'ffn_dropout': 0.4467111904409484, 'lr': 0.00033857719057883365, 'weight_decay': 6.923538277736423e-05}. Best is trial 14 with value: 0.7511987213638786.\n", - "[I 2025-02-02 15:52:19,498] Trial 37 finished with value: 0.6766265060240964 and parameters: {'d_token': 128, 'n_blocks': 4, 'attention_dropout': 0.25646445799400547, 'ffn_dropout': 0.41569404568093943, 'lr': 0.0001694941500084373, 'weight_decay': 4.150908504110461e-05}. Best is trial 14 with value: 0.7511987213638786.\n", - "[I 2025-02-02 15:52:34,709] Trial 38 finished with value: 0.6599171652093879 and parameters: {'d_token': 192, 'n_blocks': 4, 'attention_dropout': 0.240636945885518, 'ffn_dropout': 0.47317587429145236, 'lr': 0.00023673773365563876, 'weight_decay': 2.865530584633714e-05}. Best is trial 14 with value: 0.7511987213638786.\n", - "[I 2025-02-02 15:52:50,248] Trial 39 finished with value: 0.7054108216432866 and parameters: {'d_token': 192, 'n_blocks': 4, 'attention_dropout': 0.20040957141100618, 'ffn_dropout': 0.4413156611762989, 'lr': 0.00029614995530728835, 'weight_decay': 0.00011737018375863745}. Best is trial 14 with value: 0.7511987213638786.\n", - "[I 2025-02-02 15:53:08,355] Trial 40 finished with value: 0.669047619047619 and parameters: {'d_token': 192, 'n_blocks': 5, 'attention_dropout': 0.31579646247943044, 'ffn_dropout': 0.4991154146623525, 'lr': 0.0004745202231385376, 'weight_decay': 0.0004370471489831405}. Best is trial 14 with value: 0.7511987213638786.\n", - "[I 2025-02-02 15:53:23,703] Trial 41 finished with value: 0.67472306143001 and parameters: {'d_token': 192, 'n_blocks': 4, 'attention_dropout': 0.26288915185936645, 'ffn_dropout': 0.4710451809725687, 'lr': 0.0006214025038700964, 'weight_decay': 2.064974994614282e-05}. Best is trial 14 with value: 0.7511987213638786.\n", - "[I 2025-02-02 15:53:41,812] Trial 42 finished with value: 0.3887903511883647 and parameters: {'d_token': 192, 'n_blocks': 5, 'attention_dropout': 0.34329097874915243, 'ffn_dropout': 0.23704077550252764, 'lr': 0.0007399242057813222, 'weight_decay': 1.6076979534676088e-05}. Best is trial 14 with value: 0.7511987213638786.\n", - "[I 2025-02-02 15:53:57,179] Trial 43 finished with value: 0.7377124483233808 and parameters: {'d_token': 256, 'n_blocks': 4, 'attention_dropout': 0.2905336075299809, 'ffn_dropout': 0.3728812523004836, 'lr': 0.00040811140436640766, 'weight_decay': 4.236150083191819e-05}. Best is trial 14 with value: 0.7511987213638786.\n", - "[I 2025-02-02 15:54:12,240] Trial 44 finished with value: 0.718544719555331 and parameters: {'d_token': 256, 'n_blocks': 4, 'attention_dropout': 0.28847655843654574, 'ffn_dropout': 0.4015845497202453, 'lr': 0.0004032728794886698, 'weight_decay': 3.40005553116306e-05}. Best is trial 14 with value: 0.7511987213638786.\n", - "[I 2025-02-02 15:54:27,624] Trial 45 finished with value: 0.7108050847457628 and parameters: {'d_token': 256, 'n_blocks': 4, 'attention_dropout': 0.23140734639114655, 'ffn_dropout': 0.3748198632542725, 'lr': 0.00033550308733082065, 'weight_decay': 5.907611414758871e-05}. Best is trial 14 with value: 0.7511987213638786.\n", - "[I 2025-02-02 15:54:42,893] Trial 46 finished with value: 0.7417148869016307 and parameters: {'d_token': 256, 'n_blocks': 4, 'attention_dropout': 0.29449206006179474, 'ffn_dropout': 0.41949021088844474, 'lr': 0.0004558422302910519, 'weight_decay': 4.5741857621694515e-05}. Best is trial 14 with value: 0.7511987213638786.\n", - "[I 2025-02-02 15:55:09,179] Trial 47 finished with value: 0.37842696629213485 and parameters: {'d_token': 256, 'n_blocks': 8, 'attention_dropout': 0.2954843610731864, 'ffn_dropout': 0.41487866519469907, 'lr': 0.0004629369824494006, 'weight_decay': 9.014365616038466e-05}. Best is trial 14 with value: 0.7511987213638786.\n", - "[I 2025-02-02 15:55:27,086] Trial 48 finished with value: 0.6156716417910447 and parameters: {'d_token': 256, 'n_blocks': 5, 'attention_dropout': 0.3300003159606516, 'ffn_dropout': 0.3887591993621017, 'lr': 0.0005205909284889117, 'weight_decay': 7.674171394833573e-05}. Best is trial 14 with value: 0.7511987213638786.\n", - "[I 2025-02-02 15:55:42,560] Trial 49 finished with value: 0.7027328499721138 and parameters: {'d_token': 256, 'n_blocks': 4, 'attention_dropout': 0.3121871568203331, 'ffn_dropout': 0.4263500985808496, 'lr': 0.0004191199421324702, 'weight_decay': 0.000225147272905659}. Best is trial 14 with value: 0.7511987213638786.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Best trial: {'d_token': 192, 'n_blocks': 5, 'attention_dropout': 0.32657703152441825, 'ffn_dropout': 0.4178174448017821, 'lr': 0.0003267555125994075, 'weight_decay': 0.00021351678009204976}\n" - ] - } - ], - "source": [ - "import optuna\n", - "from sklearn.metrics import accuracy_score, f1_score\n", - "import torch\n", - "import torch.nn as nn\n", - "import torch.optim as optim\n", - "from ft_transformer import FTTransformer\n", - "\n", - "# 모델을 GPU로 전송\n", - "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n", - "\n", - "# 하이퍼파라미터 최적화 함수 정의\n", - "def objective(trial):\n", - " d_token = trial.suggest_categorical(\"d_token\", [64, 128, 192, 256])\n", - " n_blocks = trial.suggest_int(\"n_blocks\", 4, 8)\n", - " attention_dropout = trial.suggest_float(\"attention_dropout\", 0.2, 0.5) # Dropout 범위 증가\n", - " ffn_dropout = trial.suggest_float(\"ffn_dropout\", 0.2, 0.5)\n", - " lr = trial.suggest_float(\"lr\", 1e-4, 1e-3, log=True)\n", - " weight_decay = trial.suggest_float(\"weight_decay\", 1e-5, 1e-3, log=True) # Weight Decay 강화\n", - "\n", - " # FT-Transformer 초기화(이진분류류)\n", - " model = FTTransformer(\n", - " num_features=len(numerical_cols),\n", - " cat_cardinalities=[len(X_train[col].unique()) for col in categorical_cols],\n", - " d_token=d_token,\n", - " n_blocks=n_blocks,\n", - " attention_dropout=attention_dropout,\n", - " ffn_dropout=ffn_dropout,\n", - " num_classes=2\n", - " ).to(device)\n", - "\n", - " # 손실 함수 및 옵티마이저 설정\n", - " criterion = nn.BCELoss()\n", - " optimizer = optim.AdamW(model.parameters(), lr=lr, weight_decay=weight_decay)\n", - "\n", - " # 학습 루프\n", - " epochs = 8\n", - " for epoch in range(epochs):\n", - " model.train()\n", - " for x_num_batch, x_cat_batch, y_batch in train_loader:\n", - " x_num_batch, x_cat_batch, y_batch = (\n", - " x_num_batch.to(device),\n", - " x_cat_batch.to(device),\n", - " y_batch.to(device)\n", - " )\n", - " optimizer.zero_grad()\n", - " y_pred = model(x_num_batch, x_cat_batch)\n", - " loss = criterion(y_pred, y_batch.unsqueeze(1))\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - " # Validation 평가\n", - " model.eval()\n", - " y_pred_val, y_true_val = [], []\n", - " with torch.no_grad():\n", - " for x_num_batch, x_cat_batch, y_batch in val_loader:\n", - " x_num_batch, x_cat_batch, y_batch = (\n", - " x_num_batch.to(device),\n", - " x_cat_batch.to(device),\n", - " y_batch.to(device)\n", - " )\n", - " output = model(x_num_batch, x_cat_batch)\n", - " y_pred_val.extend((output.cpu().numpy() > 0.5).astype(int))\n", - " y_true_val.extend(y_batch.cpu().numpy())\n", - "\n", - " # f1-score 계산\n", - " val_score = f1_score(y_true_val, y_pred_val)\n", - " return val_score\n", - "\n", - "# Optuna 실행\n", - "sampler = optuna.samplers.TPESampler(seed=seed)\n", - "study = optuna.create_study(direction=\"maximize\", sampler=sampler) # f1-score를 최대화\n", - "study.optimize(objective, n_trials=50) # 50번의 탐색 시도\n", - "\n", - "# 최적의 하이퍼파라미터 출력\n", - "print(\"Best trial:\", study.best_trial.params)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### 모델 학습 루프" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Best Hyperparameters: {'d_token': 192, 'n_blocks': 5, 'attention_dropout': 0.32657703152441825, 'ffn_dropout': 0.4178174448017821, 'lr': 0.0003267555125994075, 'weight_decay': 0.00021351678009204976}\n", - "Epoch 1/50, Train Loss: 0.1531, Validation Loss: 0.1658, Validation F1-Score: 0.6962\n", - "Epoch 2/50, Train Loss: 0.1036, Validation Loss: 0.2129, Validation F1-Score: 0.6684\n", - "Epoch 3/50, Train Loss: 0.0856, Validation Loss: 0.2259, Validation F1-Score: 0.6493\n", - "Epoch 4/50, Train Loss: 0.0789, Validation Loss: 0.1908, Validation F1-Score: 0.6861\n", - "Epoch 5/50, Train Loss: 0.0726, Validation Loss: 0.1832, Validation F1-Score: 0.6957\n", - "Epoch 6/50, Train Loss: 0.0674, Validation Loss: 0.1649, Validation F1-Score: 0.7127\n", - "Epoch 7/50, Train Loss: 0.0641, Validation Loss: 0.1444, Validation F1-Score: 0.7537\n", - "Epoch 8/50, Train Loss: 0.0640, Validation Loss: 0.1749, Validation F1-Score: 0.6901\n", - "Epoch 9/50, Train Loss: 0.0593, Validation Loss: 0.1693, Validation F1-Score: 0.6952\n", - "Epoch 10/50, Train Loss: 0.0597, Validation Loss: 0.1791, Validation F1-Score: 0.7131\n", - "Epoch 11/50, Train Loss: 0.0550, Validation Loss: 0.1743, Validation F1-Score: 0.7080\n", - "Epoch 12/50, Train Loss: 0.0550, Validation Loss: 0.1761, Validation F1-Score: 0.7167\n", - "Epoch 13/50, Train Loss: 0.0544, Validation Loss: 0.1967, Validation F1-Score: 0.6750\n", - "Epoch 14/50, Train Loss: 0.0539, Validation Loss: 0.1993, Validation F1-Score: 0.6651\n", - "Epoch 15/50, Train Loss: 0.0518, Validation Loss: 0.1751, Validation F1-Score: 0.7230\n", - "Early stopping at epoch 15\n" - ] - } - ], - "source": [ - "# 최적의 하이퍼파라미터 가져오기\n", - "best_params = study.best_trial.params\n", - "print(\"Best Hyperparameters:\", best_params)\n", - "\n", - "# FT-Transformer 초기화 (최적화된 하이퍼파라미터로 설정)\n", - "ft_transformer = FTTransformer(\n", - " num_features=len(numerical_cols),\n", - " cat_cardinalities=[len(X_train[col].unique()) for col in categorical_cols],\n", - " d_token=best_params[\"d_token\"],\n", - " n_blocks=best_params[\"n_blocks\"],\n", - " attention_dropout=best_params[\"attention_dropout\"],\n", - " ffn_dropout=best_params[\"ffn_dropout\"]\n", - ").to(device)\n", - "\n", - "# 손실 함수와 옵티마이저 재설정\n", - "criterion = nn.BCELoss()\n", - "optimizer_ft = optim.AdamW(ft_transformer.parameters(), lr=best_params[\"lr\"], weight_decay=best_params[\"weight_decay\"])\n", - "\n", - "# Early Stopping 설정\n", - "best_val_f1 = -float('inf') # F1-Score는 최대화가 목표이므로 -inf로 초기화\n", - "patience = 8 # F1-Score가 개선되지 않는 Epoch 수\n", - "counter = 0 # 개선되지 않은 Epoch 수를 기록\n", - "\n", - "# 학습 루프\n", - "epochs = 50 # 최대 Epoch 수\n", - "for epoch in range(epochs):\n", - " # Training Phase\n", - " ft_transformer.train()\n", - " train_loss = 0\n", - " for x_num_batch, x_cat_batch, y_batch in train_loader:\n", - " x_num_batch, x_cat_batch, y_batch = (\n", - " x_num_batch.to(device),\n", - " x_cat_batch.to(device),\n", - " y_batch.to(device),\n", - " )\n", - " optimizer_ft.zero_grad()\n", - " y_pred = ft_transformer(x_num_batch, x_cat_batch)\n", - " loss = criterion(y_pred, y_batch.unsqueeze(1))\n", - " loss.backward()\n", - " optimizer_ft.step()\n", - " train_loss += loss.item()\n", - "\n", - " # Validation Phase\n", - " ft_transformer.eval()\n", - " val_loss = 0\n", - " y_true_val, y_pred_val = [], []\n", - " with torch.no_grad():\n", - " for x_num_batch, x_cat_batch, y_batch in val_loader:\n", - " x_num_batch, x_cat_batch, y_batch = (\n", - " x_num_batch.to(device),\n", - " x_cat_batch.to(device),\n", - " y_batch.to(device),\n", - " )\n", - " y_pred = ft_transformer(x_num_batch, x_cat_batch)\n", - " val_loss += criterion(y_pred, y_batch.unsqueeze(1)).item()\n", - " y_true_val.extend(y_batch.cpu().numpy())\n", - " y_pred_val.extend((y_pred.cpu().numpy() > 0.5).astype(int))\n", - "\n", - " # 평균 Loss 계산\n", - " train_loss /= len(train_loader)\n", - " val_loss /= len(val_loader) \n", - "\n", - " # F1-Score 계산\n", - " val_f1 = f1_score(y_true_val, y_pred_val)\n", - "\n", - " print(\n", - " f\"Epoch {epoch+1}/{epochs}, Train Loss: {train_loss:.4f}, \"\n", - " f\"Validation Loss: {val_loss:.4f}, Validation F1-Score: {val_f1:.4f}\"\n", - " )\n", - "\n", - " # Early Stopping 체크\n", - " if val_f1 > best_val_f1:\n", - " best_val_f1 = val_f1\n", - " counter = 0\n", - " # 최적 모델 저장\n", - " torch.save(ft_transformer.state_dict(), \"best_model_f1.pth\")\n", - " else:\n", - " counter += 1\n", - " if counter >= patience:\n", - " print(f\"Early stopping at epoch {epoch+1}\")\n", - " break" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### 성능 평가" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Test Accuracy: 0.9261415525114155\n", - "Test AUC: 0.9602353820488985\n", - "Test F1-Score: 0.55286800276434\n" - ] - } - ], - "source": [ - "from sklearn.metrics import accuracy_score, roc_auc_score, f1_score\n", - "\n", - "# Test 데이터 성능 평가\n", - "ft_transformer.eval()\n", - "y_pred_test = []\n", - "y_true_test = []\n", - "\n", - "with torch.no_grad():\n", - " for x_num_batch, x_cat_batch, y_batch in test_loader:\n", - " x_num_batch, x_cat_batch, y_batch = (\n", - " x_num_batch.to(device),\n", - " x_cat_batch.to(device),\n", - " y_batch.to(device),\n", - " )\n", - " output = ft_transformer(x_num_batch, x_cat_batch)\n", - " y_pred_test.extend(output.cpu().numpy())\n", - " y_true_test.extend(y_batch.cpu().numpy())\n", - "\n", - "# 연속형 예측값을 이진값으로 변환\n", - "y_pred_test_binary = (np.array(y_pred_test) > 0.5).astype(int)\n", - "\n", - "# 성능 지표 계산\n", - "print(\"Test Accuracy:\", accuracy_score(y_true_test, y_pred_test_binary))\n", - "print(\"Test AUC:\", roc_auc_score(y_true_test, y_pred_test))\n", - "print(\"Test F1-Score:\", f1_score(y_true_test, y_pred_test_binary))\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## ResNet-like" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "##### 데이터셋 준비" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# 시드 고정\n", - "seed = 42\n", - "random.seed(seed)\n", - "np.random.seed(seed)\n", - "torch.manual_seed(seed)\n", - "torch.cuda.manual_seed_all(seed)\n", - "\n", - "# DataLoader 생성\n", - "X_train, categorical_cols, numerical_cols, train_loader, val_loader, test_loader = prepare_dataloader(\"seoul\", target='binary', random_state = 42)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "##### 하이퍼파라미터 최적화" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[I 2025-02-02 15:59:20,922] A new study created in memory with name: no-name-ecb8d2cb-843f-4b8f-ba41-9b6d94be0241\n", - "[I 2025-02-02 15:59:30,115] Trial 0 finished with value: 0.5192686357243319 and parameters: {'d_main': 128, 'd_hidden': 32, 'n_blocks': 8, 'dropout_first': 0.34044600469728353, 'dropout_second': 0.21242177333881365, 'lr': 0.00010994335574766199, 'weight_decay': 0.0008123245085588687}. Best is trial 0 with value: 0.5192686357243319.\n", - "[I 2025-02-02 15:59:35,646] Trial 1 finished with value: 0.5009784735812133 and parameters: {'d_main': 64, 'd_hidden': 64, 'n_blocks': 4, 'dropout_first': 0.34474115788895177, 'dropout_second': 0.04184815819561255, 'lr': 0.0003839629299804173, 'weight_decay': 1.2562773503807034e-05}. Best is trial 0 with value: 0.5192686357243319.\n", - "[I 2025-02-02 15:59:41,174] Trial 2 finished with value: 0.19511692592974142 and parameters: {'d_main': 128, 'd_hidden': 128, 'n_blocks': 4, 'dropout_first': 0.12602063719411183, 'dropout_second': 0.28466566117599995, 'lr': 0.00853618986286683, 'weight_decay': 0.0002661901888489054}. Best is trial 0 with value: 0.5192686357243319.\n", - "[I 2025-02-02 15:59:50,093] Trial 3 finished with value: 0.6725949878738885 and parameters: {'d_main': 192, 'd_hidden': 64, 'n_blocks': 8, 'dropout_first': 0.20351199264000677, 'dropout_second': 0.19875668530619459, 'lr': 0.0004201672054372534, 'weight_decay': 3.632486956676606e-05}. Best is trial 3 with value: 0.6725949878738885.\n", - "[I 2025-02-02 15:59:59,156] Trial 4 finished with value: 0.5230160971476984 and parameters: {'d_main': 192, 'd_hidden': 32, 'n_blocks': 8, 'dropout_first': 0.1353970008207678, 'dropout_second': 0.058794858725743554, 'lr': 0.00012315571723666037, 'weight_decay': 9.462175356461487e-06}. Best is trial 3 with value: 0.6725949878738885.\n", - "[I 2025-02-02 16:00:07,468] Trial 5 finished with value: 0.0 and parameters: {'d_main': 192, 'd_hidden': 64, 'n_blocks': 7, 'dropout_first': 0.12982025747190834, 'dropout_second': 0.2960660809801552, 'lr': 0.0035033984911586884, 'weight_decay': 3.945908811099999e-06}. Best is trial 3 with value: 0.6725949878738885.\n", - "[I 2025-02-02 16:00:12,273] Trial 6 finished with value: 0.5830485304169515 and parameters: {'d_main': 128, 'd_hidden': 32, 'n_blocks': 3, 'dropout_first': 0.4452413703502375, 'dropout_second': 0.18698943804826737, 'lr': 0.0004589824181495653, 'weight_decay': 1.5512259126484753e-06}. Best is trial 3 with value: 0.6725949878738885.\n", - "[I 2025-02-02 16:00:20,540] Trial 7 finished with value: 0.19511692592974142 and parameters: {'d_main': 192, 'd_hidden': 32, 'n_blocks': 7, 'dropout_first': 0.40431401944675904, 'dropout_second': 0.16838315927084888, 'lr': 0.003482846706526885, 'weight_decay': 3.0296104428212496e-05}. Best is trial 3 with value: 0.6725949878738885.\n", - "[I 2025-02-02 16:00:28,039] Trial 8 finished with value: 0.6400343642611683 and parameters: {'d_main': 64, 'd_hidden': 64, 'n_blocks': 6, 'dropout_first': 0.4630265895704372, 'dropout_second': 0.07478766874466249, 'lr': 0.000661859559718348, 'weight_decay': 0.0001847793417351927}. Best is trial 3 with value: 0.6725949878738885.\n", - "[I 2025-02-02 16:00:36,993] Trial 9 finished with value: 0.0 and parameters: {'d_main': 192, 'd_hidden': 32, 'n_blocks': 8, 'dropout_first': 0.4214688307596458, 'dropout_second': 0.05597101766581075, 'lr': 0.006097025297491433, 'weight_decay': 4.149795789891592e-05}. Best is trial 3 with value: 0.6725949878738885.\n", - "[I 2025-02-02 16:00:44,631] Trial 10 finished with value: 0.07984031936127745 and parameters: {'d_main': 256, 'd_hidden': 128, 'n_blocks': 6, 'dropout_first': 0.23874035656264875, 'dropout_second': 0.1191627229093386, 'lr': 0.0014138916628611978, 'weight_decay': 9.186066145996928e-05}. Best is trial 3 with value: 0.6725949878738885.\n", - "[I 2025-02-02 16:00:52,142] Trial 11 finished with value: 0.6199059561128527 and parameters: {'d_main': 64, 'd_hidden': 64, 'n_blocks': 6, 'dropout_first': 0.2476670229366214, 'dropout_second': 0.10682736700049808, 'lr': 0.0006875192303983222, 'weight_decay': 0.00016773791183912636}. Best is trial 3 with value: 0.6725949878738885.\n", - "[I 2025-02-02 16:00:58,558] Trial 12 finished with value: 0.5279538904899136 and parameters: {'d_main': 64, 'd_hidden': 64, 'n_blocks': 5, 'dropout_first': 0.4992445333005462, 'dropout_second': 0.22957840083137454, 'lr': 0.00026708145837086683, 'weight_decay': 0.000544189514701203}. Best is trial 3 with value: 0.6725949878738885.\n", - "[I 2025-02-02 16:01:06,648] Trial 13 finished with value: 0.0 and parameters: {'d_main': 256, 'd_hidden': 64, 'n_blocks': 7, 'dropout_first': 0.23905337219863312, 'dropout_second': 0.010075765480341559, 'lr': 0.0012557930846273358, 'weight_decay': 4.766194018548066e-05}. Best is trial 3 with value: 0.6725949878738885.\n", - "[I 2025-02-02 16:01:12,778] Trial 14 finished with value: 0.5347852389594676 and parameters: {'d_main': 64, 'd_hidden': 64, 'n_blocks': 5, 'dropout_first': 0.19123040647428097, 'dropout_second': 0.11858708726450355, 'lr': 0.00021863781499886916, 'weight_decay': 0.00013182468471438185}. Best is trial 3 with value: 0.6725949878738885.\n", - "[I 2025-02-02 16:01:20,285] Trial 15 finished with value: 0.656 and parameters: {'d_main': 192, 'd_hidden': 64, 'n_blocks': 6, 'dropout_first': 0.29986491595140763, 'dropout_second': 0.23692395773958858, 'lr': 0.0008943218941938837, 'weight_decay': 0.0003586878521837277}. Best is trial 3 with value: 0.6725949878738885.\n", - "[I 2025-02-02 16:01:28,462] Trial 16 finished with value: 0.5844036697247706 and parameters: {'d_main': 192, 'd_hidden': 64, 'n_blocks': 7, 'dropout_first': 0.3201820298115095, 'dropout_second': 0.2509034576046632, 'lr': 0.0012368375416860133, 'weight_decay': 0.0003933843039023355}. Best is trial 3 with value: 0.6725949878738885.\n", - "[I 2025-02-02 16:01:35,404] Trial 17 finished with value: 0.0 and parameters: {'d_main': 192, 'd_hidden': 128, 'n_blocks': 5, 'dropout_first': 0.2794382614823519, 'dropout_second': 0.24944640643903046, 'lr': 0.002067442428463093, 'weight_decay': 1.7825601965382163e-05}. Best is trial 3 with value: 0.6725949878738885.\n", - "[I 2025-02-02 16:01:40,304] Trial 18 finished with value: 0.657411188240098 and parameters: {'d_main': 192, 'd_hidden': 64, 'n_blocks': 3, 'dropout_first': 0.185052145172123, 'dropout_second': 0.19465208459320812, 'lr': 0.0006509944715021355, 'weight_decay': 5.949055284431336e-06}. Best is trial 3 with value: 0.6725949878738885.\n", - "[I 2025-02-02 16:01:45,253] Trial 19 finished with value: 0.6093471280770603 and parameters: {'d_main': 192, 'd_hidden': 64, 'n_blocks': 3, 'dropout_first': 0.18071714680649648, 'dropout_second': 0.15224752924180684, 'lr': 0.00022031746107267636, 'weight_decay': 4.661801081342972e-06}. Best is trial 3 with value: 0.6725949878738885.\n", - "[I 2025-02-02 16:01:50,794] Trial 20 finished with value: 0.597237775289287 and parameters: {'d_main': 256, 'd_hidden': 128, 'n_blocks': 4, 'dropout_first': 0.1799367766346295, 'dropout_second': 0.18901655929349007, 'lr': 0.00044722710854316026, 'weight_decay': 1.1227386145926723e-06}. Best is trial 3 with value: 0.6725949878738885.\n", - "[I 2025-02-02 16:01:55,660] Trial 21 finished with value: 0.5853000674308834 and parameters: {'d_main': 192, 'd_hidden': 64, 'n_blocks': 3, 'dropout_first': 0.27955578506271855, 'dropout_second': 0.2095950254957476, 'lr': 0.0007358925470348262, 'weight_decay': 5.031388037641803e-06}. Best is trial 3 with value: 0.6725949878738885.\n", - "[I 2025-02-02 16:02:03,255] Trial 22 finished with value: 0.6617583369423993 and parameters: {'d_main': 192, 'd_hidden': 64, 'n_blocks': 6, 'dropout_first': 0.2073708534881274, 'dropout_second': 0.26298036974241923, 'lr': 0.0008285121463870491, 'weight_decay': 7.57796289720658e-05}. Best is trial 3 with value: 0.6725949878738885.\n", - "[I 2025-02-02 16:02:08,817] Trial 23 finished with value: 0.32728632878249486 and parameters: {'d_main': 192, 'd_hidden': 64, 'n_blocks': 4, 'dropout_first': 0.20001601120798612, 'dropout_second': 0.2710526705672427, 'lr': 0.0020015023761662063, 'weight_decay': 7.328055550719538e-05}. Best is trial 3 with value: 0.6725949878738885.\n", - "[I 2025-02-02 16:02:18,081] Trial 24 finished with value: 0.6281701131486539 and parameters: {'d_main': 192, 'd_hidden': 64, 'n_blocks': 8, 'dropout_first': 0.15734642099337648, 'dropout_second': 0.20467077434159242, 'lr': 0.0003676186285957462, 'weight_decay': 8.484877321047985e-06}. Best is trial 3 with value: 0.6725949878738885.\n", - "[I 2025-02-02 16:02:25,099] Trial 25 finished with value: 0.5965045592705167 and parameters: {'d_main': 192, 'd_hidden': 64, 'n_blocks': 5, 'dropout_first': 0.2170998437398996, 'dropout_second': 0.16083184686850863, 'lr': 0.0005365589635455455, 'weight_decay': 2.3038620151242498e-05}. Best is trial 3 with value: 0.6725949878738885.\n", - "[I 2025-02-02 16:02:33,316] Trial 26 finished with value: 0.5584905660377358 and parameters: {'d_main': 192, 'd_hidden': 64, 'n_blocks': 7, 'dropout_first': 0.10927125312095436, 'dropout_second': 0.2613601094556424, 'lr': 0.00030485793870362976, 'weight_decay': 6.250253353769138e-05}. Best is trial 3 with value: 0.6725949878738885.\n", - "[I 2025-02-02 16:02:40,952] Trial 27 finished with value: 0.19378427787934185 and parameters: {'d_main': 192, 'd_hidden': 64, 'n_blocks': 6, 'dropout_first': 0.15738248329924776, 'dropout_second': 0.18576686380696483, 'lr': 0.0009320900444178502, 'weight_decay': 2.1693900847788226e-06}. Best is trial 3 with value: 0.6725949878738885.\n", - "[I 2025-02-02 16:02:46,415] Trial 28 finished with value: 0.4736171297476421 and parameters: {'d_main': 128, 'd_hidden': 128, 'n_blocks': 4, 'dropout_first': 0.22284105915494618, 'dropout_second': 0.2275215422482748, 'lr': 0.00017006611735943575, 'weight_decay': 1.5091449536138288e-05}. Best is trial 3 with value: 0.6725949878738885.\n", - "[I 2025-02-02 16:02:55,324] Trial 29 finished with value: 0.19511692592974142 and parameters: {'d_main': 256, 'd_hidden': 64, 'n_blocks': 8, 'dropout_first': 0.2649726067425218, 'dropout_second': 0.1424722036991971, 'lr': 0.0022738815322358146, 'weight_decay': 2.694620758307688e-05}. Best is trial 3 with value: 0.6725949878738885.\n", - "[I 2025-02-02 16:03:00,544] Trial 30 finished with value: 0.45915354330708663 and parameters: {'d_main': 128, 'd_hidden': 32, 'n_blocks': 3, 'dropout_first': 0.35103917717404076, 'dropout_second': 0.20872104345048048, 'lr': 0.00010012078005753846, 'weight_decay': 0.00010485220093419964}. Best is trial 3 with value: 0.6725949878738885.\n", - "[I 2025-02-02 16:03:08,269] Trial 31 finished with value: 0.680594243268338 and parameters: {'d_main': 192, 'd_hidden': 64, 'n_blocks': 6, 'dropout_first': 0.3122622733458627, 'dropout_second': 0.22959715904464012, 'lr': 0.0009538992562057897, 'weight_decay': 0.0008351622710745379}. Best is trial 31 with value: 0.680594243268338.\n", - "[I 2025-02-02 16:03:15,309] Trial 32 finished with value: 0.6067335243553008 and parameters: {'d_main': 192, 'd_hidden': 64, 'n_blocks': 5, 'dropout_first': 0.35611919589547497, 'dropout_second': 0.22615889083016483, 'lr': 0.0005685888485986714, 'weight_decay': 0.0009550197959107674}. Best is trial 31 with value: 0.680594243268338.\n", - "[I 2025-02-02 16:03:23,049] Trial 33 finished with value: 0.335 and parameters: {'d_main': 192, 'd_hidden': 64, 'n_blocks': 6, 'dropout_first': 0.3796369811324729, 'dropout_second': 0.2781657506788394, 'lr': 0.0016247278936636989, 'weight_decay': 4.528464672102566e-05}. Best is trial 31 with value: 0.680594243268338.\n", - "[I 2025-02-02 16:03:31,423] Trial 34 finished with value: 0.6100415251038127 and parameters: {'d_main': 192, 'd_hidden': 64, 'n_blocks': 7, 'dropout_first': 0.31457726271891395, 'dropout_second': 0.24629443338977997, 'lr': 0.0009916395396574313, 'weight_decay': 8.468038690363233e-06}. Best is trial 31 with value: 0.680594243268338.\n", - "[I 2025-02-02 16:03:40,440] Trial 35 finished with value: 0.5792826014079785 and parameters: {'d_main': 192, 'd_hidden': 64, 'n_blocks': 8, 'dropout_first': 0.1609595903458735, 'dropout_second': 0.29949544909552134, 'lr': 0.0003598006595715833, 'weight_decay': 0.000611366018985009}. Best is trial 31 with value: 0.680594243268338.\n", - "[I 2025-02-02 16:03:47,549] Trial 36 finished with value: 0.55859375 and parameters: {'d_main': 192, 'd_hidden': 64, 'n_blocks': 5, 'dropout_first': 0.21368649308842594, 'dropout_second': 0.17650165280665778, 'lr': 0.0006969248287278233, 'weight_decay': 1.157759133419454e-05}. Best is trial 31 with value: 0.680594243268338.\n", - "[I 2025-02-02 16:03:55,811] Trial 37 finished with value: 0.5915492957746479 and parameters: {'d_main': 128, 'd_hidden': 64, 'n_blocks': 7, 'dropout_first': 0.26284056200741807, 'dropout_second': 0.19804880051449372, 'lr': 0.0004867907132712468, 'weight_decay': 2.508603520472002e-06}. Best is trial 31 with value: 0.680594243268338.\n", - "[I 2025-02-02 16:04:03,584] Trial 38 finished with value: 0.6141324454310025 and parameters: {'d_main': 192, 'd_hidden': 32, 'n_blocks': 6, 'dropout_first': 0.1421050626216277, 'dropout_second': 0.21908827524827684, 'lr': 0.00084224196723038, 'weight_decay': 0.00023303438320237155}. Best is trial 31 with value: 0.680594243268338.\n", - "[I 2025-02-02 16:04:09,139] Trial 39 finished with value: 0.0 and parameters: {'d_main': 192, 'd_hidden': 128, 'n_blocks': 4, 'dropout_first': 0.10160616201392712, 'dropout_second': 0.1391621008584947, 'lr': 0.0032379140293698006, 'weight_decay': 6.269443823194281e-06}. Best is trial 31 with value: 0.680594243268338.\n", - "[I 2025-02-02 16:04:17,430] Trial 40 finished with value: 0.6051752921535893 and parameters: {'d_main': 192, 'd_hidden': 32, 'n_blocks': 7, 'dropout_first': 0.2935358343914325, 'dropout_second': 0.26846075784783446, 'lr': 0.0012398778873158685, 'weight_decay': 3.157930756708663e-06}. Best is trial 31 with value: 0.680594243268338.\n", - "[I 2025-02-02 16:04:25,135] Trial 41 finished with value: 0.6241379310344828 and parameters: {'d_main': 192, 'd_hidden': 64, 'n_blocks': 6, 'dropout_first': 0.3190351182045631, 'dropout_second': 0.23969591667705323, 'lr': 0.0008625263450648776, 'weight_decay': 0.00034440568782878036}. Best is trial 31 with value: 0.680594243268338.\n", - "[I 2025-02-02 16:04:32,838] Trial 42 finished with value: 0.591715976331361 and parameters: {'d_main': 192, 'd_hidden': 64, 'n_blocks': 6, 'dropout_first': 0.36771124174149844, 'dropout_second': 0.2864919167273482, 'lr': 0.0005796725863182776, 'weight_decay': 0.0006693113444106173}. Best is trial 31 with value: 0.680594243268338.\n", - "[I 2025-02-02 16:04:40,467] Trial 43 finished with value: 0.19511692592974142 and parameters: {'d_main': 192, 'd_hidden': 64, 'n_blocks': 6, 'dropout_first': 0.4014188726382219, 'dropout_second': 0.261409827181683, 'lr': 0.0016241904532108644, 'weight_decay': 0.00041403989414200593}. Best is trial 31 with value: 0.680594243268338.\n", - "[I 2025-02-02 16:04:47,372] Trial 44 finished with value: 0.6356275303643725 and parameters: {'d_main': 64, 'd_hidden': 64, 'n_blocks': 5, 'dropout_first': 0.25708001150138227, 'dropout_second': 0.23696362420229689, 'lr': 0.0010726485646066052, 'weight_decay': 0.0002394830883280549}. Best is trial 31 with value: 0.680594243268338.\n", - "[I 2025-02-02 16:04:55,037] Trial 45 finished with value: 0.5898423817863397 and parameters: {'d_main': 192, 'd_hidden': 64, 'n_blocks': 6, 'dropout_first': 0.2939883848541707, 'dropout_second': 0.17573780024632835, 'lr': 0.0004283351035616762, 'weight_decay': 0.00015968844593287096}. Best is trial 31 with value: 0.680594243268338.\n", - "[I 2025-02-02 16:05:03,328] Trial 46 finished with value: 0.6297833102812356 and parameters: {'d_main': 256, 'd_hidden': 64, 'n_blocks': 7, 'dropout_first': 0.33761088583094895, 'dropout_second': 0.19784084013167183, 'lr': 0.0008029538747768709, 'weight_decay': 0.0007561108161069606}. Best is trial 31 with value: 0.680594243268338.\n", - "[I 2025-02-02 16:05:10,927] Trial 47 finished with value: 0.6577017114914425 and parameters: {'d_main': 128, 'd_hidden': 64, 'n_blocks': 6, 'dropout_first': 0.224510896201851, 'dropout_second': 0.22227813212160902, 'lr': 0.000623544967315617, 'weight_decay': 0.00010858010427724432}. Best is trial 31 with value: 0.680594243268338.\n", - "[I 2025-02-02 16:05:19,107] Trial 48 finished with value: 0.6053475935828877 and parameters: {'d_main': 128, 'd_hidden': 32, 'n_blocks': 7, 'dropout_first': 0.20179287097188797, 'dropout_second': 0.21472047222478008, 'lr': 0.0003075040219603553, 'weight_decay': 6.360962731072777e-05}. Best is trial 31 with value: 0.680594243268338.\n", - "[I 2025-02-02 16:05:26,114] Trial 49 finished with value: 0.6677727084197429 and parameters: {'d_main': 128, 'd_hidden': 64, 'n_blocks': 5, 'dropout_first': 0.23962863264554043, 'dropout_second': 0.09087448546487817, 'lr': 0.0005968243273186192, 'weight_decay': 3.526800870958912e-05}. Best is trial 31 with value: 0.680594243268338.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Best Hyperparameters: {'d_main': 192, 'd_hidden': 64, 'n_blocks': 6, 'dropout_first': 0.3122622733458627, 'dropout_second': 0.22959715904464012, 'lr': 0.0009538992562057897, 'weight_decay': 0.0008351622710745379}\n" - ] - } - ], - "source": [ - "import optuna\n", - "from sklearn.metrics import accuracy_score, f1_score\n", - "import torch\n", - "import torch.nn as nn\n", - "import torch.optim as optim\n", - "from resnet_like import ResNetLike # ResNetLike 모델 불러오기\n", - "\n", - "# GPU 설정\n", - "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", - "\n", - "# Optuna 하이퍼파라미터 최적화 함수 정의\n", - "def objective(trial):\n", - " # 하이퍼파라미터 탐색 공간 정의\n", - " d_main = trial.suggest_categorical(\"d_main\", [64, 128, 192, 256])\n", - " d_hidden = trial.suggest_categorical(\"d_hidden\", [32, 64, 128])\n", - " n_blocks = trial.suggest_int(\"n_blocks\", 3, 8) # ResNet 블록 수\n", - " dropout_first = trial.suggest_float(\"dropout_first\", 0.1, 0.5) # 첫 번째 Dropout\n", - " dropout_second = trial.suggest_float(\"dropout_second\", 0.0, 0.3) # 두 번째 Dropout\n", - " lr = trial.suggest_float(\"lr\", 1e-4, 1e-2, log=True) # 학습률\n", - " weight_decay = trial.suggest_float(\"weight_decay\", 1e-6, 1e-3, log=True) # L2 정규화\n", - "\n", - " # 연속형 변수 + 범주형 변수 개수 반영하여 모델 입력 크기 설정\n", - " input_dim = len(numerical_cols) + len(categorical_cols)\n", - "\n", - " # 모델 초기화 및 GPU로 이동\n", - " model = ResNetLike(\n", - " input_dim=input_dim,\n", - " d_main=d_main, \n", - " d_hidden=d_hidden, \n", - " n_blocks=n_blocks, \n", - " dropout_first=dropout_first, \n", - " dropout_second=dropout_second\n", - " ).to(device)\n", - "\n", - " # 손실 함수 및 옵티마이저 설정\n", - " criterion = nn.BCELoss()\n", - " optimizer = optim.AdamW(model.parameters(), lr=lr, weight_decay=weight_decay)\n", - "\n", - " # 학습 루프 (Epoch 설정)\n", - " epochs = 8 # 빠른 탐색을 위해 Epoch을 줄임\n", - " for epoch in range(epochs):\n", - " model.train()\n", - " for x_num_batch, x_cat_batch, y_batch in train_loader: \n", - " x_num_batch, x_cat_batch, y_batch = (\n", - " x_num_batch.to(device),\n", - " x_cat_batch.to(device),\n", - " y_batch.to(device)\n", - " )\n", - "\n", - " optimizer.zero_grad()\n", - " y_pred = model(x_num_batch, x_cat_batch) # 두 개의 입력을 개별적으로 전달\n", - " loss = criterion(y_pred, y_batch.unsqueeze(1))\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - " # Validation 평가\n", - " model.eval()\n", - " y_pred_val, y_true_val = [], []\n", - " with torch.no_grad():\n", - " for x_num_batch, x_cat_batch, y_batch in val_loader: \n", - " x_num_batch, x_cat_batch, y_batch = (\n", - " x_num_batch.to(device),\n", - " x_cat_batch.to(device),\n", - " y_batch.to(device)\n", - " )\n", - "\n", - " output = model(x_num_batch, x_cat_batch) # 두 개의 입력을 개별적으로 전달\n", - " y_pred_val.extend((output.cpu().numpy() > 0.5).astype(int))\n", - " y_true_val.extend(y_batch.cpu().numpy())\n", - "\n", - " # f1-score 계산 (최적화 지표)\n", - " val_score = f1_score(y_true_val, y_pred_val)\n", - " return val_score\n", - "\n", - "# Optuna 실행 (최적화 시작)\n", - "sampler = optuna.samplers.TPESampler(seed=seed)\n", - "study = optuna.create_study(direction=\"maximize\", sampler=sampler) # f1-score를 최대화\n", - "study.optimize(objective, n_trials=50) # 50번의 탐색 시도\n", - "\n", - "# 최적의 하이퍼파라미터 출력\n", - "best_params = study.best_trial.params\n", - "print(\"Best Hyperparameters:\", best_params)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "##### 모델 학습 루프" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 1/50, Train Loss: 0.5364, Validation Loss: 0.6911, Validation F1-Score: 0.6280\n", - "Epoch 2/50, Train Loss: 0.5197, Validation Loss: 0.7168, Validation F1-Score: 0.5862\n", - "Epoch 3/50, Train Loss: 0.5181, Validation Loss: 0.7134, Validation F1-Score: 0.6050\n", - "Epoch 4/50, Train Loss: 0.5160, Validation Loss: 0.6864, Validation F1-Score: 0.6337\n", - "Epoch 5/50, Train Loss: 0.5116, Validation Loss: 0.6981, Validation F1-Score: 0.6288\n", - "Epoch 6/50, Train Loss: 0.5077, Validation Loss: 0.6961, Validation F1-Score: 0.6237\n", - "Epoch 7/50, Train Loss: 0.5122, Validation Loss: 0.6937, Validation F1-Score: 0.6505\n", - "Epoch 8/50, Train Loss: 0.5121, Validation Loss: 0.6881, Validation F1-Score: 0.5837\n", - "Epoch 9/50, Train Loss: 0.5204, Validation Loss: 0.6984, Validation F1-Score: 0.5593\n", - "Epoch 10/50, Train Loss: 0.5312, Validation Loss: 0.8063, Validation F1-Score: 0.4385\n", - "Epoch 11/50, Train Loss: 0.5422, Validation Loss: 0.6892, Validation F1-Score: 0.6088\n", - "Epoch 12/50, Train Loss: 0.5183, Validation Loss: 0.6928, Validation F1-Score: 0.5327\n", - "Epoch 13/50, Train Loss: 0.5440, Validation Loss: 0.6985, Validation F1-Score: 0.4962\n", - "Epoch 14/50, Train Loss: 0.5612, Validation Loss: 0.7790, Validation F1-Score: 0.4641\n", - "Epoch 15/50, Train Loss: 0.5612, Validation Loss: 0.7481, Validation F1-Score: 0.5057\n", - "Early stopping at epoch 15\n" - ] - } - ], - "source": [ - "import os\n", - "import torch\n", - "import torch.nn as nn\n", - "import torch.optim as optim\n", - "from sklearn.metrics import f1_score\n", - "\n", - "# 최적의 하이퍼파라미터 적용\n", - "best_model = ResNetLike(\n", - " input_dim=len(numerical_cols) + len(categorical_cols), # 입력 차원\n", - " d_main=best_params[\"d_main\"],\n", - " d_hidden=best_params[\"d_hidden\"],\n", - " n_blocks=best_params[\"n_blocks\"],\n", - " dropout_first=best_params[\"dropout_first\"],\n", - " dropout_second=best_params[\"dropout_second\"]\n", - ").to(device)\n", - "\n", - "# 손실 함수 및 옵티마이저 설정\n", - "criterion = nn.BCEWithLogitsLoss()\n", - "optimizer = optim.AdamW(best_model.parameters(), lr=best_params[\"lr\"], weight_decay=best_params[\"weight_decay\"])\n", - "\n", - "# Early Stopping 설정\n", - "best_val_f1 = -float('inf') # F1-Score 최대화 목표\n", - "patience = 8 # F1-Score가 개선되지 않는 Epoch 수\n", - "counter = 0\n", - "\n", - "# 모델 저장 폴더 생성\n", - "os.makedirs(\"./models\", exist_ok=True)\n", - "\n", - "# 모델 학습 루프\n", - "epochs = 50 # 최적의 하이퍼파라미터로 충분한 학습\n", - "for epoch in range(epochs):\n", - " best_model.train()\n", - " train_loss = 0\n", - "\n", - " # Training Loop (x_batch 두 개를 나눠서 받음)\n", - " for x_num_batch, x_cat_batch, y_batch in train_loader:\n", - " x_num_batch, x_cat_batch, y_batch = (\n", - " x_num_batch.to(device),\n", - " x_cat_batch.to(device),\n", - " y_batch.to(device)\n", - " )\n", - "\n", - " optimizer.zero_grad()\n", - " y_pred = best_model(x_num_batch, x_cat_batch) # 두 개의 입력을 개별적으로 전달\n", - " loss = criterion(y_pred, y_batch.unsqueeze(1)) # y_batch 차원 조정\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - " train_loss += loss.item()\n", - "\n", - " # Validation Loop\n", - " best_model.eval()\n", - " val_loss = 0\n", - " y_pred_val, y_true_val = [], []\n", - " with torch.no_grad():\n", - " for x_num_batch, x_cat_batch, y_batch in val_loader:\n", - " x_num_batch, x_cat_batch, y_batch = (\n", - " x_num_batch.to(device),\n", - " x_cat_batch.to(device),\n", - " y_batch.to(device)\n", - " )\n", - "\n", - " output = best_model(x_num_batch, x_cat_batch)\n", - " loss = criterion(output, y_batch.unsqueeze(1)) # 차원 맞춤\n", - " val_loss += loss.item()\n", - "\n", - " y_pred_val.extend((torch.sigmoid(output).cpu().numpy() > 0.5).astype(int))\n", - " y_true_val.extend(y_batch.cpu().numpy())\n", - "\n", - " # 평균 손실 계산\n", - " train_loss /= len(train_loader)\n", - " val_loss /= len(val_loader)\n", - "\n", - " # f1-score 계산\n", - " val_f1 = f1_score(y_true_val, y_pred_val)\n", - "\n", - " print(\n", - " f\"Epoch {epoch+1}/{epochs}, Train Loss: {train_loss:.4f}, \"\n", - " f\"Validation Loss: {val_loss:.4f}, Validation F1-Score: {val_f1:.4f}\"\n", - " )\n", - "\n", - " # Early Stopping 체크\n", - " if val_f1 > best_val_f1:\n", - " best_val_f1 = val_f1\n", - " counter = 0\n", - " torch.save(best_model.state_dict(), \"./models/best_resnet_model.pth\") # 모델 저장\n", - " else:\n", - " counter += 1\n", - " if counter >= patience:\n", - " print(f\"Early stopping at epoch {epoch+1}\")\n", - " break # 성능 개선이 없으면 학습 조기 종료" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "##### 성능 평가" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Test Accuracy: 0.9118721461187215\n", - "Test AUC: 0.867653555709893\n", - "Test F1-Score: 0.5113924050632911\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/tmp/ipykernel_1311698/1784587916.py:5: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n", - " best_model.load_state_dict(torch.load(\"./models/best_resnet_model.pth\")) # 모델 저장 경로 수정\n" - ] - } - ], - "source": [ - "import torch\n", - "from sklearn.metrics import accuracy_score, roc_auc_score, f1_score\n", - "\n", - "# 저장된 최적 모델 불러오기\n", - "best_model.load_state_dict(torch.load(\"./models/best_resnet_model.pth\")) # 모델 저장 경로 수정\n", - "best_model.to(device)\n", - "best_model.eval()\n", - "\n", - "# Test 데이터 평가\n", - "y_pred_test, y_true_test = [], []\n", - "with torch.no_grad():\n", - " for x_num_batch, x_cat_batch, y_batch in test_loader: # 3개 데이터 받기\n", - " x_num_batch, x_cat_batch, y_batch = (\n", - " x_num_batch.to(device),\n", - " x_cat_batch.to(device),\n", - " y_batch.to(device)\n", - " )\n", - "\n", - " output = best_model(x_num_batch, x_cat_batch) # 두 개의 입력 전달\n", - " y_pred_test.extend((torch.sigmoid(output).cpu().numpy() > 0.5).astype(int))\n", - " y_true_test.extend(y_batch.cpu().numpy())\n", - "\n", - "# Test 성능 출력\n", - "print(\"Test Accuracy:\", accuracy_score(y_true_test, y_pred_test))\n", - "print(\"Test AUC:\", roc_auc_score(y_true_test, y_pred_test))\n", - "print(\"Test F1-Score:\", f1_score(y_true_test, y_pred_test))\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## DeepGBM" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "import numpy as np\n", - "from sklearn.preprocessing import StandardScaler, LabelEncoder\n", - "import torch\n", - "from torch.utils.data import DataLoader, TensorDataset\n", - "import random\n", - "\n", - "# 시드 고정\n", - "seed = 42\n", - "random.seed(seed)\n", - "np.random.seed(seed)\n", - "torch.manual_seed(seed)\n", - "torch.cuda.manual_seed_all(seed)\n", - "\n", - "# 데이터셋 불러오기\n", - "seoul = pd.read_csv('../data/data_for_modeling/seoul_train.csv', index_col=0)\n", - "seoul_train = pd.read_csv('../data/data_oversampled/ctgan10000/ctgan10000_3_seoul.csv')\n", - "seoul_val = seoul.loc[seoul['year'].isin([2018]),:]\n", - "seoul_test = pd.read_csv('../data/data_for_modeling/seoul_test.csv')\n", - "drop_col = ['binary_class','multi_class','visi','year']\n", - "target_col = 'binary_class'\n", - "\n", - "# 모든 데이터셋의 컬럼을 같은 순서로 정렬\n", - "common_columns = seoul_train.columns.to_list() # 공통 변수 이름 \n", - "train_data = seoul_train[common_columns]\n", - "val_data = seoul_val[common_columns]\n", - "test_data = seoul_test[common_columns]\n", - "\n", - "# 설명변수와 반응변수 분리\n", - "X_train = train_data.drop(columns=drop_col)\n", - "y_train = train_data[target_col]\n", - "X_val = val_data.drop(columns=drop_col)\n", - "y_val = val_data[target_col]\n", - "X_test = test_data.drop(columns=drop_col)\n", - "y_test = test_data[target_col]\n", - "\n", - "# 범주형 변수와 연속형 변수 분리\n", - "categorical_cols = X_train.select_dtypes(include=['object', 'category', 'int64']).columns\n", - "numerical_cols = X_train.select_dtypes(include=['float64']).columns\n", - "\n", - "# 범주형 변수 인코딩\n", - "label_encoders = {}\n", - "for col in categorical_cols:\n", - " le = LabelEncoder()\n", - " le.fit(X_train[col]) # Train 데이터 기준으로 학습\n", - " label_encoders[col] = le\n", - "\n", - "# 각 데이터셋에 변환 적용\n", - "for col in categorical_cols:\n", - " X_train[col] = label_encoders[col].transform(X_train[col])\n", - " X_val[col] = label_encoders[col].transform(X_val[col])\n", - " X_test[col] = label_encoders[col].transform(X_test[col])\n", - "\n", - "# 연속형 변수 스케일링\n", - "scaler = StandardScaler()\n", - "scaler.fit(X_train[numerical_cols]) # Train 데이터만 사용\n", - "\n", - "# 각 데이터셋에 변환 적용\n", - "X_train[numerical_cols] = scaler.transform(X_train[numerical_cols])\n", - "X_val[numerical_cols] = scaler.transform(X_val[numerical_cols])\n", - "X_test[numerical_cols] = scaler.transform(X_test[numerical_cols]) # Test 데이터는 변환만\n", - "\n", - "# 연속형 변수와 범주형 변수 분리\n", - "X_train_num = torch.tensor(X_train[numerical_cols].values, dtype=torch.float32)\n", - "X_train_cat = torch.tensor(X_train[categorical_cols].values, dtype=torch.long)\n", - "\n", - "X_val_num = torch.tensor(X_val[numerical_cols].values, dtype=torch.float32)\n", - "X_val_cat = torch.tensor(X_val[categorical_cols].values, dtype=torch.long)\n", - "\n", - "X_test_num = torch.tensor(X_test[numerical_cols].values, dtype=torch.float32)\n", - "X_test_cat = torch.tensor(X_test[categorical_cols].values, dtype=torch.long)\n", - "\n", - "y_train_tensor = torch.tensor(y_train.values, dtype=torch.float32) # 이진 분류\n", - "y_val_tensor = torch.tensor(y_val.values, dtype=torch.float32)\n", - "y_test_tensor = torch.tensor(y_test.values, dtype=torch.float32)\n", - "\n", - "# TensorDataset 생성\n", - "train_dataset = TensorDataset(X_train_num, X_train_cat, y_train_tensor)\n", - "val_dataset = TensorDataset(X_val_num, X_val_cat, y_val_tensor)\n", - "test_dataset = TensorDataset(X_test_num, X_test_cat, y_test_tensor)\n", - "\n", - "# DataLoader 생성\n", - "train_loader = DataLoader(train_dataset, batch_size=64, shuffle=True, generator=torch.Generator().manual_seed(seed))\n", - "val_loader = DataLoader(val_dataset, batch_size=64, shuffle=False)\n", - "test_loader = DataLoader(test_dataset, batch_size=64, shuffle=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n", - "[I 2025-02-02 16:37:59,547] A new study created in memory with name: no-name-d43cf13c-34f9-4811-955c-28691341b8a5\n", - "[I 2025-02-02 16:38:12,621] Trial 0 finished with value: 0.6808309726156752 and parameters: {'d_main': 128, 'd_hidden': 32, 'n_blocks': 8, 'dropout': 0.34044600469728353, 'lr': 0.0005105903209394755, 'weight_decay': 1.0994335574766187e-05}. Best is trial 0 with value: 0.6808309726156752.\n", - "[I 2025-02-02 16:38:22,577] Trial 1 finished with value: 0.6743371685842922 and parameters: {'d_main': 64, 'd_hidden': 128, 'n_blocks': 5, 'dropout': 0.21649165607921678, 'lr': 0.00040912205744437856, 'weight_decay': 1.9010245319870364e-05}. Best is trial 0 with value: 0.6808309726156752.\n", - "[I 2025-02-02 16:38:30,407] Trial 2 finished with value: 0.6716981132075471 and parameters: {'d_main': 256, 'd_hidden': 128, 'n_blocks': 3, 'dropout': 0.34301794076057535, 'lr': 0.00014808945119975197, 'weight_decay': 1.3492834268013232e-05}. Best is trial 0 with value: 0.6808309726156752.\n", - "[I 2025-02-02 16:38:37,637] Trial 3 finished with value: 0.6577981651376147 and parameters: {'d_main': 128, 'd_hidden': 64, 'n_blocks': 3, 'dropout': 0.29807076404450805, 'lr': 0.00010824018381500966, 'weight_decay': 0.000658628931758311}. Best is trial 0 with value: 0.6808309726156752.\n", - "[I 2025-02-02 16:38:48,942] Trial 4 finished with value: 0.6935050993022007 and parameters: {'d_main': 128, 'd_hidden': 128, 'n_blocks': 7, 'dropout': 0.4757995766256756, 'lr': 0.0007849235338159361, 'weight_decay': 0.0001569639638866114}. Best is trial 4 with value: 0.6935050993022007.\n", - "[I 2025-02-02 16:39:00,131] Trial 5 finished with value: 0.664714494875549 and parameters: {'d_main': 64, 'd_hidden': 64, 'n_blocks': 7, 'dropout': 0.2427013306774357, 'lr': 0.0001909565280104537, 'weight_decay': 0.00012172847081122418}. Best is trial 4 with value: 0.6935050993022007.\n", - "[I 2025-02-02 16:39:11,713] Trial 6 finished with value: 0.6680738786279683 and parameters: {'d_main': 256, 'd_hidden': 32, 'n_blocks': 7, 'dropout': 0.38274293753904687, 'lr': 0.0005358055009231865, 'weight_decay': 0.000348771262454593}. Best is trial 4 with value: 0.6935050993022007.\n", - "[I 2025-02-02 16:39:20,737] Trial 7 finished with value: 0.6916183447548762 and parameters: {'d_main': 256, 'd_hidden': 32, 'n_blocks': 4, 'dropout': 0.23007332881069884, 'lr': 0.0005365450324352025, 'weight_decay': 0.00018841476921545086}. Best is trial 4 with value: 0.6935050993022007.\n", - "[I 2025-02-02 16:39:30,545] Trial 8 finished with value: 0.6589769307923772 and parameters: {'d_main': 64, 'd_hidden': 128, 'n_blocks': 5, 'dropout': 0.3090931317527976, 'lr': 0.00026763383567660797, 'weight_decay': 1.1241862095793047e-05}. Best is trial 4 with value: 0.6935050993022007.\n", - "[I 2025-02-02 16:39:40,338] Trial 9 finished with value: 0.6521958606764261 and parameters: {'d_main': 192, 'd_hidden': 64, 'n_blocks': 5, 'dropout': 0.4022204554172195, 'lr': 0.00016935505549297937, 'weight_decay': 1.425475704402744e-05}. Best is trial 4 with value: 0.6935050993022007.\n", - "[I 2025-02-02 16:39:51,651] Trial 10 finished with value: 0.7065677966101694 and parameters: {'d_main': 128, 'd_hidden': 128, 'n_blocks': 7, 'dropout': 0.4820559747905796, 'lr': 0.0009630865624170769, 'weight_decay': 4.8723119722871314e-05}. Best is trial 10 with value: 0.7065677966101694.\n", - "[I 2025-02-02 16:40:03,284] Trial 11 finished with value: 0.6976274608783443 and parameters: {'d_main': 128, 'd_hidden': 128, 'n_blocks': 7, 'dropout': 0.49897739440085004, 'lr': 0.0009903735540622498, 'weight_decay': 5.1544513708209705e-05}. Best is trial 10 with value: 0.7065677966101694.\n", - "[I 2025-02-02 16:40:15,632] Trial 12 finished with value: 0.676424464192368 and parameters: {'d_main': 128, 'd_hidden': 128, 'n_blocks': 8, 'dropout': 0.48816208673575634, 'lr': 0.0009889782969316999, 'weight_decay': 4.3691424349158736e-05}. Best is trial 10 with value: 0.7065677966101694.\n", - "[I 2025-02-02 16:40:26,111] Trial 13 finished with value: 0.6580773042616452 and parameters: {'d_main': 128, 'd_hidden': 128, 'n_blocks': 6, 'dropout': 0.11054723376232306, 'lr': 0.000993563841007881, 'weight_decay': 5.090873160060654e-05}. Best is trial 10 with value: 0.7065677966101694.\n", - "[I 2025-02-02 16:40:36,749] Trial 14 finished with value: 0.6780219780219781 and parameters: {'d_main': 192, 'd_hidden': 128, 'n_blocks': 6, 'dropout': 0.4345281899318145, 'lr': 0.000703066044128388, 'weight_decay': 5.039872099693328e-05}. Best is trial 10 with value: 0.7065677966101694.\n", - "[I 2025-02-02 16:40:48,268] Trial 15 finished with value: 0.5949908368967624 and parameters: {'d_main': 128, 'd_hidden': 128, 'n_blocks': 7, 'dropout': 0.4985994407528791, 'lr': 0.0003141126260425451, 'weight_decay': 2.805094210329636e-05}. Best is trial 10 with value: 0.7065677966101694.\n", - "[I 2025-02-02 16:41:00,422] Trial 16 finished with value: 0.6973749380881624 and parameters: {'d_main': 128, 'd_hidden': 128, 'n_blocks': 8, 'dropout': 0.4405116343614309, 'lr': 0.0007296967058494006, 'weight_decay': 7.722238352424441e-05}. Best is trial 10 with value: 0.7065677966101694.\n", - "[I 2025-02-02 16:41:10,955] Trial 17 finished with value: 0.6833073322932918 and parameters: {'d_main': 128, 'd_hidden': 128, 'n_blocks': 6, 'dropout': 0.43765610083770745, 'lr': 0.0004012608786880134, 'weight_decay': 2.9186980345955912e-05}. Best is trial 10 with value: 0.7065677966101694.\n", - "[I 2025-02-02 16:41:22,403] Trial 18 finished with value: 0.6659751037344398 and parameters: {'d_main': 192, 'd_hidden': 64, 'n_blocks': 7, 'dropout': 0.3905098603146417, 'lr': 0.000794490885070064, 'weight_decay': 8.035935542498882e-05}. Best is trial 10 with value: 0.7065677966101694.\n", - "[I 2025-02-02 16:41:34,547] Trial 19 finished with value: 0.6765015806111696 and parameters: {'d_main': 128, 'd_hidden': 32, 'n_blocks': 8, 'dropout': 0.1618897082112804, 'lr': 0.0006218282540300562, 'weight_decay': 0.0003128762212736291}. Best is trial 10 with value: 0.7065677966101694.\n", - "[I 2025-02-02 16:41:45,105] Trial 20 finished with value: 0.6280295047418335 and parameters: {'d_main': 128, 'd_hidden': 128, 'n_blocks': 6, 'dropout': 0.4621158946002487, 'lr': 0.0004064042869034392, 'weight_decay': 2.7603261798192472e-05}. Best is trial 10 with value: 0.7065677966101694.\n", - "[I 2025-02-02 16:41:57,182] Trial 21 finished with value: 0.6953405017921147 and parameters: {'d_main': 128, 'd_hidden': 128, 'n_blocks': 8, 'dropout': 0.4255974822311327, 'lr': 0.0009993172418888458, 'weight_decay': 8.324553715242003e-05}. Best is trial 10 with value: 0.7065677966101694.\n", - "[I 2025-02-02 16:42:09,218] Trial 22 finished with value: 0.6933911159263272 and parameters: {'d_main': 128, 'd_hidden': 128, 'n_blocks': 8, 'dropout': 0.4553663047983129, 'lr': 0.0007358139178301945, 'weight_decay': 6.406934888466268e-05}. Best is trial 10 with value: 0.7065677966101694.\n", - "[I 2025-02-02 16:42:20,691] Trial 23 finished with value: 0.7056345444971037 and parameters: {'d_main': 128, 'd_hidden': 128, 'n_blocks': 7, 'dropout': 0.4990207539511106, 'lr': 0.0008344830972076081, 'weight_decay': 0.0001172549162060698}. Best is trial 10 with value: 0.7065677966101694.\n", - "[I 2025-02-02 16:42:32,369] Trial 24 finished with value: 0.677765843179377 and parameters: {'d_main': 128, 'd_hidden': 128, 'n_blocks': 7, 'dropout': 0.4963004424730474, 'lr': 0.0008635172258202799, 'weight_decay': 0.00011965822367646233}. Best is trial 10 with value: 0.7065677966101694.\n", - "[I 2025-02-02 16:42:42,922] Trial 25 finished with value: 0.6868686868686869 and parameters: {'d_main': 128, 'd_hidden': 128, 'n_blocks': 6, 'dropout': 0.4096404328164892, 'lr': 0.0006168881273294814, 'weight_decay': 0.00027785748995414627}. Best is trial 10 with value: 0.7065677966101694.\n", - "[I 2025-02-02 16:42:54,551] Trial 26 finished with value: 0.6903846153846154 and parameters: {'d_main': 192, 'd_hidden': 128, 'n_blocks': 7, 'dropout': 0.37583885242041665, 'lr': 0.0008818680724435528, 'weight_decay': 3.8281345408950636e-05}. Best is trial 10 with value: 0.7065677966101694.\n", - "[I 2025-02-02 16:43:05,183] Trial 27 finished with value: 0.6951156812339332 and parameters: {'d_main': 256, 'd_hidden': 128, 'n_blocks': 6, 'dropout': 0.4733076998075283, 'lr': 0.0006434339896205424, 'weight_decay': 0.00020042363494450496}. Best is trial 10 with value: 0.7065677966101694.\n", - "[I 2025-02-02 16:43:16,533] Trial 28 finished with value: 0.6837294332723949 and parameters: {'d_main': 64, 'd_hidden': 32, 'n_blocks': 7, 'dropout': 0.45810275836541026, 'lr': 0.00022850161272551156, 'weight_decay': 0.00010873128761655598}. Best is trial 10 with value: 0.7065677966101694.\n", - "[I 2025-02-02 16:43:25,562] Trial 29 finished with value: 0.6750392464678179 and parameters: {'d_main': 128, 'd_hidden': 64, 'n_blocks': 4, 'dropout': 0.33103658661547813, 'lr': 0.00046154770212266305, 'weight_decay': 0.000644143910687572}. Best is trial 10 with value: 0.7065677966101694.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Best Hyperparameters: {'d_main': 128, 'd_hidden': 128, 'n_blocks': 7, 'dropout': 0.4820559747905796, 'lr': 0.0009630865624170769, 'weight_decay': 4.8723119722871314e-05}\n" - ] - } - ], - "source": [ - "import optuna\n", - "from sklearn.metrics import f1_score\n", - "import torch\n", - "import torch.nn as nn\n", - "import torch.optim as optim\n", - "from deepgbm import DeepGBM # .py 파일에서 DeepGBM 불러오기\n", - "\n", - "# GPU 설정\n", - "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", - "\n", - "# 하이퍼파라미터 최적화 함수 정의\n", - "def objective(trial):\n", - " d_main = trial.suggest_categorical(\"d_main\", [64, 128, 192, 256])\n", - " d_hidden = trial.suggest_categorical(\"d_hidden\", [32, 64, 128])\n", - " n_blocks = trial.suggest_int(\"n_blocks\", 3, 8) # ResNet 블록 개수\n", - " dropout = trial.suggest_float(\"dropout\", 0.1, 0.5) # Dropout 비율\n", - " lr = trial.suggest_float(\"lr\", 1e-4, 1e-3, log=True) # 학습률\n", - " weight_decay = trial.suggest_float(\"weight_decay\", 1e-5, 1e-3, log=True) # 정규화\n", - "\n", - " # DeepGBM 모델 초기화 (x_num, x_cat을 따로 받는 구조)\n", - " model = DeepGBM(\n", - " num_features=len(numerical_cols),\n", - " cat_features=[len(X_train[col].unique()) for col in categorical_cols],\n", - " d_main=d_main,\n", - " d_hidden=d_hidden,\n", - " n_blocks=n_blocks,\n", - " dropout=dropout\n", - " ).to(device)\n", - "\n", - " # 손실 함수 및 옵티마이저 설정\n", - " criterion = nn.BCELoss()\n", - " optimizer = optim.AdamW(model.parameters(), lr=lr, weight_decay=weight_decay)\n", - "\n", - " # 학습 루프\n", - " epochs = 8 # 빠른 탐색을 위해 Epoch 수 조정\n", - " for epoch in range(epochs):\n", - " model.train()\n", - " for x_num_batch, x_cat_batch, y_batch in train_loader:\n", - " x_num_batch, x_cat_batch, y_batch = (\n", - " x_num_batch.to(device),\n", - " x_cat_batch.to(device),\n", - " y_batch.to(device)\n", - " )\n", - "\n", - " optimizer.zero_grad()\n", - " y_pred = model(x_num_batch, x_cat_batch) # 두 개의 입력 사용\n", - " loss = criterion(y_pred, y_batch.unsqueeze(1))\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - " # Validation 평가\n", - " model.eval()\n", - " y_pred_val, y_true_val = [], []\n", - " with torch.no_grad():\n", - " for x_num_batch, x_cat_batch, y_batch in val_loader:\n", - " x_num_batch, x_cat_batch, y_batch = (\n", - " x_num_batch.to(device),\n", - " x_cat_batch.to(device),\n", - " y_batch.to(device)\n", - " )\n", - " output = model(x_num_batch, x_cat_batch)\n", - " y_pred_val.extend((output.cpu().numpy() > 0.5).astype(int))\n", - " y_true_val.extend(y_batch.cpu().numpy())\n", - "\n", - " # f1-score 계산\n", - " val_score = f1_score(y_true_val, y_pred_val)\n", - " return val_score\n", - "\n", - "# Optuna 최적화 실행\n", - "sampler = optuna.samplers.TPESampler(seed=seed)\n", - "study = optuna.create_study(direction=\"maximize\", sampler=sampler)\n", - "study.optimize(objective, n_trials=50) # 50번의 탐색 시도\n", - "\n", - "# 최적의 하이퍼파라미터 출력\n", - "best_params = study.best_trial.params\n", - "print(\"Best Hyperparameters:\", best_params)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "##### 모델 학습 루프" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 1/50, Train Loss: 0.2099, Validation Loss: 0.1996, Validation F1-Score: 0.6706\n", - "Epoch 2/50, Train Loss: 0.1449, Validation Loss: 0.2015, Validation F1-Score: 0.6869\n", - "Epoch 3/50, Train Loss: 0.1205, Validation Loss: 0.1908, Validation F1-Score: 0.6717\n", - "Epoch 4/50, Train Loss: 0.1006, Validation Loss: 0.1875, Validation F1-Score: 0.6871\n", - "Epoch 5/50, Train Loss: 0.0901, Validation Loss: 0.1842, Validation F1-Score: 0.6961\n", - "Epoch 6/50, Train Loss: 0.0845, Validation Loss: 0.1994, Validation F1-Score: 0.6975\n", - "Epoch 7/50, Train Loss: 0.0778, Validation Loss: 0.1846, Validation F1-Score: 0.6903\n", - "Epoch 8/50, Train Loss: 0.0747, Validation Loss: 0.2222, Validation F1-Score: 0.6944\n", - "Epoch 9/50, Train Loss: 0.0697, Validation Loss: 0.2332, Validation F1-Score: 0.6693\n", - "Epoch 10/50, Train Loss: 0.0680, Validation Loss: 0.2238, Validation F1-Score: 0.6992\n", - "Epoch 11/50, Train Loss: 0.0630, Validation Loss: 0.2649, Validation F1-Score: 0.6868\n", - "Epoch 12/50, Train Loss: 0.0613, Validation Loss: 0.2403, Validation F1-Score: 0.6968\n", - "Epoch 13/50, Train Loss: 0.0565, Validation Loss: 0.2490, Validation F1-Score: 0.6834\n", - "Epoch 14/50, Train Loss: 0.0566, Validation Loss: 0.2333, Validation F1-Score: 0.6932\n", - "Epoch 15/50, Train Loss: 0.0530, Validation Loss: 0.2409, Validation F1-Score: 0.6930\n", - "Epoch 16/50, Train Loss: 0.0506, Validation Loss: 0.2452, Validation F1-Score: 0.6746\n", - "Epoch 17/50, Train Loss: 0.0487, Validation Loss: 0.2174, Validation F1-Score: 0.6831\n", - "Epoch 18/50, Train Loss: 0.0467, Validation Loss: 0.2583, Validation F1-Score: 0.7254\n", - "Epoch 19/50, Train Loss: 0.0452, Validation Loss: 0.2625, Validation F1-Score: 0.7012\n", - "Epoch 20/50, Train Loss: 0.0452, Validation Loss: 0.2315, Validation F1-Score: 0.7137\n", - "Epoch 21/50, Train Loss: 0.0428, Validation Loss: 0.2830, Validation F1-Score: 0.6999\n", - "Epoch 22/50, Train Loss: 0.0422, Validation Loss: 0.2739, Validation F1-Score: 0.6999\n", - "Epoch 23/50, Train Loss: 0.0401, Validation Loss: 0.2810, Validation F1-Score: 0.7142\n", - "Epoch 24/50, Train Loss: 0.0391, Validation Loss: 0.2781, Validation F1-Score: 0.7056\n", - "Epoch 25/50, Train Loss: 0.0373, Validation Loss: 0.2804, Validation F1-Score: 0.6988\n", - "Epoch 26/50, Train Loss: 0.0373, Validation Loss: 0.2887, Validation F1-Score: 0.7086\n", - "Early stopping at epoch 26\n" - ] - } - ], - "source": [ - "# 최적 하이퍼파라미터 적용\n", - "best_model = DeepGBM(\n", - " num_features=len(numerical_cols),\n", - " cat_features=[len(X_train[col].unique()) for col in categorical_cols],\n", - " d_main=best_params[\"d_main\"],\n", - " d_hidden=best_params[\"d_hidden\"],\n", - " n_blocks=best_params[\"n_blocks\"],\n", - " dropout=best_params[\"dropout\"]\n", - ").to(device)\n", - "\n", - "# 손실 함수 및 옵티마이저 설정\n", - "criterion = nn.BCELoss()\n", - "optimizer = optim.AdamW(best_model.parameters(), lr=best_params[\"lr\"], weight_decay=best_params[\"weight_decay\"])\n", - "\n", - "# Early Stopping 설정\n", - "best_val_f1 = -float(\"inf\") # F1-Score 최대화 목표\n", - "patience = 8 # F1-Score가 개선되지 않는 Epoch 수\n", - "counter = 0\n", - "\n", - "# 학습 루프\n", - "epochs = 50 # 충분한 학습\n", - "for epoch in range(epochs):\n", - " best_model.train()\n", - " train_loss = 0\n", - " for x_num_batch, x_cat_batch, y_batch in train_loader:\n", - " x_num_batch, x_cat_batch, y_batch = (\n", - " x_num_batch.to(device),\n", - " x_cat_batch.to(device),\n", - " y_batch.to(device),\n", - " )\n", - " optimizer.zero_grad()\n", - " y_pred = best_model(x_num_batch, x_cat_batch) # 두 개의 입력 사용\n", - " loss = criterion(y_pred, y_batch.unsqueeze(1))\n", - " loss.backward()\n", - " optimizer.step()\n", - " train_loss += loss.item()\n", - "\n", - " # Validation 평가\n", - " best_model.eval()\n", - " val_loss = 0\n", - " y_true_val, y_pred_val = [], []\n", - " with torch.no_grad():\n", - " for x_num_batch, x_cat_batch, y_batch in val_loader:\n", - " x_num_batch, x_cat_batch, y_batch = (\n", - " x_num_batch.to(device),\n", - " x_cat_batch.to(device),\n", - " y_batch.to(device),\n", - " )\n", - " y_pred = best_model(x_num_batch, x_cat_batch)\n", - " val_loss += criterion(y_pred, y_batch.unsqueeze(1)).item()\n", - " y_true_val.extend(y_batch.cpu().numpy())\n", - " y_pred_val.extend((y_pred.cpu().numpy() > 0.5).astype(int))\n", - "\n", - " # 평균 Loss 계산\n", - " train_loss /= len(train_loader)\n", - " val_loss /= len(val_loader)\n", - "\n", - " # F1-Score 계산\n", - " val_f1 = f1_score(y_true_val, y_pred_val)\n", - "\n", - " print(\n", - " f\"Epoch {epoch+1}/{epochs}, Train Loss: {train_loss:.4f}, \"\n", - " f\"Validation Loss: {val_loss:.4f}, Validation F1-Score: {val_f1:.4f}\"\n", - " )\n", - "\n", - " # Early Stopping 체크\n", - " if val_f1 > best_val_f1:\n", - " best_val_f1 = val_f1\n", - " counter = 0\n", - " torch.save(best_model.state_dict(), \"best_deepgbm_model.pth\") # 최적 모델 저장\n", - " else:\n", - " counter += 1\n", - " if counter >= patience:\n", - " print(f\"Early stopping at epoch {epoch+1}\")\n", - " break" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "##### 성능 평가" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Test Accuracy: 0.8901826484018265\n", - "Test AUC: 0.8314176113850111\n", - "Test F1-Score: 0.440046565774156\n" - ] - } - ], - "source": [ - "from sklearn.metrics import accuracy_score, roc_auc_score, f1_score\n", - "\n", - "# 저장된 최적 모델 불러오기\n", - "best_model.load_state_dict(torch.load(\"best_deepgbm_model.pth\", weights_only=True))\n", - "best_model.to(device)\n", - "best_model.eval()\n", - "\n", - "# Test 데이터 평가\n", - "y_pred_test, y_true_test = [], []\n", - "with torch.no_grad():\n", - " for x_num_batch, x_cat_batch, y_batch in test_loader:\n", - " x_num_batch, x_cat_batch, y_batch = (\n", - " x_num_batch.to(device),\n", - " x_cat_batch.to(device),\n", - " y_batch.to(device),\n", - " )\n", - " output = best_model(x_num_batch, x_cat_batch)\n", - " y_pred_test.extend((output.cpu().numpy() > 0.5).astype(int))\n", - " y_true_test.extend(y_batch.cpu().numpy())\n", - "\n", - "# Test 성능 출력\n", - "print(\"Test Accuracy:\", accuracy_score(y_true_test, y_pred_test))\n", - "print(\"Test AUC:\", roc_auc_score(y_true_test, y_pred_test))\n", - "print(\"Test F1-Score:\", f1_score(y_true_test, y_pred_test))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## TabNet" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Collecting pytorch-tabnet\n", - " Downloading pytorch_tabnet-4.1.0-py3-none-any.whl.metadata (15 kB)\n", - "Requirement already satisfied: numpy>=1.17 in /root/miniconda3/lib/python3.12/site-packages (from pytorch-tabnet) (1.26.4)\n", - "Requirement already satisfied: scikit_learn>0.21 in /root/miniconda3/lib/python3.12/site-packages (from pytorch-tabnet) (1.5.2)\n", - "Requirement already satisfied: scipy>1.4 in /root/miniconda3/lib/python3.12/site-packages (from pytorch-tabnet) (1.15.0)\n", - "Requirement already satisfied: torch>=1.3 in /root/miniconda3/lib/python3.12/site-packages (from pytorch-tabnet) (2.5.1)\n", - "Requirement already satisfied: tqdm>=4.36 in /root/miniconda3/lib/python3.12/site-packages (from pytorch-tabnet) (4.66.4)\n", - "Requirement already satisfied: joblib>=1.2.0 in /root/miniconda3/lib/python3.12/site-packages (from scikit_learn>0.21->pytorch-tabnet) (1.4.2)\n", - "Requirement already satisfied: threadpoolctl>=3.1.0 in /root/miniconda3/lib/python3.12/site-packages (from scikit_learn>0.21->pytorch-tabnet) (3.5.0)\n", - "Requirement already satisfied: filelock in /root/miniconda3/lib/python3.12/site-packages (from torch>=1.3->pytorch-tabnet) (3.16.1)\n", - "Requirement already satisfied: typing-extensions>=4.8.0 in /root/miniconda3/lib/python3.12/site-packages (from torch>=1.3->pytorch-tabnet) (4.12.2)\n", - "Requirement already satisfied: networkx in /root/miniconda3/lib/python3.12/site-packages (from torch>=1.3->pytorch-tabnet) (3.4.1)\n", - "Requirement already satisfied: jinja2 in /root/miniconda3/lib/python3.12/site-packages (from torch>=1.3->pytorch-tabnet) (3.1.4)\n", - "Requirement already satisfied: fsspec in /root/miniconda3/lib/python3.12/site-packages (from torch>=1.3->pytorch-tabnet) (2024.9.0)\n", - "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.4.127 in /root/miniconda3/lib/python3.12/site-packages (from torch>=1.3->pytorch-tabnet) (12.4.127)\n", - "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.4.127 in /root/miniconda3/lib/python3.12/site-packages (from torch>=1.3->pytorch-tabnet) (12.4.127)\n", - "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.4.127 in /root/miniconda3/lib/python3.12/site-packages (from torch>=1.3->pytorch-tabnet) (12.4.127)\n", - "Requirement already satisfied: nvidia-cudnn-cu12==9.1.0.70 in /root/miniconda3/lib/python3.12/site-packages (from torch>=1.3->pytorch-tabnet) (9.1.0.70)\n", - "Requirement already satisfied: nvidia-cublas-cu12==12.4.5.8 in /root/miniconda3/lib/python3.12/site-packages (from torch>=1.3->pytorch-tabnet) (12.4.5.8)\n", - "Requirement already satisfied: nvidia-cufft-cu12==11.2.1.3 in /root/miniconda3/lib/python3.12/site-packages (from torch>=1.3->pytorch-tabnet) (11.2.1.3)\n", - "Requirement already satisfied: nvidia-curand-cu12==10.3.5.147 in /root/miniconda3/lib/python3.12/site-packages (from torch>=1.3->pytorch-tabnet) (10.3.5.147)\n", - "Requirement already satisfied: nvidia-cusolver-cu12==11.6.1.9 in /root/miniconda3/lib/python3.12/site-packages (from torch>=1.3->pytorch-tabnet) (11.6.1.9)\n", - "Requirement already satisfied: nvidia-cusparse-cu12==12.3.1.170 in /root/miniconda3/lib/python3.12/site-packages (from torch>=1.3->pytorch-tabnet) (12.3.1.170)\n", - "Requirement already satisfied: nvidia-nccl-cu12==2.21.5 in /root/miniconda3/lib/python3.12/site-packages (from torch>=1.3->pytorch-tabnet) (2.21.5)\n", - "Requirement already satisfied: nvidia-nvtx-cu12==12.4.127 in /root/miniconda3/lib/python3.12/site-packages (from torch>=1.3->pytorch-tabnet) (12.4.127)\n", - "Requirement already satisfied: nvidia-nvjitlink-cu12==12.4.127 in /root/miniconda3/lib/python3.12/site-packages (from torch>=1.3->pytorch-tabnet) (12.4.127)\n", - "Requirement already satisfied: triton==3.1.0 in /root/miniconda3/lib/python3.12/site-packages (from torch>=1.3->pytorch-tabnet) (3.1.0)\n", - "Requirement already satisfied: setuptools in /root/miniconda3/lib/python3.12/site-packages (from torch>=1.3->pytorch-tabnet) (69.5.1)\n", - "Requirement already satisfied: sympy==1.13.1 in /root/miniconda3/lib/python3.12/site-packages (from torch>=1.3->pytorch-tabnet) (1.13.1)\n", - "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /root/miniconda3/lib/python3.12/site-packages (from sympy==1.13.1->torch>=1.3->pytorch-tabnet) (1.3.0)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in /root/miniconda3/lib/python3.12/site-packages (from jinja2->torch>=1.3->pytorch-tabnet) (3.0.1)\n", - "Downloading pytorch_tabnet-4.1.0-py3-none-any.whl (44 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m44.5/44.5 kB\u001b[0m \u001b[31m3.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hInstalling collected packages: pytorch-tabnet\n", - "Successfully installed pytorch-tabnet-4.1.0\n", - "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n", - "\u001b[0mNote: you may need to restart the kernel to use updated packages.\n" - ] - } - ], - "source": [ - "'''%pip install pytorch-tabnet'''" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "##### 데이터셋 준비" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "import numpy as np\n", - "from sklearn.preprocessing import StandardScaler, LabelEncoder\n", - "import torch\n", - "import random\n", - "\n", - "# 시드 고정\n", - "seed = 42\n", - "random.seed(seed)\n", - "np.random.seed(seed)\n", - "torch.manual_seed(seed)\n", - "torch.cuda.manual_seed_all(seed)\n", - "\n", - "# 데이터셋 불러오기\n", - "X_train, X_val, X_test, y_train, y_val, y_test, categorical_cols, numerical_cols = prepare_dataset(\"seoul\", target = \"binary\")" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[I 2025-02-02 17:45:02,478] A new study created in memory with name: no-name-30a8858a-054c-46b2-8eb6-150ef0380bd0\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.51162 | valid_accuracy: 0.6097 | 0:00:01s\n", - "epoch 1 | loss: 0.35934 | valid_accuracy: 0.80126 | 0:00:02s\n", - "epoch 2 | loss: 0.38031 | valid_accuracy: 0.88516 | 0:00:03s\n", - "epoch 3 | loss: 0.32778 | valid_accuracy: 0.83447 | 0:00:04s\n", - "epoch 4 | loss: 0.2564 | valid_accuracy: 0.86701 | 0:00:05s\n", - "epoch 5 | loss: 0.25241 | valid_accuracy: 0.8984 | 0:00:06s\n", - "epoch 6 | loss: 0.23158 | valid_accuracy: 0.90228 | 0:00:07s\n", - "epoch 7 | loss: 0.2121 | valid_accuracy: 0.89772 | 0:00:08s\n", - "epoch 8 | loss: 0.21777 | valid_accuracy: 0.9024 | 0:00:09s\n", - "epoch 9 | loss: 0.21399 | valid_accuracy: 0.89167 | 0:00:10s\n", - "epoch 10 | loss: 0.20104 | valid_accuracy: 0.91804 | 0:00:11s\n", - "epoch 11 | loss: 0.19229 | valid_accuracy: 0.91256 | 0:00:12s\n", - "epoch 12 | loss: 0.18519 | valid_accuracy: 0.90354 | 0:00:13s\n", - "epoch 13 | loss: 0.17756 | valid_accuracy: 0.90354 | 0:00:14s\n", - "epoch 14 | loss: 0.18859 | valid_accuracy: 0.90925 | 0:00:15s\n", - "epoch 15 | loss: 0.18694 | valid_accuracy: 0.90856 | 0:00:16s\n", - "epoch 16 | loss: 0.17478 | valid_accuracy: 0.91792 | 0:00:17s\n", - "epoch 17 | loss: 0.16563 | valid_accuracy: 0.92146 | 0:00:18s\n", - "epoch 18 | loss: 0.15951 | valid_accuracy: 0.90879 | 0:00:19s\n", - "epoch 19 | loss: 0.15968 | valid_accuracy: 0.90457 | 0:00:20s\n", - "epoch 20 | loss: 0.15753 | valid_accuracy: 0.91769 | 0:00:21s\n", - "epoch 21 | loss: 0.15174 | valid_accuracy: 0.91473 | 0:00:22s\n", - "epoch 22 | loss: 0.15183 | valid_accuracy: 0.92546 | 0:00:23s\n", - "epoch 23 | loss: 0.14962 | valid_accuracy: 0.90811 | 0:00:24s\n", - "epoch 24 | loss: 0.15239 | valid_accuracy: 0.91347 | 0:00:25s\n", - "epoch 25 | loss: 0.16776 | valid_accuracy: 0.91347 | 0:00:26s\n", - "epoch 26 | loss: 0.16609 | valid_accuracy: 0.92043 | 0:00:27s\n", - "epoch 27 | loss: 0.15284 | valid_accuracy: 0.92523 | 0:00:28s\n", - "epoch 28 | loss: 0.16267 | valid_accuracy: 0.92511 | 0:00:29s\n", - "epoch 29 | loss: 0.15016 | valid_accuracy: 0.92763 | 0:00:30s\n", - "Stop training because you reached max_epochs = 30 with best_epoch = 29 and best_valid_accuracy = 0.92763\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:45:33,592] Trial 0 finished with value: 0.6752049180327869 and parameters: {'n_d': 24, 'n_a': 64, 'n_steps': 8, 'gamma': 1.5986584841970366, 'lambda_sparse': 0.0002051338263087451, 'momentum': 0.07083786293111903}. Best is trial 0 with value: 0.6752049180327869.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.51572 | valid_accuracy: 0.7105 | 0:00:00s\n", - "epoch 1 | loss: 0.31607 | valid_accuracy: 0.8145 | 0:00:01s\n", - "epoch 2 | loss: 0.26815 | valid_accuracy: 0.85731 | 0:00:02s\n", - "epoch 3 | loss: 0.24246 | valid_accuracy: 0.8129 | 0:00:03s\n", - "epoch 4 | loss: 0.22795 | valid_accuracy: 0.88653 | 0:00:04s\n", - "epoch 5 | loss: 0.21193 | valid_accuracy: 0.88105 | 0:00:05s\n", - "epoch 6 | loss: 0.20544 | valid_accuracy: 0.91039 | 0:00:06s\n", - "epoch 7 | loss: 0.19844 | valid_accuracy: 0.89247 | 0:00:07s\n", - "epoch 8 | loss: 0.18712 | valid_accuracy: 0.91929 | 0:00:08s\n", - "epoch 9 | loss: 0.177 | valid_accuracy: 0.91427 | 0:00:09s\n", - "epoch 10 | loss: 0.17666 | valid_accuracy: 0.91438 | 0:00:09s\n", - "epoch 11 | loss: 0.17366 | valid_accuracy: 0.90982 | 0:00:10s\n", - "epoch 12 | loss: 0.17071 | valid_accuracy: 0.91324 | 0:00:11s\n", - "epoch 13 | loss: 0.18261 | valid_accuracy: 0.9137 | 0:00:12s\n", - "epoch 14 | loss: 0.17591 | valid_accuracy: 0.9266 | 0:00:13s\n", - "epoch 15 | loss: 0.16704 | valid_accuracy: 0.91872 | 0:00:14s\n", - "epoch 16 | loss: 0.16162 | valid_accuracy: 0.92317 | 0:00:15s\n", - "epoch 17 | loss: 0.16996 | valid_accuracy: 0.93105 | 0:00:16s\n", - "epoch 18 | loss: 0.17147 | valid_accuracy: 0.9234 | 0:00:17s\n", - "epoch 19 | loss: 0.15642 | valid_accuracy: 0.92626 | 0:00:17s\n", - "epoch 20 | loss: 0.14691 | valid_accuracy: 0.93242 | 0:00:18s\n", - "epoch 21 | loss: 0.15126 | valid_accuracy: 0.92934 | 0:00:19s\n", - "epoch 22 | loss: 0.14708 | valid_accuracy: 0.92831 | 0:00:20s\n", - "epoch 23 | loss: 0.13999 | valid_accuracy: 0.91758 | 0:00:21s\n", - "epoch 24 | loss: 0.13341 | valid_accuracy: 0.93139 | 0:00:22s\n", - "epoch 25 | loss: 0.12963 | valid_accuracy: 0.92957 | 0:00:23s\n", - "epoch 26 | loss: 0.129 | valid_accuracy: 0.93174 | 0:00:24s\n", - "epoch 27 | loss: 0.12625 | valid_accuracy: 0.9218 | 0:00:25s\n", - "epoch 28 | loss: 0.12765 | valid_accuracy: 0.93059 | 0:00:26s\n", - "\n", - "Early stopping occurred at epoch 28 with best_epoch = 20 and best_valid_accuracy = 0.93242\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:46:00,402] Trial 1 finished with value: 0.682062298603652 and parameters: {'n_d': 8, 'n_a': 56, 'n_steps': 7, 'gamma': 1.7080725777960455, 'lambda_sparse': 0.00010994335574766199, 'momentum': 0.3882648423431778}. Best is trial 1 with value: 0.682062298603652.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.41085 | valid_accuracy: 0.83607 | 0:00:00s\n", - "epoch 1 | loss: 0.20081 | valid_accuracy: 0.88893 | 0:00:01s\n", - "epoch 2 | loss: 0.14401 | valid_accuracy: 0.91039 | 0:00:01s\n", - "epoch 3 | loss: 0.12132 | valid_accuracy: 0.92454 | 0:00:02s\n", - "epoch 4 | loss: 0.11407 | valid_accuracy: 0.92945 | 0:00:03s\n", - "epoch 5 | loss: 0.10872 | valid_accuracy: 0.93653 | 0:00:03s\n", - "epoch 6 | loss: 0.10113 | valid_accuracy: 0.9403 | 0:00:04s\n", - "epoch 7 | loss: 0.10132 | valid_accuracy: 0.93596 | 0:00:04s\n", - "epoch 8 | loss: 0.09937 | valid_accuracy: 0.93607 | 0:00:05s\n", - "epoch 9 | loss: 0.09028 | valid_accuracy: 0.93219 | 0:00:06s\n", - "epoch 10 | loss: 0.08605 | valid_accuracy: 0.94326 | 0:00:06s\n", - "epoch 11 | loss: 0.07732 | valid_accuracy: 0.94292 | 0:00:07s\n", - "epoch 12 | loss: 0.07636 | valid_accuracy: 0.94132 | 0:00:07s\n", - "epoch 13 | loss: 0.07348 | valid_accuracy: 0.93607 | 0:00:08s\n", - "epoch 14 | loss: 0.06838 | valid_accuracy: 0.93779 | 0:00:09s\n", - "epoch 15 | loss: 0.06585 | valid_accuracy: 0.94829 | 0:00:09s\n", - "epoch 16 | loss: 0.06846 | valid_accuracy: 0.94178 | 0:00:10s\n", - "epoch 17 | loss: 0.06709 | valid_accuracy: 0.9468 | 0:00:11s\n", - "epoch 18 | loss: 0.06514 | valid_accuracy: 0.95194 | 0:00:11s\n", - "epoch 19 | loss: 0.06273 | valid_accuracy: 0.94304 | 0:00:12s\n", - "epoch 20 | loss: 0.06262 | valid_accuracy: 0.94406 | 0:00:12s\n", - "epoch 21 | loss: 0.06462 | valid_accuracy: 0.94646 | 0:00:13s\n", - "epoch 22 | loss: 0.0566 | valid_accuracy: 0.93607 | 0:00:14s\n", - "epoch 23 | loss: 0.05701 | valid_accuracy: 0.94098 | 0:00:14s\n", - "epoch 24 | loss: 0.05561 | valid_accuracy: 0.94167 | 0:00:15s\n", - "epoch 25 | loss: 0.05261 | valid_accuracy: 0.94326 | 0:00:15s\n", - "epoch 26 | loss: 0.05365 | valid_accuracy: 0.94441 | 0:00:16s\n", - "\n", - "Early stopping occurred at epoch 26 with best_epoch = 18 and best_valid_accuracy = 0.95194\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:46:17,235] Trial 2 finished with value: 0.7617430673457838 and parameters: {'n_d': 56, 'n_a': 16, 'n_steps': 4, 'gamma': 1.1834045098534338, 'lambda_sparse': 0.0004059611610484307, 'momentum': 0.21465500833657278}. Best is trial 2 with value: 0.7617430673457838.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.44898 | valid_accuracy: 0.81632 | 0:00:00s\n", - "epoch 1 | loss: 0.26007 | valid_accuracy: 0.84087 | 0:00:01s\n", - "epoch 2 | loss: 0.23179 | valid_accuracy: 0.88482 | 0:00:02s\n", - "epoch 3 | loss: 0.19977 | valid_accuracy: 0.92466 | 0:00:03s\n", - "epoch 4 | loss: 0.16338 | valid_accuracy: 0.90148 | 0:00:04s\n", - "epoch 5 | loss: 0.15101 | valid_accuracy: 0.92979 | 0:00:05s\n", - "epoch 6 | loss: 0.13924 | valid_accuracy: 0.93596 | 0:00:06s\n", - "epoch 7 | loss: 0.13496 | valid_accuracy: 0.93779 | 0:00:07s\n", - "epoch 8 | loss: 0.12218 | valid_accuracy: 0.93813 | 0:00:08s\n", - "epoch 9 | loss: 0.11479 | valid_accuracy: 0.93379 | 0:00:09s\n", - "epoch 10 | loss: 0.11308 | valid_accuracy: 0.94429 | 0:00:09s\n", - "epoch 11 | loss: 0.11463 | valid_accuracy: 0.94441 | 0:00:10s\n", - "epoch 12 | loss: 0.10708 | valid_accuracy: 0.94429 | 0:00:11s\n", - "epoch 13 | loss: 0.11045 | valid_accuracy: 0.94441 | 0:00:12s\n", - "epoch 14 | loss: 0.10571 | valid_accuracy: 0.94874 | 0:00:13s\n", - "epoch 15 | loss: 0.10262 | valid_accuracy: 0.94429 | 0:00:14s\n", - "epoch 16 | loss: 0.09813 | valid_accuracy: 0.94817 | 0:00:15s\n", - "epoch 17 | loss: 0.10055 | valid_accuracy: 0.94281 | 0:00:16s\n", - "epoch 18 | loss: 0.09636 | valid_accuracy: 0.94509 | 0:00:17s\n", - "epoch 19 | loss: 0.09546 | valid_accuracy: 0.93436 | 0:00:17s\n", - "epoch 20 | loss: 0.09955 | valid_accuracy: 0.94247 | 0:00:18s\n", - "epoch 21 | loss: 0.10707 | valid_accuracy: 0.93653 | 0:00:19s\n", - "epoch 22 | loss: 0.10258 | valid_accuracy: 0.94144 | 0:00:20s\n", - "\n", - "Early stopping occurred at epoch 22 with best_epoch = 14 and best_valid_accuracy = 0.94874\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:46:38,544] Trial 3 finished with value: 0.7623080995235575 and parameters: {'n_d': 32, 'n_a': 24, 'n_steps': 7, 'gamma': 1.139493860652042, 'lambda_sparse': 0.0003839629299804173, 'momentum': 0.15288111888453979}. Best is trial 3 with value: 0.7623080995235575.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.40326 | valid_accuracy: 0.78071 | 0:00:00s\n", - "epoch 1 | loss: 0.21984 | valid_accuracy: 0.89646 | 0:00:01s\n", - "epoch 2 | loss: 0.19744 | valid_accuracy: 0.89703 | 0:00:01s\n", - "epoch 3 | loss: 0.17036 | valid_accuracy: 0.93801 | 0:00:02s\n", - "epoch 4 | loss: 0.1596 | valid_accuracy: 0.92774 | 0:00:03s\n", - "epoch 5 | loss: 0.14734 | valid_accuracy: 0.94053 | 0:00:03s\n", - "epoch 6 | loss: 0.1261 | valid_accuracy: 0.90765 | 0:00:04s\n", - "epoch 7 | loss: 0.13111 | valid_accuracy: 0.94361 | 0:00:04s\n", - "epoch 8 | loss: 0.12254 | valid_accuracy: 0.93185 | 0:00:05s\n", - "epoch 9 | loss: 0.11484 | valid_accuracy: 0.93893 | 0:00:06s\n", - "epoch 10 | loss: 0.12082 | valid_accuracy: 0.92146 | 0:00:06s\n", - "epoch 11 | loss: 0.11907 | valid_accuracy: 0.91347 | 0:00:07s\n", - "epoch 12 | loss: 0.10386 | valid_accuracy: 0.93904 | 0:00:07s\n", - "epoch 13 | loss: 0.09116 | valid_accuracy: 0.94521 | 0:00:08s\n", - "epoch 14 | loss: 0.08679 | valid_accuracy: 0.93596 | 0:00:09s\n", - "epoch 15 | loss: 0.0809 | valid_accuracy: 0.9363 | 0:00:09s\n", - "epoch 16 | loss: 0.07893 | valid_accuracy: 0.93573 | 0:00:10s\n", - "epoch 17 | loss: 0.07731 | valid_accuracy: 0.93984 | 0:00:10s\n", - "epoch 18 | loss: 0.07423 | valid_accuracy: 0.93744 | 0:00:11s\n", - "epoch 19 | loss: 0.07371 | valid_accuracy: 0.93151 | 0:00:12s\n", - "epoch 20 | loss: 0.06917 | valid_accuracy: 0.94053 | 0:00:12s\n", - "epoch 21 | loss: 0.06548 | valid_accuracy: 0.93219 | 0:00:13s\n", - "\n", - "Early stopping occurred at epoch 21 with best_epoch = 13 and best_valid_accuracy = 0.94521\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:46:52,266] Trial 4 finished with value: 0.7433155080213903 and parameters: {'n_d': 32, 'n_a': 56, 'n_steps': 4, 'gamma': 1.5142344384136117, 'lambda_sparse': 0.0015304852121831463, 'momentum': 0.028115660960799115}. Best is trial 3 with value: 0.7623080995235575.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.34256 | valid_accuracy: 0.83813 | 0:00:00s\n", - "epoch 1 | loss: 0.19041 | valid_accuracy: 0.92409 | 0:00:00s\n", - "epoch 2 | loss: 0.15125 | valid_accuracy: 0.91781 | 0:00:01s\n", - "epoch 3 | loss: 0.13014 | valid_accuracy: 0.92911 | 0:00:01s\n", - "epoch 4 | loss: 0.11999 | valid_accuracy: 0.93059 | 0:00:02s\n", - "epoch 5 | loss: 0.10766 | valid_accuracy: 0.92763 | 0:00:03s\n", - "epoch 6 | loss: 0.09502 | valid_accuracy: 0.92934 | 0:00:03s\n", - "epoch 7 | loss: 0.09133 | valid_accuracy: 0.93482 | 0:00:04s\n", - "epoch 8 | loss: 0.08396 | valid_accuracy: 0.93345 | 0:00:04s\n", - "epoch 9 | loss: 0.08269 | valid_accuracy: 0.9274 | 0:00:04s\n", - "epoch 10 | loss: 0.08525 | valid_accuracy: 0.93447 | 0:00:05s\n", - "epoch 11 | loss: 0.07883 | valid_accuracy: 0.9363 | 0:00:05s\n", - "epoch 12 | loss: 0.07277 | valid_accuracy: 0.94258 | 0:00:06s\n", - "epoch 13 | loss: 0.06871 | valid_accuracy: 0.94292 | 0:00:06s\n", - "epoch 14 | loss: 0.06777 | valid_accuracy: 0.93756 | 0:00:07s\n", - "epoch 15 | loss: 0.06583 | valid_accuracy: 0.9363 | 0:00:07s\n", - "epoch 16 | loss: 0.06456 | valid_accuracy: 0.94155 | 0:00:08s\n", - "epoch 17 | loss: 0.06711 | valid_accuracy: 0.94155 | 0:00:08s\n", - "epoch 18 | loss: 0.06725 | valid_accuracy: 0.93824 | 0:00:09s\n", - "epoch 19 | loss: 0.06817 | valid_accuracy: 0.93539 | 0:00:09s\n", - "epoch 20 | loss: 0.06471 | valid_accuracy: 0.94224 | 0:00:10s\n", - "epoch 21 | loss: 0.06237 | valid_accuracy: 0.94167 | 0:00:10s\n", - "\n", - "Early stopping occurred at epoch 21 with best_epoch = 13 and best_valid_accuracy = 0.94292\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:47:03,493] Trial 5 finished with value: 0.6940024479804161 and parameters: {'n_d': 40, 'n_a': 16, 'n_steps': 3, 'gamma': 1.9488855372533331, 'lambda_sparse': 0.00853618986286683, 'momentum': 0.32527496576541987}. Best is trial 3 with value: 0.7623080995235575.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.54642 | valid_accuracy: 0.73162 | 0:00:00s\n", - "epoch 1 | loss: 0.34982 | valid_accuracy: 0.775 | 0:00:01s\n", - "epoch 2 | loss: 0.33705 | valid_accuracy: 0.82009 | 0:00:02s\n", - "epoch 3 | loss: 0.36378 | valid_accuracy: 0.84658 | 0:00:03s\n", - "epoch 4 | loss: 0.26389 | valid_accuracy: 0.87466 | 0:00:04s\n", - "epoch 5 | loss: 0.26708 | valid_accuracy: 0.8879 | 0:00:05s\n", - "epoch 6 | loss: 0.2209 | valid_accuracy: 0.91564 | 0:00:06s\n", - "epoch 7 | loss: 0.20343 | valid_accuracy: 0.90183 | 0:00:07s\n", - "epoch 8 | loss: 0.20446 | valid_accuracy: 0.90457 | 0:00:08s\n", - "epoch 9 | loss: 0.18992 | valid_accuracy: 0.91724 | 0:00:09s\n", - "epoch 10 | loss: 0.1826 | valid_accuracy: 0.9226 | 0:00:10s\n", - "epoch 11 | loss: 0.18397 | valid_accuracy: 0.92546 | 0:00:11s\n", - "epoch 12 | loss: 0.16944 | valid_accuracy: 0.92203 | 0:00:12s\n", - "epoch 13 | loss: 0.16622 | valid_accuracy: 0.92477 | 0:00:13s\n", - "epoch 14 | loss: 0.15952 | valid_accuracy: 0.92306 | 0:00:14s\n", - "epoch 15 | loss: 0.15906 | valid_accuracy: 0.92717 | 0:00:15s\n", - "epoch 16 | loss: 0.15773 | valid_accuracy: 0.90457 | 0:00:16s\n", - "epoch 17 | loss: 0.15543 | valid_accuracy: 0.92386 | 0:00:17s\n", - "epoch 18 | loss: 0.15167 | valid_accuracy: 0.92534 | 0:00:18s\n", - "epoch 19 | loss: 0.14806 | valid_accuracy: 0.92306 | 0:00:19s\n", - "epoch 20 | loss: 0.14644 | valid_accuracy: 0.92717 | 0:00:20s\n", - "epoch 21 | loss: 0.14616 | valid_accuracy: 0.93014 | 0:00:21s\n", - "epoch 22 | loss: 0.1509 | valid_accuracy: 0.93082 | 0:00:22s\n", - "epoch 23 | loss: 0.15154 | valid_accuracy: 0.91792 | 0:00:23s\n", - "epoch 24 | loss: 0.14669 | valid_accuracy: 0.92511 | 0:00:24s\n", - "epoch 25 | loss: 0.1441 | valid_accuracy: 0.92591 | 0:00:25s\n", - "epoch 26 | loss: 0.14843 | valid_accuracy: 0.92934 | 0:00:26s\n", - "epoch 27 | loss: 0.15011 | valid_accuracy: 0.91861 | 0:00:27s\n", - "epoch 28 | loss: 0.16114 | valid_accuracy: 0.92854 | 0:00:28s\n", - "epoch 29 | loss: 0.15804 | valid_accuracy: 0.92728 | 0:00:29s\n", - "Stop training because you reached max_epochs = 30 with best_epoch = 22 and best_valid_accuracy = 0.93082\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:47:34,042] Trial 6 finished with value: 0.6926977687626775 and parameters: {'n_d': 24, 'n_a': 8, 'n_steps': 8, 'gamma': 1.4401524937396013, 'lambda_sparse': 0.00017541893487450815, 'momentum': 0.2031189949433954}. Best is trial 3 with value: 0.7623080995235575.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.48696 | valid_accuracy: 0.68527 | 0:00:00s\n", - "epoch 1 | loss: 0.29365 | valid_accuracy: 0.82511 | 0:00:01s\n", - "epoch 2 | loss: 0.25789 | valid_accuracy: 0.86975 | 0:00:02s\n", - "epoch 3 | loss: 0.20216 | valid_accuracy: 0.8992 | 0:00:02s\n", - "epoch 4 | loss: 0.1853 | valid_accuracy: 0.92511 | 0:00:03s\n", - "epoch 5 | loss: 0.17174 | valid_accuracy: 0.91199 | 0:00:04s\n", - "epoch 6 | loss: 0.16428 | valid_accuracy: 0.92386 | 0:00:05s\n", - "epoch 7 | loss: 0.14458 | valid_accuracy: 0.92466 | 0:00:05s\n", - "epoch 8 | loss: 0.13132 | valid_accuracy: 0.93379 | 0:00:06s\n", - "epoch 9 | loss: 0.12207 | valid_accuracy: 0.92534 | 0:00:07s\n", - "epoch 10 | loss: 0.12534 | valid_accuracy: 0.92021 | 0:00:07s\n", - "epoch 11 | loss: 0.11432 | valid_accuracy: 0.92295 | 0:00:08s\n", - "epoch 12 | loss: 0.1261 | valid_accuracy: 0.9274 | 0:00:09s\n", - "epoch 13 | loss: 0.11539 | valid_accuracy: 0.92432 | 0:00:09s\n", - "epoch 14 | loss: 0.10864 | valid_accuracy: 0.93402 | 0:00:10s\n", - "epoch 15 | loss: 0.10136 | valid_accuracy: 0.93733 | 0:00:11s\n", - "epoch 16 | loss: 0.09272 | valid_accuracy: 0.9379 | 0:00:12s\n", - "epoch 17 | loss: 0.09546 | valid_accuracy: 0.93847 | 0:00:12s\n", - "epoch 18 | loss: 0.10111 | valid_accuracy: 0.9339 | 0:00:13s\n", - "epoch 19 | loss: 0.09387 | valid_accuracy: 0.93071 | 0:00:14s\n", - "epoch 20 | loss: 0.09709 | valid_accuracy: 0.93573 | 0:00:15s\n", - "epoch 21 | loss: 0.09374 | valid_accuracy: 0.92717 | 0:00:15s\n", - "epoch 22 | loss: 0.09057 | valid_accuracy: 0.93185 | 0:00:16s\n", - "epoch 23 | loss: 0.09584 | valid_accuracy: 0.91849 | 0:00:17s\n", - "epoch 24 | loss: 0.10278 | valid_accuracy: 0.93425 | 0:00:17s\n", - "epoch 25 | loss: 0.08888 | valid_accuracy: 0.93276 | 0:00:18s\n", - "\n", - "Early stopping occurred at epoch 25 with best_epoch = 17 and best_valid_accuracy = 0.93847\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:47:53,089] Trial 7 finished with value: 0.6980392156862745 and parameters: {'n_d': 8, 'n_a': 64, 'n_steps': 5, 'gamma': 1.662522284353982, 'lambda_sparse': 0.0004201672054372534, 'momentum': 0.21282652825934623}. Best is trial 3 with value: 0.7623080995235575.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.95852 | valid_accuracy: 0.55479 | 0:00:01s\n", - "epoch 1 | loss: 0.44115 | valid_accuracy: 0.49543 | 0:00:02s\n", - "epoch 2 | loss: 0.42561 | valid_accuracy: 0.80183 | 0:00:03s\n", - "epoch 3 | loss: 0.53733 | valid_accuracy: 0.74589 | 0:00:04s\n", - "epoch 4 | loss: 0.31668 | valid_accuracy: 0.86473 | 0:00:06s\n", - "epoch 5 | loss: 0.22462 | valid_accuracy: 0.84155 | 0:00:07s\n", - "epoch 6 | loss: 0.22422 | valid_accuracy: 0.88379 | 0:00:08s\n", - "epoch 7 | loss: 0.19977 | valid_accuracy: 0.88619 | 0:00:09s\n", - "epoch 8 | loss: 0.20833 | valid_accuracy: 0.82386 | 0:00:10s\n", - "epoch 9 | loss: 0.20295 | valid_accuracy: 0.89646 | 0:00:12s\n", - "epoch 10 | loss: 0.17829 | valid_accuracy: 0.89555 | 0:00:13s\n", - "epoch 11 | loss: 0.18023 | valid_accuracy: 0.92374 | 0:00:14s\n", - "epoch 12 | loss: 0.17958 | valid_accuracy: 0.89441 | 0:00:15s\n", - "epoch 13 | loss: 0.17927 | valid_accuracy: 0.89737 | 0:00:16s\n", - "epoch 14 | loss: 0.17239 | valid_accuracy: 0.92078 | 0:00:18s\n", - "epoch 15 | loss: 0.15836 | valid_accuracy: 0.91176 | 0:00:19s\n", - "epoch 16 | loss: 0.16379 | valid_accuracy: 0.93002 | 0:00:20s\n", - "epoch 17 | loss: 0.15064 | valid_accuracy: 0.92352 | 0:00:21s\n", - "epoch 18 | loss: 0.15415 | valid_accuracy: 0.9113 | 0:00:22s\n", - "epoch 19 | loss: 0.15037 | valid_accuracy: 0.92705 | 0:00:24s\n", - "epoch 20 | loss: 0.146 | valid_accuracy: 0.91735 | 0:00:25s\n", - "epoch 21 | loss: 0.14947 | valid_accuracy: 0.92637 | 0:00:26s\n", - "epoch 22 | loss: 0.14708 | valid_accuracy: 0.92066 | 0:00:27s\n", - "epoch 23 | loss: 0.15086 | valid_accuracy: 0.92386 | 0:00:29s\n", - "epoch 24 | loss: 0.13848 | valid_accuracy: 0.92774 | 0:00:30s\n", - "\n", - "Early stopping occurred at epoch 24 with best_epoch = 16 and best_valid_accuracy = 0.93002\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:48:24,135] Trial 8 finished with value: 0.6751457339692634 and parameters: {'n_d': 40, 'n_a': 16, 'n_steps': 10, 'gamma': 1.7751328233611146, 'lambda_sparse': 0.007568292060167618, 'momentum': 0.3589826666667831}. Best is trial 3 with value: 0.7623080995235575.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.2977 | valid_accuracy: 0.8282 | 0:00:00s\n", - "epoch 1 | loss: 0.13881 | valid_accuracy: 0.91792 | 0:00:01s\n", - "epoch 2 | loss: 0.12205 | valid_accuracy: 0.92854 | 0:00:01s\n", - "epoch 3 | loss: 0.10262 | valid_accuracy: 0.93196 | 0:00:02s\n", - "epoch 4 | loss: 0.09549 | valid_accuracy: 0.93824 | 0:00:02s\n", - "epoch 5 | loss: 0.0872 | valid_accuracy: 0.93413 | 0:00:03s\n", - "epoch 6 | loss: 0.08352 | valid_accuracy: 0.9387 | 0:00:03s\n", - "epoch 7 | loss: 0.07734 | valid_accuracy: 0.93916 | 0:00:04s\n", - "epoch 8 | loss: 0.07214 | valid_accuracy: 0.94441 | 0:00:04s\n", - "epoch 9 | loss: 0.07138 | valid_accuracy: 0.94623 | 0:00:05s\n", - "epoch 10 | loss: 0.0698 | valid_accuracy: 0.93984 | 0:00:05s\n", - "epoch 11 | loss: 0.06457 | valid_accuracy: 0.94007 | 0:00:06s\n", - "epoch 12 | loss: 0.0611 | valid_accuracy: 0.94144 | 0:00:06s\n", - "epoch 13 | loss: 0.06191 | valid_accuracy: 0.94429 | 0:00:07s\n", - "epoch 14 | loss: 0.06013 | valid_accuracy: 0.94178 | 0:00:07s\n", - "epoch 15 | loss: 0.05827 | valid_accuracy: 0.94212 | 0:00:08s\n", - "epoch 16 | loss: 0.05663 | valid_accuracy: 0.93573 | 0:00:08s\n", - "epoch 17 | loss: 0.05921 | valid_accuracy: 0.94075 | 0:00:09s\n", - "\n", - "Early stopping occurred at epoch 17 with best_epoch = 9 and best_valid_accuracy = 0.94623\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:48:33,712] Trial 9 finished with value: 0.729776247848537 and parameters: {'n_d': 40, 'n_a': 64, 'n_steps': 3, 'gamma': 1.1959828624191453, 'lambda_sparse': 0.00012315571723666037, 'momentum': 0.13687882899767312}. Best is trial 3 with value: 0.7623080995235575.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.64705 | valid_accuracy: 0.76416 | 0:00:01s\n", - "epoch 1 | loss: 0.29235 | valid_accuracy: 0.88938 | 0:00:02s\n", - "epoch 2 | loss: 0.17376 | valid_accuracy: 0.89977 | 0:00:03s\n", - "epoch 3 | loss: 0.13063 | valid_accuracy: 0.92146 | 0:00:04s\n", - "epoch 4 | loss: 0.12107 | valid_accuracy: 0.92922 | 0:00:06s\n", - "epoch 5 | loss: 0.1248 | valid_accuracy: 0.92374 | 0:00:07s\n", - "epoch 6 | loss: 0.10799 | valid_accuracy: 0.93916 | 0:00:08s\n", - "epoch 7 | loss: 0.10338 | valid_accuracy: 0.9226 | 0:00:09s\n", - "epoch 8 | loss: 0.11006 | valid_accuracy: 0.94372 | 0:00:10s\n", - "epoch 9 | loss: 0.09386 | valid_accuracy: 0.93893 | 0:00:12s\n", - "epoch 10 | loss: 0.09263 | valid_accuracy: 0.9347 | 0:00:13s\n", - "epoch 11 | loss: 0.09154 | valid_accuracy: 0.9403 | 0:00:14s\n", - "epoch 12 | loss: 0.09004 | valid_accuracy: 0.93824 | 0:00:15s\n", - "epoch 13 | loss: 0.08856 | valid_accuracy: 0.93733 | 0:00:17s\n", - "epoch 14 | loss: 0.09239 | valid_accuracy: 0.93539 | 0:00:18s\n", - "epoch 15 | loss: 0.08891 | valid_accuracy: 0.92751 | 0:00:19s\n", - "epoch 16 | loss: 0.08882 | valid_accuracy: 0.93596 | 0:00:20s\n", - "\n", - "Early stopping occurred at epoch 16 with best_epoch = 8 and best_valid_accuracy = 0.94372\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:48:55,252] Trial 10 finished with value: 0.7336574824419233 and parameters: {'n_d': 64, 'n_a': 32, 'n_steps': 10, 'gamma': 1.0179618756148194, 'lambda_sparse': 0.0015695029402008024, 'momentum': 0.2791963586919666}. Best is trial 3 with value: 0.7623080995235575.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.50981 | valid_accuracy: 0.76267 | 0:00:00s\n", - "epoch 1 | loss: 0.24604 | valid_accuracy: 0.8508 | 0:00:01s\n", - "epoch 2 | loss: 0.22524 | valid_accuracy: 0.87432 | 0:00:02s\n", - "epoch 3 | loss: 0.19477 | valid_accuracy: 0.904 | 0:00:03s\n", - "epoch 4 | loss: 0.17533 | valid_accuracy: 0.9274 | 0:00:03s\n", - "epoch 5 | loss: 0.15164 | valid_accuracy: 0.9339 | 0:00:04s\n", - "epoch 6 | loss: 0.14632 | valid_accuracy: 0.92443 | 0:00:05s\n", - "epoch 7 | loss: 0.15291 | valid_accuracy: 0.92968 | 0:00:06s\n", - "epoch 8 | loss: 0.13765 | valid_accuracy: 0.93505 | 0:00:07s\n", - "epoch 9 | loss: 0.13135 | valid_accuracy: 0.93242 | 0:00:07s\n", - "epoch 10 | loss: 0.1237 | valid_accuracy: 0.94007 | 0:00:08s\n", - "epoch 11 | loss: 0.12006 | valid_accuracy: 0.93333 | 0:00:09s\n", - "epoch 12 | loss: 0.11845 | valid_accuracy: 0.93664 | 0:00:10s\n", - "epoch 13 | loss: 0.12189 | valid_accuracy: 0.93607 | 0:00:11s\n", - "epoch 14 | loss: 0.11624 | valid_accuracy: 0.93345 | 0:00:12s\n", - "epoch 15 | loss: 0.11998 | valid_accuracy: 0.9355 | 0:00:12s\n", - "epoch 16 | loss: 0.12215 | valid_accuracy: 0.93311 | 0:00:13s\n", - "epoch 17 | loss: 0.1115 | valid_accuracy: 0.93231 | 0:00:14s\n", - "epoch 18 | loss: 0.11118 | valid_accuracy: 0.92888 | 0:00:15s\n", - "\n", - "Early stopping occurred at epoch 18 with best_epoch = 10 and best_valid_accuracy = 0.94007\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:49:11,078] Trial 11 finished with value: 0.714828897338403 and parameters: {'n_d': 64, 'n_a': 32, 'n_steps': 6, 'gamma': 1.2765899964395564, 'lambda_sparse': 0.0005836712223868827, 'momentum': 0.16881777729239383}. Best is trial 3 with value: 0.7623080995235575.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.54443 | valid_accuracy: 0.68653 | 0:00:00s\n", - "epoch 1 | loss: 0.19888 | valid_accuracy: 0.92934 | 0:00:01s\n", - "epoch 2 | loss: 0.15859 | valid_accuracy: 0.92763 | 0:00:02s\n", - "epoch 3 | loss: 0.14323 | valid_accuracy: 0.91747 | 0:00:03s\n", - "epoch 4 | loss: 0.13449 | valid_accuracy: 0.93539 | 0:00:03s\n", - "epoch 5 | loss: 0.11473 | valid_accuracy: 0.93219 | 0:00:04s\n", - "epoch 6 | loss: 0.11382 | valid_accuracy: 0.93687 | 0:00:05s\n", - "epoch 7 | loss: 0.10431 | valid_accuracy: 0.93676 | 0:00:06s\n", - "epoch 8 | loss: 0.0963 | valid_accuracy: 0.93927 | 0:00:07s\n", - "epoch 9 | loss: 0.09182 | valid_accuracy: 0.93744 | 0:00:08s\n", - "epoch 10 | loss: 0.09092 | valid_accuracy: 0.93094 | 0:00:08s\n", - "epoch 11 | loss: 0.09416 | valid_accuracy: 0.93527 | 0:00:09s\n", - "epoch 12 | loss: 0.08802 | valid_accuracy: 0.93676 | 0:00:10s\n", - "epoch 13 | loss: 0.08599 | valid_accuracy: 0.93676 | 0:00:11s\n", - "epoch 14 | loss: 0.0855 | valid_accuracy: 0.93607 | 0:00:12s\n", - "epoch 15 | loss: 0.08633 | valid_accuracy: 0.93128 | 0:00:12s\n", - "epoch 16 | loss: 0.08624 | valid_accuracy: 0.93904 | 0:00:13s\n", - "\n", - "Early stopping occurred at epoch 16 with best_epoch = 8 and best_valid_accuracy = 0.93927\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:49:25,330] Trial 12 finished with value: 0.6885245901639344 and parameters: {'n_d': 56, 'n_a': 24, 'n_steps': 6, 'gamma': 1.017145473653846, 'lambda_sparse': 0.0004630932602208763, 'momentum': 0.12821396325505197}. Best is trial 3 with value: 0.7623080995235575.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.39021 | valid_accuracy: 0.8605 | 0:00:00s\n", - "epoch 1 | loss: 0.20744 | valid_accuracy: 0.90434 | 0:00:01s\n", - "epoch 2 | loss: 0.16345 | valid_accuracy: 0.91587 | 0:00:02s\n", - "epoch 3 | loss: 0.14347 | valid_accuracy: 0.93059 | 0:00:02s\n", - "epoch 4 | loss: 0.13227 | valid_accuracy: 0.92534 | 0:00:03s\n", - "epoch 5 | loss: 0.1338 | valid_accuracy: 0.92477 | 0:00:04s\n", - "epoch 6 | loss: 0.13381 | valid_accuracy: 0.93059 | 0:00:05s\n", - "epoch 7 | loss: 0.13119 | valid_accuracy: 0.92705 | 0:00:05s\n", - "epoch 8 | loss: 0.12643 | valid_accuracy: 0.9218 | 0:00:06s\n", - "epoch 9 | loss: 0.11881 | valid_accuracy: 0.93219 | 0:00:07s\n", - "epoch 10 | loss: 0.11992 | valid_accuracy: 0.93847 | 0:00:07s\n", - "epoch 11 | loss: 0.11171 | valid_accuracy: 0.93174 | 0:00:08s\n", - "epoch 12 | loss: 0.10676 | valid_accuracy: 0.93824 | 0:00:09s\n", - "epoch 13 | loss: 0.10288 | valid_accuracy: 0.93847 | 0:00:09s\n", - "epoch 14 | loss: 0.10689 | valid_accuracy: 0.94315 | 0:00:10s\n", - "epoch 15 | loss: 0.10905 | valid_accuracy: 0.93002 | 0:00:11s\n", - "epoch 16 | loss: 0.1084 | valid_accuracy: 0.93345 | 0:00:12s\n", - "epoch 17 | loss: 0.10403 | valid_accuracy: 0.93699 | 0:00:12s\n", - "epoch 18 | loss: 0.10135 | valid_accuracy: 0.9379 | 0:00:13s\n", - "epoch 19 | loss: 0.09833 | valid_accuracy: 0.93174 | 0:00:14s\n", - "epoch 20 | loss: 0.097 | valid_accuracy: 0.93779 | 0:00:14s\n", - "epoch 21 | loss: 0.09527 | valid_accuracy: 0.94007 | 0:00:15s\n", - "epoch 22 | loss: 0.09524 | valid_accuracy: 0.93733 | 0:00:16s\n", - "\n", - "Early stopping occurred at epoch 22 with best_epoch = 14 and best_valid_accuracy = 0.94315\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:49:42,178] Trial 13 finished with value: 0.7233333333333334 and parameters: {'n_d': 48, 'n_a': 40, 'n_steps': 5, 'gamma': 1.24080793750091, 'lambda_sparse': 0.0008657857585898337, 'momentum': 0.25667540149110746}. Best is trial 3 with value: 0.7623080995235575.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.64359 | valid_accuracy: 0.75297 | 0:00:00s\n", - "epoch 1 | loss: 0.29294 | valid_accuracy: 0.75856 | 0:00:01s\n", - "epoch 2 | loss: 0.28205 | valid_accuracy: 0.83973 | 0:00:02s\n", - "epoch 3 | loss: 0.25178 | valid_accuracy: 0.89749 | 0:00:03s\n", - "epoch 4 | loss: 0.23547 | valid_accuracy: 0.9105 | 0:00:04s\n", - "epoch 5 | loss: 0.21306 | valid_accuracy: 0.88995 | 0:00:05s\n", - "epoch 6 | loss: 0.18191 | valid_accuracy: 0.91644 | 0:00:06s\n", - "epoch 7 | loss: 0.17079 | valid_accuracy: 0.92158 | 0:00:07s\n", - "epoch 8 | loss: 0.1593 | valid_accuracy: 0.92066 | 0:00:08s\n", - "epoch 9 | loss: 0.15135 | valid_accuracy: 0.92705 | 0:00:09s\n", - "epoch 10 | loss: 0.14464 | valid_accuracy: 0.93379 | 0:00:11s\n", - "epoch 11 | loss: 0.14458 | valid_accuracy: 0.9347 | 0:00:12s\n", - "epoch 12 | loss: 0.14334 | valid_accuracy: 0.93242 | 0:00:13s\n", - "epoch 13 | loss: 0.14632 | valid_accuracy: 0.92854 | 0:00:14s\n", - "epoch 14 | loss: 0.13872 | valid_accuracy: 0.93014 | 0:00:14s\n", - "epoch 15 | loss: 0.13632 | valid_accuracy: 0.92774 | 0:00:15s\n", - "epoch 16 | loss: 0.1355 | valid_accuracy: 0.92466 | 0:00:16s\n", - "epoch 17 | loss: 0.12837 | valid_accuracy: 0.92043 | 0:00:18s\n", - "epoch 18 | loss: 0.13153 | valid_accuracy: 0.92272 | 0:00:19s\n", - "epoch 19 | loss: 0.138 | valid_accuracy: 0.92568 | 0:00:20s\n", - "\n", - "Early stopping occurred at epoch 19 with best_epoch = 11 and best_valid_accuracy = 0.9347\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:50:02,919] Trial 14 finished with value: 0.7204301075268817 and parameters: {'n_d': 24, 'n_a': 8, 'n_steps': 8, 'gamma': 1.3427992882663162, 'lambda_sparse': 0.00030304348224437444, 'momentum': 0.08946331230660341}. Best is trial 3 with value: 0.7623080995235575.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.3892 | valid_accuracy: 0.6855 | 0:00:00s\n", - "epoch 1 | loss: 0.16781 | valid_accuracy: 0.90982 | 0:00:01s\n", - "epoch 2 | loss: 0.13605 | valid_accuracy: 0.92409 | 0:00:02s\n", - "epoch 3 | loss: 0.12093 | valid_accuracy: 0.92021 | 0:00:02s\n", - "epoch 4 | loss: 0.11758 | valid_accuracy: 0.92979 | 0:00:03s\n", - "epoch 5 | loss: 0.10738 | valid_accuracy: 0.93276 | 0:00:04s\n", - "epoch 6 | loss: 0.10967 | valid_accuracy: 0.93139 | 0:00:05s\n", - "epoch 7 | loss: 0.10541 | valid_accuracy: 0.93676 | 0:00:05s\n", - "epoch 8 | loss: 0.09431 | valid_accuracy: 0.94064 | 0:00:06s\n", - "epoch 9 | loss: 0.09774 | valid_accuracy: 0.94269 | 0:00:07s\n", - "epoch 10 | loss: 0.08963 | valid_accuracy: 0.94521 | 0:00:07s\n", - "epoch 11 | loss: 0.0905 | valid_accuracy: 0.94212 | 0:00:08s\n", - "epoch 12 | loss: 0.08551 | valid_accuracy: 0.94543 | 0:00:09s\n", - "epoch 13 | loss: 0.08468 | valid_accuracy: 0.94258 | 0:00:10s\n", - "epoch 14 | loss: 0.07902 | valid_accuracy: 0.94463 | 0:00:10s\n", - "epoch 15 | loss: 0.08157 | valid_accuracy: 0.94258 | 0:00:11s\n", - "epoch 16 | loss: 0.07662 | valid_accuracy: 0.94874 | 0:00:12s\n", - "epoch 17 | loss: 0.07576 | valid_accuracy: 0.94281 | 0:00:12s\n", - "epoch 18 | loss: 0.07322 | valid_accuracy: 0.946 | 0:00:13s\n", - "epoch 19 | loss: 0.07388 | valid_accuracy: 0.94132 | 0:00:14s\n", - "epoch 20 | loss: 0.07447 | valid_accuracy: 0.94155 | 0:00:15s\n", - "epoch 21 | loss: 0.07069 | valid_accuracy: 0.93927 | 0:00:15s\n", - "epoch 22 | loss: 0.06996 | valid_accuracy: 0.94372 | 0:00:16s\n", - "epoch 23 | loss: 0.07028 | valid_accuracy: 0.94463 | 0:00:17s\n", - "epoch 24 | loss: 0.07007 | valid_accuracy: 0.94406 | 0:00:17s\n", - "\n", - "Early stopping occurred at epoch 24 with best_epoch = 16 and best_valid_accuracy = 0.94874\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:50:21,362] Trial 15 finished with value: 0.7595072308516336 and parameters: {'n_d': 56, 'n_a': 24, 'n_steps': 5, 'gamma': 1.1232319960378998, 'lambda_sparse': 0.0023793537960598537, 'momentum': 0.2415450242787598}. Best is trial 3 with value: 0.7623080995235575.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.52921 | valid_accuracy: 0.646 | 0:00:00s\n", - "epoch 1 | loss: 0.30409 | valid_accuracy: 0.75308 | 0:00:01s\n", - "epoch 2 | loss: 0.24622 | valid_accuracy: 0.85925 | 0:00:02s\n", - "epoch 3 | loss: 0.24997 | valid_accuracy: 0.92089 | 0:00:03s\n", - "epoch 4 | loss: 0.2286 | valid_accuracy: 0.92089 | 0:00:04s\n", - "epoch 5 | loss: 0.20017 | valid_accuracy: 0.92317 | 0:00:05s\n", - "epoch 6 | loss: 0.17618 | valid_accuracy: 0.9145 | 0:00:06s\n", - "epoch 7 | loss: 0.16827 | valid_accuracy: 0.92237 | 0:00:07s\n", - "epoch 8 | loss: 0.16989 | valid_accuracy: 0.91221 | 0:00:08s\n", - "epoch 9 | loss: 0.16782 | valid_accuracy: 0.92934 | 0:00:09s\n", - "epoch 10 | loss: 0.16692 | valid_accuracy: 0.92945 | 0:00:09s\n", - "epoch 11 | loss: 0.1592 | valid_accuracy: 0.93025 | 0:00:10s\n", - "epoch 12 | loss: 0.15833 | valid_accuracy: 0.93208 | 0:00:11s\n", - "epoch 13 | loss: 0.15862 | valid_accuracy: 0.93037 | 0:00:12s\n", - "epoch 14 | loss: 0.15143 | valid_accuracy: 0.93105 | 0:00:13s\n", - "epoch 15 | loss: 0.15252 | valid_accuracy: 0.93562 | 0:00:14s\n", - "epoch 16 | loss: 0.14167 | valid_accuracy: 0.92922 | 0:00:15s\n", - "epoch 17 | loss: 0.14061 | valid_accuracy: 0.92945 | 0:00:16s\n", - "epoch 18 | loss: 0.13331 | valid_accuracy: 0.93299 | 0:00:17s\n", - "epoch 19 | loss: 0.12791 | valid_accuracy: 0.94326 | 0:00:18s\n", - "epoch 20 | loss: 0.12528 | valid_accuracy: 0.93744 | 0:00:18s\n", - "epoch 21 | loss: 0.12422 | valid_accuracy: 0.93539 | 0:00:19s\n", - "epoch 22 | loss: 0.12001 | valid_accuracy: 0.93413 | 0:00:20s\n", - "epoch 23 | loss: 0.1228 | valid_accuracy: 0.9403 | 0:00:21s\n", - "epoch 24 | loss: 0.1167 | valid_accuracy: 0.94292 | 0:00:22s\n", - "epoch 25 | loss: 0.11684 | valid_accuracy: 0.94247 | 0:00:23s\n", - "epoch 26 | loss: 0.11316 | valid_accuracy: 0.94463 | 0:00:24s\n", - "epoch 27 | loss: 0.10914 | valid_accuracy: 0.93847 | 0:00:25s\n", - "epoch 28 | loss: 0.10715 | valid_accuracy: 0.93916 | 0:00:26s\n", - "epoch 29 | loss: 0.10973 | valid_accuracy: 0.93619 | 0:00:27s\n", - "Stop training because you reached max_epochs = 30 with best_epoch = 26 and best_valid_accuracy = 0.94463\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:50:49,085] Trial 16 finished with value: 0.723961297666477 and parameters: {'n_d': 48, 'n_a': 40, 'n_steps': 7, 'gamma': 1.3942734108004724, 'lambda_sparse': 0.0009265347442301014, 'momentum': 0.30312923567815764}. Best is trial 3 with value: 0.7623080995235575.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.37941 | valid_accuracy: 0.86963 | 0:00:00s\n", - "epoch 1 | loss: 0.18481 | valid_accuracy: 0.92477 | 0:00:01s\n", - "epoch 2 | loss: 0.14904 | valid_accuracy: 0.92968 | 0:00:01s\n", - "epoch 3 | loss: 0.13069 | valid_accuracy: 0.91963 | 0:00:02s\n", - "epoch 4 | loss: 0.11683 | valid_accuracy: 0.93196 | 0:00:03s\n", - "epoch 5 | loss: 0.10633 | valid_accuracy: 0.93288 | 0:00:03s\n", - "epoch 6 | loss: 0.09759 | valid_accuracy: 0.94144 | 0:00:04s\n", - "epoch 7 | loss: 0.09288 | valid_accuracy: 0.93242 | 0:00:04s\n", - "epoch 8 | loss: 0.08709 | valid_accuracy: 0.93858 | 0:00:05s\n", - "epoch 9 | loss: 0.08528 | valid_accuracy: 0.94132 | 0:00:06s\n", - "epoch 10 | loss: 0.08487 | valid_accuracy: 0.93801 | 0:00:06s\n", - "epoch 11 | loss: 0.0881 | valid_accuracy: 0.9339 | 0:00:07s\n", - "epoch 12 | loss: 0.08164 | valid_accuracy: 0.93721 | 0:00:07s\n", - "epoch 13 | loss: 0.07956 | valid_accuracy: 0.94235 | 0:00:08s\n", - "epoch 14 | loss: 0.07609 | valid_accuracy: 0.94692 | 0:00:09s\n", - "epoch 15 | loss: 0.07549 | valid_accuracy: 0.94578 | 0:00:09s\n", - "epoch 16 | loss: 0.06854 | valid_accuracy: 0.9411 | 0:00:10s\n", - "epoch 17 | loss: 0.06796 | valid_accuracy: 0.93938 | 0:00:10s\n", - "epoch 18 | loss: 0.06769 | valid_accuracy: 0.93858 | 0:00:11s\n", - "epoch 19 | loss: 0.0698 | valid_accuracy: 0.94189 | 0:00:12s\n", - "epoch 20 | loss: 0.06964 | valid_accuracy: 0.94543 | 0:00:12s\n", - "epoch 21 | loss: 0.06632 | valid_accuracy: 0.93059 | 0:00:13s\n", - "epoch 22 | loss: 0.06611 | valid_accuracy: 0.93961 | 0:00:13s\n", - "\n", - "Early stopping occurred at epoch 22 with best_epoch = 14 and best_valid_accuracy = 0.94692\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:51:03,377] Trial 17 finished with value: 0.7446457990115322 and parameters: {'n_d': 32, 'n_a': 24, 'n_steps': 4, 'gamma': 1.1229176751032102, 'lambda_sparse': 0.003743309561243855, 'momentum': 0.1774269738540521}. Best is trial 3 with value: 0.7623080995235575.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.624 | valid_accuracy: 0.69977 | 0:00:01s\n", - "epoch 1 | loss: 0.27796 | valid_accuracy: 0.81005 | 0:00:02s\n", - "epoch 2 | loss: 0.2435 | valid_accuracy: 0.9113 | 0:00:03s\n", - "epoch 3 | loss: 0.22003 | valid_accuracy: 0.9153 | 0:00:04s\n", - "epoch 4 | loss: 0.21892 | valid_accuracy: 0.91906 | 0:00:05s\n", - "epoch 5 | loss: 0.20274 | valid_accuracy: 0.90479 | 0:00:06s\n", - "epoch 6 | loss: 0.17367 | valid_accuracy: 0.91781 | 0:00:07s\n", - "epoch 7 | loss: 0.16973 | valid_accuracy: 0.91062 | 0:00:08s\n", - "epoch 8 | loss: 0.16077 | valid_accuracy: 0.9234 | 0:00:10s\n", - "epoch 9 | loss: 0.14759 | valid_accuracy: 0.93105 | 0:00:11s\n", - "epoch 10 | loss: 0.14162 | valid_accuracy: 0.93151 | 0:00:12s\n", - "epoch 11 | loss: 0.14227 | valid_accuracy: 0.93505 | 0:00:13s\n", - "epoch 12 | loss: 0.13655 | valid_accuracy: 0.93653 | 0:00:14s\n", - "epoch 13 | loss: 0.13343 | valid_accuracy: 0.93231 | 0:00:15s\n", - "epoch 14 | loss: 0.13448 | valid_accuracy: 0.92454 | 0:00:16s\n", - "epoch 15 | loss: 0.12931 | valid_accuracy: 0.93333 | 0:00:17s\n", - "epoch 16 | loss: 0.12407 | valid_accuracy: 0.9411 | 0:00:18s\n", - "epoch 17 | loss: 0.12118 | valid_accuracy: 0.93413 | 0:00:19s\n", - "epoch 18 | loss: 0.12116 | valid_accuracy: 0.92979 | 0:00:21s\n", - "epoch 19 | loss: 0.12082 | valid_accuracy: 0.93505 | 0:00:22s\n", - "epoch 20 | loss: 0.12163 | valid_accuracy: 0.93539 | 0:00:23s\n", - "epoch 21 | loss: 0.11732 | valid_accuracy: 0.93756 | 0:00:24s\n", - "epoch 22 | loss: 0.11412 | valid_accuracy: 0.93573 | 0:00:25s\n", - "epoch 23 | loss: 0.10798 | valid_accuracy: 0.92945 | 0:00:26s\n", - "epoch 24 | loss: 0.10756 | valid_accuracy: 0.94395 | 0:00:27s\n", - "epoch 25 | loss: 0.10692 | valid_accuracy: 0.94212 | 0:00:28s\n", - "epoch 26 | loss: 0.10502 | valid_accuracy: 0.93425 | 0:00:29s\n", - "epoch 27 | loss: 0.1035 | valid_accuracy: 0.94155 | 0:00:30s\n", - "epoch 28 | loss: 0.10731 | valid_accuracy: 0.94247 | 0:00:31s\n", - "epoch 29 | loss: 0.10359 | valid_accuracy: 0.93824 | 0:00:33s\n", - "Stop training because you reached max_epochs = 30 with best_epoch = 24 and best_valid_accuracy = 0.94395\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:51:37,278] Trial 18 finished with value: 0.7384123601491742 and parameters: {'n_d': 16, 'n_a': 16, 'n_steps': 9, 'gamma': 1.1342940581443615, 'lambda_sparse': 0.0003186419211415119, 'momentum': 0.12004229695216845}. Best is trial 3 with value: 0.7623080995235575.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.35191 | valid_accuracy: 0.7661 | 0:00:00s\n", - "epoch 1 | loss: 0.22988 | valid_accuracy: 0.88995 | 0:00:01s\n", - "epoch 2 | loss: 0.19959 | valid_accuracy: 0.91279 | 0:00:01s\n", - "epoch 3 | loss: 0.17566 | valid_accuracy: 0.92203 | 0:00:02s\n", - "epoch 4 | loss: 0.14589 | valid_accuracy: 0.9274 | 0:00:03s\n", - "epoch 5 | loss: 0.12829 | valid_accuracy: 0.93881 | 0:00:03s\n", - "epoch 6 | loss: 0.11731 | valid_accuracy: 0.93699 | 0:00:04s\n", - "epoch 7 | loss: 0.11036 | valid_accuracy: 0.93927 | 0:00:04s\n", - "epoch 8 | loss: 0.11062 | valid_accuracy: 0.93858 | 0:00:05s\n", - "epoch 9 | loss: 0.09907 | valid_accuracy: 0.9403 | 0:00:06s\n", - "epoch 10 | loss: 0.09521 | valid_accuracy: 0.94486 | 0:00:06s\n", - "epoch 11 | loss: 0.08668 | valid_accuracy: 0.94304 | 0:00:07s\n", - "epoch 12 | loss: 0.08419 | valid_accuracy: 0.93744 | 0:00:07s\n", - "epoch 13 | loss: 0.07734 | valid_accuracy: 0.94018 | 0:00:08s\n", - "epoch 14 | loss: 0.0732 | valid_accuracy: 0.93779 | 0:00:09s\n", - "epoch 15 | loss: 0.07284 | valid_accuracy: 0.93368 | 0:00:09s\n", - "epoch 16 | loss: 0.07146 | valid_accuracy: 0.93995 | 0:00:10s\n", - "epoch 17 | loss: 0.07099 | valid_accuracy: 0.94018 | 0:00:10s\n", - "epoch 18 | loss: 0.06689 | valid_accuracy: 0.94064 | 0:00:11s\n", - "\n", - "Early stopping occurred at epoch 18 with best_epoch = 10 and best_valid_accuracy = 0.94486\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:51:49,153] Trial 19 finished with value: 0.7410187667560322 and parameters: {'n_d': 48, 'n_a': 48, 'n_steps': 4, 'gamma': 1.313378794087976, 'lambda_sparse': 0.0006934858536024428, 'momentum': 0.030467246352504904}. Best is trial 3 with value: 0.7623080995235575.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.45767 | valid_accuracy: 0.78322 | 0:00:00s\n", - "epoch 1 | loss: 0.18894 | valid_accuracy: 0.86495 | 0:00:01s\n", - "epoch 2 | loss: 0.14092 | valid_accuracy: 0.93185 | 0:00:02s\n", - "epoch 3 | loss: 0.12462 | valid_accuracy: 0.93527 | 0:00:03s\n", - "epoch 4 | loss: 0.11243 | valid_accuracy: 0.92637 | 0:00:03s\n", - "epoch 5 | loss: 0.09825 | valid_accuracy: 0.92557 | 0:00:04s\n", - "epoch 6 | loss: 0.09195 | valid_accuracy: 0.93916 | 0:00:05s\n", - "epoch 7 | loss: 0.08764 | valid_accuracy: 0.93779 | 0:00:06s\n", - "epoch 8 | loss: 0.0824 | valid_accuracy: 0.94007 | 0:00:07s\n", - "epoch 9 | loss: 0.08075 | valid_accuracy: 0.93653 | 0:00:08s\n", - "epoch 10 | loss: 0.0789 | valid_accuracy: 0.93779 | 0:00:08s\n", - "epoch 11 | loss: 0.08072 | valid_accuracy: 0.94098 | 0:00:09s\n", - "epoch 12 | loss: 0.0745 | valid_accuracy: 0.94189 | 0:00:10s\n", - "epoch 13 | loss: 0.07463 | valid_accuracy: 0.94258 | 0:00:11s\n", - "epoch 14 | loss: 0.07059 | valid_accuracy: 0.94201 | 0:00:12s\n", - "epoch 15 | loss: 0.07092 | valid_accuracy: 0.94121 | 0:00:12s\n", - "epoch 16 | loss: 0.06723 | valid_accuracy: 0.93916 | 0:00:13s\n", - "epoch 17 | loss: 0.06698 | valid_accuracy: 0.94155 | 0:00:14s\n", - "epoch 18 | loss: 0.06673 | valid_accuracy: 0.94372 | 0:00:15s\n", - "epoch 19 | loss: 0.06547 | valid_accuracy: 0.94132 | 0:00:16s\n", - "epoch 20 | loss: 0.06276 | valid_accuracy: 0.93801 | 0:00:16s\n", - "epoch 21 | loss: 0.06269 | valid_accuracy: 0.94098 | 0:00:17s\n", - "epoch 22 | loss: 0.05956 | valid_accuracy: 0.94281 | 0:00:18s\n", - "epoch 23 | loss: 0.05975 | valid_accuracy: 0.94475 | 0:00:19s\n", - "epoch 24 | loss: 0.0656 | valid_accuracy: 0.93904 | 0:00:20s\n", - "epoch 25 | loss: 0.0627 | valid_accuracy: 0.94315 | 0:00:20s\n", - "epoch 26 | loss: 0.06008 | valid_accuracy: 0.94452 | 0:00:21s\n", - "epoch 27 | loss: 0.06152 | valid_accuracy: 0.9411 | 0:00:22s\n", - "epoch 28 | loss: 0.0625 | valid_accuracy: 0.93756 | 0:00:23s\n", - "epoch 29 | loss: 0.05881 | valid_accuracy: 0.94361 | 0:00:24s\n", - "Stop training because you reached max_epochs = 30 with best_epoch = 23 and best_valid_accuracy = 0.94475\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:52:13,807] Trial 20 finished with value: 0.7259343148357871 and parameters: {'n_d': 56, 'n_a': 8, 'n_steps': 6, 'gamma': 1.010033289129168, 'lambda_sparse': 0.00024163285968083325, 'momentum': 0.21395506234129033}. Best is trial 3 with value: 0.7623080995235575.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.40211 | valid_accuracy: 0.76473 | 0:00:00s\n", - "epoch 1 | loss: 0.18057 | valid_accuracy: 0.92203 | 0:00:01s\n", - "epoch 2 | loss: 0.15386 | valid_accuracy: 0.9121 | 0:00:02s\n", - "epoch 3 | loss: 0.14088 | valid_accuracy: 0.9226 | 0:00:02s\n", - "epoch 4 | loss: 0.13926 | valid_accuracy: 0.93082 | 0:00:03s\n", - "epoch 5 | loss: 0.12709 | valid_accuracy: 0.93025 | 0:00:04s\n", - "epoch 6 | loss: 0.12436 | valid_accuracy: 0.93573 | 0:00:05s\n", - "epoch 7 | loss: 0.11683 | valid_accuracy: 0.93276 | 0:00:05s\n", - "epoch 8 | loss: 0.10997 | valid_accuracy: 0.93733 | 0:00:06s\n", - "epoch 9 | loss: 0.10217 | valid_accuracy: 0.93539 | 0:00:07s\n", - "epoch 10 | loss: 0.09985 | valid_accuracy: 0.92991 | 0:00:07s\n", - "epoch 11 | loss: 0.09574 | valid_accuracy: 0.93596 | 0:00:08s\n", - "epoch 12 | loss: 0.09224 | valid_accuracy: 0.93813 | 0:00:09s\n", - "epoch 13 | loss: 0.08921 | valid_accuracy: 0.93094 | 0:00:10s\n", - "epoch 14 | loss: 0.08516 | valid_accuracy: 0.93082 | 0:00:10s\n", - "epoch 15 | loss: 0.08748 | valid_accuracy: 0.93813 | 0:00:11s\n", - "epoch 16 | loss: 0.08493 | valid_accuracy: 0.93345 | 0:00:12s\n", - "epoch 17 | loss: 0.08119 | valid_accuracy: 0.93493 | 0:00:12s\n", - "epoch 18 | loss: 0.0818 | valid_accuracy: 0.93824 | 0:00:13s\n", - "epoch 19 | loss: 0.07903 | valid_accuracy: 0.93744 | 0:00:14s\n", - "epoch 20 | loss: 0.07899 | valid_accuracy: 0.93037 | 0:00:14s\n", - "epoch 21 | loss: 0.07789 | valid_accuracy: 0.93756 | 0:00:15s\n", - "epoch 22 | loss: 0.07829 | valid_accuracy: 0.93813 | 0:00:16s\n", - "epoch 23 | loss: 0.07602 | valid_accuracy: 0.93539 | 0:00:17s\n", - "epoch 24 | loss: 0.07435 | valid_accuracy: 0.93493 | 0:00:17s\n", - "epoch 25 | loss: 0.07385 | valid_accuracy: 0.93904 | 0:00:18s\n", - "epoch 26 | loss: 0.07141 | valid_accuracy: 0.94155 | 0:00:19s\n", - "epoch 27 | loss: 0.06869 | valid_accuracy: 0.93847 | 0:00:20s\n", - "epoch 28 | loss: 0.06624 | valid_accuracy: 0.93938 | 0:00:20s\n", - "epoch 29 | loss: 0.067 | valid_accuracy: 0.93174 | 0:00:21s\n", - "Stop training because you reached max_epochs = 30 with best_epoch = 26 and best_valid_accuracy = 0.94155\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:52:35,793] Trial 21 finished with value: 0.7229437229437229 and parameters: {'n_d': 56, 'n_a': 24, 'n_steps': 5, 'gamma': 1.1633563297268914, 'lambda_sparse': 0.0032084615724975586, 'momentum': 0.2449184141264009}. Best is trial 3 with value: 0.7623080995235575.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.41115 | valid_accuracy: 0.72078 | 0:00:00s\n", - "epoch 1 | loss: 0.17945 | valid_accuracy: 0.88881 | 0:00:01s\n", - "epoch 2 | loss: 0.14713 | valid_accuracy: 0.91324 | 0:00:02s\n", - "epoch 3 | loss: 0.13612 | valid_accuracy: 0.92352 | 0:00:02s\n", - "epoch 4 | loss: 0.12366 | valid_accuracy: 0.93242 | 0:00:03s\n", - "epoch 5 | loss: 0.1203 | valid_accuracy: 0.92979 | 0:00:04s\n", - "epoch 6 | loss: 0.11271 | valid_accuracy: 0.93265 | 0:00:04s\n", - "epoch 7 | loss: 0.10493 | valid_accuracy: 0.93447 | 0:00:05s\n", - "epoch 8 | loss: 0.10303 | valid_accuracy: 0.9371 | 0:00:06s\n", - "epoch 9 | loss: 0.10188 | valid_accuracy: 0.93927 | 0:00:07s\n", - "epoch 10 | loss: 0.10548 | valid_accuracy: 0.93653 | 0:00:07s\n", - "epoch 11 | loss: 0.09854 | valid_accuracy: 0.93721 | 0:00:08s\n", - "epoch 12 | loss: 0.09401 | valid_accuracy: 0.93756 | 0:00:09s\n", - "epoch 13 | loss: 0.09029 | valid_accuracy: 0.94189 | 0:00:09s\n", - "epoch 14 | loss: 0.0828 | valid_accuracy: 0.9411 | 0:00:10s\n", - "epoch 15 | loss: 0.08584 | valid_accuracy: 0.93916 | 0:00:11s\n", - "epoch 16 | loss: 0.08464 | valid_accuracy: 0.94247 | 0:00:12s\n", - "epoch 17 | loss: 0.08248 | valid_accuracy: 0.93527 | 0:00:12s\n", - "epoch 18 | loss: 0.08326 | valid_accuracy: 0.94247 | 0:00:13s\n", - "epoch 19 | loss: 0.07868 | valid_accuracy: 0.94247 | 0:00:14s\n", - "epoch 20 | loss: 0.08013 | valid_accuracy: 0.94349 | 0:00:15s\n", - "epoch 21 | loss: 0.07806 | valid_accuracy: 0.94053 | 0:00:15s\n", - "epoch 22 | loss: 0.07372 | valid_accuracy: 0.94064 | 0:00:16s\n", - "epoch 23 | loss: 0.0723 | valid_accuracy: 0.94132 | 0:00:17s\n", - "epoch 24 | loss: 0.07422 | valid_accuracy: 0.93733 | 0:00:17s\n", - "epoch 25 | loss: 0.08092 | valid_accuracy: 0.92854 | 0:00:18s\n", - "epoch 26 | loss: 0.07504 | valid_accuracy: 0.94018 | 0:00:19s\n", - "epoch 27 | loss: 0.07517 | valid_accuracy: 0.94167 | 0:00:20s\n", - "epoch 28 | loss: 0.06732 | valid_accuracy: 0.93938 | 0:00:20s\n", - "\n", - "Early stopping occurred at epoch 28 with best_epoch = 20 and best_valid_accuracy = 0.94349\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:52:57,022] Trial 22 finished with value: 0.7173043974871502 and parameters: {'n_d': 64, 'n_a': 24, 'n_steps': 5, 'gamma': 1.0961922879933907, 'lambda_sparse': 0.0016155377070117766, 'momentum': 0.17094765515429317}. Best is trial 3 with value: 0.7623080995235575.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.41821 | valid_accuracy: 0.84372 | 0:00:00s\n", - "epoch 1 | loss: 0.20584 | valid_accuracy: 0.87728 | 0:00:01s\n", - "epoch 2 | loss: 0.16455 | valid_accuracy: 0.92226 | 0:00:01s\n", - "epoch 3 | loss: 0.13399 | valid_accuracy: 0.92466 | 0:00:02s\n", - "epoch 4 | loss: 0.11364 | valid_accuracy: 0.93276 | 0:00:03s\n", - "epoch 5 | loss: 0.10375 | valid_accuracy: 0.93584 | 0:00:03s\n", - "epoch 6 | loss: 0.10204 | valid_accuracy: 0.93345 | 0:00:04s\n", - "epoch 7 | loss: 0.09306 | valid_accuracy: 0.92717 | 0:00:04s\n", - "epoch 8 | loss: 0.09096 | valid_accuracy: 0.93368 | 0:00:05s\n", - "epoch 9 | loss: 0.09034 | valid_accuracy: 0.93676 | 0:00:06s\n", - "epoch 10 | loss: 0.08428 | valid_accuracy: 0.93231 | 0:00:06s\n", - "epoch 11 | loss: 0.08575 | valid_accuracy: 0.94269 | 0:00:07s\n", - "epoch 12 | loss: 0.0817 | valid_accuracy: 0.94189 | 0:00:07s\n", - "epoch 13 | loss: 0.07896 | valid_accuracy: 0.93676 | 0:00:08s\n", - "epoch 14 | loss: 0.07372 | valid_accuracy: 0.9395 | 0:00:09s\n", - "epoch 15 | loss: 0.07335 | valid_accuracy: 0.93813 | 0:00:09s\n", - "epoch 16 | loss: 0.07094 | valid_accuracy: 0.93436 | 0:00:10s\n", - "epoch 17 | loss: 0.06639 | valid_accuracy: 0.92808 | 0:00:10s\n", - "epoch 18 | loss: 0.06913 | valid_accuracy: 0.93425 | 0:00:11s\n", - "epoch 19 | loss: 0.07044 | valid_accuracy: 0.93333 | 0:00:12s\n", - "\n", - "Early stopping occurred at epoch 19 with best_epoch = 11 and best_valid_accuracy = 0.94269\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:53:09,553] Trial 23 finished with value: 0.732409381663113 and parameters: {'n_d': 48, 'n_a': 32, 'n_steps': 4, 'gamma': 1.242298444344926, 'lambda_sparse': 0.002445878111790842, 'momentum': 0.24208438843370517}. Best is trial 3 with value: 0.7623080995235575.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.4941 | valid_accuracy: 0.69349 | 0:00:00s\n", - "epoch 1 | loss: 0.22395 | valid_accuracy: 0.88105 | 0:00:01s\n", - "epoch 2 | loss: 0.18918 | valid_accuracy: 0.90365 | 0:00:02s\n", - "epoch 3 | loss: 0.15504 | valid_accuracy: 0.91655 | 0:00:03s\n", - "epoch 4 | loss: 0.14261 | valid_accuracy: 0.92785 | 0:00:04s\n", - "epoch 5 | loss: 0.14548 | valid_accuracy: 0.92763 | 0:00:05s\n", - "epoch 6 | loss: 0.12783 | valid_accuracy: 0.93082 | 0:00:06s\n", - "epoch 7 | loss: 0.12422 | valid_accuracy: 0.92614 | 0:00:07s\n", - "epoch 8 | loss: 0.11942 | valid_accuracy: 0.94007 | 0:00:08s\n", - "epoch 9 | loss: 0.11568 | valid_accuracy: 0.94018 | 0:00:09s\n", - "epoch 10 | loss: 0.10884 | valid_accuracy: 0.93687 | 0:00:09s\n", - "epoch 11 | loss: 0.1049 | valid_accuracy: 0.93345 | 0:00:10s\n", - "epoch 12 | loss: 0.1031 | valid_accuracy: 0.93128 | 0:00:11s\n", - "epoch 13 | loss: 0.09914 | valid_accuracy: 0.93607 | 0:00:12s\n", - "epoch 14 | loss: 0.10053 | valid_accuracy: 0.94201 | 0:00:13s\n", - "epoch 15 | loss: 0.09411 | valid_accuracy: 0.93836 | 0:00:14s\n", - "epoch 16 | loss: 0.09745 | valid_accuracy: 0.93493 | 0:00:15s\n", - "epoch 17 | loss: 0.09518 | valid_accuracy: 0.93938 | 0:00:16s\n", - "epoch 18 | loss: 0.09151 | valid_accuracy: 0.93881 | 0:00:17s\n", - "epoch 19 | loss: 0.08873 | valid_accuracy: 0.9371 | 0:00:18s\n", - "epoch 20 | loss: 0.08906 | valid_accuracy: 0.92979 | 0:00:18s\n", - "epoch 21 | loss: 0.08373 | valid_accuracy: 0.93733 | 0:00:19s\n", - "epoch 22 | loss: 0.0792 | valid_accuracy: 0.93562 | 0:00:20s\n", - "\n", - "Early stopping occurred at epoch 22 with best_epoch = 14 and best_valid_accuracy = 0.94201\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:53:30,887] Trial 24 finished with value: 0.7180910099889012 and parameters: {'n_d': 56, 'n_a': 16, 'n_steps': 7, 'gamma': 1.0929277549886702, 'lambda_sparse': 0.005229229230684199, 'momentum': 0.28408237254426966}. Best is trial 3 with value: 0.7623080995235575.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.34691 | valid_accuracy: 0.82911 | 0:00:00s\n", - "epoch 1 | loss: 0.18238 | valid_accuracy: 0.88413 | 0:00:00s\n", - "epoch 2 | loss: 0.14372 | valid_accuracy: 0.91416 | 0:00:01s\n", - "epoch 3 | loss: 0.12771 | valid_accuracy: 0.93185 | 0:00:02s\n", - "epoch 4 | loss: 0.11084 | valid_accuracy: 0.93128 | 0:00:02s\n", - "epoch 5 | loss: 0.0963 | valid_accuracy: 0.93539 | 0:00:03s\n", - "epoch 6 | loss: 0.09497 | valid_accuracy: 0.93379 | 0:00:03s\n", - "epoch 7 | loss: 0.08821 | valid_accuracy: 0.92751 | 0:00:04s\n", - "epoch 8 | loss: 0.08247 | valid_accuracy: 0.9371 | 0:00:04s\n", - "epoch 9 | loss: 0.08307 | valid_accuracy: 0.93425 | 0:00:05s\n", - "epoch 10 | loss: 0.07629 | valid_accuracy: 0.93311 | 0:00:05s\n", - "epoch 11 | loss: 0.06965 | valid_accuracy: 0.93196 | 0:00:06s\n", - "epoch 12 | loss: 0.0691 | valid_accuracy: 0.93219 | 0:00:06s\n", - "epoch 13 | loss: 0.0746 | valid_accuracy: 0.94247 | 0:00:07s\n", - "epoch 14 | loss: 0.06728 | valid_accuracy: 0.92877 | 0:00:07s\n", - "epoch 15 | loss: 0.06603 | valid_accuracy: 0.93744 | 0:00:08s\n", - "epoch 16 | loss: 0.06397 | valid_accuracy: 0.93584 | 0:00:08s\n", - "epoch 17 | loss: 0.0613 | valid_accuracy: 0.9282 | 0:00:09s\n", - "epoch 18 | loss: 0.06263 | valid_accuracy: 0.93436 | 0:00:09s\n", - "epoch 19 | loss: 0.05693 | valid_accuracy: 0.9363 | 0:00:10s\n", - "epoch 20 | loss: 0.05689 | valid_accuracy: 0.93356 | 0:00:10s\n", - "epoch 21 | loss: 0.0577 | valid_accuracy: 0.93813 | 0:00:11s\n", - "\n", - "Early stopping occurred at epoch 21 with best_epoch = 13 and best_valid_accuracy = 0.94247\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:53:42,283] Trial 25 finished with value: 0.7230769230769231 and parameters: {'n_d': 32, 'n_a': 24, 'n_steps': 3, 'gamma': 1.4670609710214029, 'lambda_sparse': 0.0012766533397569697, 'momentum': 0.15133588150594873}. Best is trial 3 with value: 0.7623080995235575.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.46262 | valid_accuracy: 0.79795 | 0:00:00s\n", - "epoch 1 | loss: 0.23106 | valid_accuracy: 0.8282 | 0:00:01s\n", - "epoch 2 | loss: 0.18701 | valid_accuracy: 0.86221 | 0:00:02s\n", - "epoch 3 | loss: 0.17376 | valid_accuracy: 0.89543 | 0:00:03s\n", - "epoch 4 | loss: 0.15922 | valid_accuracy: 0.92911 | 0:00:03s\n", - "epoch 5 | loss: 0.14441 | valid_accuracy: 0.93413 | 0:00:04s\n", - "epoch 6 | loss: 0.1343 | valid_accuracy: 0.94189 | 0:00:05s\n", - "epoch 7 | loss: 0.13649 | valid_accuracy: 0.9363 | 0:00:06s\n", - "epoch 8 | loss: 0.1306 | valid_accuracy: 0.93322 | 0:00:07s\n", - "epoch 9 | loss: 0.1252 | valid_accuracy: 0.93002 | 0:00:07s\n", - "epoch 10 | loss: 0.12126 | valid_accuracy: 0.93699 | 0:00:08s\n", - "epoch 11 | loss: 0.12 | valid_accuracy: 0.93071 | 0:00:09s\n", - "epoch 12 | loss: 0.11542 | valid_accuracy: 0.93151 | 0:00:10s\n", - "epoch 13 | loss: 0.11732 | valid_accuracy: 0.93447 | 0:00:11s\n", - "epoch 14 | loss: 0.1122 | valid_accuracy: 0.93425 | 0:00:11s\n", - "\n", - "Early stopping occurred at epoch 14 with best_epoch = 6 and best_valid_accuracy = 0.94189\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:53:54,775] Trial 26 finished with value: 0.7342036553524804 and parameters: {'n_d': 40, 'n_a': 16, 'n_steps': 6, 'gamma': 1.1950236581346791, 'lambda_sparse': 0.0023400383905811677, 'momentum': 0.19507415200606232}. Best is trial 3 with value: 0.7623080995235575.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.46816 | valid_accuracy: 0.77009 | 0:00:00s\n", - "epoch 1 | loss: 0.23461 | valid_accuracy: 0.87968 | 0:00:01s\n", - "epoch 2 | loss: 0.19834 | valid_accuracy: 0.89281 | 0:00:02s\n", - "epoch 3 | loss: 0.1735 | valid_accuracy: 0.93459 | 0:00:02s\n", - "epoch 4 | loss: 0.16171 | valid_accuracy: 0.92626 | 0:00:03s\n", - "epoch 5 | loss: 0.15309 | valid_accuracy: 0.92135 | 0:00:04s\n", - "epoch 6 | loss: 0.14653 | valid_accuracy: 0.91655 | 0:00:04s\n", - "epoch 7 | loss: 0.15426 | valid_accuracy: 0.92991 | 0:00:05s\n", - "epoch 8 | loss: 0.1395 | valid_accuracy: 0.93231 | 0:00:06s\n", - "epoch 9 | loss: 0.13185 | valid_accuracy: 0.93037 | 0:00:07s\n", - "epoch 10 | loss: 0.12759 | valid_accuracy: 0.93607 | 0:00:07s\n", - "epoch 11 | loss: 0.11971 | valid_accuracy: 0.93447 | 0:00:08s\n", - "epoch 12 | loss: 0.11881 | valid_accuracy: 0.93984 | 0:00:09s\n", - "epoch 13 | loss: 0.11442 | valid_accuracy: 0.9355 | 0:00:10s\n", - "epoch 14 | loss: 0.10914 | valid_accuracy: 0.93242 | 0:00:10s\n", - "epoch 15 | loss: 0.11241 | valid_accuracy: 0.93744 | 0:00:11s\n", - "epoch 16 | loss: 0.12271 | valid_accuracy: 0.93607 | 0:00:12s\n", - "epoch 17 | loss: 0.11658 | valid_accuracy: 0.93276 | 0:00:12s\n", - "epoch 18 | loss: 0.11444 | valid_accuracy: 0.93584 | 0:00:13s\n", - "epoch 19 | loss: 0.10761 | valid_accuracy: 0.92774 | 0:00:14s\n", - "epoch 20 | loss: 0.10325 | valid_accuracy: 0.93596 | 0:00:15s\n", - "\n", - "Early stopping occurred at epoch 20 with best_epoch = 12 and best_valid_accuracy = 0.93984\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:54:10,273] Trial 27 finished with value: 0.7083563918096292 and parameters: {'n_d': 56, 'n_a': 32, 'n_steps': 5, 'gamma': 1.3594035424449564, 'lambda_sparse': 0.0004512588696281365, 'momentum': 0.09526450591717059}. Best is trial 3 with value: 0.7623080995235575.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.34807 | valid_accuracy: 0.87489 | 0:00:00s\n", - "epoch 1 | loss: 0.17781 | valid_accuracy: 0.92797 | 0:00:01s\n", - "epoch 2 | loss: 0.13428 | valid_accuracy: 0.92694 | 0:00:01s\n", - "epoch 3 | loss: 0.12184 | valid_accuracy: 0.93984 | 0:00:02s\n", - "epoch 4 | loss: 0.10939 | valid_accuracy: 0.93333 | 0:00:03s\n", - "epoch 5 | loss: 0.09965 | valid_accuracy: 0.93893 | 0:00:03s\n", - "epoch 6 | loss: 0.09667 | valid_accuracy: 0.94087 | 0:00:04s\n", - "epoch 7 | loss: 0.09394 | valid_accuracy: 0.9403 | 0:00:04s\n", - "epoch 8 | loss: 0.0925 | valid_accuracy: 0.93744 | 0:00:05s\n", - "epoch 9 | loss: 0.08903 | valid_accuracy: 0.94189 | 0:00:06s\n", - "epoch 10 | loss: 0.08595 | valid_accuracy: 0.9339 | 0:00:06s\n", - "epoch 11 | loss: 0.0921 | valid_accuracy: 0.9363 | 0:00:07s\n", - "epoch 12 | loss: 0.09322 | valid_accuracy: 0.93938 | 0:00:07s\n", - "epoch 13 | loss: 0.08945 | valid_accuracy: 0.93447 | 0:00:08s\n", - "epoch 14 | loss: 0.09022 | valid_accuracy: 0.92991 | 0:00:09s\n", - "epoch 15 | loss: 0.08631 | valid_accuracy: 0.92854 | 0:00:09s\n", - "epoch 16 | loss: 0.08414 | valid_accuracy: 0.92957 | 0:00:10s\n", - "epoch 17 | loss: 0.08354 | valid_accuracy: 0.93584 | 0:00:10s\n", - "\n", - "Early stopping occurred at epoch 17 with best_epoch = 9 and best_valid_accuracy = 0.94189\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:54:21,479] Trial 28 finished with value: 0.7279529663281668 and parameters: {'n_d': 16, 'n_a': 40, 'n_steps': 4, 'gamma': 1.0692005047122888, 'lambda_sparse': 0.0006747716169120555, 'momentum': 0.22245931557841697}. Best is trial 3 with value: 0.7623080995235575.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.59666 | valid_accuracy: 0.79943 | 0:00:01s\n", - "epoch 1 | loss: 0.45964 | valid_accuracy: 0.69498 | 0:00:02s\n", - "epoch 2 | loss: 0.38141 | valid_accuracy: 0.89121 | 0:00:03s\n", - "epoch 3 | loss: 0.25975 | valid_accuracy: 0.86758 | 0:00:04s\n", - "epoch 4 | loss: 0.22724 | valid_accuracy: 0.89589 | 0:00:05s\n", - "epoch 5 | loss: 0.20368 | valid_accuracy: 0.91701 | 0:00:06s\n", - "epoch 6 | loss: 0.18894 | valid_accuracy: 0.89874 | 0:00:07s\n", - "epoch 7 | loss: 0.1854 | valid_accuracy: 0.92295 | 0:00:08s\n", - "epoch 8 | loss: 0.16135 | valid_accuracy: 0.91849 | 0:00:10s\n", - "epoch 9 | loss: 0.15377 | valid_accuracy: 0.91119 | 0:00:11s\n", - "epoch 10 | loss: 0.1584 | valid_accuracy: 0.89829 | 0:00:12s\n", - "epoch 11 | loss: 0.15127 | valid_accuracy: 0.93128 | 0:00:13s\n", - "epoch 12 | loss: 0.13917 | valid_accuracy: 0.93562 | 0:00:14s\n", - "epoch 13 | loss: 0.13326 | valid_accuracy: 0.93881 | 0:00:15s\n", - "epoch 14 | loss: 0.12677 | valid_accuracy: 0.93025 | 0:00:16s\n", - "epoch 15 | loss: 0.1305 | valid_accuracy: 0.93596 | 0:00:17s\n", - "epoch 16 | loss: 0.12925 | valid_accuracy: 0.9266 | 0:00:18s\n", - "epoch 17 | loss: 0.13814 | valid_accuracy: 0.93379 | 0:00:20s\n", - "epoch 18 | loss: 0.13049 | valid_accuracy: 0.93619 | 0:00:21s\n", - "epoch 19 | loss: 0.12173 | valid_accuracy: 0.93162 | 0:00:22s\n", - "epoch 20 | loss: 0.1178 | valid_accuracy: 0.93356 | 0:00:23s\n", - "epoch 21 | loss: 0.11358 | valid_accuracy: 0.93653 | 0:00:24s\n", - "\n", - "Early stopping occurred at epoch 21 with best_epoch = 13 and best_valid_accuracy = 0.93881\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:54:46,785] Trial 29 finished with value: 0.7102702702702702 and parameters: {'n_d': 48, 'n_a': 24, 'n_steps': 9, 'gamma': 1.5343779318950561, 'lambda_sparse': 0.00016300834886699597, 'momentum': 0.2649384617298923}. Best is trial 3 with value: 0.7623080995235575.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.58329 | valid_accuracy: 0.81941 | 0:00:00s\n", - "epoch 1 | loss: 0.29285 | valid_accuracy: 0.86336 | 0:00:01s\n", - "epoch 2 | loss: 0.22696 | valid_accuracy: 0.91461 | 0:00:02s\n", - "epoch 3 | loss: 0.18988 | valid_accuracy: 0.90925 | 0:00:03s\n", - "epoch 4 | loss: 0.18638 | valid_accuracy: 0.90811 | 0:00:04s\n", - "epoch 5 | loss: 0.16789 | valid_accuracy: 0.93516 | 0:00:05s\n", - "epoch 6 | loss: 0.16765 | valid_accuracy: 0.88025 | 0:00:06s\n", - "epoch 7 | loss: 0.15636 | valid_accuracy: 0.9355 | 0:00:07s\n", - "epoch 8 | loss: 0.14617 | valid_accuracy: 0.92957 | 0:00:08s\n", - "epoch 9 | loss: 0.14327 | valid_accuracy: 0.92363 | 0:00:09s\n", - "epoch 10 | loss: 0.14005 | valid_accuracy: 0.92203 | 0:00:09s\n", - "epoch 11 | loss: 0.14772 | valid_accuracy: 0.93185 | 0:00:10s\n", - "epoch 12 | loss: 0.1399 | valid_accuracy: 0.93927 | 0:00:11s\n", - "epoch 13 | loss: 0.12887 | valid_accuracy: 0.94258 | 0:00:12s\n", - "epoch 14 | loss: 0.12591 | valid_accuracy: 0.93779 | 0:00:13s\n", - "epoch 15 | loss: 0.13704 | valid_accuracy: 0.93516 | 0:00:14s\n", - "epoch 16 | loss: 0.12821 | valid_accuracy: 0.9347 | 0:00:15s\n", - "epoch 17 | loss: 0.12955 | valid_accuracy: 0.9355 | 0:00:16s\n", - "epoch 18 | loss: 0.1195 | valid_accuracy: 0.93881 | 0:00:17s\n", - "epoch 19 | loss: 0.11225 | valid_accuracy: 0.93116 | 0:00:18s\n", - "epoch 20 | loss: 0.11565 | valid_accuracy: 0.93413 | 0:00:19s\n", - "epoch 21 | loss: 0.10814 | valid_accuracy: 0.93984 | 0:00:19s\n", - "\n", - "Early stopping occurred at epoch 21 with best_epoch = 13 and best_valid_accuracy = 0.94258\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:55:07,377] Trial 30 finished with value: 0.7237781438769907 and parameters: {'n_d': 64, 'n_a': 8, 'n_steps': 7, 'gamma': 1.2706701528592617, 'lambda_sparse': 0.00020591233059887278, 'momentum': 0.3242256066504274}. Best is trial 3 with value: 0.7623080995235575.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.38446 | valid_accuracy: 0.83938 | 0:00:00s\n", - "epoch 1 | loss: 0.18263 | valid_accuracy: 0.9105 | 0:00:01s\n", - "epoch 2 | loss: 0.14896 | valid_accuracy: 0.92329 | 0:00:01s\n", - "epoch 3 | loss: 0.13453 | valid_accuracy: 0.92032 | 0:00:02s\n", - "epoch 4 | loss: 0.12652 | valid_accuracy: 0.92911 | 0:00:03s\n", - "epoch 5 | loss: 0.12708 | valid_accuracy: 0.93413 | 0:00:03s\n", - "epoch 6 | loss: 0.11877 | valid_accuracy: 0.94018 | 0:00:04s\n", - "epoch 7 | loss: 0.10601 | valid_accuracy: 0.93299 | 0:00:04s\n", - "epoch 8 | loss: 0.09763 | valid_accuracy: 0.93105 | 0:00:05s\n", - "epoch 9 | loss: 0.09013 | valid_accuracy: 0.9403 | 0:00:06s\n", - "epoch 10 | loss: 0.09183 | valid_accuracy: 0.93938 | 0:00:06s\n", - "epoch 11 | loss: 0.09007 | valid_accuracy: 0.93756 | 0:00:07s\n", - "epoch 12 | loss: 0.08186 | valid_accuracy: 0.94304 | 0:00:07s\n", - "epoch 13 | loss: 0.08119 | valid_accuracy: 0.93699 | 0:00:08s\n", - "epoch 14 | loss: 0.08249 | valid_accuracy: 0.94361 | 0:00:09s\n", - "epoch 15 | loss: 0.0798 | valid_accuracy: 0.9411 | 0:00:09s\n", - "epoch 16 | loss: 0.07149 | valid_accuracy: 0.93699 | 0:00:10s\n", - "epoch 17 | loss: 0.07026 | valid_accuracy: 0.9379 | 0:00:10s\n", - "epoch 18 | loss: 0.06834 | valid_accuracy: 0.93447 | 0:00:11s\n", - "epoch 19 | loss: 0.06959 | valid_accuracy: 0.93573 | 0:00:12s\n", - "epoch 20 | loss: 0.06952 | valid_accuracy: 0.94155 | 0:00:12s\n", - "epoch 21 | loss: 0.0714 | valid_accuracy: 0.93961 | 0:00:13s\n", - "epoch 22 | loss: 0.06892 | valid_accuracy: 0.93562 | 0:00:13s\n", - "\n", - "Early stopping occurred at epoch 22 with best_epoch = 14 and best_valid_accuracy = 0.94361\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:55:21,705] Trial 31 finished with value: 0.7443064182194618 and parameters: {'n_d': 32, 'n_a': 24, 'n_steps': 4, 'gamma': 1.1322179042515514, 'lambda_sparse': 0.004321765994323485, 'momentum': 0.17628878892831942}. Best is trial 3 with value: 0.7623080995235575.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.3318 | valid_accuracy: 0.86998 | 0:00:00s\n", - "epoch 1 | loss: 0.18827 | valid_accuracy: 0.91884 | 0:00:01s\n", - "epoch 2 | loss: 0.15179 | valid_accuracy: 0.91564 | 0:00:01s\n", - "epoch 3 | loss: 0.12275 | valid_accuracy: 0.92534 | 0:00:02s\n", - "epoch 4 | loss: 0.10362 | valid_accuracy: 0.92842 | 0:00:02s\n", - "epoch 5 | loss: 0.09638 | valid_accuracy: 0.93676 | 0:00:03s\n", - "epoch 6 | loss: 0.08977 | valid_accuracy: 0.93733 | 0:00:03s\n", - "epoch 7 | loss: 0.08381 | valid_accuracy: 0.93927 | 0:00:04s\n", - "epoch 8 | loss: 0.07998 | valid_accuracy: 0.93995 | 0:00:04s\n", - "epoch 9 | loss: 0.0772 | valid_accuracy: 0.93938 | 0:00:05s\n", - "epoch 10 | loss: 0.07595 | valid_accuracy: 0.9403 | 0:00:05s\n", - "epoch 11 | loss: 0.07569 | valid_accuracy: 0.93368 | 0:00:06s\n", - "epoch 12 | loss: 0.07672 | valid_accuracy: 0.94452 | 0:00:06s\n", - "epoch 13 | loss: 0.07309 | valid_accuracy: 0.93733 | 0:00:07s\n", - "epoch 14 | loss: 0.06878 | valid_accuracy: 0.93276 | 0:00:07s\n", - "epoch 15 | loss: 0.0674 | valid_accuracy: 0.94132 | 0:00:07s\n", - "epoch 16 | loss: 0.07026 | valid_accuracy: 0.94315 | 0:00:08s\n", - "epoch 17 | loss: 0.06721 | valid_accuracy: 0.94247 | 0:00:08s\n", - "epoch 18 | loss: 0.06498 | valid_accuracy: 0.93356 | 0:00:09s\n", - "epoch 19 | loss: 0.06452 | valid_accuracy: 0.93801 | 0:00:09s\n", - "epoch 20 | loss: 0.06448 | valid_accuracy: 0.94167 | 0:00:10s\n", - "\n", - "Early stopping occurred at epoch 20 with best_epoch = 12 and best_valid_accuracy = 0.94452\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:55:32,520] Trial 32 finished with value: 0.7527975584944049 and parameters: {'n_d': 24, 'n_a': 16, 'n_steps': 3, 'gamma': 1.1745885111041938, 'lambda_sparse': 0.0055991752089406465, 'momentum': 0.19256679452155392}. Best is trial 3 with value: 0.7623080995235575.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.41156 | valid_accuracy: 0.8645 | 0:00:00s\n", - "epoch 1 | loss: 0.19191 | valid_accuracy: 0.91039 | 0:00:00s\n", - "epoch 2 | loss: 0.15266 | valid_accuracy: 0.91918 | 0:00:01s\n", - "epoch 3 | loss: 0.14502 | valid_accuracy: 0.92705 | 0:00:01s\n", - "epoch 4 | loss: 0.13591 | valid_accuracy: 0.93573 | 0:00:02s\n", - "epoch 5 | loss: 0.12342 | valid_accuracy: 0.94041 | 0:00:02s\n", - "epoch 6 | loss: 0.1181 | valid_accuracy: 0.93493 | 0:00:03s\n", - "epoch 7 | loss: 0.11095 | valid_accuracy: 0.93893 | 0:00:03s\n", - "epoch 8 | loss: 0.10741 | valid_accuracy: 0.93927 | 0:00:04s\n", - "epoch 9 | loss: 0.10457 | valid_accuracy: 0.9363 | 0:00:04s\n", - "epoch 10 | loss: 0.1003 | valid_accuracy: 0.93744 | 0:00:05s\n", - "epoch 11 | loss: 0.09707 | valid_accuracy: 0.94406 | 0:00:05s\n", - "epoch 12 | loss: 0.09323 | valid_accuracy: 0.9411 | 0:00:06s\n", - "epoch 13 | loss: 0.09063 | valid_accuracy: 0.93779 | 0:00:06s\n", - "epoch 14 | loss: 0.08585 | valid_accuracy: 0.93664 | 0:00:07s\n", - "epoch 15 | loss: 0.08433 | valid_accuracy: 0.94292 | 0:00:07s\n", - "epoch 16 | loss: 0.08005 | valid_accuracy: 0.93961 | 0:00:08s\n", - "epoch 17 | loss: 0.07712 | valid_accuracy: 0.93596 | 0:00:08s\n", - "epoch 18 | loss: 0.07722 | valid_accuracy: 0.94144 | 0:00:09s\n", - "epoch 19 | loss: 0.07351 | valid_accuracy: 0.94212 | 0:00:09s\n", - "\n", - "Early stopping occurred at epoch 19 with best_epoch = 11 and best_valid_accuracy = 0.94406\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:55:42,705] Trial 33 finished with value: 0.7219069239500567 and parameters: {'n_d': 16, 'n_a': 16, 'n_steps': 3, 'gamma': 1.1913150299734105, 'lambda_sparse': 0.00619159825684386, 'momentum': 0.22821635612221458}. Best is trial 3 with value: 0.7623080995235575.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.31715 | valid_accuracy: 0.86689 | 0:00:00s\n", - "epoch 1 | loss: 0.17315 | valid_accuracy: 0.91826 | 0:00:00s\n", - "epoch 2 | loss: 0.1247 | valid_accuracy: 0.92979 | 0:00:01s\n", - "epoch 3 | loss: 0.1067 | valid_accuracy: 0.93584 | 0:00:01s\n", - "epoch 4 | loss: 0.09597 | valid_accuracy: 0.93436 | 0:00:02s\n", - "epoch 5 | loss: 0.0925 | valid_accuracy: 0.9339 | 0:00:03s\n", - "epoch 6 | loss: 0.08782 | valid_accuracy: 0.93573 | 0:00:03s\n", - "epoch 7 | loss: 0.08932 | valid_accuracy: 0.9371 | 0:00:04s\n", - "epoch 8 | loss: 0.08297 | valid_accuracy: 0.93299 | 0:00:04s\n", - "epoch 9 | loss: 0.07792 | valid_accuracy: 0.93071 | 0:00:04s\n", - "epoch 10 | loss: 0.07672 | valid_accuracy: 0.93162 | 0:00:05s\n", - "epoch 11 | loss: 0.07467 | valid_accuracy: 0.93128 | 0:00:05s\n", - "epoch 12 | loss: 0.07016 | valid_accuracy: 0.93162 | 0:00:06s\n", - "epoch 13 | loss: 0.06865 | valid_accuracy: 0.93105 | 0:00:06s\n", - "epoch 14 | loss: 0.0669 | valid_accuracy: 0.93447 | 0:00:07s\n", - "epoch 15 | loss: 0.06509 | valid_accuracy: 0.92854 | 0:00:07s\n", - "\n", - "Early stopping occurred at epoch 15 with best_epoch = 7 and best_valid_accuracy = 0.9371\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:55:50,919] Trial 34 finished with value: 0.728437654016757 and parameters: {'n_d': 24, 'n_a': 16, 'n_steps': 3, 'gamma': 1.0580799204136884, 'lambda_sparse': 0.0021396821263543835, 'momentum': 0.19302606110846154}. Best is trial 3 with value: 0.7623080995235575.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.47847 | valid_accuracy: 0.85811 | 0:00:00s\n", - "epoch 1 | loss: 0.26958 | valid_accuracy: 0.89235 | 0:00:01s\n", - "epoch 2 | loss: 0.23715 | valid_accuracy: 0.91336 | 0:00:01s\n", - "epoch 3 | loss: 0.21046 | valid_accuracy: 0.91998 | 0:00:02s\n", - "epoch 4 | loss: 0.19762 | valid_accuracy: 0.92717 | 0:00:02s\n", - "epoch 5 | loss: 0.18819 | valid_accuracy: 0.90685 | 0:00:03s\n", - "epoch 6 | loss: 0.17835 | valid_accuracy: 0.91735 | 0:00:04s\n", - "epoch 7 | loss: 0.16374 | valid_accuracy: 0.9266 | 0:00:04s\n", - "epoch 8 | loss: 0.15243 | valid_accuracy: 0.91884 | 0:00:05s\n", - "epoch 9 | loss: 0.14681 | valid_accuracy: 0.93253 | 0:00:05s\n", - "epoch 10 | loss: 0.14221 | valid_accuracy: 0.92352 | 0:00:06s\n", - "epoch 11 | loss: 0.13792 | valid_accuracy: 0.92443 | 0:00:07s\n", - "epoch 12 | loss: 0.13522 | valid_accuracy: 0.94064 | 0:00:07s\n", - "epoch 13 | loss: 0.12487 | valid_accuracy: 0.92546 | 0:00:08s\n", - "epoch 14 | loss: 0.12912 | valid_accuracy: 0.93493 | 0:00:08s\n", - "epoch 15 | loss: 0.11491 | valid_accuracy: 0.93847 | 0:00:09s\n", - "epoch 16 | loss: 0.10628 | valid_accuracy: 0.9355 | 0:00:10s\n", - "epoch 17 | loss: 0.10426 | valid_accuracy: 0.93744 | 0:00:10s\n", - "epoch 18 | loss: 0.1046 | valid_accuracy: 0.9387 | 0:00:11s\n", - "epoch 19 | loss: 0.10161 | valid_accuracy: 0.91941 | 0:00:11s\n", - "epoch 20 | loss: 0.0975 | valid_accuracy: 0.93402 | 0:00:12s\n", - "\n", - "Early stopping occurred at epoch 20 with best_epoch = 12 and best_valid_accuracy = 0.94064\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:56:03,769] Trial 35 finished with value: 0.7161572052401747 and parameters: {'n_d': 24, 'n_a': 8, 'n_steps': 4, 'gamma': 1.9739236222012595, 'lambda_sparse': 0.001147872267664542, 'momentum': 0.15342249904964475}. Best is trial 3 with value: 0.7623080995235575.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.33235 | valid_accuracy: 0.85137 | 0:00:00s\n", - "epoch 1 | loss: 0.17157 | valid_accuracy: 0.92466 | 0:00:00s\n", - "epoch 2 | loss: 0.1364 | valid_accuracy: 0.93402 | 0:00:01s\n", - "epoch 3 | loss: 0.11848 | valid_accuracy: 0.92671 | 0:00:01s\n", - "epoch 4 | loss: 0.11056 | valid_accuracy: 0.93493 | 0:00:02s\n", - "epoch 5 | loss: 0.09953 | valid_accuracy: 0.93562 | 0:00:02s\n", - "epoch 6 | loss: 0.09568 | valid_accuracy: 0.93893 | 0:00:03s\n", - "epoch 7 | loss: 0.08778 | valid_accuracy: 0.94292 | 0:00:03s\n", - "epoch 8 | loss: 0.08715 | valid_accuracy: 0.94018 | 0:00:04s\n", - "epoch 9 | loss: 0.08656 | valid_accuracy: 0.93961 | 0:00:04s\n", - "epoch 10 | loss: 0.08587 | valid_accuracy: 0.94635 | 0:00:05s\n", - "epoch 11 | loss: 0.0755 | valid_accuracy: 0.94064 | 0:00:05s\n", - "epoch 12 | loss: 0.07821 | valid_accuracy: 0.94041 | 0:00:06s\n", - "epoch 13 | loss: 0.07822 | valid_accuracy: 0.94132 | 0:00:06s\n", - "epoch 14 | loss: 0.0733 | valid_accuracy: 0.93779 | 0:00:07s\n", - "epoch 15 | loss: 0.07769 | valid_accuracy: 0.94189 | 0:00:07s\n", - "epoch 16 | loss: 0.07335 | valid_accuracy: 0.93642 | 0:00:08s\n", - "epoch 17 | loss: 0.06728 | valid_accuracy: 0.94053 | 0:00:08s\n", - "epoch 18 | loss: 0.06699 | valid_accuracy: 0.93916 | 0:00:09s\n", - "\n", - "Early stopping occurred at epoch 18 with best_epoch = 10 and best_valid_accuracy = 0.94635\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:56:13,542] Trial 36 finished with value: 0.7486631016042781 and parameters: {'n_d': 32, 'n_a': 16, 'n_steps': 3, 'gamma': 1.222807949141764, 'lambda_sparse': 0.00971405536579209, 'momentum': 0.0627213067269411}. Best is trial 3 with value: 0.7623080995235575.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.60726 | valid_accuracy: 0.65959 | 0:00:00s\n", - "epoch 1 | loss: 0.32327 | valid_accuracy: 0.78071 | 0:00:01s\n", - "epoch 2 | loss: 0.26496 | valid_accuracy: 0.87352 | 0:00:02s\n", - "epoch 3 | loss: 0.23906 | valid_accuracy: 0.8903 | 0:00:02s\n", - "epoch 4 | loss: 0.22749 | valid_accuracy: 0.9016 | 0:00:03s\n", - "epoch 5 | loss: 0.21507 | valid_accuracy: 0.92237 | 0:00:04s\n", - "epoch 6 | loss: 0.20581 | valid_accuracy: 0.91005 | 0:00:04s\n", - "epoch 7 | loss: 0.19077 | valid_accuracy: 0.90445 | 0:00:05s\n", - "epoch 8 | loss: 0.19085 | valid_accuracy: 0.92443 | 0:00:06s\n", - "epoch 9 | loss: 0.18194 | valid_accuracy: 0.92215 | 0:00:07s\n", - "epoch 10 | loss: 0.17387 | valid_accuracy: 0.92203 | 0:00:07s\n", - "epoch 11 | loss: 0.16277 | valid_accuracy: 0.92249 | 0:00:08s\n", - "epoch 12 | loss: 0.16102 | valid_accuracy: 0.91918 | 0:00:09s\n", - "epoch 13 | loss: 0.15843 | valid_accuracy: 0.92317 | 0:00:09s\n", - "epoch 14 | loss: 0.15387 | valid_accuracy: 0.90925 | 0:00:10s\n", - "epoch 15 | loss: 0.15106 | valid_accuracy: 0.93105 | 0:00:11s\n", - "epoch 16 | loss: 0.14664 | valid_accuracy: 0.92454 | 0:00:11s\n", - "epoch 17 | loss: 0.14664 | valid_accuracy: 0.925 | 0:00:12s\n", - "epoch 18 | loss: 0.14471 | valid_accuracy: 0.93151 | 0:00:13s\n", - "epoch 19 | loss: 0.13268 | valid_accuracy: 0.93356 | 0:00:13s\n", - "epoch 20 | loss: 0.13501 | valid_accuracy: 0.91427 | 0:00:14s\n", - "epoch 21 | loss: 0.13932 | valid_accuracy: 0.93664 | 0:00:15s\n", - "epoch 22 | loss: 0.13938 | valid_accuracy: 0.93196 | 0:00:15s\n", - "epoch 23 | loss: 0.13442 | valid_accuracy: 0.93025 | 0:00:16s\n", - "epoch 24 | loss: 0.14439 | valid_accuracy: 0.93082 | 0:00:17s\n", - "epoch 25 | loss: 0.13891 | valid_accuracy: 0.93288 | 0:00:18s\n", - "epoch 26 | loss: 0.12943 | valid_accuracy: 0.93048 | 0:00:18s\n", - "epoch 27 | loss: 0.1252 | valid_accuracy: 0.93493 | 0:00:19s\n", - "epoch 28 | loss: 0.13083 | valid_accuracy: 0.94053 | 0:00:20s\n", - "epoch 29 | loss: 0.12587 | valid_accuracy: 0.9371 | 0:00:20s\n", - "Stop training because you reached max_epochs = 30 with best_epoch = 28 and best_valid_accuracy = 0.94053\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:56:34,902] Trial 37 finished with value: 0.71326362135388 and parameters: {'n_d': 8, 'n_a': 32, 'n_steps': 5, 'gamma': 1.8874432378189303, 'lambda_sparse': 0.0003092858649558244, 'momentum': 0.1132397079238248}. Best is trial 3 with value: 0.7623080995235575.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.53619 | valid_accuracy: 0.57557 | 0:00:00s\n", - "epoch 1 | loss: 0.31732 | valid_accuracy: 0.90674 | 0:00:01s\n", - "epoch 2 | loss: 0.29928 | valid_accuracy: 0.83345 | 0:00:02s\n", - "epoch 3 | loss: 0.25874 | valid_accuracy: 0.87089 | 0:00:03s\n", - "epoch 4 | loss: 0.25692 | valid_accuracy: 0.87705 | 0:00:04s\n", - "epoch 5 | loss: 0.26225 | valid_accuracy: 0.88995 | 0:00:05s\n", - "epoch 6 | loss: 0.23708 | valid_accuracy: 0.90034 | 0:00:06s\n", - "epoch 7 | loss: 0.22675 | valid_accuracy: 0.90114 | 0:00:07s\n", - "epoch 8 | loss: 0.22047 | valid_accuracy: 0.91632 | 0:00:08s\n", - "epoch 9 | loss: 0.2018 | valid_accuracy: 0.91724 | 0:00:08s\n", - "epoch 10 | loss: 0.19331 | valid_accuracy: 0.90959 | 0:00:09s\n", - "epoch 11 | loss: 0.19135 | valid_accuracy: 0.925 | 0:00:10s\n", - "epoch 12 | loss: 0.17713 | valid_accuracy: 0.92511 | 0:00:11s\n", - "epoch 13 | loss: 0.16346 | valid_accuracy: 0.92089 | 0:00:12s\n", - "epoch 14 | loss: 0.15398 | valid_accuracy: 0.92249 | 0:00:13s\n", - "epoch 15 | loss: 0.15063 | valid_accuracy: 0.91792 | 0:00:14s\n", - "epoch 16 | loss: 0.14622 | valid_accuracy: 0.93116 | 0:00:15s\n", - "epoch 17 | loss: 0.13301 | valid_accuracy: 0.93059 | 0:00:16s\n", - "epoch 18 | loss: 0.13113 | valid_accuracy: 0.92477 | 0:00:17s\n", - "epoch 19 | loss: 0.13038 | valid_accuracy: 0.9242 | 0:00:17s\n", - "epoch 20 | loss: 0.12463 | valid_accuracy: 0.92397 | 0:00:18s\n", - "epoch 21 | loss: 0.12433 | valid_accuracy: 0.92683 | 0:00:19s\n", - "epoch 22 | loss: 0.12402 | valid_accuracy: 0.92432 | 0:00:20s\n", - "epoch 23 | loss: 0.11884 | valid_accuracy: 0.91963 | 0:00:21s\n", - "epoch 24 | loss: 0.11529 | valid_accuracy: 0.93253 | 0:00:22s\n", - "epoch 25 | loss: 0.10892 | valid_accuracy: 0.92603 | 0:00:23s\n", - "epoch 26 | loss: 0.11023 | valid_accuracy: 0.92877 | 0:00:24s\n", - "epoch 27 | loss: 0.11299 | valid_accuracy: 0.92922 | 0:00:25s\n", - "epoch 28 | loss: 0.11595 | valid_accuracy: 0.92523 | 0:00:26s\n", - "epoch 29 | loss: 0.11495 | valid_accuracy: 0.93425 | 0:00:26s\n", - "Stop training because you reached max_epochs = 30 with best_epoch = 29 and best_valid_accuracy = 0.93425\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:57:02,460] Trial 38 finished with value: 0.700312174817898 and parameters: {'n_d': 40, 'n_a': 16, 'n_steps': 7, 'gamma': 1.5873343488710119, 'lambda_sparse': 0.0030518355038422927, 'momentum': 0.2357369293781781}. Best is trial 3 with value: 0.7623080995235575.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.48542 | valid_accuracy: 0.65194 | 0:00:01s\n", - "epoch 1 | loss: 0.29712 | valid_accuracy: 0.73813 | 0:00:02s\n", - "epoch 2 | loss: 0.30147 | valid_accuracy: 0.80537 | 0:00:03s\n", - "epoch 3 | loss: 0.2837 | valid_accuracy: 0.84132 | 0:00:04s\n", - "epoch 4 | loss: 0.28887 | valid_accuracy: 0.81906 | 0:00:05s\n", - "epoch 5 | loss: 0.25672 | valid_accuracy: 0.89772 | 0:00:06s\n", - "epoch 6 | loss: 0.2351 | valid_accuracy: 0.86438 | 0:00:07s\n", - "epoch 7 | loss: 0.22729 | valid_accuracy: 0.87534 | 0:00:08s\n", - "epoch 8 | loss: 0.21406 | valid_accuracy: 0.8887 | 0:00:09s\n", - "epoch 9 | loss: 0.20939 | valid_accuracy: 0.90103 | 0:00:10s\n", - "epoch 10 | loss: 0.19016 | valid_accuracy: 0.92158 | 0:00:11s\n", - "epoch 11 | loss: 0.17315 | valid_accuracy: 0.91712 | 0:00:12s\n", - "epoch 12 | loss: 0.17264 | valid_accuracy: 0.92534 | 0:00:13s\n", - "epoch 13 | loss: 0.15628 | valid_accuracy: 0.921 | 0:00:14s\n", - "epoch 14 | loss: 0.1504 | valid_accuracy: 0.91096 | 0:00:15s\n", - "epoch 15 | loss: 0.15163 | valid_accuracy: 0.89943 | 0:00:16s\n", - "epoch 16 | loss: 0.14703 | valid_accuracy: 0.92249 | 0:00:17s\n", - "epoch 17 | loss: 0.14098 | valid_accuracy: 0.93208 | 0:00:18s\n", - "epoch 18 | loss: 0.13393 | valid_accuracy: 0.93459 | 0:00:19s\n", - "epoch 19 | loss: 0.13433 | valid_accuracy: 0.94098 | 0:00:20s\n", - "epoch 20 | loss: 0.1287 | valid_accuracy: 0.92021 | 0:00:21s\n", - "epoch 21 | loss: 0.13776 | valid_accuracy: 0.9347 | 0:00:22s\n", - "epoch 22 | loss: 0.14637 | valid_accuracy: 0.93699 | 0:00:23s\n", - "epoch 23 | loss: 0.15078 | valid_accuracy: 0.90959 | 0:00:24s\n", - "epoch 24 | loss: 0.15007 | valid_accuracy: 0.92432 | 0:00:25s\n", - "epoch 25 | loss: 0.14907 | valid_accuracy: 0.92477 | 0:00:26s\n", - "epoch 26 | loss: 0.14999 | valid_accuracy: 0.92477 | 0:00:27s\n", - "epoch 27 | loss: 0.15182 | valid_accuracy: 0.91062 | 0:00:28s\n", - "\n", - "Early stopping occurred at epoch 27 with best_epoch = 19 and best_valid_accuracy = 0.94098\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:57:31,273] Trial 39 finished with value: 0.7067498581962564 and parameters: {'n_d': 16, 'n_a': 8, 'n_steps': 8, 'gamma': 1.4138397892874734, 'lambda_sparse': 0.005707090935690004, 'momentum': 0.3896524847355101}. Best is trial 3 with value: 0.7623080995235575.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.31977 | valid_accuracy: 0.87443 | 0:00:00s\n", - "epoch 1 | loss: 0.14673 | valid_accuracy: 0.91553 | 0:00:01s\n", - "epoch 2 | loss: 0.11759 | valid_accuracy: 0.91701 | 0:00:01s\n", - "epoch 3 | loss: 0.10481 | valid_accuracy: 0.92203 | 0:00:01s\n", - "epoch 4 | loss: 0.1001 | valid_accuracy: 0.93573 | 0:00:02s\n", - "epoch 5 | loss: 0.0901 | valid_accuracy: 0.93653 | 0:00:02s\n", - "epoch 6 | loss: 0.08319 | valid_accuracy: 0.93699 | 0:00:03s\n", - "epoch 7 | loss: 0.07801 | valid_accuracy: 0.93311 | 0:00:04s\n", - "epoch 8 | loss: 0.074 | valid_accuracy: 0.93687 | 0:00:04s\n", - "epoch 9 | loss: 0.07202 | valid_accuracy: 0.94155 | 0:00:05s\n", - "epoch 10 | loss: 0.07003 | valid_accuracy: 0.94007 | 0:00:05s\n", - "epoch 11 | loss: 0.0669 | valid_accuracy: 0.93082 | 0:00:06s\n", - "epoch 12 | loss: 0.066 | valid_accuracy: 0.94189 | 0:00:06s\n", - "epoch 13 | loss: 0.06317 | valid_accuracy: 0.93995 | 0:00:07s\n", - "epoch 14 | loss: 0.06402 | valid_accuracy: 0.94041 | 0:00:07s\n", - "epoch 15 | loss: 0.06095 | valid_accuracy: 0.9387 | 0:00:08s\n", - "epoch 16 | loss: 0.05869 | valid_accuracy: 0.93596 | 0:00:08s\n", - "epoch 17 | loss: 0.05896 | valid_accuracy: 0.94338 | 0:00:09s\n", - "epoch 18 | loss: 0.05731 | valid_accuracy: 0.93676 | 0:00:09s\n", - "epoch 19 | loss: 0.05453 | valid_accuracy: 0.93938 | 0:00:10s\n", - "epoch 20 | loss: 0.05415 | valid_accuracy: 0.94406 | 0:00:10s\n", - "epoch 21 | loss: 0.05257 | valid_accuracy: 0.94189 | 0:00:11s\n", - "epoch 22 | loss: 0.0537 | valid_accuracy: 0.9476 | 0:00:11s\n", - "epoch 23 | loss: 0.05393 | valid_accuracy: 0.93973 | 0:00:12s\n", - "epoch 24 | loss: 0.05513 | valid_accuracy: 0.94007 | 0:00:12s\n", - "epoch 25 | loss: 0.05287 | valid_accuracy: 0.93847 | 0:00:13s\n", - "epoch 26 | loss: 0.05136 | valid_accuracy: 0.94521 | 0:00:13s\n", - "epoch 27 | loss: 0.04965 | valid_accuracy: 0.93687 | 0:00:14s\n", - "epoch 28 | loss: 0.05255 | valid_accuracy: 0.93836 | 0:00:14s\n", - "epoch 29 | loss: 0.04759 | valid_accuracy: 0.94372 | 0:00:15s\n", - "Stop training because you reached max_epochs = 30 with best_epoch = 22 and best_valid_accuracy = 0.9476\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:57:46,859] Trial 40 finished with value: 0.7402376910016978 and parameters: {'n_d': 24, 'n_a': 48, 'n_steps': 3, 'gamma': 1.1667191264583925, 'lambda_sparse': 0.00010306424305630605, 'momentum': 0.1985963870918099}. Best is trial 3 with value: 0.7623080995235575.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.33659 | valid_accuracy: 0.81541 | 0:00:00s\n", - "epoch 1 | loss: 0.16148 | valid_accuracy: 0.921 | 0:00:00s\n", - "epoch 2 | loss: 0.13764 | valid_accuracy: 0.9355 | 0:00:01s\n", - "epoch 3 | loss: 0.11773 | valid_accuracy: 0.93858 | 0:00:01s\n", - "epoch 4 | loss: 0.10368 | valid_accuracy: 0.93904 | 0:00:02s\n", - "epoch 5 | loss: 0.08849 | valid_accuracy: 0.93642 | 0:00:02s\n", - "epoch 6 | loss: 0.08623 | valid_accuracy: 0.93916 | 0:00:03s\n", - "epoch 7 | loss: 0.08374 | valid_accuracy: 0.94406 | 0:00:03s\n", - "epoch 8 | loss: 0.08064 | valid_accuracy: 0.94155 | 0:00:04s\n", - "epoch 9 | loss: 0.07317 | valid_accuracy: 0.94543 | 0:00:05s\n", - "epoch 10 | loss: 0.07671 | valid_accuracy: 0.94452 | 0:00:05s\n", - "epoch 11 | loss: 0.07501 | valid_accuracy: 0.94509 | 0:00:05s\n", - "epoch 12 | loss: 0.07546 | valid_accuracy: 0.9387 | 0:00:06s\n", - "epoch 13 | loss: 0.07465 | valid_accuracy: 0.9355 | 0:00:07s\n", - "epoch 14 | loss: 0.06778 | valid_accuracy: 0.94315 | 0:00:07s\n", - "epoch 15 | loss: 0.0673 | valid_accuracy: 0.94498 | 0:00:07s\n", - "epoch 16 | loss: 0.07388 | valid_accuracy: 0.93379 | 0:00:08s\n", - "epoch 17 | loss: 0.07704 | valid_accuracy: 0.93961 | 0:00:08s\n", - "\n", - "Early stopping occurred at epoch 17 with best_epoch = 9 and best_valid_accuracy = 0.94543\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:57:56,146] Trial 41 finished with value: 0.749475890985325 and parameters: {'n_d': 32, 'n_a': 16, 'n_steps': 3, 'gamma': 1.228568193669895, 'lambda_sparse': 0.00877988970783116, 'momentum': 0.04866248413822538}. Best is trial 3 with value: 0.7623080995235575.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.33777 | valid_accuracy: 0.8629 | 0:00:00s\n", - "epoch 1 | loss: 0.16269 | valid_accuracy: 0.88311 | 0:00:00s\n", - "epoch 2 | loss: 0.13035 | valid_accuracy: 0.92454 | 0:00:01s\n", - "epoch 3 | loss: 0.10987 | valid_accuracy: 0.929 | 0:00:02s\n", - "epoch 4 | loss: 0.10458 | valid_accuracy: 0.92694 | 0:00:02s\n", - "epoch 5 | loss: 0.09602 | valid_accuracy: 0.93322 | 0:00:03s\n", - "epoch 6 | loss: 0.0927 | valid_accuracy: 0.93938 | 0:00:03s\n", - "epoch 7 | loss: 0.08976 | valid_accuracy: 0.92763 | 0:00:04s\n", - "epoch 8 | loss: 0.08873 | valid_accuracy: 0.9371 | 0:00:04s\n", - "epoch 9 | loss: 0.08053 | valid_accuracy: 0.93881 | 0:00:05s\n", - "epoch 10 | loss: 0.07759 | valid_accuracy: 0.93459 | 0:00:05s\n", - "epoch 11 | loss: 0.0775 | valid_accuracy: 0.93002 | 0:00:06s\n", - "epoch 12 | loss: 0.08402 | valid_accuracy: 0.92968 | 0:00:06s\n", - "epoch 13 | loss: 0.08826 | valid_accuracy: 0.93322 | 0:00:06s\n", - "epoch 14 | loss: 0.08031 | valid_accuracy: 0.93539 | 0:00:07s\n", - "\n", - "Early stopping occurred at epoch 14 with best_epoch = 6 and best_valid_accuracy = 0.93938\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:58:03,957] Trial 42 finished with value: 0.7324937027707809 and parameters: {'n_d': 32, 'n_a': 24, 'n_steps': 3, 'gamma': 1.2742240557233138, 'lambda_sparse': 0.007683519076360558, 'momentum': 0.011013535481353275}. Best is trial 3 with value: 0.7623080995235575.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.31833 | valid_accuracy: 0.76621 | 0:00:00s\n", - "epoch 1 | loss: 0.15936 | valid_accuracy: 0.90491 | 0:00:01s\n", - "epoch 2 | loss: 0.11391 | valid_accuracy: 0.92192 | 0:00:01s\n", - "epoch 3 | loss: 0.09934 | valid_accuracy: 0.9339 | 0:00:02s\n", - "epoch 4 | loss: 0.09147 | valid_accuracy: 0.93516 | 0:00:02s\n", - "epoch 5 | loss: 0.08413 | valid_accuracy: 0.93539 | 0:00:03s\n", - "epoch 6 | loss: 0.08536 | valid_accuracy: 0.9355 | 0:00:04s\n", - "epoch 7 | loss: 0.08246 | valid_accuracy: 0.9339 | 0:00:04s\n", - "epoch 8 | loss: 0.08266 | valid_accuracy: 0.94703 | 0:00:05s\n", - "epoch 9 | loss: 0.07919 | valid_accuracy: 0.94132 | 0:00:06s\n", - "epoch 10 | loss: 0.07745 | valid_accuracy: 0.93584 | 0:00:06s\n", - "epoch 11 | loss: 0.07453 | valid_accuracy: 0.93938 | 0:00:07s\n", - "epoch 12 | loss: 0.07515 | valid_accuracy: 0.93584 | 0:00:07s\n", - "epoch 13 | loss: 0.0729 | valid_accuracy: 0.93836 | 0:00:08s\n", - "epoch 14 | loss: 0.07833 | valid_accuracy: 0.9395 | 0:00:08s\n", - "epoch 15 | loss: 0.07302 | valid_accuracy: 0.93984 | 0:00:09s\n", - "epoch 16 | loss: 0.06809 | valid_accuracy: 0.93973 | 0:00:10s\n", - "\n", - "Early stopping occurred at epoch 16 with best_epoch = 8 and best_valid_accuracy = 0.94703\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:58:14,556] Trial 43 finished with value: 0.7668341708542713 and parameters: {'n_d': 32, 'n_a': 16, 'n_steps': 4, 'gamma': 1.0637727829168055, 'lambda_sparse': 0.0067730056400632, 'momentum': 0.04700564113367184}. Best is trial 43 with value: 0.7668341708542713.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.33348 | valid_accuracy: 0.88847 | 0:00:00s\n", - "epoch 1 | loss: 0.17942 | valid_accuracy: 0.88014 | 0:00:01s\n", - "epoch 2 | loss: 0.13183 | valid_accuracy: 0.91712 | 0:00:01s\n", - "epoch 3 | loss: 0.11399 | valid_accuracy: 0.92568 | 0:00:02s\n", - "epoch 4 | loss: 0.10637 | valid_accuracy: 0.93619 | 0:00:03s\n", - "epoch 5 | loss: 0.10003 | valid_accuracy: 0.93219 | 0:00:03s\n", - "epoch 6 | loss: 0.09927 | valid_accuracy: 0.93082 | 0:00:04s\n", - "epoch 7 | loss: 0.09199 | valid_accuracy: 0.9355 | 0:00:04s\n", - "epoch 8 | loss: 0.08574 | valid_accuracy: 0.93938 | 0:00:05s\n", - "epoch 9 | loss: 0.08799 | valid_accuracy: 0.93299 | 0:00:06s\n", - "epoch 10 | loss: 0.0914 | valid_accuracy: 0.93699 | 0:00:06s\n", - "epoch 11 | loss: 0.0822 | valid_accuracy: 0.93539 | 0:00:07s\n", - "epoch 12 | loss: 0.07794 | valid_accuracy: 0.9379 | 0:00:07s\n", - "epoch 13 | loss: 0.07885 | valid_accuracy: 0.93048 | 0:00:08s\n", - "epoch 14 | loss: 0.07626 | valid_accuracy: 0.94121 | 0:00:09s\n", - "epoch 15 | loss: 0.07289 | valid_accuracy: 0.94087 | 0:00:09s\n", - "epoch 16 | loss: 0.06912 | valid_accuracy: 0.93014 | 0:00:10s\n", - "epoch 17 | loss: 0.07172 | valid_accuracy: 0.9371 | 0:00:10s\n", - "epoch 18 | loss: 0.0686 | valid_accuracy: 0.94075 | 0:00:11s\n", - "epoch 19 | loss: 0.0701 | valid_accuracy: 0.94007 | 0:00:12s\n", - "epoch 20 | loss: 0.06486 | valid_accuracy: 0.94281 | 0:00:12s\n", - "epoch 21 | loss: 0.0695 | valid_accuracy: 0.94189 | 0:00:13s\n", - "epoch 22 | loss: 0.06483 | valid_accuracy: 0.93984 | 0:00:13s\n", - "epoch 23 | loss: 0.07076 | valid_accuracy: 0.9379 | 0:00:14s\n", - "epoch 24 | loss: 0.07237 | valid_accuracy: 0.93413 | 0:00:15s\n", - "epoch 25 | loss: 0.06788 | valid_accuracy: 0.93699 | 0:00:15s\n", - "epoch 26 | loss: 0.06267 | valid_accuracy: 0.93459 | 0:00:16s\n", - "epoch 27 | loss: 0.06219 | valid_accuracy: 0.93733 | 0:00:16s\n", - "epoch 28 | loss: 0.06379 | valid_accuracy: 0.94212 | 0:00:17s\n", - "\n", - "Early stopping occurred at epoch 28 with best_epoch = 20 and best_valid_accuracy = 0.94281\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:58:32,364] Trial 44 finished with value: 0.7125645438898451 and parameters: {'n_d': 40, 'n_a': 16, 'n_steps': 4, 'gamma': 1.0553647445745977, 'lambda_sparse': 0.004260884118394998, 'momentum': 0.09213632154541491}. Best is trial 43 with value: 0.7668341708542713.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.37053 | valid_accuracy: 0.85308 | 0:00:00s\n", - "epoch 1 | loss: 0.18977 | valid_accuracy: 0.91861 | 0:00:01s\n", - "epoch 2 | loss: 0.16849 | valid_accuracy: 0.92203 | 0:00:02s\n", - "epoch 3 | loss: 0.14414 | valid_accuracy: 0.93105 | 0:00:02s\n", - "epoch 4 | loss: 0.12881 | valid_accuracy: 0.93642 | 0:00:03s\n", - "epoch 5 | loss: 0.1242 | valid_accuracy: 0.93368 | 0:00:04s\n", - "epoch 6 | loss: 0.10982 | valid_accuracy: 0.93836 | 0:00:04s\n", - "epoch 7 | loss: 0.10474 | valid_accuracy: 0.93847 | 0:00:05s\n", - "epoch 8 | loss: 0.0969 | valid_accuracy: 0.94167 | 0:00:06s\n", - "epoch 9 | loss: 0.09312 | valid_accuracy: 0.93995 | 0:00:06s\n", - "epoch 10 | loss: 0.09079 | valid_accuracy: 0.93927 | 0:00:07s\n", - "epoch 11 | loss: 0.08816 | valid_accuracy: 0.93858 | 0:00:08s\n", - "epoch 12 | loss: 0.09465 | valid_accuracy: 0.94326 | 0:00:09s\n", - "epoch 13 | loss: 0.0861 | valid_accuracy: 0.94269 | 0:00:09s\n", - "epoch 14 | loss: 0.08523 | valid_accuracy: 0.94167 | 0:00:10s\n", - "epoch 15 | loss: 0.08406 | valid_accuracy: 0.94532 | 0:00:11s\n", - "epoch 16 | loss: 0.08572 | valid_accuracy: 0.93676 | 0:00:11s\n", - "epoch 17 | loss: 0.08231 | valid_accuracy: 0.93836 | 0:00:12s\n", - "epoch 18 | loss: 0.08121 | valid_accuracy: 0.94075 | 0:00:13s\n", - "epoch 19 | loss: 0.07751 | valid_accuracy: 0.94155 | 0:00:13s\n", - "epoch 20 | loss: 0.08151 | valid_accuracy: 0.94132 | 0:00:14s\n", - "epoch 21 | loss: 0.08475 | valid_accuracy: 0.93916 | 0:00:15s\n", - "epoch 22 | loss: 0.08126 | valid_accuracy: 0.94372 | 0:00:15s\n", - "epoch 23 | loss: 0.0778 | valid_accuracy: 0.93539 | 0:00:16s\n", - "\n", - "Early stopping occurred at epoch 23 with best_epoch = 15 and best_valid_accuracy = 0.94532\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:58:49,459] Trial 45 finished with value: 0.7474960463890353 and parameters: {'n_d': 24, 'n_a': 8, 'n_steps': 5, 'gamma': 1.0055647304698914, 'lambda_sparse': 0.00637503128174428, 'momentum': 0.14768674158804773}. Best is trial 43 with value: 0.7668341708542713.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.32825 | valid_accuracy: 0.87911 | 0:00:00s\n", - "epoch 1 | loss: 0.15042 | valid_accuracy: 0.92374 | 0:00:01s\n", - "epoch 2 | loss: 0.11754 | valid_accuracy: 0.91769 | 0:00:01s\n", - "epoch 3 | loss: 0.10857 | valid_accuracy: 0.92911 | 0:00:02s\n", - "epoch 4 | loss: 0.09521 | valid_accuracy: 0.92751 | 0:00:02s\n", - "epoch 5 | loss: 0.09114 | valid_accuracy: 0.93516 | 0:00:03s\n", - "epoch 6 | loss: 0.0875 | valid_accuracy: 0.93311 | 0:00:04s\n", - "epoch 7 | loss: 0.08057 | valid_accuracy: 0.93162 | 0:00:04s\n", - "epoch 8 | loss: 0.08178 | valid_accuracy: 0.92922 | 0:00:05s\n", - "epoch 9 | loss: 0.07724 | valid_accuracy: 0.93493 | 0:00:06s\n", - "epoch 10 | loss: 0.0752 | valid_accuracy: 0.92763 | 0:00:06s\n", - "epoch 11 | loss: 0.07963 | valid_accuracy: 0.93059 | 0:00:07s\n", - "epoch 12 | loss: 0.0779 | valid_accuracy: 0.93653 | 0:00:07s\n", - "epoch 13 | loss: 0.07947 | valid_accuracy: 0.93482 | 0:00:08s\n", - "epoch 14 | loss: 0.07601 | valid_accuracy: 0.93858 | 0:00:09s\n", - "epoch 15 | loss: 0.08031 | valid_accuracy: 0.94235 | 0:00:09s\n", - "epoch 16 | loss: 0.07341 | valid_accuracy: 0.93162 | 0:00:10s\n", - "epoch 17 | loss: 0.07114 | valid_accuracy: 0.93733 | 0:00:10s\n", - "epoch 18 | loss: 0.06835 | valid_accuracy: 0.93995 | 0:00:11s\n", - "epoch 19 | loss: 0.06766 | valid_accuracy: 0.93984 | 0:00:12s\n", - "epoch 20 | loss: 0.06208 | valid_accuracy: 0.9395 | 0:00:12s\n", - "epoch 21 | loss: 0.06305 | valid_accuracy: 0.93094 | 0:00:13s\n", - "epoch 22 | loss: 0.06895 | valid_accuracy: 0.9411 | 0:00:14s\n", - "epoch 23 | loss: 0.06489 | valid_accuracy: 0.93721 | 0:00:14s\n", - "\n", - "Early stopping occurred at epoch 23 with best_epoch = 15 and best_valid_accuracy = 0.94235\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:59:04,514] Trial 46 finished with value: 0.7174034695019585 and parameters: {'n_d': 40, 'n_a': 24, 'n_steps': 4, 'gamma': 1.1521457363834404, 'lambda_sparse': 0.0018429919548465087, 'momentum': 0.3563162284282082}. Best is trial 43 with value: 0.7668341708542713.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.45096 | valid_accuracy: 0.75845 | 0:00:00s\n", - "epoch 1 | loss: 0.25648 | valid_accuracy: 0.79201 | 0:00:01s\n", - "epoch 2 | loss: 0.22042 | valid_accuracy: 0.83664 | 0:00:02s\n", - "epoch 3 | loss: 0.20895 | valid_accuracy: 0.89041 | 0:00:03s\n", - "epoch 4 | loss: 0.19748 | valid_accuracy: 0.90228 | 0:00:03s\n", - "epoch 5 | loss: 0.18834 | valid_accuracy: 0.9145 | 0:00:04s\n", - "epoch 6 | loss: 0.17576 | valid_accuracy: 0.92979 | 0:00:05s\n", - "epoch 7 | loss: 0.16898 | valid_accuracy: 0.93094 | 0:00:06s\n", - "epoch 8 | loss: 0.1635 | valid_accuracy: 0.92854 | 0:00:07s\n", - "epoch 9 | loss: 0.1589 | valid_accuracy: 0.93447 | 0:00:08s\n", - "epoch 10 | loss: 0.15276 | valid_accuracy: 0.92409 | 0:00:08s\n", - "epoch 11 | loss: 0.14862 | valid_accuracy: 0.92808 | 0:00:09s\n", - "epoch 12 | loss: 0.1534 | valid_accuracy: 0.93276 | 0:00:10s\n", - "epoch 13 | loss: 0.14868 | valid_accuracy: 0.93288 | 0:00:11s\n", - "epoch 14 | loss: 0.14521 | valid_accuracy: 0.9371 | 0:00:11s\n", - "epoch 15 | loss: 0.16208 | valid_accuracy: 0.94155 | 0:00:12s\n", - "epoch 16 | loss: 0.14509 | valid_accuracy: 0.94224 | 0:00:13s\n", - "epoch 17 | loss: 0.13395 | valid_accuracy: 0.93447 | 0:00:14s\n", - "epoch 18 | loss: 0.12431 | valid_accuracy: 0.93653 | 0:00:15s\n", - "epoch 19 | loss: 0.12072 | valid_accuracy: 0.94247 | 0:00:15s\n", - "epoch 20 | loss: 0.12043 | valid_accuracy: 0.93801 | 0:00:16s\n", - "epoch 21 | loss: 0.12123 | valid_accuracy: 0.93162 | 0:00:17s\n", - "epoch 22 | loss: 0.12482 | valid_accuracy: 0.93493 | 0:00:18s\n", - "epoch 23 | loss: 0.1135 | valid_accuracy: 0.94555 | 0:00:19s\n", - "epoch 24 | loss: 0.11734 | valid_accuracy: 0.93014 | 0:00:19s\n", - "epoch 25 | loss: 0.11518 | valid_accuracy: 0.93459 | 0:00:20s\n", - "epoch 26 | loss: 0.11282 | valid_accuracy: 0.93801 | 0:00:21s\n", - "epoch 27 | loss: 0.1146 | valid_accuracy: 0.9363 | 0:00:22s\n", - "epoch 28 | loss: 0.11062 | valid_accuracy: 0.93619 | 0:00:23s\n", - "epoch 29 | loss: 0.10881 | valid_accuracy: 0.93231 | 0:00:23s\n", - "Stop training because you reached max_epochs = 30 with best_epoch = 23 and best_valid_accuracy = 0.94555\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:59:29,003] Trial 47 finished with value: 0.7406199021207178 and parameters: {'n_d': 24, 'n_a': 32, 'n_steps': 6, 'gamma': 1.3148856617955356, 'lambda_sparse': 0.0005546394043418927, 'momentum': 0.26455657959525036}. Best is trial 43 with value: 0.7668341708542713.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.37015 | valid_accuracy: 0.69977 | 0:00:00s\n", - "epoch 1 | loss: 0.25558 | valid_accuracy: 0.79418 | 0:00:01s\n", - "epoch 2 | loss: 0.22682 | valid_accuracy: 0.86941 | 0:00:01s\n", - "epoch 3 | loss: 0.20077 | valid_accuracy: 0.88128 | 0:00:02s\n", - "epoch 4 | loss: 0.16969 | valid_accuracy: 0.90285 | 0:00:02s\n", - "epoch 5 | loss: 0.14816 | valid_accuracy: 0.90913 | 0:00:03s\n", - "epoch 6 | loss: 0.1425 | valid_accuracy: 0.91313 | 0:00:04s\n", - "epoch 7 | loss: 0.12966 | valid_accuracy: 0.91792 | 0:00:04s\n", - "epoch 8 | loss: 0.11946 | valid_accuracy: 0.91027 | 0:00:05s\n", - "epoch 9 | loss: 0.1147 | valid_accuracy: 0.92477 | 0:00:06s\n", - "epoch 10 | loss: 0.11491 | valid_accuracy: 0.92249 | 0:00:06s\n", - "epoch 11 | loss: 0.1068 | valid_accuracy: 0.91941 | 0:00:07s\n", - "epoch 12 | loss: 0.10981 | valid_accuracy: 0.90468 | 0:00:07s\n", - "epoch 13 | loss: 0.10048 | valid_accuracy: 0.93151 | 0:00:08s\n", - "epoch 14 | loss: 0.09698 | valid_accuracy: 0.92842 | 0:00:08s\n", - "epoch 15 | loss: 0.09362 | valid_accuracy: 0.92831 | 0:00:09s\n", - "epoch 16 | loss: 0.08679 | valid_accuracy: 0.92808 | 0:00:10s\n", - "epoch 17 | loss: 0.08355 | valid_accuracy: 0.9266 | 0:00:10s\n", - "epoch 18 | loss: 0.08247 | valid_accuracy: 0.9242 | 0:00:11s\n", - "epoch 19 | loss: 0.08347 | valid_accuracy: 0.92671 | 0:00:11s\n", - "epoch 20 | loss: 0.08235 | valid_accuracy: 0.93128 | 0:00:12s\n", - "epoch 21 | loss: 0.08107 | valid_accuracy: 0.93265 | 0:00:13s\n", - "epoch 22 | loss: 0.07809 | valid_accuracy: 0.93151 | 0:00:13s\n", - "epoch 23 | loss: 0.08124 | valid_accuracy: 0.92968 | 0:00:14s\n", - "epoch 24 | loss: 0.0735 | valid_accuracy: 0.93059 | 0:00:14s\n", - "epoch 25 | loss: 0.07251 | valid_accuracy: 0.9242 | 0:00:15s\n", - "epoch 26 | loss: 0.07171 | valid_accuracy: 0.93059 | 0:00:16s\n", - "epoch 27 | loss: 0.07134 | valid_accuracy: 0.92991 | 0:00:16s\n", - "epoch 28 | loss: 0.07244 | valid_accuracy: 0.92968 | 0:00:17s\n", - "epoch 29 | loss: 0.06894 | valid_accuracy: 0.93174 | 0:00:17s\n", - "\n", - "Early stopping occurred at epoch 29 with best_epoch = 21 and best_valid_accuracy = 0.93265\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 17:59:47,384] Trial 48 finished with value: 0.6995926680244399 and parameters: {'n_d': 32, 'n_a': 16, 'n_steps': 4, 'gamma': 1.7151786447984039, 'lambda_sparse': 0.004674209747462921, 'momentum': 0.2814624393570832}. Best is trial 43 with value: 0.7668341708542713.\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.3458 | valid_accuracy: 0.79475 | 0:00:00s\n", - "epoch 1 | loss: 0.18822 | valid_accuracy: 0.87991 | 0:00:01s\n", - "epoch 2 | loss: 0.14911 | valid_accuracy: 0.91187 | 0:00:02s\n", - "epoch 3 | loss: 0.13122 | valid_accuracy: 0.91918 | 0:00:02s\n", - "epoch 4 | loss: 0.12069 | valid_accuracy: 0.92877 | 0:00:03s\n", - "epoch 5 | loss: 0.11757 | valid_accuracy: 0.93596 | 0:00:04s\n", - "epoch 6 | loss: 0.10929 | valid_accuracy: 0.93276 | 0:00:05s\n", - "epoch 7 | loss: 0.106 | valid_accuracy: 0.93299 | 0:00:05s\n", - "epoch 8 | loss: 0.11179 | valid_accuracy: 0.92683 | 0:00:06s\n", - "epoch 9 | loss: 0.1022 | valid_accuracy: 0.93276 | 0:00:07s\n", - "epoch 10 | loss: 0.09173 | valid_accuracy: 0.93881 | 0:00:07s\n", - "epoch 11 | loss: 0.09782 | valid_accuracy: 0.9371 | 0:00:08s\n", - "epoch 12 | loss: 0.09728 | valid_accuracy: 0.93436 | 0:00:09s\n", - "epoch 13 | loss: 0.09453 | valid_accuracy: 0.93527 | 0:00:09s\n", - "epoch 14 | loss: 0.08701 | valid_accuracy: 0.93413 | 0:00:10s\n", - "epoch 15 | loss: 0.08408 | valid_accuracy: 0.93881 | 0:00:11s\n", - "epoch 16 | loss: 0.09053 | valid_accuracy: 0.94064 | 0:00:12s\n", - "epoch 17 | loss: 0.08787 | valid_accuracy: 0.93505 | 0:00:12s\n", - "epoch 18 | loss: 0.08235 | valid_accuracy: 0.94235 | 0:00:13s\n", - "epoch 19 | loss: 0.07899 | valid_accuracy: 0.94098 | 0:00:14s\n", - "epoch 20 | loss: 0.07536 | valid_accuracy: 0.93687 | 0:00:14s\n", - "epoch 21 | loss: 0.07645 | valid_accuracy: 0.93687 | 0:00:15s\n", - "epoch 22 | loss: 0.07528 | valid_accuracy: 0.94201 | 0:00:16s\n", - "epoch 23 | loss: 0.07414 | valid_accuracy: 0.93779 | 0:00:17s\n", - "epoch 24 | loss: 0.07399 | valid_accuracy: 0.93756 | 0:00:17s\n", - "epoch 25 | loss: 0.07219 | valid_accuracy: 0.93596 | 0:00:18s\n", - "epoch 26 | loss: 0.07177 | valid_accuracy: 0.93505 | 0:00:19s\n", - "\n", - "Early stopping occurred at epoch 26 with best_epoch = 18 and best_valid_accuracy = 0.94235\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n", - "[I 2025-02-02 18:00:07,052] Trial 49 finished with value: 0.7167694896242288 and parameters: {'n_d': 48, 'n_a': 24, 'n_steps': 5, 'gamma': 1.1028661256752443, 'lambda_sparse': 0.003345545315056804, 'momentum': 0.20788217904522843}. Best is trial 43 with value: 0.7668341708542713.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Best Hyperparameters: {'n_d': 32, 'n_a': 16, 'n_steps': 4, 'gamma': 1.0637727829168055, 'lambda_sparse': 0.0067730056400632, 'momentum': 0.04700564113367184}\n" - ] - } - ], - "source": [ - "import optuna\n", - "from pytorch_tabnet.tab_model import TabNetClassifier\n", - "from sklearn.metrics import f1_score\n", - "\n", - "# Optuna 최적화 함수\n", - "def objective(trial):\n", - " # 하이퍼파라미터 탐색 범위 설정\n", - " n_d = trial.suggest_int(\"n_d\", 8, 64, step=8)\n", - " n_a = trial.suggest_int(\"n_a\", 8, 64, step=8)\n", - " n_steps = trial.suggest_int(\"n_steps\", 3, 10)\n", - " gamma = trial.suggest_float(\"gamma\", 1.0, 2.0)\n", - " lambda_sparse = trial.suggest_float(\"lambda_sparse\", 0.0001, 0.01, log=True)\n", - " momentum = trial.suggest_float(\"momentum\", 0.01, 0.4)\n", - "\n", - " # TabNet 모델 생성\n", - " model = TabNetClassifier(\n", - " n_d=n_d, n_a=n_a, n_steps=n_steps,\n", - " gamma=gamma, lambda_sparse=lambda_sparse,\n", - " momentum=momentum,\n", - " seed=42\n", - " )\n", - "\n", - " # 모델 학습\n", - " model.fit(\n", - " X_train.to_numpy(), y_train.to_numpy(),\n", - " eval_set=[(X_val.to_numpy(), y_val.to_numpy())],\n", - " eval_name=['valid'],\n", - " eval_metric=['balanced_accuracy'],\n", - " max_epochs=25,\n", - " patience=6,\n", - " batch_size=1024,\n", - " virtual_batch_size=128,\n", - " verbose=0\n", - " )\n", - "\n", - " # Validation 데이터로 F1-score 계산\n", - " y_pred_val = model.predict(X_val.to_numpy())\n", - " val_score = f1_score(y_val.to_numpy(), y_pred_val)\n", - "\n", - " return val_score # 최적화를 F1-score 기준으로 진행\n", - "\n", - "# Optuna 실행\n", - "sampler = optuna.samplers.TPESampler(seed=seed)\n", - "study = optuna.create_study(direction=\"maximize\", sampler=sampler)\n", - "study.optimize(objective, n_trials=50) # 50번 탐색 실행\n", - "\n", - "# 최적 하이퍼파라미터 출력\n", - "best_params = study.best_trial.params\n", - "print(\"Best Hyperparameters:\", best_params)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "##### 모델 학습 루프" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch 0 | loss: 0.31833 | valid_balanced_accuracy: 0.82671 | 0:00:00s\n", - "epoch 1 | loss: 0.15936 | valid_balanced_accuracy: 0.87756 | 0:00:01s\n", - "epoch 2 | loss: 0.11391 | valid_balanced_accuracy: 0.87688 | 0:00:01s\n", - "epoch 3 | loss: 0.09934 | valid_balanced_accuracy: 0.82004 | 0:00:02s\n", - "epoch 4 | loss: 0.09147 | valid_balanced_accuracy: 0.82121 | 0:00:03s\n", - "epoch 5 | loss: 0.08413 | valid_balanced_accuracy: 0.7819 | 0:00:03s\n", - "epoch 6 | loss: 0.08536 | valid_balanced_accuracy: 0.85017 | 0:00:04s\n", - "epoch 7 | loss: 0.08246 | valid_balanced_accuracy: 0.76575 | 0:00:04s\n", - "epoch 8 | loss: 0.08266 | valid_balanced_accuracy: 0.88493 | 0:00:05s\n", - "epoch 9 | loss: 0.07919 | valid_balanced_accuracy: 0.82698 | 0:00:06s\n", - "epoch 10 | loss: 0.07745 | valid_balanced_accuracy: 0.80953 | 0:00:06s\n", - "epoch 11 | loss: 0.07453 | valid_balanced_accuracy: 0.81522 | 0:00:07s\n", - "epoch 12 | loss: 0.07515 | valid_balanced_accuracy: 0.85778 | 0:00:07s\n", - "epoch 13 | loss: 0.0729 | valid_balanced_accuracy: 0.82253 | 0:00:08s\n", - "epoch 14 | loss: 0.07833 | valid_balanced_accuracy: 0.86493 | 0:00:09s\n", - "epoch 15 | loss: 0.07302 | valid_balanced_accuracy: 0.8062 | 0:00:09s\n", - "epoch 16 | loss: 0.06809 | valid_balanced_accuracy: 0.82516 | 0:00:10s\n", - "\n", - "Early stopping occurred at epoch 16 with best_epoch = 8 and best_valid_balanced_accuracy = 0.88493\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/callbacks.py:172: UserWarning: Best weights from best epoch are automatically used!\n", - " warnings.warn(wrn_msg)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Successfully saved model at ./models/tabnet_model.zip\n", - "Validation F1-Score: 0.7668\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:454: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n", - " saved_state_dict = torch.load(f, map_location=self.device)\n", - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:82: UserWarning: Device used : cuda\n", - " warnings.warn(f\"Device used : {self.device}\")\n" - ] - } - ], - "source": [ - "from pytorch_tabnet.tab_model import TabNetClassifier\n", - "from sklearn.metrics import accuracy_score, f1_score, roc_auc_score\n", - "import numpy as np\n", - "\n", - "# 최적 하이퍼파라미터 적용\n", - "best_model = TabNetClassifier(\n", - " n_d=best_params[\"n_d\"],\n", - " n_a=best_params[\"n_a\"],\n", - " n_steps=best_params[\"n_steps\"],\n", - " gamma=best_params[\"gamma\"],\n", - " lambda_sparse=best_params[\"lambda_sparse\"],\n", - " momentum=best_params[\"momentum\"],\n", - " seed=42 # 시드 고정\n", - ")\n", - "\n", - "# Early Stopping 설정\n", - "best_val_f1 = -float(\"inf\") # F1-score는 최대화가 목표\n", - "patience = 8 # F1-score가 개선되지 않는 Epoch 수\n", - "counter = 0\n", - "\n", - "# 모델 학습\n", - "best_model.fit(\n", - " X_train.to_numpy(), y_train.to_numpy(),\n", - " eval_set=[(X_val.to_numpy(), y_val.to_numpy())],\n", - " eval_name=['valid'],\n", - " eval_metric=['balanced_accuracy'], # TabNet은 F1-score를 지원하지 않음 (balanced_accuracy 사용)\n", - " max_epochs=50, # 최대 Epoch\n", - " patience=patience, # Early stopping 적용\n", - " batch_size=1024,\n", - " virtual_batch_size=128\n", - ")\n", - "\n", - "# 학습이 끝난 후 모델 저장 (경로 명확히 지정)\n", - "best_model.save_model(\"./models/tabnet_model\")\n", - "\n", - "# 저장된 모델 불러오기\n", - "best_model.load_model(\"./models/tabnet_model.zip\")\n", - "\n", - "# Validation 데이터 평가\n", - "y_pred_val = best_model.predict(X_val.to_numpy())\n", - "val_f1 = f1_score(y_val.to_numpy(), y_pred_val)\n", - "\n", - "print(f\"Validation F1-Score: {val_f1:.4f}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "##### 성능 평가" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Test Accuracy: 0.8888127853881278\n", - "Test AUC: 0.8782745963906583\n", - "Test F1-Score: 0.46775956284153003\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/miniconda3/lib/python3.12/site-packages/pytorch_tabnet/abstract_model.py:454: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n", - " saved_state_dict = torch.load(f, map_location=self.device)\n" - ] - } - ], - "source": [ - "from sklearn.metrics import accuracy_score, roc_auc_score, f1_score\n", - "\n", - "# 저장된 모델 불러오기\n", - "loaded_model = TabNetClassifier()\n", - "loaded_model.load_model(\"./models/tabnet_model.zip\")\n", - "\n", - "# Test 데이터 평가\n", - "y_pred_test = loaded_model.predict(X_test.to_numpy())\n", - "\n", - "# 성능 평가 지표 계산\n", - "print(\"Test Accuracy:\", accuracy_score(y_test, y_pred_test))\n", - "print(\"Test AUC:\", roc_auc_score(y_test, y_pred_test))\n", - "print(\"Test F1-Score:\", f1_score(y_test, y_pred_test))" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.10" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -}