Yair fixed categorical, ID, temporal cols and pre-processing [no more errors for XGBoost]

data_loader.py

@@ -11,14 +11,15 @@ from imblearn.over_sampling import SMOTE
 #  CONFIGURATION
 # ===========================
 
-TRAIN_PATH = "~/Downloads/train_dataset_full - train_dataset_full (1).csv"
-TEST_PATH = "~/Downloads/X_test_1st (1).csv"  # Replace with actual test dataset path
+TRAIN_PATH = "data/train_dataset_full - train_dataset_full.csv"
+# TRAIN_PATH = "data/train_dataset_full - train_dataset_partial_for_testing.csv"
+TEST_PATH = "data/X_test_1st.csv"  # Replace with actual test dataset path
 
-CATEGORICAL_COLUMNS = ["gender", "product", "campaign_id", "webpage_id"]
+CATEGORICAL_COLUMNS = ["gender", "product",]
+IDS_COLUMNS = [ "user_id", "session_id", "campaign_id", "webpage_id"]
 TARGET_COLUMN = "is_click"
-
 FEATURE_COLUMNS = [
-    "age_level", "gender", "product", "campaign_id", "webpage_id",
+    "age_level", "gender", "product",
     "product_category_1", "product_category_2", "user_group_id",
     "user_depth", "city_development_index", "var_1"
 ]
@@ -30,6 +31,7 @@ AGGREGATED_COLUMNS = [
     "unique_campaigns_city_age_prod", "unique_webpages_city_age_prod"
 ]
 
+TEMPORAL_COLUMNS = ["year", "month", "day", "hour", "minute", "weekday"]
 # ===========================
 #  LOAD DATASETS
 # ===========================
@@ -37,8 +39,37 @@ AGGREGATED_COLUMNS = [
 def load_data(train_path=TRAIN_PATH, test_path=TEST_PATH):
     """Load train & test datasets, handling missing values."""
     train_df = pd.read_csv(train_path)
+    y_train = train_df[TARGET_COLUMN]
+    train_df = train_df[~y_train.isnull()]
+
+
     test_df = pd.read_csv(test_path)
 
+    train_df["DateTime"] = pd.to_datetime(train_df["DateTime"])
+    test_df["DateTime"] = pd.to_datetime(test_df["DateTime"])
+    train_df["DateTime"].fillna(train_df["DateTime"].mode()[0], inplace=True)
+    test_df["DateTime"].fillna(test_df["DateTime"].mode()[0], inplace=True)
+
+    if "DateTime" in train_df.columns:
+        train_df["DateTime"] = pd.to_datetime(train_df["DateTime"])
+        train_df["year"] = train_df["DateTime"].dt.year
+        train_df["month"] = train_df["DateTime"].dt.month
+        train_df["day"] = train_df["DateTime"].dt.day
+        train_df["hour"] = train_df["DateTime"].dt.hour
+        train_df["minute"] = train_df["DateTime"].dt.minute
+        train_df["weekday"] = train_df["DateTime"].dt.weekday
+        train_df.drop("DateTime", axis=1, inplace=True)
+
+    if "DateTime" in test_df.columns:
+        test_df["DateTime"] = pd.to_datetime(test_df["DateTime"])
+        test_df["year"] = test_df["DateTime"].dt.year
+        test_df["month"] = test_df["DateTime"].dt.month
+        test_df["day"] = test_df["DateTime"].dt.day
+        test_df["hour"] = test_df["DateTime"].dt.hour
+        test_df["minute"] = test_df["DateTime"].dt.minute
+        test_df["weekday"] = test_df["DateTime"].dt.weekday
+        test_df.drop("DateTime", axis=1, inplace=True)
+
     # Fill missing values
     train_df.fillna(-1, inplace=True)
     test_df.fillna(-1, inplace=True)
@@ -108,11 +139,12 @@ def preprocess_data(df, test_df, categorical_columns):
 
     numerical_columns = [col for col in FEATURE_COLUMNS + AGGREGATED_COLUMNS if col not in categorical_columns]
 
-    scaler = StandardScaler()
-    df[numerical_columns] = scaler.fit_transform(df[numerical_columns])
-    test_df[numerical_columns] = scaler.transform(test_df[numerical_columns])
+    # scaler = StandardScaler()
+    # df[numerical_columns] = scaler.fit_transform(df[numerical_columns])
+    # test_df[numerical_columns] = scaler.transform(test_df[numerical_columns])
+
 
-    return df, test_df, label_encoders, scaler
+    return df, test_df, label_encoders,# scaler
 
 
 # ===========================
@@ -122,7 +154,7 @@ def preprocess_data(df, test_df, categorical_columns):
 def split_and_balance_data(df, target_column):
     """Splits data into training and validation sets, applies SMOTE to balance classes."""
 
-    X = df[FEATURE_COLUMNS + AGGREGATED_COLUMNS]
+    X = df[IDS_COLUMNS + FEATURE_COLUMNS + AGGREGATED_COLUMNS + TEMPORAL_COLUMNS]
     y = df[target_column]
 
     # Handle class imbalance using SMOTE
@@ -172,7 +204,7 @@ def load_and_process_data():
 
     df, test_df = load_data()
     df, test_df = add_aggregated_features(df, test_df)
-    df, test_df, label_encoders, scaler = preprocess_data(df, test_df, CATEGORICAL_COLUMNS)
+    df, test_df, label_encoders = preprocess_data(df, test_df, CATEGORICAL_COLUMNS)
     X_train, X_val, y_train, y_val = split_and_balance_data(df, TARGET_COLUMN)
 
     return X_train, X_val, y_train, y_val, test_df
@@ -182,3 +214,4 @@ if __name__ == "__main__":
     print("🔹 Loading and processing data...")
     X_train, X_val, y_train, y_val, test_df = load_and_process_data()
     print("✅ Data successfully loaded and processed!")
+