Update src

src/data_processing.py

@@ -14,73 +14,119 @@ VOL_WINDOWS = [20, 50]
 LAGS = 3
 
 def load_data(symbols, market_symbol, start_date, end_date):
-    print(f"Downloading data from {start_date} to {end_date}...")
+    print("Downloading data for AAPL and market index (auto_adjust=True)...")
+    
     df_market = yf.download(market_symbol, start=start_date, end=end_date, auto_adjust=True, progress=False)
+
     if isinstance(df_market.columns, pd.MultiIndex):
         df_market.columns = df_market.columns.droplevel(1)
     
     df_market = df_market.reset_index()[['Date', 'Close']].rename(columns={'Close': 'Market_Close'})
     dfs = []
-    
     for symbol in symbols:
-        try:
-            df = yf.download(symbol, start=start_date, end=end_date, auto_adjust=True, progress=False)
-            if isinstance(df.columns, pd.MultiIndex):
-                df.columns = df.columns.droplevel(1)
-            df = df.reset_index()[['Date', 'Open', 'High', 'Low', 'Close', 'Volume']]
-            df['Ticker'] = symbol
-            df = pd.merge(df, df_market, on='Date', how='left')
-            dfs.append(df)
-        except Exception as e:
-            print(f"Lỗi khi tải dữ liệu {symbol}: {e}")
+        df = yf.download(symbol, start=start_date, end=end_date, auto_adjust=True, progress=False)
+        if isinstance(df.columns, pd.MultiIndex):
+            df.columns = df.columns.droplevel(1)
+        df = df.reset_index()[['Date', 'Open', 'High', 'Low', 'Close', 'Volume']]
+        df['Ticker'] = symbol
+    
+        df = pd.merge(df, df_market, on='Date', how='left')
+        dfs.append(df)
 
-    df_concat = pd.concat(dfs, ignore_index=True)
-    df_concat = df_concat.sort_values(['Ticker', 'Date']).reset_index(drop=True)
-    return df_concat
+    df = pd.concat(dfs, ignore_index = True)
+    df = df.sort_values(['Ticker', 'Date']).reset_index(drop=True)
+    print(f"Loaded raw panel data: {len(df)} rows | {len(symbols)} tickers | "
+          f"from {df['Date'].min().date()} to {df['Date'].max().date()}")
+    return df
 
 def clean_data(df):
     cleaned_dfs = []
     for ticker, group in df.groupby('Ticker'):
         group = group.set_index('Date').sort_index()
-        start_dt, end_dt = group.index.min(), group.index.max()
+        start_dt = group.index.min()
+        end_dt = group.index.max()
         all_business_days = pd.date_range(start=start_dt, end=end_dt, freq="B")
-        group = group.reindex(all_business_days).ffill().reset_index().rename(columns={'index': 'Date'})
+
+        group = group.reindex(all_business_days)
+        group = group.ffill()
+        group = group.reset_index().rename(columns={'index': 'Date'})
         group['Ticker'] = ticker
         cleaned_dfs.append(group)
     
-    df_cleaned = pd.concat(cleaned_dfs, ignore_index=True).sort_values(['Ticker', 'Date']).reset_index(drop=True)
-    return df_cleaned
+    df_cleaned = pd.concat(cleaned_dfs, ignore_index = True)
+    df_cleaned = df_cleaned.sort_values(['Ticker', 'Date']).reset_index(drop=True)
+
+    print(f"Data cleaned: {len(df_cleaned)} rows | "
+          f"from {df_cleaned['Date'].min().date()} to {df_cleaned['Date'].max().date()}")
+    return df
 
 def validate_data(df, stage="pre_feature"):
+
+    print(f"Validating data at stage: {stage}...")
     num_cols = df.select_dtypes(include=[np.number]).columns
-    if df[num_cols].isna().sum().sum() > 0:
-        print(f"WARNING: Có NaN values tại {stage}")
+    
+    nan_count = df[num_cols].isna().sum().sum()
+    inf_count = np.isinf(df[num_cols]).sum().sum()
+    
+    if nan_count > 0:
+        print(f"WARNING: Tìm thấy {nan_count} NaN values tại stage {stage}")
+    if inf_count > 0:
+        print(f"WARNING: Tìm thấy {inf_count} Inf values tại stage {stage}")
+
+    if 'Date' in df.columns and 'Market_Return' in df.columns:
+        market_std_per_date = df.groupby('Date')['Market_Return'].std(ddof=0).max()
+        if pd.notna(market_std_per_date) and market_std_per_date > 1e-8:
+            print(f"WARNING: Cross-ticker contamination detected! "
+                  f"Max std of Market_Return per date: {market_std_per_date:.2e}")
+    
+    # Kiểm tra nhanh variance của returns (nên > 0)
+    if 'Daily_Return' in df.columns:
+        for ticker, grp in df.groupby('Ticker'):
+            if len(grp) > 1 and grp['Daily_Return'].std(ddof=0) == 0:
+                print(f"WARNING: Ticker {ticker} has zero variance in Daily_Return!")
+    
+    print(f"Validation passed at {stage} (no critical issues).")
     return df
 
 def generate_technical_features(df, is_inference=False):
+    """
+    Feature Engineering hoàn toàn mới theo 5 yêu cầu:
+    1. Corporate actions đã được xử lý ở load_data (auto_adjust=True)
+    2. TẤT CẢ features được chuyển sang dạng stationary (ratio, pct distance, normalized, position 0-1)
+    3. Multi-timeframe: nhiều windows để Linear_Regression tự chọn tín hiệu mạnh
+    4. Market Regime & Volatility: ATR normalized + rolling volatility
+    5. Gọi validate_data ngay trước khi return
+    """
     data = df.copy()
 
     def add_features(group):
         g = group.copy()
+    
+        # === 1. BASIC RETURNS (luôn stationary) ===
         g['Daily_Return'] = g['Close'].pct_change()
         g['Log_Return'] = np.log(1 + g['Daily_Return'])
         g['Market_Return'] = g['Market_Close'].pct_change()
         g['Market_Log_Return'] = np.log(1 + g['Market_Return'])
     
+        # === 2. LAGGED FEATURES – CHỈ lag returns (KHÔNG lag Close raw) ===
+        # Lý do: Close raw và SMA raw là non-stationary → Linear_Regression sẽ học nhầm trend dài hạn thay vì pattern thực sự.
         for i in range(1, LAGS + 1):
             g[f'Return_Lag_{i}'] = g['Daily_Return'].shift(i)
             g[f'Market_Return_Lag_{i}'] = g['Market_Return'].shift(i)
     
+        # === 3. MULTI-TIMEFRAME TECHNICAL INDICATORS (Stationary version) ===
+        # SMA & EMA → Ratio + % Distance (thay vì giá trị tuyệt đối)
         for w in SMA_WINDOWS:
             sma = g['Close'].rolling(window=w).mean()
             g[f'SMA_{w}_Ratio'] = g['Close'] / sma
-            g[f'SMA_{w}_Distance_pct'] = (g['Close'] - sma) / sma * 100
+            g[f'SMA_{w}_Distance_pct'] = (g['Close'] - sma) / sma * 100   # % distance từ giá đến SMA
         
         for w in EMA_WINDOWS:
             ema = g['Close'].ewm(span=w, adjust=False).mean()
             g[f'EMA_{w}_Ratio'] = g['Close'] / ema
             g[f'EMA_{w}_Distance_pct'] = (g['Close'] - ema) / ema * 100
     
+        # RSI multi-window (đã stationary tự nhiên 0-100)
         for w in RSI_WINDOWS:
             delta = g['Close'].diff()
             gain = delta.where(delta > 0, 0).rolling(w).mean()
@@ -88,46 +134,68 @@ def generate_technical_features(df, is_inference=False):
             rs = gain / loss
             g[f'RSI_{w}'] = 100 - (100 / (1 + rs))
     
+        # MACD: giữ cấu trúc gốc nhưng normalize Hist theo % giá (stationary)
         ema_fast = g['Close'].ewm(span=12, adjust=False).mean()
         ema_slow = g['Close'].ewm(span=26, adjust=False).mean()
         g['MACD_Line'] = ema_fast - ema_slow
         g['MACD_Signal'] = g['MACD_Line'].ewm(span=9, adjust=False).mean()
-        g['MACD_Hist'] = (g['MACD_Line'] - g['MACD_Signal'])
-        g['MACD_Hist_Normalized'] = g['MACD_Hist'] / g['Close'] * 100
+        g['MACD_Hist'] = g['MACD_Line'] - g['MACD_Signal']
+        g['MACD_Hist_Normalized'] = g['MACD_Hist'] / g['Close'] * 100   # % của giá → stationary
     
+        # Bollinger Bands: Width % + Position (0-1) thay vì Upper/Lower tuyệt đối
         for w in BB_WINDOWS:
             middle = g['Close'].rolling(w).mean()
             std_dev = g['Close'].rolling(w).std()
-            bb_range = (middle + 2 * std_dev) - (middle - 2 * std_dev)
-            g[f'BB_Width_{w}_pct'] = (bb_range / middle * 100)
-            g[f'BB_Position_{w}'] = (g['Close'] - (middle - 2 * std_dev)) / bb_range.where(bb_range > 0, 1)
+            upper = middle + 2 * std_dev
+            lower = middle - 2 * std_dev
+            bb_range = upper - lower
+            
+            g[f'BB_Width_{w}_pct'] = (bb_range / middle * 100)                  # % width (stationary)
+            g[f'BB_Position_{w}'] = (g['Close'] - lower) / bb_range.where(bb_range > 0, 1)  # 0-1 position
         
-        tr = pd.concat([g['High'] - g['Low'], 
-                        abs(g['High'] - g['Close'].shift(1)), 
-                        abs(g['Low'] - g['Close'].shift(1))], axis=1).max(axis=1)
+        # === 4. VOLATILITY & MARKET REGIME FEATURES ===
+        # True Range & ATR normalized
+        def calculate_true_range(high, low, close):
+            tr1 = high - low
+            tr2 = abs(high - close.shift(1))
+            tr3 = abs(low - close.shift(1))
+            return pd.concat([tr1, tr2, tr3], axis=1).max(axis=1)
         
+        tr = calculate_true_range(g['High'], g['Low'], g['Close'])
         for w in ATR_WINDOWS:
             atr = tr.rolling(w).mean()
-            g[f'ATR_Normalized_{w}'] = atr / g['Close']
             g[f'ATR_{w}'] = atr
+            g[f'ATR_Normalized_{w}'] = atr / g['Close']   # Relative volatility → stationary
         
+        # Rolling volatility (market regime detection)
         for w in VOL_WINDOWS:
             g[f'Market_Rolling_Vol_{w}'] = g['Market_Return'].rolling(w).std()
             g[f'AAPL_Rolling_Vol_{w}'] = g['Daily_Return'].rolling(w).std()
         
+        # Relative volume
         g['Rel_Volume_20'] = g['Volume'] / g['Volume'].rolling(20).mean()
         return g
         
+    # Xóa NaN (do rolling + lag)
     data_list = [add_features(group) for _, group in data.groupby('Ticker')]
     data = pd.concat(data_list, ignore_index=True)
     
     if not is_inference:
         data['Target_Return'] = data.groupby('Ticker')['Daily_Return'].shift(-1)
         data = data.dropna().reset_index(drop=True)
-        data = validate_data(data, stage="post_feature")
-        X = data.drop(columns=['Date', 'Ticker', 'Market_Close', 'Target_Return'], errors='ignore')
-        y = data['Target_Return']
-        return data, X, y
+        # === 5. DATA VALIDATION TRƯỚC KHI TRẢ VỀ ===
+        data = validate_data(data, stage="post_feature_engineering")
+
+        df_backtest = data.copy()
+        drop_cols = ['Date', 'Ticker', 'Market_Close', 'Target_Return']
+        X = data.drop(columns=drop_cols, errors='ignore')
+        y = data['Target_Return'].copy()
+
+        print(f"Generated stationary features & prepared ML data:\n"
+            f"   • Total rows: {len(data)} | Tickers: {data['Ticker'].nunique()}\n"
+            f"   • Features: {X.shape[1]} | X shape: {X.shape} | y shape: {y.shape}")
+        
+        return df_backtest, X, y
     else:
         # Nếu là predict, dòng cuối cùng của mỗi ticker sẽ chứa feature đầy đủ và không bị loại bỏ do thiếu target
         data = data.dropna().reset_index(drop=True)

src/train.py

@@ -32,7 +32,10 @@ def main():
     print("3. Tối ưu siêu tham số Ridge Regression...")
     def objective_lr(trial):
         alpha = trial.suggest_float('alpha', 1e-4, 1e4, log=True)
+        
+        tscv = TimeSeriesSplit(n_splits=5)
         fold_scores = []
+        
         for train_idx, val_idx in tscv.split(X):
             X_train, X_val = X.iloc[train_idx], X.iloc[val_idx]
             y_train, y_val = y.iloc[train_idx], y.iloc[val_idx]
@@ -44,7 +47,10 @@ def main():
             model = Ridge(alpha=alpha, random_state=42)
             model.fit(X_train_scaled, y_train)
             preds = model.predict(X_val_scaled)
-            fold_scores.append(np.sqrt(mean_squared_error(y_val, preds)))
+            
+            rmse = np.sqrt(mean_squared_error(y_val, preds))
+            fold_scores.append(rmse)
+        
         return np.mean(fold_scores)
 
     study_lr = optuna.create_study(direction='minimize')
@@ -54,9 +60,17 @@ def main():
     # === TỐI ƯU SVR ===
     print("4. Tối ưu siêu tham số SVR...")
     def objective_svr(trial):
-        C = trial.suggest_float('C', 1e-3, 10.0, log=True)
+        # Chỉ tối ưu siêu tham số SVR
+        kernel = trial.suggest_categorical('kernel', ['linear', 'rbf'])
+        C = trial.suggest_float('C', 1e-3, 100.0, log=True)
         epsilon = trial.suggest_float('epsilon', 1e-3, 1.0, log=True)
+        gamma = trial.suggest_categorical('gamma', ['scale', 'auto']) if kernel == 'rbf' else 'scale'
+        
+        # Chuẩn bị data với feature cố định
+        
+        tscv = TimeSeriesSplit(n_splits=5)
         fold_scores = []
+        
         for train_idx, val_idx in tscv.split(X):
             X_train, X_val = X.iloc[train_idx], X.iloc[val_idx]
             y_train, y_val = y.iloc[train_idx], y.iloc[val_idx]
@@ -65,11 +79,18 @@ def main():
             X_train_scaled = scaler.fit_transform(X_train)
             X_val_scaled = scaler.transform(X_val)
             
-            # Khống chế max_iter để SVR chạy nhanh hơn trong quá trình tìm kiếm
-            model = SVR(kernel='rbf', C=C, epsilon=epsilon, gamma='scale', max_iter=2000)
+            X_train_scaled = X_train_scaled.astype('float32')
+            X_val_scaled   = X_val_scaled.astype('float32')
+            y_train_f32    = y_train.values.astype('float32')
+            y_val_f32      = y_val.values.astype('float32')
+            
+            model = SVR(kernel=kernel, C=C, epsilon=epsilon, gamma=gamma, max_iter=5000)
             model.fit(X_train_scaled, y_train)
             preds = model.predict(X_val_scaled)
-            fold_scores.append(np.sqrt(mean_squared_error(y_val, preds)))
+            
+            rmse = np.sqrt(mean_squared_error(y_val, preds))
+            fold_scores.append(rmse)
+        
         return np.mean(fold_scores)
 
     study_svr = optuna.create_study(direction='minimize')