""" Module 1: Exploratory Data Analysis (EDA) Generates comprehensive analysis and figures for the credit card fraud dataset. """ import os import numpy as np import pandas as pd import matplotlib matplotlib.use('Agg') import matplotlib.pyplot as plt import matplotlib.gridspec as gridspec import seaborn as sns from datasets import load_dataset import warnings warnings.filterwarnings('ignore') from config import FIGURES_DIR, FIG_DPI, FIG_BG, DATASET_ID, DATA_DIR, SEED # Style plt.style.use('seaborn-v0_8-whitegrid') sns.set_palette("husl") def load_data(): """Load the credit card fraud dataset from HuggingFace Hub.""" print("=" * 60) print("LOADING DATASET") print("=" * 60) ds = load_dataset(DATASET_ID, split="train") df = ds.to_pandas() # Save raw data df.to_csv(os.path.join(DATA_DIR, "creditcard.csv"), index=False) print(f"Dataset shape: {df.shape}") print(f"Columns: {list(df.columns)}") return df def basic_statistics(df): """Print basic dataset statistics.""" print("\n" + "=" * 60) print("BASIC STATISTICS") print("=" * 60) print(f"\nShape: {df.shape[0]} rows, {df.shape[1]} columns") print(f"\nData types:\n{df.dtypes.value_counts()}") print(f"\nMissing values: {df.isnull().sum().sum()}") print(f"\nDuplicate rows: {df.duplicated().sum()}") print(f"\nBasic stats for Amount:") print(df['Amount'].describe()) print(f"\nBasic stats for Time:") print(df['Time'].describe()) return df.describe() def class_distribution_analysis(df): """Analyze and visualize class distribution.""" print("\n" + "=" * 60) print("CLASS DISTRIBUTION ANALYSIS") print("=" * 60) class_counts = df['Class'].value_counts() fraud_ratio = class_counts[1] / len(df) * 100 print(f"\nClass 0 (Legitimate): {class_counts[0]:,} ({100 - fraud_ratio:.3f}%)") print(f"Class 1 (Fraud): {class_counts[1]:,} ({fraud_ratio:.3f}%)") print(f"Imbalance ratio: 1:{class_counts[0] // class_counts[1]}") # Figure: Class Distribution fig, axes = plt.subplots(1, 2, figsize=(14, 5), facecolor=FIG_BG) # Bar plot colors = ['#2ecc71', '#e74c3c'] bars = axes[0].bar(['Legitimate\n(Class 0)', 'Fraud\n(Class 1)'], class_counts.values, color=colors, edgecolor='black', linewidth=0.5) axes[0].set_ylabel('Number of Transactions', fontsize=12) axes[0].set_title('Transaction Class Distribution', fontsize=14, fontweight='bold') for bar, count in zip(bars, class_counts.values): axes[0].text(bar.get_x() + bar.get_width()/2., bar.get_height() + 1000, f'{count:,}', ha='center', va='bottom', fontsize=11, fontweight='bold') axes[0].set_yscale('log') axes[0].set_ylabel('Number of Transactions (log scale)', fontsize=12) # Pie chart axes[1].pie(class_counts.values, labels=['Legitimate', 'Fraud'], colors=colors, autopct='%1.3f%%', startangle=90, explode=(0, 0.1), shadow=True, textprops={'fontsize': 12}) axes[1].set_title('Fraud Ratio', fontsize=14, fontweight='bold') plt.tight_layout() plt.savefig(os.path.join(FIGURES_DIR, "class_distribution.png"), dpi=FIG_DPI, bbox_inches='tight', facecolor=FIG_BG) plt.savefig(os.path.join(FIGURES_DIR, "class_distribution.pdf"), bbox_inches='tight', facecolor=FIG_BG) plt.close() print("Saved: class_distribution.png/pdf") return class_counts, fraud_ratio def transaction_amount_analysis(df): """Analyze transaction amounts by class.""" print("\n" + "=" * 60) print("TRANSACTION AMOUNT ANALYSIS") print("=" * 60) for cls, label in [(0, 'Legitimate'), (1, 'Fraud')]: subset = df[df['Class'] == cls]['Amount'] print(f"\n{label} Transactions:") print(f" Mean: ${subset.mean():.2f}") print(f" Median: ${subset.median():.2f}") print(f" Std: ${subset.std():.2f}") print(f" Min: ${subset.min():.2f}") print(f" Max: ${subset.max():.2f}") print(f" Q25: ${subset.quantile(0.25):.2f}") print(f" Q75: ${subset.quantile(0.75):.2f}") fig, axes = plt.subplots(2, 2, figsize=(14, 10), facecolor=FIG_BG) # Amount distribution - Legitimate axes[0, 0].hist(df[df['Class'] == 0]['Amount'], bins=100, color='#2ecc71', alpha=0.7, edgecolor='black', linewidth=0.3) axes[0, 0].set_title('Legitimate Transaction Amounts', fontsize=12, fontweight='bold') axes[0, 0].set_xlabel('Amount ($)') axes[0, 0].set_ylabel('Frequency') axes[0, 0].set_xlim(0, 2500) # Amount distribution - Fraud axes[0, 1].hist(df[df['Class'] == 1]['Amount'], bins=50, color='#e74c3c', alpha=0.7, edgecolor='black', linewidth=0.3) axes[0, 1].set_title('Fraudulent Transaction Amounts', fontsize=12, fontweight='bold') axes[0, 1].set_xlabel('Amount ($)') axes[0, 1].set_ylabel('Frequency') # Log-scaled comparison for cls, color, label in [(0, '#2ecc71', 'Legitimate'), (1, '#e74c3c', 'Fraud')]: subset = df[df['Class'] == cls]['Amount'] axes[1, 0].hist(np.log1p(subset), bins=50, color=color, alpha=0.6, label=label, edgecolor='black', linewidth=0.3) axes[1, 0].set_title('Log-Scaled Amount Distribution', fontsize=12, fontweight='bold') axes[1, 0].set_xlabel('log(1 + Amount)') axes[1, 0].set_ylabel('Frequency') axes[1, 0].legend() # Box plot comparison df_plot = df[['Amount', 'Class']].copy() df_plot['Class'] = df_plot['Class'].map({0: 'Legitimate', 1: 'Fraud'}) sns.boxplot(data=df_plot, x='Class', y='Amount', palette=['#2ecc71', '#e74c3c'], ax=axes[1, 1]) axes[1, 1].set_title('Amount by Class (Box Plot)', fontsize=12, fontweight='bold') axes[1, 1].set_ylim(0, 500) plt.tight_layout() plt.savefig(os.path.join(FIGURES_DIR, "amount_analysis.png"), dpi=FIG_DPI, bbox_inches='tight', facecolor=FIG_BG) plt.savefig(os.path.join(FIGURES_DIR, "amount_analysis.pdf"), bbox_inches='tight', facecolor=FIG_BG) plt.close() print("Saved: amount_analysis.png/pdf") def time_analysis(df): """Analyze temporal patterns.""" print("\n" + "=" * 60) print("TEMPORAL ANALYSIS") print("=" * 60) df_temp = df.copy() df_temp['Hour'] = (df_temp['Time'] / 3600) % 24 fig, axes = plt.subplots(1, 2, figsize=(14, 5), facecolor=FIG_BG) # Transaction density over time for cls, color, label in [(0, '#2ecc71', 'Legitimate'), (1, '#e74c3c', 'Fraud')]: subset = df_temp[df_temp['Class'] == cls] axes[0].hist(subset['Hour'], bins=48, color=color, alpha=0.6, label=label, density=True) axes[0].set_title('Transaction Density by Hour of Day', fontsize=12, fontweight='bold') axes[0].set_xlabel('Hour of Day') axes[0].set_ylabel('Density') axes[0].legend() # Fraud rate by hour hourly_fraud = df_temp.groupby(df_temp['Hour'].astype(int))['Class'].mean() * 100 axes[1].bar(hourly_fraud.index, hourly_fraud.values, color='#e74c3c', alpha=0.7, edgecolor='black', linewidth=0.3) axes[1].set_title('Fraud Rate by Hour', fontsize=12, fontweight='bold') axes[1].set_xlabel('Hour of Day') axes[1].set_ylabel('Fraud Rate (%)') plt.tight_layout() plt.savefig(os.path.join(FIGURES_DIR, "time_analysis.png"), dpi=FIG_DPI, bbox_inches='tight', facecolor=FIG_BG) plt.savefig(os.path.join(FIGURES_DIR, "time_analysis.pdf"), bbox_inches='tight', facecolor=FIG_BG) plt.close() print("Saved: time_analysis.png/pdf") def correlation_heatmap(df): """Generate correlation heatmap.""" print("\n" + "=" * 60) print("CORRELATION ANALYSIS") print("=" * 60) # Correlation with target correlations = df.corr()['Class'].drop('Class').sort_values() print("\nTop 10 features positively correlated with Fraud:") print(correlations.tail(10)) print("\nTop 10 features negatively correlated with Fraud:") print(correlations.head(10)) fig, axes = plt.subplots(1, 2, figsize=(18, 7), facecolor=FIG_BG) # Correlation with Class colors = ['#e74c3c' if v < 0 else '#2ecc71' for v in correlations.values] axes[0].barh(correlations.index, correlations.values, color=colors, edgecolor='black', linewidth=0.3) axes[0].set_title('Feature Correlation with Fraud (Class)', fontsize=12, fontweight='bold') axes[0].set_xlabel('Pearson Correlation') axes[0].axvline(x=0, color='black', linewidth=0.5) # Full heatmap (subset of important features) important_features = list(correlations.head(5).index) + list(correlations.tail(5).index) + ['Amount', 'Time', 'Class'] corr_matrix = df[important_features].corr() sns.heatmap(corr_matrix, annot=True, fmt='.2f', cmap='RdBu_r', center=0, ax=axes[1], square=True, linewidths=0.5) axes[1].set_title('Correlation Heatmap (Top Features)', fontsize=12, fontweight='bold') plt.tight_layout() plt.savefig(os.path.join(FIGURES_DIR, "correlation_heatmap.png"), dpi=FIG_DPI, bbox_inches='tight', facecolor=FIG_BG) plt.savefig(os.path.join(FIGURES_DIR, "correlation_heatmap.pdf"), bbox_inches='tight', facecolor=FIG_BG) plt.close() print("Saved: correlation_heatmap.png/pdf") return correlations def feature_distributions(df): """Plot distributions of key PCA features by class.""" print("\n" + "=" * 60) print("FEATURE DISTRIBUTIONS") print("=" * 60) # Select most discriminative features corr_with_class = df.corr()['Class'].drop('Class').abs().sort_values(ascending=False) top_features = corr_with_class.head(12).index.tolist() fig, axes = plt.subplots(3, 4, figsize=(20, 12), facecolor=FIG_BG) axes = axes.ravel() for i, feat in enumerate(top_features): for cls, color, label in [(0, '#2ecc71', 'Legit'), (1, '#e74c3c', 'Fraud')]: subset = df[df['Class'] == cls][feat] axes[i].hist(subset, bins=50, color=color, alpha=0.5, label=label, density=True) axes[i].set_title(f'{feat}', fontsize=10, fontweight='bold') axes[i].legend(fontsize=8) plt.suptitle('Distribution of Top 12 Discriminative Features by Class', fontsize=14, fontweight='bold', y=1.02) plt.tight_layout() plt.savefig(os.path.join(FIGURES_DIR, "feature_distributions.png"), dpi=FIG_DPI, bbox_inches='tight', facecolor=FIG_BG) plt.savefig(os.path.join(FIGURES_DIR, "feature_distributions.pdf"), bbox_inches='tight', facecolor=FIG_BG) plt.close() print("Saved: feature_distributions.png/pdf") def missing_values_analysis(df): """Check for missing values.""" print("\n" + "=" * 60) print("MISSING VALUES ANALYSIS") print("=" * 60) missing = df.isnull().sum() missing_pct = (missing / len(df)) * 100 if missing.sum() == 0: print("No missing values found in the dataset.") else: missing_report = pd.DataFrame({'Missing Count': missing, 'Percentage': missing_pct}) missing_report = missing_report[missing_report['Missing Count'] > 0] print(missing_report) return missing def key_observations(df, class_counts, fraud_ratio, correlations): """Generate 5 key observations from the data.""" print("\n" + "=" * 60) print("5 KEY OBSERVATIONS") print("=" * 60) observations = [] # 1. Extreme class imbalance obs1 = (f"1. EXTREME CLASS IMBALANCE: Only {fraud_ratio:.3f}% of transactions are fraudulent " f"({class_counts[1]:,} out of {len(df):,}). The imbalance ratio is approximately " f"1:{class_counts[0] // class_counts[1]}, making accuracy a misleading metric.") observations.append(obs1) # 2. Amount patterns fraud_amt = df[df['Class'] == 1]['Amount'] legit_amt = df[df['Class'] == 0]['Amount'] obs2 = (f"2. AMOUNT PATTERNS: Fraudulent transactions have a mean of ${fraud_amt.mean():.2f} " f"(median: ${fraud_amt.median():.2f}) vs legitimate mean of ${legit_amt.mean():.2f} " f"(median: ${legit_amt.median():.2f}). Fraud tends to involve smaller amounts to " f"avoid detection triggers.") observations.append(obs2) # 3. Temporal patterns df_temp = df.copy() df_temp['Hour'] = (df_temp['Time'] / 3600) % 24 night_fraud = df_temp[(df_temp['Hour'] >= 0) & (df_temp['Hour'] <= 6)] night_fraud_rate = night_fraud['Class'].mean() * 100 day_fraud_rate = df_temp[(df_temp['Hour'] >= 7) & (df_temp['Hour'] <= 23)]['Class'].mean() * 100 obs3 = (f"3. TEMPORAL PATTERNS: Night-time (0-6h) fraud rate is {night_fraud_rate:.3f}% " f"vs daytime (7-23h) rate of {day_fraud_rate:.3f}%. " f"Fraudsters are more active during low-activity periods.") observations.append(obs3) # 4. PCA features top_neg = correlations.head(3) top_pos = correlations.tail(3) obs4 = (f"4. KEY DISCRIMINATIVE FEATURES: Most negatively correlated with fraud: " f"{list(top_neg.index)} (r={top_neg.values[0]:.3f} to {top_neg.values[2]:.3f}). " f"Most positively correlated: {list(top_pos.index)} " f"(r={top_pos.values[0]:.3f} to {top_pos.values[2]:.3f}).") observations.append(obs4) # 5. No missing values obs5 = (f"5. DATA QUALITY: The dataset has no missing values and {df.duplicated().sum()} " f"duplicate rows. All V1-V28 features are PCA-transformed, ensuring no " f"multicollinearity in the principal components. Only 'Time' and 'Amount' are " f"in original scale and need normalization.") observations.append(obs5) for obs in observations: print(f"\n{obs}") return observations def run_eda(): """Run the complete EDA pipeline.""" print("=" * 60) print("FRAUD DETECTION SYSTEM - EXPLORATORY DATA ANALYSIS") print("=" * 60) # Load data df = load_data() # Basic stats stats = basic_statistics(df) # Class distribution class_counts, fraud_ratio = class_distribution_analysis(df) # Amount analysis transaction_amount_analysis(df) # Time analysis time_analysis(df) # Correlation correlations = correlation_heatmap(df) # Feature distributions feature_distributions(df) # Missing values missing = missing_values_analysis(df) # Key observations observations = key_observations(df, class_counts, fraud_ratio, correlations) print("\n" + "=" * 60) print("EDA COMPLETE - All figures saved to:", FIGURES_DIR) print("=" * 60) return df, stats, class_counts, fraud_ratio, correlations, observations if __name__ == "__main__": df, stats, class_counts, fraud_ratio, correlations, observations = run_eda()