Upload training/generate_dataset.py with huggingface_hub

training/generate_dataset.py

@@ -0,0 +1,361 @@
+"""
+ALWAS Synthetic Dataset Generator
+Generates realistic analog IC layout block data for ML model training.
+Covers: block metadata, stage transitions, hours, bottleneck labels.
+"""
+import numpy as np
+import pandas as pd
+import json
+from datetime import datetime, timedelta
+import random
+
+np.random.seed(42)
+random.seed(42)
+
+# === Domain Constants ===
+TECH_NODES = ['5nm', '7nm', '12nm', '14nm', '22nm', '28nm', '45nm', '65nm']
+TECH_NODE_COMPLEXITY = {'5nm': 1.6, '7nm': 1.4, '12nm': 1.2, '14nm': 1.1, '22nm': 0.9, '28nm': 0.8, '45nm': 0.6, '65nm': 0.5}
+TECH_NODE_WEIGHTS = [0.05, 0.15, 0.2, 0.15, 0.15, 0.15, 0.1, 0.05]
+
+BLOCK_TYPES = ['ADC', 'DAC', 'PLL', 'LDO', 'BGR', 'OTA', 'Comparator', 'SerDes', 
+               'VCO', 'Mixer', 'LNA', 'PA', 'TIA', 'SampleHold', 'LVDS_Driver',
+               'BandgapRef', 'CurrentMirror', 'DiffAmp', 'Oscillator', 'PowerDetector']
+BLOCK_TYPE_COMPLEXITY = {
+    'ADC': 1.5, 'DAC': 1.3, 'PLL': 1.7, 'LDO': 0.8, 'BGR': 0.7, 'OTA': 0.6,
+    'Comparator': 0.5, 'SerDes': 1.8, 'VCO': 1.2, 'Mixer': 1.1, 'LNA': 1.0,
+    'PA': 1.3, 'TIA': 0.9, 'SampleHold': 0.7, 'LVDS_Driver': 1.0,
+    'BandgapRef': 0.6, 'CurrentMirror': 0.4, 'DiffAmp': 0.5, 'Oscillator': 1.1,
+    'PowerDetector': 0.8
+}
+BLOCK_TYPE_WEIGHTS = [0.1, 0.08, 0.08, 0.1, 0.06, 0.08, 0.07, 0.04, 0.06, 0.05,
+                      0.05, 0.04, 0.04, 0.03, 0.03, 0.02, 0.02, 0.02, 0.02, 0.01]
+
+PRIORITIES = ['P1-Critical', 'P2-High', 'P3-Medium', 'P4-Low']
+PRIORITY_WEIGHTS = [0.1, 0.25, 0.45, 0.2]
+PRIORITY_FACTOR = {'P1-Critical': 0.85, 'P2-High': 0.95, 'P3-Medium': 1.0, 'P4-Low': 1.1}
+
+STAGES = ['Not Started', 'In Progress', 'DRC', 'LVS', 'ERC', 'Review', 'Completed']
+STAGE_IDX = {s: i for i, s in enumerate(STAGES)}
+
+ENGINEERS = [f'eng_{i:03d}' for i in range(1, 51)]
+ENGINEER_SKILL = {e: np.clip(np.random.normal(1.0, 0.2), 0.5, 1.5) for e in ENGINEERS}
+
+# === Helper Functions ===
+def estimate_transistor_count(block_type, tech_node):
+    base = {
+        'ADC': 50000, 'DAC': 35000, 'PLL': 80000, 'LDO': 8000, 'BGR': 5000,
+        'OTA': 3000, 'Comparator': 2000, 'SerDes': 120000, 'VCO': 15000,
+        'Mixer': 10000, 'LNA': 6000, 'PA': 20000, 'TIA': 4000, 'SampleHold': 3500,
+        'LVDS_Driver': 8000, 'BandgapRef': 3000, 'CurrentMirror': 1500,
+        'DiffAmp': 2500, 'Oscillator': 12000, 'PowerDetector': 5000
+    }
+    node_scale = {'5nm': 2.0, '7nm': 1.7, '12nm': 1.3, '14nm': 1.2, '22nm': 1.0, '28nm': 0.9, '45nm': 0.7, '65nm': 0.5}
+    count = base.get(block_type, 10000) * node_scale.get(tech_node, 1.0)
+    return int(count * np.random.lognormal(0, 0.3))
+
+def compute_true_hours(block_type, tech_node, transistor_count, priority, engineer, 
+                        has_dependencies, constraint_complexity):
+    """Physics-inspired hour estimation with noise."""
+    base = 20
+    type_mult = BLOCK_TYPE_COMPLEXITY.get(block_type, 1.0)
+    node_mult = TECH_NODE_COMPLEXITY.get(tech_node, 1.0)
+    size_mult = np.log1p(transistor_count) / np.log1p(10000)
+    priority_mult = PRIORITY_FACTOR.get(priority, 1.0)
+    skill_mult = 1.0 / ENGINEER_SKILL.get(engineer, 1.0)
+    dep_mult = 1.15 if has_dependencies else 1.0
+    constraint_mult = 1 + 0.2 * constraint_complexity
+    
+    hours = base * type_mult * node_mult * size_mult * priority_mult * skill_mult * dep_mult * constraint_mult
+    noise = np.random.lognormal(0, 0.15)
+    return max(4, round(hours * noise, 1))
+
+def compute_complexity_label(hours, transistor_count, tech_node):
+    """Derive complexity label from multiple signals."""
+    node_score = TECH_NODE_COMPLEXITY.get(tech_node, 1.0)
+    size_score = np.log1p(transistor_count) / np.log1p(100000)
+    combined = 0.5 * (hours / 100) + 0.3 * node_score + 0.2 * size_score
+    if combined < 0.35:
+        return 'Low'
+    elif combined < 0.65:
+        return 'Medium'
+    else:
+        return 'High'
+
+def generate_stage_transitions(block, start_date):
+    """Generate realistic stage transition events with timestamps."""
+    transitions = []
+    current_date = start_date
+    total_hours = block['actual_hours']
+    stage_proportions = {
+        'Not Started': 0.0, 'In Progress': 0.35, 'DRC': 0.2, 
+        'LVS': 0.15, 'ERC': 0.15, 'Review': 0.1, 'Completed': 0.05
+    }
+    
+    for i, stage in enumerate(STAGES):
+        if stage == 'Not Started':
+            transitions.append({
+                'stage': stage, 'timestamp': current_date.isoformat(),
+                'hours_in_stage': 0, 'drc_violations': 0, 'lvs_mismatches': 0
+            })
+            current_date += timedelta(hours=np.random.exponential(4))
+            continue
+        
+        proportion = stage_proportions.get(stage, 0.1)
+        stage_hours = total_hours * proportion * np.random.uniform(0.7, 1.3)
+        stage_hours = max(1, stage_hours)
+        
+        drc_violations = 0
+        lvs_mismatches = 0
+        
+        if stage == 'DRC':
+            if block['tech_node'] in ['5nm', '7nm', '12nm']:
+                drc_violations = int(np.random.exponential(8) + np.random.poisson(3))
+            else:
+                drc_violations = int(np.random.exponential(3) + np.random.poisson(1))
+        
+        if stage == 'LVS':
+            lvs_mismatches = int(np.random.exponential(2))
+        
+        # Days to complete this stage (8 hours/day)
+        days = max(0.5, stage_hours / 8)
+        # Add some variance for weekends, blocked time
+        if np.random.random() < 0.15:
+            days *= np.random.uniform(1.5, 3.0)  # delays
+        
+        transitions.append({
+            'stage': stage,
+            'timestamp': current_date.isoformat(),
+            'hours_in_stage': round(stage_hours, 1),
+            'days_in_stage': round(days, 1),
+            'drc_violations': drc_violations,
+            'lvs_mismatches': lvs_mismatches
+        })
+        current_date += timedelta(days=days)
+        
+        if i >= block.get('final_stage_idx', len(STAGES) - 1):
+            break
+    
+    return transitions
+
+def generate_block(block_id, is_completed=True):
+    """Generate a single block with all features."""
+    tech_node = np.random.choice(TECH_NODES, p=TECH_NODE_WEIGHTS)
+    block_type = np.random.choice(BLOCK_TYPES, p=BLOCK_TYPE_WEIGHTS)
+    priority = np.random.choice(PRIORITIES, p=PRIORITY_WEIGHTS)
+    engineer = np.random.choice(ENGINEERS)
+    transistor_count = estimate_transistor_count(block_type, tech_node)
+    has_dependencies = np.random.random() < 0.35
+    num_dependencies = int(np.random.exponential(1.5)) if has_dependencies else 0
+    constraint_complexity = np.random.uniform(0, 3)  # analog constraint score
+    
+    actual_hours = compute_true_hours(
+        block_type, tech_node, transistor_count, priority, engineer,
+        has_dependencies, constraint_complexity
+    )
+    
+    # Estimated hours (simulating AI/human estimate — noisy version of actual)
+    estimation_noise = np.random.normal(0, 0.25)
+    estimated_hours = max(4, round(actual_hours * np.exp(estimation_noise), 1))
+    
+    complexity = compute_complexity_label(actual_hours, transistor_count, tech_node)
+    
+    # Determine final stage
+    if is_completed:
+        final_stage = 'Completed'
+        final_stage_idx = 6
+    else:
+        # In-progress blocks stop at various stages
+        final_stage_idx = np.random.choice(range(1, 6), p=[0.3, 0.25, 0.2, 0.15, 0.1])
+        final_stage = STAGES[final_stage_idx]
+    
+    # Start date: random in last 2 years
+    start_date = datetime(2024, 1, 1) + timedelta(days=np.random.randint(0, 730))
+    
+    # DRC iterations (for completed blocks)
+    drc_iterations = max(1, int(np.random.exponential(2) + 1))
+    if tech_node in ['5nm', '7nm']:
+        drc_iterations = max(1, int(np.random.exponential(3) + 2))
+    
+    # Hours logged so far
+    hours_logged = actual_hours if is_completed else round(actual_hours * np.random.uniform(0.1, 0.9), 1)
+    
+    # Bottleneck risk label
+    hours_ratio = hours_logged / max(estimated_hours, 1)
+    days_in_current = np.random.exponential(3) if not is_completed else 0
+    
+    if hours_ratio > 1.3 or days_in_current > 5:
+        bottleneck_risk = 'High'
+    elif hours_ratio > 1.0 or days_in_current > 3:
+        bottleneck_risk = 'Medium'
+    else:
+        bottleneck_risk = 'Low'
+    
+    block = {
+        'block_id': f'BLK-{block_id:05d}',
+        'block_name': f'{block_type}_{tech_node}_{block_id}',
+        'block_type': block_type,
+        'tech_node': tech_node,
+        'priority': priority,
+        'priority_numeric': PRIORITIES.index(priority) + 1,
+        'engineer_id': engineer,
+        'engineer_skill_factor': round(ENGINEER_SKILL[engineer], 3),
+        'transistor_count': transistor_count,
+        'transistor_count_log': round(np.log1p(transistor_count), 4),
+        'has_dependencies': int(has_dependencies),
+        'num_dependencies': num_dependencies,
+        'constraint_complexity': round(constraint_complexity, 2),
+        'estimated_hours': estimated_hours,
+        'actual_hours': actual_hours,
+        'hours_logged': hours_logged,
+        'hours_over_estimate_ratio': round(hours_logged / max(estimated_hours, 1), 3),
+        'drc_iterations': drc_iterations,
+        'drc_violations_total': 0,  # filled from transitions
+        'lvs_mismatches_total': 0,
+        'current_stage': final_stage,
+        'current_stage_idx': STAGE_IDX[final_stage],
+        'days_in_current_stage': round(days_in_current, 1),
+        'is_completed': int(is_completed),
+        'complexity': complexity,
+        'bottleneck_risk': bottleneck_risk,
+        'start_date': start_date.strftime('%Y-%m-%d'),
+        'final_stage_idx': final_stage_idx,
+    }
+    
+    # Generate transitions
+    transitions = generate_stage_transitions(block, start_date)
+    block['transitions'] = json.dumps(transitions)
+    block['num_stage_transitions'] = len(transitions)
+    
+    # Aggregate DRC/LVS from transitions
+    block['drc_violations_total'] = sum(t.get('drc_violations', 0) for t in transitions)
+    block['lvs_mismatches_total'] = sum(t.get('lvs_mismatches', 0) for t in transitions)
+    
+    # Compute total days from transitions
+    if len(transitions) > 1:
+        block['total_days'] = sum(t.get('days_in_stage', 0) for t in transitions)
+    else:
+        block['total_days'] = round(actual_hours / 8, 1)
+    
+    # Due date and overdue status
+    due_days = max(int(block['total_days'] * np.random.uniform(0.8, 1.5)), 3)
+    block['due_date'] = (start_date + timedelta(days=due_days)).strftime('%Y-%m-%d')
+    if is_completed:
+        block['is_overdue'] = int(block['total_days'] > due_days)
+    else:
+        elapsed = (datetime.now() - start_date).days
+        block['is_overdue'] = int(elapsed > due_days)
+    
+    return block
+
+def generate_dataset(n_completed=3000, n_in_progress=1000):
+    """Generate full dataset."""
+    print(f"Generating {n_completed} completed + {n_in_progress} in-progress blocks...")
+    blocks = []
+    
+    for i in range(n_completed):
+        blocks.append(generate_block(i + 1, is_completed=True))
+    
+    for i in range(n_in_progress):
+        blocks.append(generate_block(n_completed + i + 1, is_completed=False))
+    
+    df = pd.DataFrame(blocks)
+    return df
+
+# === Generate SFT Dataset for LLM Fine-tuning ===
+def generate_sft_dataset(df, n_samples=2000):
+    """Generate conversational dataset for complexity estimation SFT."""
+    sft_data = []
+    sampled = df.sample(n=min(n_samples, len(df)), random_state=42)
+    
+    for _, row in sampled.iterrows():
+        user_msg = (
+            f"Estimate the complexity and required hours for this analog IC layout block:\n"
+            f"- Block Type: {row['block_type']}\n"
+            f"- Technology Node: {row['tech_node']}\n"
+            f"- Priority: {row['priority']}\n"
+            f"- Estimated Transistor Count: ~{row['transistor_count']:,}\n"
+            f"- Has Dependencies: {'Yes' if row['has_dependencies'] else 'No'}"
+            + (f" ({row['num_dependencies']} blocks)\n" if row['has_dependencies'] else "\n") +
+            f"- Constraint Complexity Score: {row['constraint_complexity']:.1f}/3.0\n"
+            f"- DRC Iterations Expected: {row['drc_iterations']}"
+        )
+        
+        # Build a realistic explanation
+        reasons = []
+        if row['complexity'] == 'High':
+            if row['tech_node'] in ['5nm', '7nm', '12nm']:
+                reasons.append(f"Advanced {row['tech_node']} node requires extensive DRC/LVS iterations")
+            if row['transistor_count'] > 50000:
+                reasons.append(f"Large transistor count (~{row['transistor_count']:,}) increases layout complexity")
+            if row['block_type'] in ['PLL', 'SerDes', 'ADC']:
+                reasons.append(f"{row['block_type']} blocks require precision matching and careful routing")
+            if row['has_dependencies']:
+                reasons.append(f"Inter-block dependencies ({row['num_dependencies']}) add integration overhead")
+        elif row['complexity'] == 'Medium':
+            reasons.append(f"{row['block_type']} at {row['tech_node']} has moderate layout challenges")
+            if row['constraint_complexity'] > 1.5:
+                reasons.append("Analog constraints require careful floor planning")
+        else:
+            reasons.append(f"{row['block_type']} at {row['tech_node']} is a well-characterized block")
+            if row['transistor_count'] < 10000:
+                reasons.append("Small transistor count allows straightforward layout")
+        
+        if not reasons:
+            reasons.append(f"Standard {row['block_type']} layout at {row['tech_node']}")
+        
+        risk_level = 'low' if row['complexity'] == 'Low' else ('medium' if row['complexity'] == 'Medium' else 'high')
+        
+        assistant_msg = (
+            f'{{"complexity": "{row["complexity"]}", '
+            f'"estimated_hours": {row["actual_hours"]}, '
+            f'"confidence": {round(np.random.uniform(0.7, 0.95), 2)}, '
+            f'"risk_level": "{risk_level}", '
+            f'"reasoning": "{"; ".join(reasons)}", '
+            f'"recommended_drc_iterations": {row["drc_iterations"]}, '
+            f'"suggested_engineer_skill_level": "{"senior" if row["complexity"] == "High" else "mid" if row["complexity"] == "Medium" else "junior"}"}}'
+        )
+        
+        sft_data.append({
+            "messages": [
+                {"role": "system", "content": "You are ALWAS AI, an analog IC layout complexity estimation assistant. Given block metadata, estimate complexity (Low/Medium/High), required hours, and provide reasoning. Respond in JSON format."},
+                {"role": "user", "content": user_msg},
+                {"role": "assistant", "content": assistant_msg}
+            ]
+        })
+    
+    return sft_data
+
+
+if __name__ == '__main__':
+    # Generate main tabular dataset
+    df = generate_dataset(n_completed=3000, n_in_progress=1000)
+    
+    # Save tabular data
+    df.to_csv('/app/alwas_blocks_dataset.csv', index=False)
+    df.to_parquet('/app/alwas_blocks_dataset.parquet', index=False)
+    
+    # Generate SFT dataset
+    sft_data = generate_sft_dataset(df, n_samples=2000)
+    with open('/app/alwas_sft_dataset.json', 'w') as f:
+        json.dump(sft_data, f, indent=2)
+    
+    # Print dataset stats
+    print(f"\n=== Dataset Statistics ===")
+    print(f"Total blocks: {len(df)}")
+    print(f"Completed: {df['is_completed'].sum()}")
+    print(f"In-progress: {(~df['is_completed'].astype(bool)).sum()}")
+    print(f"\nComplexity distribution:")
+    print(df['complexity'].value_counts())
+    print(f"\nBottleneck risk distribution:")
+    print(df['bottleneck_risk'].value_counts())
+    print(f"\nBlock type distribution:")
+    print(df['block_type'].value_counts().head(10))
+    print(f"\nTech node distribution:")
+    print(df['tech_node'].value_counts())
+    print(f"\nHours statistics:")
+    print(df['actual_hours'].describe())
+    print(f"\nSFT samples: {len(sft_data)}")
+    print(f"\nFiles saved:")
+    print(f"  /app/alwas_blocks_dataset.csv")
+    print(f"  /app/alwas_blocks_dataset.parquet")
+    print(f"  /app/alwas_sft_dataset.json")