Upload notebook_manager.py

notebook_manager.py

@@ -0,0 +1,459 @@
+# ╔══════════════════════════════════════════════════════════════════════════════╗
+# ║  NOTEBOOK : Analyse Vacances - VERSION MANAGER                               ║
+# ║  Graphes, tableaux markdown, explications métier                             ║
+# ╚══════════════════════════════════════════════════════════════════════════════╝
+
+# ════════════════════════════════════════════════════════════════════════════════
+# CELLULE 1 — Imports
+# ════════════════════════════════════════════════════════════════════════════════
+import pandas as pd
+import numpy as np
+from datetime import date
+import warnings
+warnings.filterwarnings("ignore")
+
+try:
+    import matplotlib.pyplot as plt
+    MATPLOTLIB_OK = True
+except ImportError:
+    MATPLOTLIB_OK = False
+    print("⚠️ matplotlib non installé → pas de graphes. pip install matplotlib")
+
+# ════════════════════════════════════════════════════════════════════════════════
+# CELLULE 2 — Calendrier vacances scolaires (2023-2027)
+# ════════════════════════════════════════════════════════════════════════════════
+
+VACANCES = {
+    "2023-2024": {
+        "A": [(date(2023,10,21),date(2023,11,5)), (date(2023,12,23),date(2024,1,7)),
+              (date(2024,2,17),date(2024,3,3)), (date(2024,4,13),date(2024,4,28)),
+              (date(2024,7,6),date(2024,9,1))],
+        "B": [(date(2023,10,21),date(2023,11,5)), (date(2023,12,23),date(2024,1,7)),
+              (date(2024,2,24),date(2024,3,10)), (date(2024,4,20),date(2024,5,5)),
+              (date(2024,7,6),date(2024,9,1))],
+        "C": [(date(2023,10,21),date(2023,11,5)), (date(2023,12,23),date(2024,1,7)),
+              (date(2024,2,10),date(2024,2,25)), (date(2024,4,6),date(2024,4,21)),
+              (date(2024,7,6),date(2024,9,1))],
+    },
+    "2024-2025": {
+        "A": [(date(2024,10,19),date(2024,11,3)), (date(2024,12,21),date(2025,1,5)),
+              (date(2025,2,8),date(2025,2,23)), (date(2025,4,5),date(2025,4,20)),
+              (date(2025,7,5),date(2025,8,31))],
+        "B": [(date(2024,10,19),date(2024,11,3)), (date(2024,12,21),date(2025,1,5)),
+              (date(2025,2,22),date(2025,3,9)), (date(2025,4,19),date(2025,5,4)),
+              (date(2025,7,5),date(2025,8,31))],
+        "C": [(date(2024,10,19),date(2024,11,3)), (date(2024,12,21),date(2025,1,5)),
+              (date(2025,2,15),date(2025,3,2)), (date(2025,4,12),date(2025,4,27)),
+              (date(2025,7,5),date(2025,8,31))],
+    },
+    "2025-2026": {
+        "A": [(date(2025,10,18),date(2025,11,2)), (date(2025,12,20),date(2026,1,4)),
+              (date(2026,2,14),date(2026,3,1)), (date(2026,4,4),date(2026,4,19)),
+              (date(2026,7,4),date(2026,8,31))],
+        "B": [(date(2025,10,18),date(2025,11,2)), (date(2025,12,20),date(2026,1,4)),
+              (date(2026,2,21),date(2026,3,8)), (date(2026,4,11),date(2026,4,26)),
+              (date(2026,7,4),date(2026,8,31))],
+        "C": [(date(2025,10,18),date(2025,11,2)), (date(2025,12,20),date(2026,1,4)),
+              (date(2026,2,7),date(2026,2,22)), (date(2026,3,28),date(2026,4,12)),
+              (date(2026,7,4),date(2026,8,31))],
+    },
+    "2026-2027": {
+        "A": [(date(2026,10,17),date(2026,11,1)), (date(2026,12,19),date(2027,1,3)),
+              (date(2027,2,14),date(2027,3,1)), (date(2027,4,4),date(2027,4,19)),
+              (date(2027,7,3),date(2027,8,31))],
+        "B": [(date(2026,10,17),date(2026,11,1)), (date(2026,12,19),date(2027,1,3)),
+              (date(2027,2,21),date(2027,3,8)), (date(2027,4,11),date(2027,4,26)),
+              (date(2027,7,3),date(2027,8,31))],
+        "C": [(date(2026,10,17),date(2026,11,1)), (date(2026,12,19),date(2027,1,3)),
+              (date(2027,2,7),date(2027,2,22)), (date(2027,3,28),date(2027,4,12)),
+              (date(2027,7,3),date(2027,8,31))],
+    },
+}
+
+DR_TO_ZONE = {
+    "Besancon": "A", "Bordeaux": "A", "Clermont-Ferrand": "A",
+    "Dijon": "A", "Grenoble": "A", "Lyon": "A", "Limoges": "A", "Poitiers": "A",
+    "Aix-Marseille": "B", "Amiens": "B", "Caen": "B", "Lille": "B",
+    "Nantes": "B", "Nice": "B", "Orleans-Tours": "B", "Reims": "B",
+    "Rennes": "B", "Rouen": "B", "Strasbourg": "B",
+    "Creteil": "C", "Montpellier": "C", "Nancy-Metz": "C",
+    "Paris": "C", "Toulouse": "C", "Versailles": "C",
+    "AFC": "C",
+}
+
+def get_zone(dr): return DR_TO_ZONE.get(dr, "C")
+
+def is_vacances(d, zone, vac):
+    for debut, fin in vac.get(zone, []):
+        if debut <= d <= fin: return True
+    return False
+
+def get_annee_scolaire(d):
+    return f"{d.year}-{d.year+1}" if d.month >= 9 else f"{d.year-1}-{d.year}"
+
+def get_periode_vacances(d, vac):
+    for zone in ["A","B","C"]:
+        for debut, fin in vac.get(zone, []):
+            if debut <= d <= fin:
+                m = d.month
+                if m in [10,11]: return "Toussaint"
+                elif m in [12,1]: return "Noel"
+                elif m in [2,3]: return "Hiver"
+                elif m in [4,5]: return "Printemps"
+                elif m in [7,8]: return "Ete"
+    return "Hors_vacances"
+
+def add_vacances(df):
+    df = df.copy()
+    df["Date"] = pd.to_datetime(df["Date"]).dt.tz_localize(None)
+    df["zone_vacances"] = df["DR"].apply(get_zone)
+    df["annee_scolaire"] = df["Date"].apply(lambda d: get_annee_scolaire(d.date()))
+    def _vac(row):
+        d = row["Date"].date()
+        return is_vacances(d, row["zone_vacances"], VACANCES.get(row["annee_scolaire"], {}))
+    def _per(row):
+        d = row["Date"].date()
+        return get_periode_vacances(d, VACANCES.get(row["annee_scolaire"], {}))
+    df["is_vacances_zone"] = df.apply(_vac, axis=1)
+    df["periode_vacances"] = df.apply(_per, axis=1)
+    return df
+
+# ════════════════════════════════════════════════════════════════════════════════
+# CELLULE 3 — Métriques (vocabulaire métier)
+# ════════════════════════════════════════════════════════════════════════════════
+
+def ecart_absolu(y_true, y_pred):
+    return np.mean(np.abs(np.asarray(y_true) - np.asarray(y_pred)))
+
+def ecart_relatif_pct(y_true, y_pred):
+    yt, yp = np.asarray(y_true), np.asarray(y_pred)
+    return np.mean(np.abs((yt - yp) / np.maximum(yt, 1))) * 100
+
+# ════════════════════════════════════════════════════════════════════════════════
+# CELLULE 4 — Analyse globale avec explications
+# ════════════════════════════════════════════════════════════════════════════════
+
+def analyse_globale(df):
+    dfp = df[(df["count"] > 0) & (df["prediction_XGB"].notna())].copy()
+    if len(dfp) == 0:
+        print("❌ Aucune donnée passée avec prédiction valide.")
+        return None
+    
+    mask_v = dfp["is_vacances_zone"]
+    mask_h = ~mask_v
+    
+    rows = []
+    for mask, label in [(mask_v, "Vacances scolaires"), (mask_h, "Hors vacances")]:
+        sub = dfp[mask]
+        if len(sub) == 0: continue
+        yt, yp = sub["count"].values, sub["prediction_XGB"].values
+        rows.append({
+            "Periode": label,
+            "Nb_jours": len(sub),
+            "Vol_reel": round(yt.mean(), 1),
+            "Vol_pred": round(yp.mean(), 1),
+            "Surprediction_%": round(((yp.mean() - yt.mean()) / max(yt.mean(), 1)) * 100, 1),
+            "Ecart_Absolu": round(ecart_absolu(yt, yp), 1),
+            "Ecart_Relatif_%": round(ecart_relatif_pct(yt, yp), 1),
+        })
+    
+    df_res = pd.DataFrame(rows)
+    
+    print("\n" + "=" * 75)
+    print("📊 EFFET VACANCES SCOLAIRES — RÉSULTATS AVANT CORRECTION")
+    print("=" * 75)
+    print("""
+PROCÉDURE :
+1. Identification des jours de vacances scolaires par zone (A/B/C)
+2. Comparaison volume réel d'appels vs prédiction XGBoost
+3. Métriques :
+   • Ecart_Absolu    = erreur moyenne en nombre d'appels/jour
+   • Ecart_Relatif_% = erreur moyenne relative (% du volume réel)
+4. Correction = ajustement multiplicatif uniquement sur jours de vacances
+    """)
+    
+    print("\n📋 TABLEAU RÉCAPITULATIF (markdown)\n")
+    print(df_res.to_markdown(index=False))
+    
+    if len(df_res) >= 2:
+        row_v = df_res[df_res["Periode"] == "Vacances scolaires"].iloc[0]
+        row_h = df_res[df_res["Periode"] == "Hors vacances"].iloc[0]
+        baisse = ((row_v["Vol_reel"] - row_h["Vol_reel"]) / max(row_h["Vol_reel"], 1)) * 100
+        
+        print(f"""
+📈 INTERPRÉTATION MÉTIER :
+
+→ Pendant les vacances scolaires, le volume baisse de {abs(baisse):.1f}%
+  ({row_v['Vol_reel']:.0f} appels/jour vs {row_h['Vol_reel']:.0f} hors vacances)
+
+→ Le modèle {'sur-prédit' if row_v['Surprediction_%'] > 0 else 'sous-prédit'} 
+  de {abs(row_v['Surprediction_%']):.1f}% en période de vacances
+  → Il ne capte pas complètement cette baisse
+
+→ Ecart_Absolu = {row_v['Ecart_Absolu']:.1f} appels/jour en vacances
+  (marge d'erreur de {row_v['Ecart_Relatif_%']:.1f}% du volume réel)
+        """)
+    
+    return df_res
+
+# ════════════════════════════════════════════════════════════════════════════════
+# CELLULE 5 — Analyse par sous-type d'accueil
+# ════════════════════════════════════════════════════════════════════════════════
+
+def analyse_par_sous_type(df):
+    dfp = df[(df["count"] > 0) & (df["prediction_XGB"].notna())].copy()
+    
+    rows = []
+    for st in sorted(dfp["sous_type_accueil"].dropna().unique()):
+        for periode_label, mask_base in [
+            ("Vacances", dfp["is_vacances_zone"]),
+            ("Hors_vacances", ~dfp["is_vacances_zone"])
+        ]:
+            mask = mask_base & (dfp["sous_type_accueil"] == st)
+            if mask.sum() < 5: continue
+            sub = dfp[mask]
+            yt, yp = sub["count"].values, sub["prediction_XGB"].values
+            rows.append({
+                "Sous_type": st,
+                "Periode": periode_label,
+                "Nb_jours": len(sub),
+                "Vol_reel": round(yt.mean(), 1),
+                "Vol_pred": round(yp.mean(), 1),
+                "Surprediction_%": round(((yp.mean() - yt.mean()) / max(yt.mean(), 1)) * 100, 1),
+                "Ecart_Absolu": round(ecart_absolu(yt, yp), 1),
+                "Ecart_Relatif_%": round(ecart_relatif_pct(yt, yp), 1),
+            })
+    
+    df_st = pd.DataFrame(rows)
+    
+    print("\n" + "=" * 75)
+    print("📊 ANALYSE PAR SOUS-TYPE D'ACCUEIL")
+    print("=" * 75)
+    
+    if len(df_st) > 0:
+        print("\n📋 Détail par sous-type (markdown)\n")
+        print(df_st.to_markdown(index=False))
+        
+        print("\n📋 Synthèse par sous-type :\n")
+        for st in sorted(df_st["Sous_type"].unique()):
+            sub = df_st[df_st["Sous_type"] == st]
+            vac = sub[sub["Periode"] == "Vacances"]
+            hors = sub[sub["Periode"] == "Hors_vacances"]
+            if len(vac) > 0 and len(hors) > 0:
+                baisse = ((vac.iloc[0]["Vol_reel"] - hors.iloc[0]["Vol_reel"]) 
+                          / max(hors.iloc[0]["Vol_reel"], 1)) * 100
+                print(f"  • {st:<25} : baisse vacances = {baisse:+.1f}% | "
+                      f"Ecart vac = {vac.iloc[0]['Ecart_Absolu']:.1f} appels "
+                      f"({vac.iloc[0]['Ecart_Relatif_%']:.1f}%)")
+    
+    return df_st
+
+# ════════════════════════════════════════════════════════════════════════════════
+# CELLULE 6 — Calcul facteurs + correction
+# ════════════════════════════════════════════════════════════════════════════════
+
+def calcule_facteurs(df):
+    dfp = df[(df["count"] > 0) & (df["prediction_XGB"].notna())].copy()
+    facteurs = {}
+    m_v = dfp["is_vacances_zone"]
+    if m_v.sum() > 0:
+        facteurs[("GLOBAL", "ALL")] = dfp.loc[m_v, "count"].mean() / max(dfp.loc[m_v, "prediction_XGB"].mean(), 1)
+    for zone in ["A", "B", "C"]:
+        for st in dfp["sous_type_accueil"].dropna().unique():
+            m = (dfp["zone_vacances"]==zone) & (dfp["sous_type_accueil"]==st) & dfp["is_vacances_zone"]
+            if m.sum() < 3: continue
+            f = dfp.loc[m, "count"].mean() / max(dfp.loc[m, "prediction_XGB"].mean(), 1)
+            facteurs[(zone, st)] = f
+    return facteurs
+
+def corrige_predictions(df, facteurs):
+    df = df.copy()
+    df["prediction_XGB_corrige"] = df["prediction_XGB"].astype(float)
+    m_v = df["is_vacances_zone"]
+    for zone in ["A", "B", "C"]:
+        for st in df["sous_type_accueil"].dropna().unique():
+            m = m_v & (df["zone_vacances"]==zone) & (df["sous_type_accueil"]==st)
+            if not m.any(): continue
+            f = facteurs.get((zone, st), facteurs.get(("GLOBAL","ALL"), 1.0))
+            df.loc[m, "prediction_XGB_corrige"] = df.loc[m, "prediction_XGB"] * f
+    return df
+
+# ════════════════════════════════════════════════════════════════════════════════
+# CELLULE 7 — Évaluation avant/après avec markdown
+# ═══════════════════════════════��════════════════════════════════════════════════
+
+def evalue_correction(df):
+    dfp = df[(df["count"] > 0) & (df["prediction_XGB"].notna())].copy()
+    
+    rows = []
+    for label, mask in [
+        ("Toutes_periodes", pd.Series([True]*len(dfp), index=dfp.index)),
+        ("Vacances", dfp["is_vacances_zone"]),
+        ("Hors_vacances", ~dfp["is_vacances_zone"]),
+    ]:
+        if mask.sum() < 2: continue
+        yt = dfp.loc[mask, "count"].values
+        y_avant = dfp.loc[mask, "prediction_XGB"].values
+        y_apres = dfp.loc[mask, "prediction_XGB_corrige"].values
+        
+        ea_avant = ecart_absolu(yt, y_avant)
+        ea_apres = ecart_absolu(yt, y_apres)
+        er_avant = ecart_relatif_pct(yt, y_avant)
+        er_apres = ecart_relatif_pct(yt, y_apres)
+        gain = ((ea_avant - ea_apres) / max(ea_avant, 1)) * 100
+        
+        rows.append({
+            "Periode": label,
+            "Nb_jours": mask.sum(),
+            "Ecart_Absolu_avant": round(ea_avant, 2),
+            "Ecart_Absolu_apres": round(ea_apres, 2),
+            "Gain_Ecart_Absolu_%": round(gain, 1),
+            "Ecart_Relatif_%_avant": round(er_avant, 1),
+            "Ecart_Relatif_%_apres": round(er_apres, 1),
+        })
+    
+    df_eval = pd.DataFrame(rows)
+    
+    print("\n" + "=" * 75)
+    print("📊 ÉVALUATION : AVANT vs APRÈS CORRECTION")
+    print("=" * 75)
+    print("\n📋 Résultats (markdown)\n")
+    print(df_eval.to_markdown(index=False))
+    
+    vac_row = df_eval[df_eval["Periode"] == "Vacances"]
+    if len(vac_row) > 0:
+        gain_vac = vac_row.iloc[0]["Gain_Ecart_Absolu_%"]
+        ea_av = vac_row.iloc[0]["Ecart_Absolu_avant"]
+        ea_ap = vac_row.iloc[0]["Ecart_Absolu_apres"]
+        print(f"""
+📈 INTERPRÉTATION :
+
+→ Sur les jours de vacances :
+  Ecart_Absolu passe de {ea_av:.2f} à {ea_ap:.2f} appels/jour
+  → Gain de {gain_vac:.1f}% sur la précision des prédictions
+
+→ Hors vacances : aucune modification
+  → La correction ne touche QUE les jours identifiés comme vacances
+
+→ Le facteur correcteur est appliqué sans re-entraîner le modèle
+  (post-processing uniquement)
+        """)
+    
+    return df_eval
+
+# ════════════════════════════════════════════════════════════════════════════════
+# CELLULE 8 — Graphes pour le manager
+# ════════════════════════════════════════════════════════════════════════════════
+
+def graphes_manager(df, dr_filtre=None, st_filtre=None):
+    if not MATPLOTLIB_OK:
+        print("❌ matplotlib non installé. pip install matplotlib")
+        return
+    
+    dfp = df[(df["count"] > 0) & (df["prediction_XGB"].notna())].copy()
+    if dr_filtre: dfp = dfp[dfp["DR"] == dr_filtre]
+    if st_filtre: dfp = dfp[dfp["sous_type_accueil"] == st_filtre]
+    dfp = dfp.sort_values("Date")
+    
+    if len(dfp) == 0:
+        print("❌ Pas de données pour ce filtre")
+        return
+    
+    fig, axes = plt.subplots(3, 1, figsize=(14, 12))
+    titre = f"DR={dr_filtre}, Type={st_filtre}" if (dr_filtre or st_filtre) else "Global"
+    
+    # --- GRAPHE 1 : Série temporelle ---
+    ax1 = axes[0]
+    ax1.plot(dfp["Date"], dfp["count"], label="Réel", color="black", linewidth=1)
+    ax1.plot(dfp["Date"], dfp["prediction_XGB"], label="XGB avant", color="orange", alpha=0.8)
+    if "prediction_XGB_corrige" in dfp.columns:
+        ax1.plot(dfp["Date"], dfp["prediction_XGB_corrige"], label="XGB corrigé", color="green", alpha=0.8)
+    
+    vac = dfp[dfp["is_vacances_zone"]]
+    if len(vac) > 0:
+        for _, r in vac.iterrows():
+            ax1.axvline(r["Date"], color="red", alpha=0.03)
+    
+    ax1.set_title(f"Volume d'appels — {titre}")
+    ax1.set_ylabel("Appels / jour")
+    ax1.legend(loc="upper left")
+    ax1.grid(True, alpha=0.3)
+    
+    # --- GRAPHE 2 : Boxplot par période ---
+    ax2 = axes[1]
+    data_box, labels_box, colors_box = [], [], []
+    for periode in ["Toussaint", "Noel", "Hiver", "Printemps", "Ete", "Hors_vacances"]:
+        mask = dfp["periode_vacances"] == periode
+        if mask.sum() < 3: continue
+        data_box.append(dfp.loc[mask, "count"].values)
+        labels_box.append(periode)
+        colors_box.append("lightcoral" if periode != "Hors_vacances" else "lightblue")
+    
+    bp = ax2.boxplot(data_box, labels=labels_box, patch_artist=True)
+    for patch, color in zip(bp["boxes"], colors_box):
+        patch.set_facecolor(color)
+    ax2.set_title("Distribution des volumes par période")
+    ax2.set_ylabel("Appels / jour")
+    ax2.grid(True, alpha=0.3, axis="y")
+    
+    # --- GRAPHE 3 : Erreur avant/après ---
+    ax3 = axes[2]
+    periodes, ea_avant, ea_apres = [], [], []
+    for periode in ["Toussaint", "Noel", "Hiver", "Printemps", "Ete"]:
+        mask = dfp["periode_vacances"] == periode
+        if mask.sum() < 3: continue
+        yt = dfp.loc[mask, "count"].values
+        yp_av = dfp.loc[mask, "prediction_XGB"].values
+        periodes.append(periode)
+        ea_avant.append(ecart_absolu(yt, yp_av))
+        if "prediction_XGB_corrige" in dfp.columns:
+            yp_ap = dfp.loc[mask, "prediction_XGB_corrige"].values
+            ea_apres.append(ecart_absolu(yt, yp_ap))
+        else:
+            ea_apres.append(ecart_absolu(yt, yp_av))
+    
+    x = np.arange(len(periodes))
+    width = 0.35
+    ax3.bar(x - width/2, ea_avant, width, label="Avant", color="orange", alpha=0.8)
+    ax3.bar(x + width/2, ea_apres, width, label="Après", color="green", alpha=0.8)
+    ax3.set_title("Ecart Absolu par période (avant vs après correction)")
+    ax3.set_ylabel("Ecart Absolu (appels/jour)")
+    ax3.set_xticks(x)
+    ax3.set_xticklabels(periodes)
+    ax3.legend()
+    ax3.grid(True, alpha=0.3, axis="y")
+    
+    plt.tight_layout()
+    plt.show()
+    print("💾 Sauvegarde : plt.savefig('vacances_manager.png', dpi=150, bbox_inches='tight')")
+
+# ════════════════════════════════════════════════════════════════════════════════
+# CELLULE 9 — Pipeline complet
+# ════════════════════════════════════════════════════════════════════════════════
+
+def pipeline_manager(df):
+    print("\n🔵" + "═" * 73 + "🔵")
+    print("  ANALYSE VACANCES SCOLAIRES — RAPPORT MANAGER")
+    print("🔵" + "═" * 73 + "🔵")
+    
+    df_global = analyse_globale(df)
+    df_st = analyse_par_sous_type(df)
+    
+    if MATPLOTLIB_OK:
+        print("\n📊 Génération des graphes...")
+        graphes_manager(df)
+    
+    facteurs = calcule_facteurs(df)
+    print(f"\n🔧 Facteur correcteur global = {facteurs.get(('GLOBAL','ALL'), 1.0):.4f}")
+    print("   (≈ Volume_reel_vacances / Volume_pred_vacances)")
+    
+    df = corrige_predictions(df, facteurs)
+    df_eval = evalue_correction(df)
+    
+    return df, df_global, df_st, df_eval, facteurs
+
+# ════════════════════════════════════════════════════════════════════════════════
+# CELLULE 10 — Exécution
+# ════════════════════════════════════════════════════════════════════════════════
+# df = add_vacances(df)
+# df, global_res, st_res, eval_res, facteurs = pipeline_manager(df)