output/data_lineage.json

{
  "generated_at": "2026-03-22 22:14",
  "pipeline_version": "2.0",
  "sources": {
    "FRED": {
      "name": "Federal Reserve Economic Data",
      "url": "https://fred.stlouisfed.org/",
      "type": "REST API",
      "auth": "API Key (free)",
      "features_count": 12
    },
    "EIA": {
      "name": "US Energy Information Administration",
      "url": "https://www.eia.gov/opendata/",
      "type": "REST API v2",
      "auth": "API Key (free)",
      "features_count": 2
    },
    "CSV": {
      "name": "Public CSV Sources",
      "url": "Various (Baker Hughes, World Bank, OPEC, GPR)",
      "type": "Manual Download / Scheduled",
      "auth": "None",
      "features_count": 4
    }
  },
  "factor_groups": {
    "Price": [
      "WTI_spot",
      "Brent_spot",
      "natgas_spot_henry",
      "iron_ore_spot"
    ],
    "Supply": [
      "us_oil_inventory_total",
      "us_crude_production",
      "rig_count_us_new",
      "supply_saudi"
    ],
    "Demand": [
      "pmi_us_mfg",
      "ipi_us",
      "nonfarm_us",
      "usd_index",
      "cpi_us",
      "fed_funds_rate",
      "yield_spread_10y2y"
    ],
    "Risk": [
      "vix",
      "gpr_index",
      "gold_spot"
    ]
  },
  "features": {
    "WTI_spot": {
      "source": "FRED",
      "factor_group": "Price",
      "description": "WTI Crude Oil Spot Price ($/bbl)",
      "frequency": "daily→monthly",
      "release_lag_days": 1,
      "total_months": 435,
      "missing": 0,
      "missing_rate": 0.0,
      "first_valid": "1990-01-31",
      "last_valid": "2026-03-31",
      "staleness_days": -9,
      "outliers": 0,
      "latest_value": 93.39,
      "status": "OK"
    },
    "Brent_spot": {
      "source": "FRED",
      "factor_group": "Price",
      "description": "Brent Crude Oil Spot Price ($/bbl)",
      "frequency": "daily→monthly",
      "release_lag_days": 1,
      "total_months": 435,
      "missing": 0,
      "missing_rate": 0.0,
      "first_valid": "1990-01-31",
      "last_valid": "2026-03-31",
      "staleness_days": -9,
      "outliers": 0,
      "latest_value": 101.04,
      "status": "OK"
    },
    "natgas_spot_henry": {
      "source": "FRED",
      "factor_group": "Price",
      "description": "Henry Hub Natural Gas Spot Price ($/MMBtu)",
      "frequency": "daily→monthly",
      "release_lag_days": 1,
      "total_months": 435,
      "missing": 84,
      "missing_rate": 0.193,
      "first_valid": "1997-01-31",
      "last_valid": "2026-03-31",
      "staleness_days": -9,
      "outliers": 0,
      "latest_value": 3.03,
      "status": "OK"
    },
    "pmi_us_mfg": {
      "source": "FRED",
      "factor_group": "Demand",
      "description": "US Manufacturing Employment (proxy for ISM PMI)",
      "frequency": "monthly",
      "release_lag_days": 5,
      "total_months": 435,
      "missing": 1,
      "missing_rate": 0.002,
      "first_valid": "1990-01-31",
      "last_valid": "2026-02-28",
      "staleness_days": 22,
      "outliers": 1,
      "latest_value": null,
      "status": "OK"
    },
    "ipi_us": {
      "source": "FRED",
      "factor_group": "Demand",
      "description": "US Industrial Production Index",
      "frequency": "monthly",
      "release_lag_days": 15,
      "total_months": 435,
      "missing": 1,
      "missing_rate": 0.002,
      "first_valid": "1990-01-31",
      "last_valid": "2026-02-28",
      "staleness_days": 22,
      "outliers": 1,
      "latest_value": null,
      "status": "OK"
    },
    "nonfarm_us": {
      "source": "FRED",
      "factor_group": "Demand",
      "description": "US Total Nonfarm Payrolls",
      "frequency": "monthly",
      "release_lag_days": 5,
      "total_months": 435,
      "missing": 1,
      "missing_rate": 0.002,
      "first_valid": "1990-01-31",
      "last_valid": "2026-02-28",
      "staleness_days": 22,
      "outliers": 1,
      "latest_value": null,
      "status": "OK"
    },
    "usd_index": {
      "source": "FRED",
      "factor_group": "Demand",
      "description": "Trade Weighted US Dollar Index (Broad Goods & Services)",
      "frequency": "daily→monthly",
      "release_lag_days": 1,
      "total_months": 435,
      "missing": 192,
      "missing_rate": 0.441,
      "first_valid": "2006-01-31",
      "last_valid": "2026-03-31",
      "staleness_days": -9,
      "outliers": 0,
      "latest_value": 120.5518,
      "status": "HIGH_MISSING"
    },
    "cpi_us": {
      "source": "FRED",
      "factor_group": "Demand",
      "description": "US Consumer Price Index (All Urban)",
      "frequency": "monthly",
      "release_lag_days": 12,
      "total_months": 435,
      "missing": 2,
      "missing_rate": 0.005,
      "first_valid": "1990-01-31",
      "last_valid": "2026-02-28",
      "staleness_days": 22,
      "outliers": 0,
      "latest_value": null,
      "status": "OK"
    },
    "fed_funds_rate": {
      "source": "FRED",
      "factor_group": "Demand",
      "description": "Federal Funds Effective Rate",
      "frequency": "monthly",
      "release_lag_days": 1,
      "total_months": 435,
      "missing": 1,
      "missing_rate": 0.002,
      "first_valid": "1990-01-31",
      "last_valid": "2026-02-28",
      "staleness_days": 22,
      "outliers": 3,
      "latest_value": null,
      "status": "OK"
    },
    "yield_spread_10y2y": {
      "source": "FRED",
      "factor_group": "Demand",
      "description": "10Y-2Y Treasury Yield Spread",
      "frequency": "daily→monthly",
      "release_lag_days": 1,
      "total_months": 435,
      "missing": 0,
      "missing_rate": 0.0,
      "first_valid": "1990-01-31",
      "last_valid": "2026-03-31",
      "staleness_days": -9,
      "outliers": 0,
      "latest_value": 0.51,
      "status": "OK"
    },
    "vix": {
      "source": "FRED",
      "factor_group": "Risk",
      "description": "CBOE Volatility Index (VIX)",
      "frequency": "daily→monthly",
      "release_lag_days": 1,
      "total_months": 435,
      "missing": 0,
      "missing_rate": 0.0,
      "first_valid": "1990-01-31",
      "last_valid": "2026-03-31",
      "staleness_days": -9,
      "outliers": 0,
      "latest_value": 24.06,
      "status": "OK"
    },
    "iron_ore_spot": {
      "source": "CSV",
      "factor_group": "Price",
      "description": "Iron Ore Spot Price (World Bank Pink Sheet)",
      "frequency": "monthly",
      "release_lag_days": 30,
      "total_months": 435,
      "missing": 1,
      "missing_rate": 0.002,
      "first_valid": "1990-01-31",
      "last_valid": "2026-02-28",
      "staleness_days": 22,
      "outliers": 3,
      "latest_value": null,
      "status": "OK"
    },
    "us_oil_inventory_total": {
      "source": "EIA",
      "factor_group": "Supply",
      "description": "US Ending Stocks of Crude Oil (Thousand Barrels)",
      "frequency": "weekly→monthly",
      "release_lag_days": 5,
      "total_months": 435,
      "missing": 205,
      "missing_rate": 0.471,
      "first_valid": "2007-01-31",
      "last_valid": "2026-02-28",
      "staleness_days": 22,
      "outliers": 0,
      "latest_value": null,
      "status": "HIGH_MISSING"
    },
    "supply_saudi": {
      "source": "CSV",
      "factor_group": "Supply",
      "description": "Saudi Arabia Crude Oil Production (OPEC MOMR)",
      "frequency": "monthly",
      "release_lag_days": 15,
      "total_months": 435,
      "missing": 1,
      "missing_rate": 0.002,
      "first_valid": "1990-01-31",
      "last_valid": "2026-02-28",
      "staleness_days": 22,
      "outliers": 4,
      "latest_value": null,
      "status": "OUTLIER_ALERT"
    },
    "rig_count_us_new": {
      "source": "CSV",
      "factor_group": "Supply",
      "description": "Baker Hughes US Oil Rig Count",
      "frequency": "weekly→monthly",
      "release_lag_days": 3,
      "total_months": 435,
      "missing": 1,
      "missing_rate": 0.002,
      "first_valid": "1990-01-31",
      "last_valid": "2026-02-28",
      "staleness_days": 22,
      "outliers": 6,
      "latest_value": null,
      "status": "OUTLIER_ALERT"
    }
  },
  "data_flow": [
    {
      "step": 1,
      "name": "Source Fetch",
      "desc": "FRED API + EIA API + CSV files"
    },
    {
      "step": 2,
      "name": "Frequency Align",
      "desc": "Daily/Weekly → Monthly (last/mean)"
    },
    {
      "step": 3,
      "name": "Quality Check",
      "desc": "Missing, outlier, staleness detection"
    },
    {
      "step": 4,
      "name": "Point-in-Time",
      "desc": "Release lag applied; no future data"
    },
    {
      "step": 5,
      "name": "Feature Eng",
      "desc": "Lags, momentum, vol, spreads, RSI"
    },
    {
      "step": 6,
      "name": "Panel Build",
      "desc": "Final panel_monthly_live.csv"
    }
  ]
}