| { |
| "train": { |
| "train_runtime": 638.1554, |
| "train_samples_per_second": 51.077, |
| "train_steps_per_second": 3.197, |
| "total_flos": 0.0, |
| "train_loss": 0.6800105861589021, |
| "epoch": 3.0 |
| }, |
| "validation": { |
| "sentiment": { |
| "accuracy": 0.008485639686684074, |
| "macro_precision": 0.0028285465622280245, |
| "macro_recall": 0.3333333333333333, |
| "macro_f1": 0.0056094929881337656, |
| "confusion_matrix": [ |
| [ |
| 0, |
| 72, |
| 0 |
| ], |
| [ |
| 0, |
| 13, |
| 0 |
| ], |
| [ |
| 0, |
| 1447, |
| 0 |
| ] |
| ], |
| "predicted_class_counts": { |
| "bearish": 0, |
| "neutral": 1532, |
| "bullish": 0 |
| }, |
| "true_class_counts": { |
| "bearish": 72, |
| "neutral": 13, |
| "bullish": 1447 |
| }, |
| "expected_calibration_error": 0.8264312032931156, |
| "brier_score": 1.573439388720477 |
| }, |
| "direction_1d": { |
| "accuracy": 0.3413838120104439, |
| "macro_precision": 0.3335515858937544, |
| "macro_recall": 0.33669626802974273, |
| "macro_f1": 0.33331285508030406, |
| "confusion_matrix": [ |
| [ |
| 104, |
| 200, |
| 164 |
| ], |
| [ |
| 116, |
| 228, |
| 186 |
| ], |
| [ |
| 154, |
| 189, |
| 191 |
| ] |
| ], |
| "predicted_class_counts": { |
| "bearish": 374, |
| "neutral": 617, |
| "bullish": 541 |
| }, |
| "true_class_counts": { |
| "bearish": 468, |
| "neutral": 530, |
| "bullish": 534 |
| }, |
| "expected_calibration_error": 0.028834249269868946, |
| "brier_score": 0.6675477741251082 |
| }, |
| "direction_5d": { |
| "accuracy": 0.3936031331592689, |
| "macro_precision": 0.27396668808245767, |
| "macro_recall": 0.3484856246398156, |
| "macro_f1": 0.29043708459698375, |
| "confusion_matrix": [ |
| [ |
| 0, |
| 131, |
| 208 |
| ], |
| [ |
| 0, |
| 381, |
| 424 |
| ], |
| [ |
| 0, |
| 166, |
| 222 |
| ] |
| ], |
| "predicted_class_counts": { |
| "bearish": 0, |
| "neutral": 678, |
| "bullish": 854 |
| }, |
| "true_class_counts": { |
| "bearish": 339, |
| "neutral": 805, |
| "bullish": 388 |
| }, |
| "expected_calibration_error": 0.060579302202783736, |
| "brier_score": 0.6606167988480304 |
| }, |
| "volatility": { |
| "accuracy": 0.41906005221932113, |
| "macro_precision": 0.13968668407310705, |
| "macro_recall": 0.3333333333333333, |
| "macro_f1": 0.1968721251149954, |
| "confusion_matrix": [ |
| [ |
| 0, |
| 535, |
| 0 |
| ], |
| [ |
| 0, |
| 642, |
| 0 |
| ], |
| [ |
| 0, |
| 355, |
| 0 |
| ] |
| ], |
| "predicted_class_counts": { |
| "lower_vol": 0, |
| "normal_vol": 1532, |
| "higher_vol": 0 |
| }, |
| "true_class_counts": { |
| "lower_vol": 535, |
| "normal_vol": 642, |
| "higher_vol": 355 |
| }, |
| "expected_calibration_error": 0.15814054444159914, |
| "brier_score": 0.6913779982386878 |
| }, |
| "event_backtest": { |
| "split": "validation", |
| "transaction_cost_bps": 10.0, |
| "active_1d_fraction": 0.5972584856396866, |
| "hit_ratio_1d_active": 0.5398907103825137, |
| "avg_signal_return_1d_all": -0.000668341834270596, |
| "avg_signal_return_1d_all_net": -0.0012656003199102831, |
| "avg_signal_return_1d_active": -0.0011190160547568885, |
| "avg_signal_return_1d_active_net": -0.0021190160547568896, |
| "avg_abnormal_signal_return_1d_all": -0.0009099995935003474, |
| "avg_abnormal_signal_return_1d_all_net": -0.0015072580791400346, |
| "avg_abnormal_signal_return_1d_active": -0.001523627734691292, |
| "avg_abnormal_signal_return_1d_active_net": -0.002523627734691293, |
| "soft_signal_backtest_1d": { |
| "mean_abs_signal": 0.04358483850955963, |
| "active_fraction_abs_score_ge_0.10": 0.0731070496083551, |
| "hit_ratio_all_gross": 0.5071801566579635, |
| "hit_ratio_active_gross": 0.5446428571428571, |
| "avg_signal_return_all": -2.6098849048139527e-05, |
| "avg_signal_return_all_net": -6.968369416426867e-05, |
| "avg_abnormal_signal_return_all": -3.516719152685255e-05, |
| "avg_abnormal_signal_return_all_net": -7.87520402809605e-05 |
| }, |
| "active_5d_fraction": 0.5574412532637075, |
| "hit_ratio_5d_active": 0.5644028103044496, |
| "avg_signal_return_5d_all": 0.00222230488512584, |
| "avg_signal_return_5d_active": 0.003986617194394364, |
| "confidence_backtests_1d": { |
| "top_10pct_directional_confidence": { |
| "num_events": 153, |
| "mean_confidence": 0.39838218688964844, |
| "hit_ratio": 0.5424836601307189, |
| "avg_signal_return": -0.0040096313650019805, |
| "avg_signal_return_net": -0.005009631365001981, |
| "median_signal_return": 0.001708349445834756, |
| "avg_abnormal_signal_return": -0.0016484424157800104, |
| "avg_abnormal_signal_return_net": -0.002648442415780011 |
| }, |
| "top_20pct_directional_confidence": { |
| "num_events": 306, |
| "mean_confidence": 0.38756096363067627, |
| "hit_ratio": 0.5588235294117647, |
| "avg_signal_return": -0.0005135064179175138, |
| "avg_signal_return_net": -0.0015135064179175146, |
| "median_signal_return": 0.003223672043532133, |
| "avg_abnormal_signal_return": -0.0006505049740809268, |
| "avg_abnormal_signal_return_net": -0.0016505049740809278 |
| }, |
| "top_30pct_directional_confidence": { |
| "num_events": 460, |
| "mean_confidence": 0.38047993183135986, |
| "hit_ratio": 0.5543478260869565, |
| "avg_signal_return": -0.0009027605532142131, |
| "avg_signal_return_net": -0.001902760553214214, |
| "median_signal_return": 0.002927212161011994, |
| "avg_abnormal_signal_return": -0.0014120966409751892, |
| "avg_abnormal_signal_return_net": -0.0024120966409751903 |
| }, |
| "threshold_0.34": { |
| "num_events": 1040, |
| "coverage": 0.6788511749347258, |
| "hit_ratio": 0.5394230769230769, |
| "avg_signal_return": -0.0007571742408449959, |
| "avg_signal_return_net": -0.0017571742408449967, |
| "median_signal_return": 0.0012542950571514666, |
| "avg_abnormal_signal_return": -0.0012730416878412143, |
| "avg_abnormal_signal_return_net": -0.0022730416878412154 |
| }, |
| "threshold_0.36": { |
| "num_events": 494, |
| "coverage": 0.3224543080939948, |
| "hit_ratio": 0.5566801619433198, |
| "avg_signal_return": -0.0008215918299352735, |
| "avg_signal_return_net": -0.0018215918299352744, |
| "median_signal_return": 0.0024513842072337866, |
| "avg_abnormal_signal_return": -0.0013882744387551298, |
| "avg_abnormal_signal_return_net": -0.002388274438755131 |
| }, |
| "threshold_0.38": { |
| "num_events": 188, |
| "coverage": 0.1227154046997389, |
| "hit_ratio": 0.5478723404255319, |
| "avg_signal_return": -0.0020706070304669917, |
| "avg_signal_return_net": -0.0030706070304669926, |
| "median_signal_return": 0.0030422902200371027, |
| "avg_abnormal_signal_return": -0.0007841120187801239, |
| "avg_abnormal_signal_return_net": -0.0017841120187801248 |
| }, |
| "threshold_0.40": { |
| "num_events": 49, |
| "coverage": 0.03198433420365535, |
| "hit_ratio": 0.5714285714285714, |
| "avg_signal_return": -0.0023946920445435966, |
| "avg_signal_return_net": -0.0033946920445435975, |
| "median_signal_return": 0.003524972125887871, |
| "avg_abnormal_signal_return": 0.0008117058985552997, |
| "avg_abnormal_signal_return_net": -0.00018829410144470112 |
| }, |
| "threshold_0.42": { |
| "num_events": 8, |
| "coverage": 0.005221932114882507, |
| "hit_ratio": 0.625, |
| "avg_signal_return": 0.007374357082881033, |
| "avg_signal_return_net": 0.0063743570828810325, |
| "median_signal_return": 0.010437145829200745, |
| "avg_abnormal_signal_return": 0.013620887020806549, |
| "avg_abnormal_signal_return_net": 0.012620887020806548 |
| }, |
| "threshold_0.45": { |
| "num_events": 0, |
| "coverage": 0.0, |
| "hit_ratio": null, |
| "avg_signal_return": null, |
| "avg_signal_return_net": null, |
| "median_signal_return": null, |
| "avg_abnormal_signal_return": null, |
| "avg_abnormal_signal_return_net": null |
| }, |
| "threshold_0.50": { |
| "num_events": 0, |
| "coverage": 0.0, |
| "hit_ratio": null, |
| "avg_signal_return": null, |
| "avg_signal_return_net": null, |
| "median_signal_return": null, |
| "avg_abnormal_signal_return": null, |
| "avg_abnormal_signal_return_net": null |
| } |
| }, |
| "mean_text_gate_weight": 0.5071590542793274, |
| "std_text_gate_weight": 0.0016348367789760232 |
| } |
| }, |
| "test": { |
| "sentiment": { |
| "accuracy": 0.009784735812133072, |
| "macro_precision": 0.0032615786040443573, |
| "macro_recall": 0.3333333333333333, |
| "macro_f1": 0.006459948320413436, |
| "confusion_matrix": [ |
| [ |
| 0, |
| 110, |
| 0 |
| ], |
| [ |
| 0, |
| 15, |
| 0 |
| ], |
| [ |
| 0, |
| 1408, |
| 0 |
| ] |
| ], |
| "predicted_class_counts": { |
| "bearish": 0, |
| "neutral": 1533, |
| "bullish": 0 |
| }, |
| "true_class_counts": { |
| "bearish": 110, |
| "neutral": 15, |
| "bullish": 1408 |
| }, |
| "expected_calibration_error": 0.8247767172447622, |
| "brier_score": 1.5716914553389048 |
| }, |
| "direction_1d": { |
| "accuracy": 0.3333333333333333, |
| "macro_precision": 0.2797514006239241, |
| "macro_recall": 0.30693224852151285, |
| "macro_f1": 0.28541099148120913, |
| "confusion_matrix": [ |
| [ |
| 31, |
| 217, |
| 212 |
| ], |
| [ |
| 54, |
| 138, |
| 244 |
| ], |
| [ |
| 124, |
| 171, |
| 342 |
| ] |
| ], |
| "predicted_class_counts": { |
| "bearish": 209, |
| "neutral": 526, |
| "bullish": 798 |
| }, |
| "true_class_counts": { |
| "bearish": 460, |
| "neutral": 436, |
| "bullish": 637 |
| }, |
| "expected_calibration_error": 0.03496117605077468, |
| "brier_score": 0.6723385830557309 |
| }, |
| "direction_5d": { |
| "accuracy": 0.45857795172863663, |
| "macro_precision": 0.28120269843193446, |
| "macro_recall": 0.33741824434028733, |
| "macro_f1": 0.30491929241175314, |
| "confusion_matrix": [ |
| [ |
| 0, |
| 123, |
| 143 |
| ], |
| [ |
| 0, |
| 553, |
| 311 |
| ], |
| [ |
| 0, |
| 253, |
| 150 |
| ] |
| ], |
| "predicted_class_counts": { |
| "bearish": 0, |
| "neutral": 929, |
| "bullish": 604 |
| }, |
| "true_class_counts": { |
| "bearish": 266, |
| "neutral": 864, |
| "bullish": 403 |
| }, |
| "expected_calibration_error": 0.024249568105288305, |
| "brier_score": 0.633983250156182 |
| }, |
| "volatility": { |
| "accuracy": 0.39921722113502933, |
| "macro_precision": 0.13307240704500978, |
| "macro_recall": 0.3333333333333333, |
| "macro_f1": 0.1902097902097902, |
| "confusion_matrix": [ |
| [ |
| 0, |
| 534, |
| 0 |
| ], |
| [ |
| 0, |
| 612, |
| 0 |
| ], |
| [ |
| 0, |
| 387, |
| 0 |
| ] |
| ], |
| "predicted_class_counts": { |
| "lower_vol": 0, |
| "normal_vol": 1533, |
| "higher_vol": 0 |
| }, |
| "true_class_counts": { |
| "lower_vol": 534, |
| "normal_vol": 612, |
| "higher_vol": 387 |
| }, |
| "expected_calibration_error": 0.1760233100613036, |
| "brier_score": 0.712886643106662 |
| }, |
| "event_backtest": { |
| "split": "test", |
| "transaction_cost_bps": 10.0, |
| "active_1d_fraction": 0.6568819308545336, |
| "hit_ratio_1d_active": 0.5789473684210527, |
| "avg_signal_return_1d_all": 0.0023507131510749455, |
| "avg_signal_return_1d_all_net": 0.0016938312202204116, |
| "avg_signal_return_1d_active": 0.0035785931088360393, |
| "avg_signal_return_1d_active_net": 0.002578593108836038, |
| "avg_abnormal_signal_return_1d_all": 0.0007162017627715418, |
| "avg_abnormal_signal_return_1d_all_net": 5.93198319170076e-05, |
| "avg_abnormal_signal_return_1d_active": 0.0010903051661656142, |
| "avg_abnormal_signal_return_1d_active_net": 9.030516616561321e-05, |
| "soft_signal_backtest_1d": { |
| "mean_abs_signal": 0.049029190093278885, |
| "active_fraction_abs_score_ge_0.10": 0.11545988258317025, |
| "hit_ratio_all_gross": 0.532941943900848, |
| "hit_ratio_active_gross": 0.4463276836158192, |
| "avg_signal_return_all": -4.1903094825102016e-05, |
| "avg_signal_return_all_net": -9.093229164136574e-05, |
| "avg_abnormal_signal_return_all": 6.6492548285168596e-06, |
| "avg_abnormal_signal_return_all_net": -4.237994289724156e-05 |
| }, |
| "active_5d_fraction": 0.39399869536855836, |
| "hit_ratio_5d_active": 0.3708609271523179, |
| "avg_signal_return_5d_all": -0.004762698527096634, |
| "avg_signal_return_5d_active": -0.012088107354369436, |
| "confidence_backtests_1d": { |
| "top_10pct_directional_confidence": { |
| "num_events": 153, |
| "mean_confidence": 0.40472039580345154, |
| "hit_ratio": 0.48366013071895425, |
| "avg_signal_return": -0.0017291086948829782, |
| "avg_signal_return_net": -0.002729108694882979, |
| "median_signal_return": -0.0011537353275343776, |
| "avg_abnormal_signal_return": 0.0023775517351216227, |
| "avg_abnormal_signal_return_net": 0.001377551735121622 |
| }, |
| "top_20pct_directional_confidence": { |
| "num_events": 307, |
| "mean_confidence": 0.39327916502952576, |
| "hit_ratio": 0.46579804560260585, |
| "avg_signal_return": -0.0018346647964980064, |
| "avg_signal_return_net": -0.0028346647964980075, |
| "median_signal_return": -0.001494303229264915, |
| "avg_abnormal_signal_return": 0.00013626769358741198, |
| "avg_abnormal_signal_return_net": -0.0008637323064125889 |
| }, |
| "top_30pct_directional_confidence": { |
| "num_events": 460, |
| "mean_confidence": 0.3858344554901123, |
| "hit_ratio": 0.5021739130434782, |
| "avg_signal_return": 0.000168503345380684, |
| "avg_signal_return_net": -0.0008314966546193169, |
| "median_signal_return": 0.00047751690726727247, |
| "avg_abnormal_signal_return": -0.0006967808645482443, |
| "avg_abnormal_signal_return_net": -0.0016967808645482452 |
| }, |
| "threshold_0.34": { |
| "num_events": 1108, |
| "coverage": 0.7227658186562296, |
| "hit_ratio": 0.5523465703971119, |
| "avg_signal_return": 0.002400788538323842, |
| "avg_signal_return_net": 0.001400788538323841, |
| "median_signal_return": 0.0029742957558482885, |
| "avg_abnormal_signal_return": 0.00043566977963306825, |
| "avg_abnormal_signal_return_net": -0.0005643302203669326 |
| }, |
| "threshold_0.36": { |
| "num_events": 636, |
| "coverage": 0.41487279843444225, |
| "hit_ratio": 0.5471698113207547, |
| "avg_signal_return": 0.001725182487273244, |
| "avg_signal_return_net": 0.000725182487273243, |
| "median_signal_return": 0.0027859483379870653, |
| "avg_abnormal_signal_return": -0.0001395135721791607, |
| "avg_abnormal_signal_return_net": -0.0011395135721791617 |
| }, |
| "threshold_0.38": { |
| "num_events": 247, |
| "coverage": 0.16112198303979125, |
| "hit_ratio": 0.44129554655870445, |
| "avg_signal_return": -0.0028902063559107334, |
| "avg_signal_return_net": -0.0038902063559107343, |
| "median_signal_return": -0.0022296553943306208, |
| "avg_abnormal_signal_return": 0.0008437850360328761, |
| "avg_abnormal_signal_return_net": -0.00015621496396712474 |
| }, |
| "threshold_0.40": { |
| "num_events": 85, |
| "coverage": 0.055446836268754074, |
| "hit_ratio": 0.43529411764705883, |
| "avg_signal_return": -0.0034284473146887168, |
| "avg_signal_return_net": -0.004428447314688717, |
| "median_signal_return": -0.0031941309571266174, |
| "avg_abnormal_signal_return": 0.0008126156158087884, |
| "avg_abnormal_signal_return_net": -0.00018738438419121242 |
| }, |
| "threshold_0.42": { |
| "num_events": 20, |
| "coverage": 0.01304631441617743, |
| "hit_ratio": 0.25, |
| "avg_signal_return": -0.008969856356270612, |
| "avg_signal_return_net": -0.009969856356270613, |
| "median_signal_return": -0.011959618888795376, |
| "avg_abnormal_signal_return": -0.0006033775032847188, |
| "avg_abnormal_signal_return_net": -0.0016033775032847197 |
| }, |
| "threshold_0.45": { |
| "num_events": 2, |
| "coverage": 0.001304631441617743, |
| "hit_ratio": 0.5, |
| "avg_signal_return": 0.0010717622935771942, |
| "avg_signal_return_net": 7.176229357719333e-05, |
| "median_signal_return": 0.0010717622935771942, |
| "avg_abnormal_signal_return": 0.01302456425037235, |
| "avg_abnormal_signal_return_net": 0.01202456425037235 |
| }, |
| "threshold_0.50": { |
| "num_events": 0, |
| "coverage": 0.0, |
| "hit_ratio": null, |
| "avg_signal_return": null, |
| "avg_signal_return_net": null, |
| "median_signal_return": null, |
| "avg_abnormal_signal_return": null, |
| "avg_abnormal_signal_return_net": null |
| } |
| }, |
| "mean_text_gate_weight": 0.506599485874176, |
| "std_text_gate_weight": 0.0016385371563956141 |
| } |
| }, |
| "data": { |
| "num_rows": 13930, |
| "train_rows": 10865, |
| "validation_rows": 1532, |
| "test_rows": 1533, |
| "symbols": [ |
| "AAPL", |
| "ABNB", |
| "ADBE", |
| "ADI", |
| "ADP", |
| "AMAT", |
| "AMD", |
| "AMZN", |
| "ARKK", |
| "ASML", |
| "AVGO", |
| "BA", |
| "BABA", |
| "BAC", |
| "BIDU", |
| "BMY", |
| "CAT", |
| "CMCSA", |
| "COIN", |
| "COST", |
| "CRM", |
| "CSCO", |
| "CVX", |
| "DHR", |
| "DIS", |
| "ELF", |
| "GE", |
| "GM", |
| "GOOGL", |
| "HD", |
| "IBM", |
| "INTC", |
| "IOVA", |
| "JNJ", |
| "JPM", |
| "KO", |
| "LULU", |
| "MA", |
| "MCD", |
| "MDB", |
| "META", |
| "MRNA", |
| "MSFT", |
| "MU", |
| "NFLX", |
| "NKE", |
| "NVDA", |
| "ORCL", |
| "PEP", |
| "PFE", |
| "PLTR", |
| "PYPL", |
| "QCOM", |
| "RCL", |
| "SHOP", |
| "SNOW", |
| "SPOT", |
| "TGT", |
| "TSLA", |
| "TSM", |
| "TXN", |
| "UAL", |
| "UBER", |
| "UNH", |
| "UPS", |
| "WMT", |
| "XOM", |
| "ZM" |
| ], |
| "date_min": "2022-08-12 21:13:30+00:00", |
| "date_max": "2023-10-04 20:50:09+00:00", |
| "volatility_thresholds": { |
| "q33": 0.01998170108283479, |
| "q66": 0.049223492463942975 |
| } |
| } |
| } |
