Spaces:
Sleeping
Sleeping
File size: 1,347 Bytes
e078b1d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 | {
"cells": [
{
"cell_type": "markdown",
"id": "9e49735d",
"metadata": {},
"source": [
"# Model Evaluation\n",
"Generate summaries for the manual evaluation set and compare model outputs."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2c44453f",
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"from src.models.registry import summarize_text\n",
"\n",
"eval_path = '../data/processed/eval_set_with_refs.csv'\n",
"df = pd.read_csv(eval_path)\n",
"text = df.iloc[0]['Description']\n",
"print('Source:', text)\n",
"for model_name in ['lead1', 'textrank', 'flan_t5_small']:\n",
" summary = summarize_text(text, model_name)\n",
" print('\\n', model_name, ':', summary)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "235752ff",
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"from src.evaluate.eval import evaluate_outputs\n",
"from src.evaluate.tables import aggregate_metrics_table\n",
"\n",
"outputs = pd.read_csv('../data/processed/model_outputs.csv')\n",
"detailed = evaluate_outputs(outputs)\n",
"aggregate = aggregate_metrics_table(detailed)\n",
"aggregate"
]
}
],
"metadata": {},
"nbformat": 4,
"nbformat_minor": 5
}
|