| """Show count, mean and median loss per token and label.""" |
| import streamlit as st |
|
|
| from src.subpages.page import Context, Page |
| from src.utils import AgGrid, aggrid_interactive_table |
|
|
|
|
| @st.cache |
| def get_loss_by_token(df_tokens): |
| return ( |
| df_tokens.groupby("tokens")[["losses"]] |
| .agg(["count", "mean", "median", "sum"]) |
| .droplevel(level=0, axis=1) |
| .sort_values(by="sum", ascending=False) |
| .reset_index() |
| ) |
|
|
|
|
| @st.cache |
| def get_loss_by_label(df_tokens): |
| return ( |
| df_tokens.groupby("labels")[["losses"]] |
| .agg(["count", "mean", "median", "sum"]) |
| .droplevel(level=0, axis=1) |
| .sort_values(by="mean", ascending=False) |
| .reset_index() |
| ) |
|
|
|
|
| class LossesPage(Page): |
| name = "Loss by Token/Label" |
| icon = "sort-alpha-down" |
|
|
| def render(self, context: Context): |
| st.title(self.name) |
| with st.expander("💡", expanded=True): |
| st.write("Show count, mean and median loss per token and label.") |
| st.write( |
| "Look out for tokens that have a big gap between mean and median, indicating systematic labeling issues." |
| ) |
|
|
| col1, _, col2 = st.columns([8, 1, 6]) |
|
|
| with col1: |
| st.subheader("💬 Loss by Token") |
|
|
| st.session_state["_merge_tokens"] = st.checkbox( |
| "Merge tokens", value=True, key="merge_tokens" |
| ) |
| loss_by_token = ( |
| get_loss_by_token(context.df_tokens_merged) |
| if st.session_state["merge_tokens"] |
| else get_loss_by_token(context.df_tokens_cleaned) |
| ) |
| aggrid_interactive_table(loss_by_token.round(3)) |
| |
| |
| |
|
|
| st.write( |
| "_Caveat: Even though tokens have contextual representations, we average them to get these summary statistics._" |
| ) |
|
|
| with col2: |
| st.subheader("🏷️ Loss by Label") |
| loss_by_label = get_loss_by_label(context.df_tokens_cleaned) |
| AgGrid(loss_by_label.round(3), height=200) |
|
|
