vihsd.py

from typing import Any

import datasets
import evaluate

from sklearn.metrics import f1_score, accuracy_score


_DESCRIPTION = """
This metric is used to compute the accuracy and F1 score of models on the ViHSD dataset from [A Large-scale Dataset for Hate Speech Detection on Vietnamese Social Media Texts](https://arxiv.org/abs/2103.11528) by Luu et al. (2021). 
The ViHSD dataset is a large-scale dataset for hate speech detection on Vietnamese social media texts. 
It contains over 30,000 comments, each labeled as CLEAN, OFFENSIVE, or HATE. 
The dataset is used to evaluate the quality of hate speech detection models, including deep learning and transformer models.
"""

_KWARGS_DESCRIPTION = """
Args:
    predictions: list of predictions to score.
        Each translation should be tokenized into a list of tokens.
    references: list of lists of references for each translation.
        Each reference should be tokenized into a list of tokens.
Returns: depending on the GLUE subset, one or several of:
    "accuracy": Accuracy
    "micro_f1": Micro averaged F1 score
    "macro_f1": Macro averaged F1 score
    "weighted_f1": Weighted averaged F1 score
"""

_CITATION = """
@InProceedings{10.1007/978-3-030-79457-6_35,
    author="Luu, Son T.
    and Nguyen, Kiet Van
    and Nguyen, Ngan Luu-Thuy",
    editor="Fujita, Hamido
    and Selamat, Ali
    and Lin, Jerry Chun-Wei
    and Ali, Moonis",
    title="A Large-Scale Dataset for Hate Speech Detection on Vietnamese Social Media Texts",
    booktitle="Advances and Trends in Artificial Intelligence. Artificial Intelligence Practices",
    year="2021",
    publisher="Springer International Publishing",
    address="Cham",
    pages="415--426",
    abstract="In recent years, Vietnam witnesses the mass development of social network users on different social platforms such as Facebook, Youtube, Instagram, and Tiktok. On social media, hate speech has become a critical problem for social network users. To solve this problem, we introduce the ViHSD - a human-annotated dataset for automatically detecting hate speech on the social network. This dataset contains over 30,000 comments, each comment in the dataset has one of three labels: CLEAN, OFFENSIVE, or HATE. Besides, we introduce the data creation process for annotating and evaluating the quality of the dataset. Finally, we evaluate the dataset by deep learning and transformer models.",
    isbn="978-3-030-79457-6"
}
"""


def acc_and_f1(preds, labels):
    return {
        "accuracy": float(accuracy_score(y_true=labels, y_pred=preds)),
        "micro_f1": float(f1_score(y_true=labels, y_pred=preds, average="micro")),
        "macro_f1": float(f1_score(y_true=labels, y_pred=preds, average="macro")),
        "weighted_f1": float(f1_score(y_true=labels, y_pred=preds, average="weighted")),
    }


@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
class ViHSD(evaluate.Metric):
    def _info(self):
        return evaluate.MetricInfo(
            description=_DESCRIPTION,
            citation=_CITATION,
            homepage="https://github.com/sonlam1102/vihsd",
            inputs_description=_KWARGS_DESCRIPTION,
            features=datasets.Features(
                {
                    "predictions": datasets.Value("int64"),
                    "references": datasets.Value("int64"),
                }
            ),
            codebase_urls=["https://github.com/sonlam1102/vihsd"],
            reference_urls=[
                "https://github.com/sonlam1102/vihsd",
                "https://arxiv.org/abs/2103.11528",
            ],
            format="numpy",
        )


    def _compute(
        self,
        predictions: Any = None,
        references: Any = None,
        **kwargs: Any
    ):
        return acc_and_f1(predictions, references)