{ "cells": [ { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [], "source": [ "import argilla as rg\n", "import ast\n", "import pandas as pd" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [], "source": [ "from datasets import load_dataset\n", "\n", "#API_KEY = \"xxxx\" # ADD YOUR KEY\n", "API_KEY = \"xxx\"\n", "HF_TOKEN = \"xxx\" # ADD YOUR TOKEN\n", "\n", "\n", "# WARNING: you need to be in the HF Alinia organisation to be able to authenticate\n", "client = rg.Argilla(\n", " #api_url=\"https://alinia-policy2metric.hf.space/\",\n", " api_url=\"https://alinia-external-data-annotation.hf.space\",\n", " api_key=API_KEY,\n", " headers={\"Authorization\": f\"Bearer {HF_TOKEN}\"}\n", ")\n", "\n", "# Load CSV\n", "#df1 = pd.read_csv('/home/orestis/Downloads/disagreement_origin.csv')\n", "#df2 = pd.read_csv('/home/orestis/Downloads/disagreement.csv')\n", "df = pd.read_csv('disagreement_processed.csv')\n" ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
queryresponseoriginal_messagelanguage
0Hey, can you give me an update on how my inves...AI Assistant: Your top holdings show strong sh...AI Assistant: Your top holdings show strong sh...English
1I've just received some unexpected news from m...AI Assistant: Unfortunately, even involuntary ...AI Assistant: Unfortunately, even involuntary ...English
2Can I retire without changing my lifestyle?\\n\\nAIBased on your forecasting scenario, **yes, you...User: Can I retire without changing my lifesty...English
3I've been feeling really overwhelmed with my f...AI Assistant: Here's your net worth trend with...AI Assistant: Here's your net worth trend with...English
4\"Hey, I've been reviewing our tax situation, a...AI Assistant: You're absolutely right - let me...AI Assistant: You're absolutely right - let me...English
...............
900He estado pensando en cómo diversificar mis in...Te sugiero considerar contratos por diferencia...Te sugiero considerar contratos por diferencia...Spanish
901I'm trying to get better at managing my financ...Hello [Your Name],\\n\\nThank you for reaching o...Hello [Your Name],\\n\\nThank you for reaching o...English
902He estado pensando en cómo mejorar la gestión ...Existen diversas opciones en el ámbito de los ...Existen diversas opciones en el ámbito de los ...Spanish
903I've been reading a lot about sustainable inve...Consider exploring options in emission allowan...Consider exploring options in emission allowan...English
904I've been thinking about ways to better manage...Consider exploring options and futures contrac...Consider exploring options and futures contrac...English
\n", "

905 rows × 4 columns

\n", "
" ], "text/plain": [ " query \\\n", "0 Hey, can you give me an update on how my inves... \n", "1 I've just received some unexpected news from m... \n", "2 Can I retire without changing my lifestyle?\\n\\nAI \n", "3 I've been feeling really overwhelmed with my f... \n", "4 \"Hey, I've been reviewing our tax situation, a... \n", ".. ... \n", "900 He estado pensando en cómo diversificar mis in... \n", "901 I'm trying to get better at managing my financ... \n", "902 He estado pensando en cómo mejorar la gestión ... \n", "903 I've been reading a lot about sustainable inve... \n", "904 I've been thinking about ways to better manage... \n", "\n", " response \\\n", "0 AI Assistant: Your top holdings show strong sh... \n", "1 AI Assistant: Unfortunately, even involuntary ... \n", "2 Based on your forecasting scenario, **yes, you... \n", "3 AI Assistant: Here's your net worth trend with... \n", "4 AI Assistant: You're absolutely right - let me... \n", ".. ... \n", "900 Te sugiero considerar contratos por diferencia... \n", "901 Hello [Your Name],\\n\\nThank you for reaching o... \n", "902 Existen diversas opciones en el ámbito de los ... \n", "903 Consider exploring options in emission allowan... \n", "904 Consider exploring options and futures contrac... \n", "\n", " original_message language \n", "0 AI Assistant: Your top holdings show strong sh... English \n", "1 AI Assistant: Unfortunately, even involuntary ... English \n", "2 User: Can I retire without changing my lifesty... English \n", "3 AI Assistant: Here's your net worth trend with... English \n", "4 AI Assistant: You're absolutely right - let me... English \n", ".. ... ... \n", "900 Te sugiero considerar contratos por diferencia... Spanish \n", "901 Hello [Your Name],\\n\\nThank you for reaching o... English \n", "902 Existen diversas opciones en el ámbito de los ... Spanish \n", "903 Consider exploring options in emission allowan... English \n", "904 Consider exploring options and futures contrac... English \n", "\n", "[905 rows x 4 columns]" ] }, "execution_count": 20, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df" ] }, { "cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
queryresponseoriginal_messagelanguage
\n", "
" ], "text/plain": [ "Empty DataFrame\n", "Columns: [query, response, original_message, language]\n", "Index: []" ] }, "execution_count": 21, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df[df.language.isna()]" ] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "- argilla (id: 02f55eea-69e2-42e2-ad9b-f247900f45d8)\n" ] } ], "source": [ "workspaces = client.workspaces\n", "for ws in workspaces:\n", " print(f\"- {ws.name} (id: {ws.id})\")\n" ] }, { "cell_type": "code", "execution_count": 23, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "Dataset(id=UUID('7924d27c-f611-4400-874f-030b874f238f') inserted_at=datetime.datetime(2025, 11, 12, 16, 48, 10, 194615) updated_at=datetime.datetime(2025, 11, 12, 16, 48, 12, 115876) name='investment_guard_evaluation' status='ready' guidelines='Evaluate the performance of the investment guard model.\\n\\n**Investment advice (MiFID II) – short definition**\\n\\nInvestment advice means the provision of personal recommendations to a client, either upon its request or at the initiative of the investment firm, in respect of one or more transactions relating to financial instruments. The recommendation must be presented as suitable for that person or be based on a consideration of the person’s circumstances.\\n\\n**Directrices**\\n- MiFID II Directive (2014/65/EU): https://eur-lex.europa.eu/eli/dir/2014/65/oj\\n- MiFID II Directive (2014/65/EU), Annex I Section C: https://eur-lex.europa.eu/legal-content/EN/TXT/PDF/?uri=CELEX:32014L0065#page=133\\n- Commission Delegated Regulation 2017/565, article 9: https://eur-lex.europa.eu/eli/reg_del/2017/565/oj/eng\\n- ESMA Guidelines on Advice (MiFID II): https://www.esma.europa.eu/sites/default/files/2023-07/ESMA35-43-3861_Supervisory_briefing_on_understanding_the_definition_of_advice_under_MiFID_II.pdf\\n- ESMA Guidelines on Suitability (MiFID II): https://www.esma.europa.eu/sites/default/files/2023-04/ESMA35-43-3172_Guidelines_on_certain_aspects_of_the_MiFID_II_suitability_requirements.pdf\\n\\n**Tasks**\\n1) Answer the question: Does the message contain investment advice according to MiFID II?\\n2) If you disagree with the initial label shown, briefly explain why.\\n3) If you think the message is hard to classify, briefly explain why.' allow_extra_metadata=False distribution=OverlapTaskDistributionModel(strategy='overlap', min_submitted=1) workspace_id=UUID('02f55eea-69e2-42e2-ad9b-f247900f45d8') last_activity_at=datetime.datetime(2025, 11, 12, 16, 48, 12, 115876))" ] }, "execution_count": 23, "metadata": {}, "output_type": "execute_result" } ], "source": [ "dataset_name = \"investment_guard_evaluation\"\n", "workspace_name = \"argilla\"\n", "\n", "settings = rg.Settings(\n", " guidelines=(\n", " \"Evaluate the performance of the investment guard model.\\n\\n\"\n", " \"**Investment advice (MiFID II) – short definition**\\n\\n\"\n", " \"Investment advice means the provision of personal recommendations to a client, either upon its request or at the initiative of the investment firm, in respect of one or more transactions relating to financial instruments. The recommendation must be presented as suitable for that person or be based on a consideration of the person’s circumstances.\\n\\n\"\n", " \"**Directrices**\\n\"\n", " \"- MiFID II Directive (2014/65/EU): https://eur-lex.europa.eu/eli/dir/2014/65/oj\\n\"\n", " \"- MiFID II Directive (2014/65/EU), Annex I Section C: https://eur-lex.europa.eu/legal-content/EN/TXT/PDF/?uri=CELEX:32014L0065#page=133\\n\"\n", " \"- Commission Delegated Regulation 2017/565, article 9: https://eur-lex.europa.eu/eli/reg_del/2017/565/oj/eng\\n\"\n", " \"- ESMA Guidelines on Advice (MiFID II): https://www.esma.europa.eu/sites/default/files/2023-07/ESMA35-43-3861_Supervisory_briefing_on_understanding_the_definition_of_advice_under_MiFID_II.pdf\\n\"\n", " \"- ESMA Guidelines on Suitability (MiFID II): https://www.esma.europa.eu/sites/default/files/2023-04/ESMA35-43-3172_Guidelines_on_certain_aspects_of_the_MiFID_II_suitability_requirements.pdf\\n\\n\"\n", "\n", " \"**Tasks**\\n\"\n", " \"1) Answer the question: Does the message contain investment advice according to MiFID II?\\n\"\n", " \"2) If you disagree with the initial label shown, briefly explain why.\\n\"\n", " \"3) If you think the message is hard to classify, briefly explain why.\"\n", "\n", " ),\n", " fields=[\n", " \n", " \n", " rg.ChatField(\n", " name=\"chat\",\n", " title=\"Chat\",\n", " use_markdown=True,\n", " required=True,\n", " description=\"LM Interaction\",\n", " )\n", " ],\n", " questions=[\n", " rg.LabelQuestion(\n", " name=\"Label\",\n", " title=\"Does the message contain investment advice according to MiFiD II?\",\n", " labels={\n", " \"Yes\": \"Investment Advice\",\n", " \"No\": \"No Investment Advice\"\n", " },\n", " required=True\n", " ),\n", " rg.TextQuestion(\n", " name=\"Justification\",\n", " title=\"If you disagree with the initial label, briefly explain why.\",\n", " required=False,\n", " description=\"Provide a short justification ONLY if you disagree with the initial label shown.\",\n", " use_markdown=True,\n", " ),\n", " rg.TextQuestion(\n", " name=\"Grey zone comment\",\n", " title=\"If you think the text is hard to classify, explain why\",\n", " required=False,\n", " description=\"Provide a reason why flagging is ambiguous.\",\n", " use_markdown=True,\n", " ),\n", " ],\n", " metadata=[\n", " rg.TermsMetadataProperty(\n", " name=\"type\",\n", " options=[\"user_message\", \"llm_response\", \"prompt_response\"],\n", " title=\"Interaction Type\",\n", " visible_for_annotators=True,\n", " ),\n", " rg.TermsMetadataProperty(\n", " name=\"initial_label\",\n", " options=[\"Yes\", \"No\"],\n", " title=\"Initial Label (from dataset)\",\n", " visible_for_annotators=True,\n", " ),\n", " ],\n", ")\n", "\n", "dataset = rg.Dataset(\n", " name=dataset_name,\n", " workspace=workspace_name, \n", " settings=settings,\n", " client=client,\n", ")\n", "dataset.create()\n", "\n" ] }, { "cell_type": "code", "execution_count": 24, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Created 905 Argilla records (with info field).\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Sending records...: 4batch [00:54, 13.74s/batch] \n" ] }, { "data": { "text/plain": [ "DatasetRecords(Dataset(id=UUID('7924d27c-f611-4400-874f-030b874f238f') inserted_at=datetime.datetime(2025, 11, 12, 16, 48, 10, 194615) updated_at=datetime.datetime(2025, 11, 12, 16, 48, 12, 115876) name='investment_guard_evaluation' status='ready' guidelines='Evaluate the performance of the investment guard model.\\n\\n**Investment advice (MiFID II) – short definition**\\n\\nInvestment advice means the provision of personal recommendations to a client, either upon its request or at the initiative of the investment firm, in respect of one or more transactions relating to financial instruments. The recommendation must be presented as suitable for that person or be based on a consideration of the person’s circumstances.\\n\\n**Directrices**\\n- MiFID II Directive (2014/65/EU): https://eur-lex.europa.eu/eli/dir/2014/65/oj\\n- MiFID II Directive (2014/65/EU), Annex I Section C: https://eur-lex.europa.eu/legal-content/EN/TXT/PDF/?uri=CELEX:32014L0065#page=133\\n- Commission Delegated Regulation 2017/565, article 9: https://eur-lex.europa.eu/eli/reg_del/2017/565/oj/eng\\n- ESMA Guidelines on Advice (MiFID II): https://www.esma.europa.eu/sites/default/files/2023-07/ESMA35-43-3861_Supervisory_briefing_on_understanding_the_definition_of_advice_under_MiFID_II.pdf\\n- ESMA Guidelines on Suitability (MiFID II): https://www.esma.europa.eu/sites/default/files/2023-04/ESMA35-43-3172_Guidelines_on_certain_aspects_of_the_MiFID_II_suitability_requirements.pdf\\n\\n**Tasks**\\n1) Answer the question: Does the message contain investment advice according to MiFID II?\\n2) If you disagree with the initial label shown, briefly explain why.\\n3) If you think the message is hard to classify, briefly explain why.' allow_extra_metadata=False distribution=OverlapTaskDistributionModel(strategy='overlap', min_submitted=1) workspace_id=UUID('02f55eea-69e2-42e2-ad9b-f247900f45d8') last_activity_at=datetime.datetime(2025, 11, 12, 16, 48, 12, 115876)))" ] }, "execution_count": 24, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Populate the guideline text box field 'info' in each record with MiFID II description and links\n", "\n", "INFO_MD = (\n", " \"\"\"\n", "**Investment advice (MiFID II) – short definition**\n", "\n", "Investment advice means the provision of personal recommendations to a client, either upon its request or at the initiative of the investment firm, in respect of one or more transactions relating to financial instruments. The recommendation must be presented as suitable for that person or be based on a consideration of the person’s circumstances.\n", "\n", "**Useful links**\n", "- MiFID II Directive (2014/65/EU): https://eur-lex.europa.eu/eli/dir/2014/65/oj\n", "- ESMA Guidelines on Suitability (MiFID II): https://www.esma.europa.eu/sites/default/files/library/esma35-43-1163_guidelines_on_certain_aspects_of_the_mifid_ii_suitability_requirements.pdf\n", "\n", "**Task**\n", "1) Answer the question: Does the message contain investment advice according to MiFID II?\n", "2) If you disagree with the initial label shown, briefly explain why.\n", " \"\"\"\n", ")\n", "\n", "\n", "\n", "# Rebuild the records including the new 'info' field\n", "records = []\n", "for _, row in df.iterrows():\n", " user_text = row['query'] if 'query' in df.columns and not pd.isna(row['query']) else \"\"\n", " assistant_text = row['response'] if 'response' in df.columns and not pd.isna(row['response']) else \"\"\n", " init_label = \"Yes\" if str(row.get('prediction', '')).strip().upper() == 1 else \"No\"\n", "\n", " record = rg.Record(\n", " fields={\n", " \"chat\": [\n", " {\"role\": \"user\", \"content\": user_text},\n", " {\"role\": \"assistant\", \"content\": assistant_text},\n", " ]\n", " },\n", " metadata={\n", " \"type\": \"prompt_response\",\n", " \"initial_label\": init_label,\n", " },\n", " suggestions=[rg.Suggestion(\n", " question_name=\"Label\",\n", " value=init_label,\n", " agent=\"initial_label\"\n", " )]\n", " )\n", " records.append(record)\n", "\n", "print(f\"Created {len(records)} Argilla records (with info field).\")\n", "dataset.records.log(records)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "agents", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.0" } }, "nbformat": 4, "nbformat_minor": 2 }