Add model

config.json

@@ -0,0 +1,186 @@
+{
+  "architectures": [
+    "ModernBertForSequenceClassification"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "causal_mask": false,
+  "classifier_activation": "gelu",
+  "classifier_bias": false,
+  "classifier_dropout": 0.0,
+  "classifier_pooling": "mean",
+  "cls_token_id": 50281,
+  "decoder_bias": true,
+  "deterministic_flash_attn": false,
+  "dtype": "float32",
+  "embedding_dropout": 0.0,
+  "eos_token_id": null,
+  "global_attn_every_n_layers": 3,
+  "global_rope_theta": 160000.0,
+  "gradient_checkpointing": false,
+  "hidden_activation": "gelu",
+  "hidden_size": 384,
+  "id2label": {
+    "0": "aerospace",
+    "1": "agronomy",
+    "2": "artistic",
+    "3": "astronomy",
+    "4": "atmospheric_science",
+    "5": "automotive",
+    "6": "beauty",
+    "7": "biology",
+    "8": "celebrity",
+    "9": "chemistry",
+    "10": "civil_engineering",
+    "11": "communication_engineering",
+    "12": "computer_science_and_technology",
+    "13": "design",
+    "14": "drama_and_film",
+    "15": "economics",
+    "16": "electronic_science",
+    "17": "entertainment",
+    "18": "environmental_science",
+    "19": "fashion",
+    "20": "finance",
+    "21": "food",
+    "22": "gamble",
+    "23": "game",
+    "24": "geography",
+    "25": "health",
+    "26": "history",
+    "27": "hobby",
+    "28": "hydraulic_engineering",
+    "29": "instrument_science",
+    "30": "journalism_and_media_communication",
+    "31": "landscape_architecture",
+    "32": "law",
+    "33": "library",
+    "34": "literature",
+    "35": "materials_science",
+    "36": "mathematics",
+    "37": "mechanical_engineering",
+    "38": "medical",
+    "39": "mining_engineering",
+    "40": "movie",
+    "41": "music_and_dance",
+    "42": "news",
+    "43": "nuclear_science",
+    "44": "ocean_science",
+    "45": "optical_engineering",
+    "46": "painting",
+    "47": "pet",
+    "48": "petroleum_and_natural_gas_engineering",
+    "49": "philosophy",
+    "50": "photo",
+    "51": "physics",
+    "52": "politics",
+    "53": "psychology",
+    "54": "public_administration",
+    "55": "relationship",
+    "56": "religion",
+    "57": "sociology",
+    "58": "sports",
+    "59": "statistics",
+    "60": "systems_science",
+    "61": "textile_science",
+    "62": "topicality",
+    "63": "transportation_engineering",
+    "64": "travel",
+    "65": "urban_planning",
+    "66": "vulgar_language"
+  },
+  "initializer_cutoff_factor": 2.0,
+  "initializer_range": 0.02,
+  "intermediate_size": 576,
+  "is_causal": false,
+  "label2id": {
+    "aerospace": 0,
+    "agronomy": 1,
+    "artistic": 2,
+    "astronomy": 3,
+    "atmospheric_science": 4,
+    "automotive": 5,
+    "beauty": 6,
+    "biology": 7,
+    "celebrity": 8,
+    "chemistry": 9,
+    "civil_engineering": 10,
+    "communication_engineering": 11,
+    "computer_science_and_technology": 12,
+    "design": 13,
+    "drama_and_film": 14,
+    "economics": 15,
+    "electronic_science": 16,
+    "entertainment": 17,
+    "environmental_science": 18,
+    "fashion": 19,
+    "finance": 20,
+    "food": 21,
+    "gamble": 22,
+    "game": 23,
+    "geography": 24,
+    "health": 25,
+    "history": 26,
+    "hobby": 27,
+    "hydraulic_engineering": 28,
+    "instrument_science": 29,
+    "journalism_and_media_communication": 30,
+    "landscape_architecture": 31,
+    "law": 32,
+    "library": 33,
+    "literature": 34,
+    "materials_science": 35,
+    "mathematics": 36,
+    "mechanical_engineering": 37,
+    "medical": 38,
+    "mining_engineering": 39,
+    "movie": 40,
+    "music_and_dance": 41,
+    "news": 42,
+    "nuclear_science": 43,
+    "ocean_science": 44,
+    "optical_engineering": 45,
+    "painting": 46,
+    "pet": 47,
+    "petroleum_and_natural_gas_engineering": 48,
+    "philosophy": 49,
+    "photo": 50,
+    "physics": 51,
+    "politics": 52,
+    "psychology": 53,
+    "public_administration": 54,
+    "relationship": 55,
+    "religion": 56,
+    "sociology": 57,
+    "sports": 58,
+    "statistics": 59,
+    "systems_science": 60,
+    "textile_science": 61,
+    "topicality": 62,
+    "transportation_engineering": 63,
+    "travel": 64,
+    "urban_planning": 65,
+    "vulgar_language": 66
+  },
+  "layer_norm_eps": 1e-05,
+  "local_attention": 128,
+  "local_rope_theta": 160000.0,
+  "max_position_embeddings": 7999,
+  "mlp_bias": false,
+  "mlp_dropout": 0.0,
+  "model_type": "modernbert",
+  "norm_bias": false,
+  "norm_eps": 1e-05,
+  "num_attention_heads": 6,
+  "num_hidden_layers": 10,
+  "pad_token_id": 50283,
+  "position_embedding_type": "sans_pos",
+  "problem_type": "single_label_classification",
+  "repad_logits_with_grad": false,
+  "sep_token_id": 50282,
+  "sparse_pred_ignore_index": -100,
+  "sparse_prediction": false,
+  "transformers_version": "4.57.6",
+  "vocab_size": 50368
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:248bc199f8c4534ead516d88d15ce83882e1b4b30c1fbff6187427c236983c1d
+size 128233740

special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "cls_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "[PAD]",
+  "sep_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,945 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "|||IP_ADDRESS|||",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "1": {
+      "content": "<|padding|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50254": {
+      "content": "                        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50255": {
+      "content": "                       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50256": {
+      "content": "                      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50257": {
+      "content": "                     ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50258": {
+      "content": "                    ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50259": {
+      "content": "                   ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50260": {
+      "content": "                  ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50261": {
+      "content": "                 ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50262": {
+      "content": "                ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50263": {
+      "content": "               ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50264": {
+      "content": "              ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50265": {
+      "content": "             ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50266": {
+      "content": "            ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50267": {
+      "content": "           ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50268": {
+      "content": "          ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50269": {
+      "content": "         ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50270": {
+      "content": "        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50271": {
+      "content": "       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50272": {
+      "content": "      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50273": {
+      "content": "     ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50274": {
+      "content": "    ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50275": {
+      "content": "   ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50276": {
+      "content": "  ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50277": {
+      "content": "|||EMAIL_ADDRESS|||",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50278": {
+      "content": "|||PHONE_NUMBER|||",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50279": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50280": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50281": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50282": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50283": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50284": {
+      "content": "[MASK]",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50285": {
+      "content": "[unused0]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50286": {
+      "content": "[unused1]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50287": {
+      "content": "[unused2]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50288": {
+      "content": "[unused3]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50289": {
+      "content": "[unused4]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50290": {
+      "content": "[unused5]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50291": {
+      "content": "[unused6]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50292": {
+      "content": "[unused7]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50293": {
+      "content": "[unused8]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50294": {
+      "content": "[unused9]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50295": {
+      "content": "[unused10]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50296": {
+      "content": "[unused11]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50297": {
+      "content": "[unused12]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50298": {
+      "content": "[unused13]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50299": {
+      "content": "[unused14]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50300": {
+      "content": "[unused15]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50301": {
+      "content": "[unused16]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50302": {
+      "content": "[unused17]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50303": {
+      "content": "[unused18]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50304": {
+      "content": "[unused19]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50305": {
+      "content": "[unused20]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50306": {
+      "content": "[unused21]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50307": {
+      "content": "[unused22]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50308": {
+      "content": "[unused23]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50309": {
+      "content": "[unused24]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50310": {
+      "content": "[unused25]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50311": {
+      "content": "[unused26]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50312": {
+      "content": "[unused27]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50313": {
+      "content": "[unused28]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50314": {
+      "content": "[unused29]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50315": {
+      "content": "[unused30]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50316": {
+      "content": "[unused31]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50317": {
+      "content": "[unused32]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50318": {
+      "content": "[unused33]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50319": {
+      "content": "[unused34]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50320": {
+      "content": "[unused35]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50321": {
+      "content": "[unused36]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50322": {
+      "content": "[unused37]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50323": {
+      "content": "[unused38]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50324": {
+      "content": "[unused39]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50325": {
+      "content": "[unused40]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50326": {
+      "content": "[unused41]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50327": {
+      "content": "[unused42]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50328": {
+      "content": "[unused43]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50329": {
+      "content": "[unused44]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50330": {
+      "content": "[unused45]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50331": {
+      "content": "[unused46]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50332": {
+      "content": "[unused47]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50333": {
+      "content": "[unused48]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50334": {
+      "content": "[unused49]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50335": {
+      "content": "[unused50]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50336": {
+      "content": "[unused51]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50337": {
+      "content": "[unused52]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50338": {
+      "content": "[unused53]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50339": {
+      "content": "[unused54]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50340": {
+      "content": "[unused55]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50341": {
+      "content": "[unused56]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50342": {
+      "content": "[unused57]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50343": {
+      "content": "[unused58]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50344": {
+      "content": "[unused59]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50345": {
+      "content": "[unused60]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50346": {
+      "content": "[unused61]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50347": {
+      "content": "[unused62]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50348": {
+      "content": "[unused63]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50349": {
+      "content": "[unused64]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50350": {
+      "content": "[unused65]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50351": {
+      "content": "[unused66]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50352": {
+      "content": "[unused67]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50353": {
+      "content": "[unused68]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50354": {
+      "content": "[unused69]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50355": {
+      "content": "[unused70]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50356": {
+      "content": "[unused71]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50357": {
+      "content": "[unused72]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50358": {
+      "content": "[unused73]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50359": {
+      "content": "[unused74]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50360": {
+      "content": "[unused75]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50361": {
+      "content": "[unused76]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50362": {
+      "content": "[unused77]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50363": {
+      "content": "[unused78]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50364": {
+      "content": "[unused79]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50365": {
+      "content": "[unused80]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50366": {
+      "content": "[unused81]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50367": {
+      "content": "[unused82]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "extra_special_tokens": {},
+  "mask_token": "[MASK]",
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 8192,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": "[UNK]"
+}

train.py

@@ -0,0 +1,92 @@
+import warnings
+
+import numpy as np
+
+from datasets import load_dataset
+from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, average_precision_score
+from scipy.special import softmax
+from transformers import AutoModelForSequenceClassification, AutoConfig, AutoTokenizer
+from txtai import Embeddings
+from txtai.pipeline import HFTrainer
+
+
+def batchlabel(rows):
+    return {"label": [config.label2id[label] for label in rows["label"]]}
+
+def batchtext(rows):
+    texts = []
+    for x in rows["id"]:
+        results = embeddings.search("SELECT text FROM txtai WHERE id=:id", 1, parameters={"id": x})
+        texts.append(results[0]["text"])
+
+    return {"text": texts}
+
+def metrics(pred):
+    logits, labelids = pred
+    preds = logits.argmax(-1)
+
+    # Calculate accuracy, precision, recall, and F1-score
+    accuracy = accuracy_score(labelids, preds)
+    precision = precision_score(labelids, preds, average="weighted", zero_division=0)
+    recall = recall_score(labelids, preds, average="weighted", zero_division=0)
+    f1 = f1_score(labelids, preds, average="weighted", zero_division=0)
+
+    # Calculate PR AUC
+    probs = softmax(logits, axis=-1)
+    nclasses = logits.shape[1]
+    onehot = np.eye(nclasses)[labelids]
+
+    # average_precision_score doesn't support zero_division parameter
+    with warnings.catch_warnings():
+        warnings.filterwarnings("ignore", message="No positive class found in y_true")
+        prauc = average_precision_score(onehot, probs, average="weighted")
+
+    return {
+        "accuracy": accuracy,
+        "precision": precision,
+        "recall": recall,
+        "f1": f1,
+        "prauc": prauc
+    }
+
+# Embeddings database
+embeddings = Embeddings()
+embeddings.load(provider="huggingface-hub", container="neuml/txtai-wikipedia-slim")
+
+# Training dataset
+ds = load_dataset("csv", data_files="labels.csv", split="train", keep_default_na=False)
+labels = dict(enumerate(sorted(ds.unique("label"))))
+print(labels)
+
+# Base model
+path = "jhu-clsp/ettin-encoder-32m"
+
+# Model configuration
+config = AutoConfig.from_pretrained(path)
+config.num_labels = len(labels)
+config.id2label = labels
+config.label2id = {label: uid for uid, label in labels.items()}
+
+# Map label ids
+ds = ds.map(batchlabel, batched=True)
+
+# Map text
+ds = ds.map(batchtext, batched=True)
+
+# Split into train and test
+ds = ds.train_test_split(test_size=0.05, seed=42)
+training, test = ds["train"], ds["test"]
+
+# Model to train
+model = AutoModelForSequenceClassification.from_pretrained(path, config=config)
+tokenizer = AutoTokenizer.from_pretrained(path)
+
+train = HFTrainer()
+train(
+    (model, tokenizer), training, test, metrics=metrics, maxlength=512, bf16=True,
+    learning_rate=5e-5, per_device_train_batch_size=64, num_train_epochs=3,
+    warmup_ratio=0.1, lr_scheduler_type="cosine",
+    eval_strategy="steps", eval_steps=500, logging_steps=500,
+    tokenizers=True, dataloader_num_workers=20,
+    output_dir="domain-labeler"
+)

trainer_state.json

@@ -0,0 +1,216 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 4455,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.3367003367003367,
+      "grad_norm": 14.949820518493652,
+      "learning_rate": 4.9978441082243424e-05,
+      "loss": 1.6348,
+      "step": 500
+    },
+    {
+      "epoch": 0.3367003367003367,
+      "eval_accuracy": 0.7818,
+      "eval_f1": 0.7707654278149872,
+      "eval_loss": 0.7248653173446655,
+      "eval_prauc": 0.8471943467878073,
+      "eval_precision": 0.7777639682274672,
+      "eval_recall": 0.7818,
+      "eval_runtime": 6.5782,
+      "eval_samples_per_second": 760.089,
+      "eval_steps_per_second": 95.011,
+      "step": 500
+    },
+    {
+      "epoch": 0.6734006734006734,
+      "grad_norm": 8.724627494812012,
+      "learning_rate": 4.7689101828499175e-05,
+      "loss": 0.6566,
+      "step": 1000
+    },
+    {
+      "epoch": 0.6734006734006734,
+      "eval_accuracy": 0.8194,
+      "eval_f1": 0.8125511449697567,
+      "eval_loss": 0.5900622606277466,
+      "eval_prauc": 0.883177875981317,
+      "eval_precision": 0.8170544867915973,
+      "eval_recall": 0.8194,
+      "eval_runtime": 6.2222,
+      "eval_samples_per_second": 803.57,
+      "eval_steps_per_second": 100.446,
+      "step": 1000
+    },
+    {
+      "epoch": 1.0101010101010102,
+      "grad_norm": 8.932344436645508,
+      "learning_rate": 4.196084543323817e-05,
+      "loss": 0.5695,
+      "step": 1500
+    },
+    {
+      "epoch": 1.0101010101010102,
+      "eval_accuracy": 0.8298,
+      "eval_f1": 0.8214116862727279,
+      "eval_loss": 0.5388458967208862,
+      "eval_prauc": 0.8942335681618219,
+      "eval_precision": 0.824507012622034,
+      "eval_recall": 0.8298,
+      "eval_runtime": 6.9973,
+      "eval_samples_per_second": 714.564,
+      "eval_steps_per_second": 89.32,
+      "step": 1500
+    },
+    {
+      "epoch": 1.3468013468013469,
+      "grad_norm": 11.769265174865723,
+      "learning_rate": 3.366188603986574e-05,
+      "loss": 0.3954,
+      "step": 2000
+    },
+    {
+      "epoch": 1.3468013468013469,
+      "eval_accuracy": 0.832,
+      "eval_f1": 0.8276523330085487,
+      "eval_loss": 0.5264924764633179,
+      "eval_prauc": 0.8980912722084475,
+      "eval_precision": 0.828413850188029,
+      "eval_recall": 0.832,
+      "eval_runtime": 6.9005,
+      "eval_samples_per_second": 724.583,
+      "eval_steps_per_second": 90.573,
+      "step": 2000
+    },
+    {
+      "epoch": 1.6835016835016834,
+      "grad_norm": 12.866188049316406,
+      "learning_rate": 2.4050071311561374e-05,
+      "loss": 0.3735,
+      "step": 2500
+    },
+    {
+      "epoch": 1.6835016835016834,
+      "eval_accuracy": 0.843,
+      "eval_f1": 0.8369456713438264,
+      "eval_loss": 0.5062804222106934,
+      "eval_prauc": 0.9023592788997854,
+      "eval_precision": 0.8391826013500971,
+      "eval_recall": 0.843,
+      "eval_runtime": 6.7126,
+      "eval_samples_per_second": 744.87,
+      "eval_steps_per_second": 93.109,
+      "step": 2500
+    },
+    {
+      "epoch": 2.0202020202020203,
+      "grad_norm": 7.454126834869385,
+      "learning_rate": 1.4582234342913715e-05,
+      "loss": 0.3436,
+      "step": 3000
+    },
+    {
+      "epoch": 2.0202020202020203,
+      "eval_accuracy": 0.84,
+      "eval_f1": 0.8369555486562935,
+      "eval_loss": 0.5169184803962708,
+      "eval_prauc": 0.9039284056011966,
+      "eval_precision": 0.8386328079429232,
+      "eval_recall": 0.84,
+      "eval_runtime": 6.0064,
+      "eval_samples_per_second": 832.444,
+      "eval_steps_per_second": 104.055,
+      "step": 3000
+    },
+    {
+      "epoch": 2.356902356902357,
+      "grad_norm": 10.186031341552734,
+      "learning_rate": 6.693385963821353e-06,
+      "loss": 0.1677,
+      "step": 3500
+    },
+    {
+      "epoch": 2.356902356902357,
+      "eval_accuracy": 0.8408,
+      "eval_f1": 0.8384862313439404,
+      "eval_loss": 0.5824030041694641,
+      "eval_prauc": 0.9009043897343983,
+      "eval_precision": 0.8396918530177993,
+      "eval_recall": 0.8408,
+      "eval_runtime": 7.0427,
+      "eval_samples_per_second": 709.957,
+      "eval_steps_per_second": 88.745,
+      "step": 3500
+    },
+    {
+      "epoch": 2.6936026936026938,
+      "grad_norm": 8.157699584960938,
+      "learning_rate": 1.5792145767053622e-06,
+      "loss": 0.1491,
+      "step": 4000
+    },
+    {
+      "epoch": 2.6936026936026938,
+      "eval_accuracy": 0.8422,
+      "eval_f1": 0.8391634085924852,
+      "eval_loss": 0.5748035907745361,
+      "eval_prauc": 0.903134748047534,
+      "eval_precision": 0.8388717647632293,
+      "eval_recall": 0.8422,
+      "eval_runtime": 7.0678,
+      "eval_samples_per_second": 707.436,
+      "eval_steps_per_second": 88.429,
+      "step": 4000
+    },
+    {
+      "epoch": 3.0,
+      "step": 4455,
+      "total_flos": 1.1103474080022912e+16,
+      "train_loss": 0.4963774555861348,
+      "train_runtime": 674.1321,
+      "train_samples_per_second": 422.766,
+      "train_steps_per_second": 6.608
+    },
+    {
+      "epoch": 3.0,
+      "eval_accuracy": 0.8426,
+      "eval_f1": 0.8397110808761383,
+      "eval_loss": 0.5739430785179138,
+      "eval_prauc": 0.9033464177917727,
+      "eval_precision": 0.8396082701885788,
+      "eval_recall": 0.8426,
+      "eval_runtime": 6.1808,
+      "eval_samples_per_second": 808.953,
+      "eval_steps_per_second": 101.119,
+      "step": 4455
+    }
+  ],
+  "logging_steps": 500,
+  "max_steps": 4455,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.1103474080022912e+16,
+  "train_batch_size": 64,
+  "trial_name": null,
+  "trial_params": null
+}