CE fine-tuned epoch 1/3 best_val=0.5636

Browse files

Files changed (6) hide show

README.md +52 -48
config.json +24 -15
config_sentence_transformers.json +1 -1
model.safetensors +2 -2
tokenizer.json +0 -0
tokenizer_config.json +12 -9

README.md CHANGED Viewed

@@ -6,7 +6,7 @@ tags:
 - generated_from_trainer
 - dataset_size:7419
 - loss:BinaryCrossEntropyLoss
-base_model: cross-encoder/ms-marco-MiniLM-L6-v2
 pipeline_tag: text-ranking
 library_name: sentence-transformers
 metrics:
@@ -18,7 +18,7 @@ metrics:
 - recall
 - average_precision
 model-index:
-- name: CrossEncoder based on cross-encoder/ms-marco-MiniLM-L6-v2
   results:
   - task:
       type: cross-encoder-classification
@@ -28,38 +28,38 @@ model-index:
       type: ce-val
     metrics:
     - type: accuracy
-      value: 0.5587878787878788
       name: Accuracy
     - type: accuracy_threshold
-      value: 0.892870306968689
       name: Accuracy Threshold
     - type: f1
-      value: 0.6677471636952999
       name: F1
     - type: f1_threshold
-      value: -8.320409774780273
       name: F1 Threshold
     - type: precision
-      value: 0.5018270401948843
       name: Precision
     - type: recall
-      value: 0.9975786924939467
       name: Recall
     - type: average_precision
-      value: 0.5638184038907261
       name: Average Precision
 ---
-# CrossEncoder based on cross-encoder/ms-marco-MiniLM-L6-v2
-This is a [Cross Encoder](https://www.sbert.net/docs/cross_encoder/usage/usage.html) model finetuned from [cross-encoder/ms-marco-MiniLM-L6-v2](https://huggingface.co/cross-encoder/ms-marco-MiniLM-L6-v2) using the [sentence-transformers](https://www.SBERT.net) library. It computes scores for pairs of texts, which can be used for text reranking and semantic search.
 ## Model Details
 ### Model Description
 - **Model Type:** Cross Encoder
-- **Base model:** [cross-encoder/ms-marco-MiniLM-L6-v2](https://huggingface.co/cross-encoder/ms-marco-MiniLM-L6-v2) <!-- at revision c5ee24cb16019beea0893ab7796b1df96625c6b8 -->
-- **Maximum Sequence Length:** 512 tokens
 - **Number of Output Labels:** 1 label
 - **Supported Modality:** Text
 <!-- - **Training Dataset:** Unknown -->
@@ -77,7 +77,7 @@ This is a [Cross Encoder](https://www.sbert.net/docs/cross_encoder/usage/usage.h
 ```
 CrossEncoder(
-  (0): Transformer({'transformer_task': 'sequence-classification', 'modality_config': {'text': {'method': 'forward', 'method_output_name': 'logits'}}, 'module_output_name': 'scores', 'architecture': 'BertForSequenceClassification'})
 )
 ```
@@ -99,25 +99,25 @@ from sentence_transformers import CrossEncoder
 model = CrossEncoder("cross_encoder_model_id")
 # Get scores for pairs of inputs
 pairs = [
-    ['The last time the planet was even four degrees warmer, Peter Brannen points out in The Ends of the World, his new history of the planet’s major extinction events, the oceans were hundreds of feet higher.', 'Almost all scientists acknowledge that the rate of species loss is greater now than at any time in human history, with extinctions occurring at rates hundreds of times higher than background extinction rates.'],
-    ['[S]unspot activity on the surface of our star has dropped to a new low.', "Patches of the star's surface with a lower temperature and luminosity than average are known as starspots."],
-    ['More money is dedicated within the Department of Homeland Security to climate change than what\'s spent combating "Islamist terrorists radicalizing over the Internet in the United States of America."', "The center works on the Internet's routing infrastructure (the SPRI program) and Domain Name System (DNSSEC), identity theft and other online criminal activity (ITTC), Internet traffic and networks research (PREDICT datasets and the DETER testbed), Department of Defense and HSARPA exercises (Livewire and Determined Promise), and wireless security in cooperation with Canada."],
-    ['Worst-case global heating scenarios may need to be revised upwards in light of a better understanding of the role of clouds, scientists have said.', 'With this information, scientists can produce scenarios of how greenhouse gas emissions may vary in the future.'],
-    ['Prof Adam Scaife, a climate modelling expert at the UK’s Met Office, said the evidence for a link to shrinking Arctic ice was now good: ‘The consensus points towards that being a real effect.’”', 'Some models of modern climate exhibit Arctic amplification without changes in snow and ice cover.'],
 ]
 scores = model.predict(pairs)
 print(scores)
-# [-0.0067  0.3615 -3.1055 -0.8462  0.1024]
 # Or rank different texts based on similarity to a single text
 ranks = model.rank(
-    'The last time the planet was even four degrees warmer, Peter Brannen points out in The Ends of the World, his new history of the planet’s major extinction events, the oceans were hundreds of feet higher.',
     [
-        'Almost all scientists acknowledge that the rate of species loss is greater now than at any time in human history, with extinctions occurring at rates hundreds of times higher than background extinction rates.',
-        "Patches of the star's surface with a lower temperature and luminosity than average are known as starspots.",
-        "The center works on the Internet's routing infrastructure (the SPRI program) and Domain Name System (DNSSEC), identity theft and other online criminal activity (ITTC), Internet traffic and networks research (PREDICT datasets and the DETER testbed), Department of Defense and HSARPA exercises (Livewire and Determined Promise), and wireless security in cooperation with Canada.",
-        'With this information, scientists can produce scenarios of how greenhouse gas emissions may vary in the future.',
-        'Some models of modern climate exhibit Arctic amplification without changes in snow and ice cover.',
     ]
 )
 # [{'corpus_id': ..., 'score': ...}, {'corpus_id': ..., 'score': ...}, ...]
@@ -158,13 +158,13 @@ You can finetune this model on your own dataset.
 | Metric                | Value      |
 |:----------------------|:-----------|
-| accuracy              | 0.5588     |
-| accuracy_threshold    | 0.8929     |
-| f1                    | 0.6677     |
-| f1_threshold          | -8.3204    |
-| precision             | 0.5018     |
-| recall                | 0.9976     |
-| **average_precision** | **0.5638** |
 <!--
 ## Bias, Risks and Limitations
@@ -190,13 +190,13 @@ You can finetune this model on your own dataset.
   |         | sentence_0                                                                        | sentence_1                                                                         | label                                                          |
   |:--------|:----------------------------------------------------------------------------------|:-----------------------------------------------------------------------------------|:---------------------------------------------------------------|
   | type    | string                                                                            | string                                                                             | float                                                          |
-  | details | <ul><li>min: 7 tokens</li><li>mean: 27.57 tokens</li><li>max: 82 tokens</li></ul> | <ul><li>min: 6 tokens</li><li>mean: 33.03 tokens</li><li>max: 333 tokens</li></ul> | <ul><li>min: 0.0</li><li>mean: 0.52</li><li>max: 1.0</li></ul> |
 * Samples:
-  | sentence_0                                                                                                                                                                                                               | sentence_1                                                                                                                                                                                                                                                                                                                                                                                            | label            |
-  |:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-----------------|
-  | <code>The last time the planet was even four degrees warmer, Peter Brannen points out in The Ends of the World, his new history of the planet’s major extinction events, the oceans were hundreds of feet higher.</code> | <code>Almost all scientists acknowledge that the rate of species loss is greater now than at any time in human history, with extinctions occurring at rates hundreds of times higher than background extinction rates.</code>                                                                                                                                                                         | <code>0.0</code> |
-  | <code>[S]unspot activity on the surface of our star has dropped to a new low.</code>                                                                                                                                     | <code>Patches of the star's surface with a lower temperature and luminosity than average are known as starspots.</code>                                                                                                                                                                                                                                                                               | <code>1.0</code> |
-  | <code>More money is dedicated within the Department of Homeland Security to climate change than what's spent combating "Islamist terrorists radicalizing over the Internet in the United States of America."</code>      | <code>The center works on the Internet's routing infrastructure (the SPRI program) and Domain Name System (DNSSEC), identity theft and other online criminal activity (ITTC), Internet traffic and networks research (PREDICT datasets and the DETER testbed), Department of Defense and HSARPA exercises (Livewire and Determined Promise), and wireless security in cooperation with Canada.</code> | <code>1.0</code> |
 * Loss: [<code>BinaryCrossEntropyLoss</code>](https://sbert.net/docs/package_reference/cross_encoder/losses.html#binarycrossentropyloss) with these parameters:
   ```json
   {
@@ -208,17 +208,18 @@ You can finetune this model on your own dataset.
 ### Training Hyperparameters
 #### Non-Default Hyperparameters
-- `per_device_train_batch_size`: 16
-- `per_device_eval_batch_size`: 16
 - `num_train_epochs`: 1
 #### All Hyperparameters
 <details><summary>Click to expand</summary>
 - `do_predict`: False
 - `prediction_loss_only`: True
-- `per_device_train_batch_size`: 16
-- `per_device_eval_batch_size`: 16
 - `gradient_accumulation_steps`: 1
 - `eval_accumulation_steps`: None
 - `torch_empty_cache_steps`: None
@@ -246,7 +247,7 @@ You can finetune this model on your own dataset.
 - `seed`: 42
 - `data_seed`: None
 - `bf16`: False
-- `fp16`: False
 - `bf16_full_eval`: False
 - `fp16_full_eval`: False
 - `tf32`: None
@@ -315,13 +316,16 @@ You can finetune this model on your own dataset.
 </details>
 ### Training Logs
-| Epoch | Step | ce-val_average_precision |
-|:-----:|:----:|:------------------------:|
-| -1    | -1   | 0.5638                   |
 ### Training Time
-- **Training**: 33.0 seconds
 ### Framework Versions
 - Python: 3.12.13

 - generated_from_trainer
 - dataset_size:7419
 - loss:BinaryCrossEntropyLoss
+base_model: cross-encoder/nli-deberta-v3-base
 pipeline_tag: text-ranking
 library_name: sentence-transformers
 metrics:
 - recall
 - average_precision
 model-index:
+- name: CrossEncoder based on cross-encoder/nli-deberta-v3-base
   results:
   - task:
       type: cross-encoder-classification
       type: ce-val
     metrics:
     - type: accuracy
+      value: 0.5636363636363636
       name: Accuracy
     - type: accuracy_threshold
+      value: 0.512444794178009
       name: Accuracy Threshold
     - type: f1
+      value: 0.6711297071129707
       name: F1
     - type: f1_threshold
+      value: 0.4493926167488098
       name: F1 Threshold
     - type: precision
+      value: 0.5127877237851662
       name: Precision
     - type: recall
+      value: 0.9709443099273608
       name: Recall
     - type: average_precision
+      value: 0.565654586279988
       name: Average Precision
 ---
+# CrossEncoder based on cross-encoder/nli-deberta-v3-base
+This is a [Cross Encoder](https://www.sbert.net/docs/cross_encoder/usage/usage.html) model finetuned from [cross-encoder/nli-deberta-v3-base](https://huggingface.co/cross-encoder/nli-deberta-v3-base) using the [sentence-transformers](https://www.SBERT.net) library. It computes scores for pairs of texts, which can be used for text reranking and semantic search.
 ## Model Details
 ### Model Description
 - **Model Type:** Cross Encoder
+- **Base model:** [cross-encoder/nli-deberta-v3-base](https://huggingface.co/cross-encoder/nli-deberta-v3-base) <!-- at revision 6c749ce3425cd33b46d187e45b92bbf96ee12ec7 -->
+- **Maximum Sequence Length:** 256 tokens
 - **Number of Output Labels:** 1 label
 - **Supported Modality:** Text
 <!-- - **Training Dataset:** Unknown -->
 ```
 CrossEncoder(
+  (0): Transformer({'transformer_task': 'sequence-classification', 'modality_config': {'text': {'method': 'forward', 'method_output_name': 'logits'}}, 'module_output_name': 'scores', 'architecture': 'DebertaV2ForSequenceClassification'})
 )
 ```
 model = CrossEncoder("cross_encoder_model_id")
 # Get scores for pairs of inputs
 pairs = [
+    ['An independent inquiry found CRU is a small research unit with limited resources and their rigour and honesty are not in doubt.', 'The media and other scientific organisations were criticised for having "sometimes neglected" to reflect the uncertainties, doubts and assumptions of the work done by the CRU.'],
+    ['As president, Obama will immediately close the Mississippi River Gulf Outlet, which experts say funneled floodwater into New Orleans.', 'Levees along the MRGO and the Intracoastal Waterway were breached in approximately 20 places, directly flooding most of St. Bernard Parish and New Orleans East.'],
+    ['If we double atmospheric carbon dioxide[…] we’d only raise global surface temperatures by about a degree Celsius.', 'Not only do increasing carbon dioxide concentrations lead to increases in global surface temperature, but increasing global temperatures also cause increasing concentrations of carbon dioxide.'],
+    ['But as that upper layer warms up, the oxygen-rich waters are less likely to mix down into cooler layers of the ocean because the warm waters are less dense and do not sink as readily.', 'Water that is saltier or cooler will be denser, and will sink in relation to the surrounding water.'],
+    ['Less than half of published scientists endorse global warming.', 'Scientists Reach 100% Consensus on Anthropogenic Global Warming.'],
 ]
 scores = model.predict(pairs)
 print(scores)
+# [0.5286 0.4566 0.49   0.5111 0.6522]
 # Or rank different texts based on similarity to a single text
 ranks = model.rank(
+    'An independent inquiry found CRU is a small research unit with limited resources and their rigour and honesty are not in doubt.',
     [
+        'The media and other scientific organisations were criticised for having "sometimes neglected" to reflect the uncertainties, doubts and assumptions of the work done by the CRU.',
+        'Levees along the MRGO and the Intracoastal Waterway were breached in approximately 20 places, directly flooding most of St. Bernard Parish and New Orleans East.',
+        'Not only do increasing carbon dioxide concentrations lead to increases in global surface temperature, but increasing global temperatures also cause increasing concentrations of carbon dioxide.',
+        'Water that is saltier or cooler will be denser, and will sink in relation to the surrounding water.',
+        'Scientists Reach 100% Consensus on Anthropogenic Global Warming.',
     ]
 )
 # [{'corpus_id': ..., 'score': ...}, {'corpus_id': ..., 'score': ...}, ...]
 | Metric                | Value      |
 |:----------------------|:-----------|
+| accuracy              | 0.5636     |
+| accuracy_threshold    | 0.5124     |
+| f1                    | 0.6711     |
+| f1_threshold          | 0.4494     |
+| precision             | 0.5128     |
+| recall                | 0.9709     |
+| **average_precision** | **0.5657** |
 <!--
 ## Bias, Risks and Limitations
   |         | sentence_0                                                                        | sentence_1                                                                         | label                                                          |
   |:--------|:----------------------------------------------------------------------------------|:-----------------------------------------------------------------------------------|:---------------------------------------------------------------|
   | type    | string                                                                            | string                                                                             | float                                                          |
+  | details | <ul><li>min: 7 tokens</li><li>mean: 26.73 tokens</li><li>max: 66 tokens</li></ul> | <ul><li>min: 6 tokens</li><li>mean: 31.55 tokens</li><li>max: 133 tokens</li></ul> | <ul><li>min: 0.0</li><li>mean: 0.53</li><li>max: 1.0</li></ul> |
 * Samples:
+  | sentence_0                                                                                                                                         | sentence_1                                                                                                                                                                                                    | label            |
+  |:---------------------------------------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-----------------|
+  | <code>An independent inquiry found CRU is a small research unit with limited resources and their rigour and honesty are not in doubt.</code>       | <code>The media and other scientific organisations were criticised for having "sometimes neglected" to reflect the uncertainties, doubts and assumptions of the work done by the CRU.</code>                  | <code>0.0</code> |
+  | <code>As president, Obama will immediately close the Mississippi River Gulf Outlet, which experts say funneled floodwater into New Orleans.</code> | <code>Levees along the MRGO and the Intracoastal Waterway were breached in approximately 20 places, directly flooding most of St. Bernard Parish and New Orleans East.</code>                                 | <code>1.0</code> |
+  | <code>If we double atmospheric carbon dioxide[…] we’d only raise global surface temperatures by about a degree Celsius.</code>                     | <code>Not only do increasing carbon dioxide concentrations lead to increases in global surface temperature, but increasing global temperatures also cause increasing concentrations of carbon dioxide.</code> | <code>0.0</code> |
 * Loss: [<code>BinaryCrossEntropyLoss</code>](https://sbert.net/docs/package_reference/cross_encoder/losses.html#binarycrossentropyloss) with these parameters:
   ```json
   {
 ### Training Hyperparameters
 #### Non-Default Hyperparameters
+- `per_device_train_batch_size`: 4
+- `per_device_eval_batch_size`: 4
 - `num_train_epochs`: 1
+- `fp16`: True
 #### All Hyperparameters
 <details><summary>Click to expand</summary>
 - `do_predict`: False
 - `prediction_loss_only`: True
+- `per_device_train_batch_size`: 4
+- `per_device_eval_batch_size`: 4
 - `gradient_accumulation_steps`: 1
 - `eval_accumulation_steps`: None
 - `torch_empty_cache_steps`: None
 - `seed`: 42
 - `data_seed`: None
 - `bf16`: False
+- `fp16`: True
 - `bf16_full_eval`: False
 - `fp16_full_eval`: False
 - `tf32`: None
 </details>
 ### Training Logs
+| Epoch  | Step | Training Loss | ce-val_average_precision |
+|:------:|:----:|:-------------:|:------------------------:|
+| 0.2695 | 500  | 0.7103        | -                        |
+| 0.5391 | 1000 | 0.6983        | -                        |
+| 0.8086 | 1500 | 0.6982        | -                        |
+| -1     | -1   | -             | 0.5657                   |
 ### Training Time
+- **Training**: 7.1 minutes
 ### Framework Versions
 - Python: 3.12.13

config.json CHANGED Viewed

@@ -1,36 +1,45 @@
 {
-  "add_cross_attention": false,
   "architectures": [
-    "BertForSequenceClassification"
   ],
   "attention_probs_dropout_prob": 0.1,
-  "bos_token_id": null,
-  "classifier_dropout": null,
   "dtype": "float32",
-  "eos_token_id": null,
-  "gradient_checkpointing": false,
   "hidden_act": "gelu",
   "hidden_dropout_prob": 0.1,
-  "hidden_size": 384,
   "id2label": {
     "0": "LABEL_0"
   },
   "initializer_range": 0.02,
-  "intermediate_size": 1536,
-  "is_decoder": false,
   "label2id": {
     "LABEL_0": 0
   },
-  "layer_norm_eps": 1e-12,
   "max_position_embeddings": 512,
-  "model_type": "bert",
   "num_attention_heads": 12,
-  "num_hidden_layers": 6,
   "pad_token_id": 0,
-  "position_embedding_type": "absolute",
   "tie_word_embeddings": true,
   "transformers_version": "5.0.0",
-  "type_vocab_size": 2,
   "use_cache": false,
-  "vocab_size": 30522
 }

 {
   "architectures": [
+    "DebertaV2ForSequenceClassification"
   ],
   "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 1,
   "dtype": "float32",
+  "eos_token_id": 2,
   "hidden_act": "gelu",
   "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
   "id2label": {
     "0": "LABEL_0"
   },
   "initializer_range": 0.02,
+  "intermediate_size": 3072,
   "label2id": {
     "LABEL_0": 0
   },
+  "layer_norm_eps": 1e-07,
+  "legacy": true,
   "max_position_embeddings": 512,
+  "max_relative_positions": -1,
+  "model_type": "deberta-v2",
+  "norm_rel_ebd": "layer_norm",
   "num_attention_heads": 12,
+  "num_hidden_layers": 12,
   "pad_token_id": 0,
+  "pooler_dropout": 0,
+  "pooler_hidden_act": "gelu",
+  "pooler_hidden_size": 768,
+  "pos_att_type": [
+    "p2c",
+    "c2p"
+  ],
+  "position_biased_input": false,
+  "position_buckets": 256,
+  "relative_attention": true,
+  "share_att_key": true,
   "tie_word_embeddings": true,
   "transformers_version": "5.0.0",
+  "type_vocab_size": 0,
   "use_cache": false,
+  "vocab_size": 128100
 }

config_sentence_transformers.json CHANGED Viewed

@@ -4,7 +4,7 @@
     "sentence_transformers": "5.4.1",
     "transformers": "5.0.0"
   },
-  "activation_fn": "torch.nn.modules.linear.Identity",
   "default_prompt_name": null,
   "model_type": "CrossEncoder",
   "prompts": {}

     "sentence_transformers": "5.4.1",
     "transformers": "5.0.0"
   },
+  "activation_fn": "torch.nn.modules.activation.Sigmoid",
   "default_prompt_name": null,
   "model_type": "CrossEncoder",
   "prompts": {}

model.safetensors CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5677755caa216f568eba3369d5320dff550650bc5e66300950b7802b4b550ac5
-size 90866404

 version https://git-lfs.github.com/spec/v1
+oid sha256:922a849772df9d09c87547738e9d2e009866a84a654b460b49743b396fc81a74
+size 737716172

tokenizer.json CHANGED Viewed

The diff for this file is too large to render. See raw diff

tokenizer_config.json CHANGED Viewed

@@ -1,18 +1,21 @@
 {
   "backend": "tokenizers",
-  "clean_up_tokenization_spaces": true,
   "cls_token": "[CLS]",
-  "do_basic_tokenize": true,
-  "do_lower_case": true,
   "is_local": false,
   "mask_token": "[MASK]",
-  "model_max_length": 512,
   "model_specific_special_tokens": {},
-  "never_split": null,
   "pad_token": "[PAD]",
   "sep_token": "[SEP]",
-  "strip_accents": null,
-  "tokenize_chinese_chars": true,
-  "tokenizer_class": "BertTokenizer",
-  "unk_token": "[UNK]"
 }

 {
+  "add_prefix_space": true,
   "backend": "tokenizers",
+  "bos_token": "[CLS]",
+  "clean_up_tokenization_spaces": false,
   "cls_token": "[CLS]",
+  "do_lower_case": false,
+  "eos_token": "[SEP]",
   "is_local": false,
   "mask_token": "[MASK]",
+  "model_max_length": 256,
   "model_specific_special_tokens": {},
   "pad_token": "[PAD]",
   "sep_token": "[SEP]",
+  "sp_model_kwargs": {},
+  "split_by_punct": false,
+  "tokenizer_class": "DebertaV2Tokenizer",
+  "unk_id": 3,
+  "unk_token": "[UNK]",
+  "vocab_type": "spm"
 }