upstream-archive byte-perfect snapshot of Helsinki-NLP/opus-mt-eo-caenes (ADR-039 Phase D)

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+source.spm filter=lfs diff=lfs merge=lfs -text
+target.spm filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,80 @@
+---
+language:
+- eo
+- en
+- es
+- ca
+tags:
+- translation
+- machine-translation
+- marian
+- opus-mt
+- multilingual
+license: cc-by-4.0
+pipeline_tag: translation
+metrics:
+- bleu
+- chrf
+---
+
+# Esperanto -> Catalan, English, Spanish MT Model
+
+## Model description
+
+This repository contains a **multilingual MarianMT** model for **Esperanto → (English, Spanish, Catalan)** translation using language tags.
+
+## Usage
+
+The model is loaded and used with `transformers` as:
+
+```python
+from transformers import MarianMTModel, MarianTokenizer
+import torch
+
+model_name = "Helsinki-NLP/opus-mt-eo-caenes"
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+model = MarianMTModel.from_pretrained(model_name).to(device)
+tokenizer = MarianTokenizer.from_pretrained(model_name)
+
+source_texts = [
+    ">>spa<< Saluton, kiel vi fartas?",
+    ">>eng<< Saluton, kiel vi fartas?",
+    ">>cat<< Saluton, kiel vi fartas?"
+]
+
+inputs = tokenizer(source_texts, return_tensors="pt", padding=True, truncation=True)
+inputs = {k: v.to(device) for k, v in inputs.items()}
+
+translated_ids = model.generate(inputs["input_ids"])
+translated_texts = tokenizer.batch_decode(translated_ids, skip_special_tokens=True)
+
+for src, tgt in zip(source_texts, translated_texts):
+    print(f"Source: {src} => Translated: {tgt}")
+````
+
+### Supported target languages (via tags)
+
+You control the target language by prefixing the source sentence with one of the following tags:
+
+* `>>eng<<` → English
+* `>>spa<<` → Spanish
+* `>>cat<<` → Catalan
+
+## Training data
+
+The model was trained using **Tatoeba** parallel data, with **FLORES-200** used as the development set.
+
+Training sentence-pair counts:
+
+* **ca-eo**: 672,931
+* **es-eo**: 4,677,945
+* **eo-en**: 5,000,000
+
+## Evaluation on FLORES
+
+| Language Pair |  BLEU |  ChrF++ |
+| ------------- | ----: | ----: |
+| epo-spa       | 19.98 | 49.11 |
+| epo-cat       | 28.35 | 55.42 |
+| epo-eng       | 37.47 | 63.09 |

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<s>": 32001
+}

config.json

@@ -0,0 +1,39 @@
+{
+  "activation_dropout": 0.0,
+  "activation_function": "relu",
+  "architectures": [
+    "MarianMTModel"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "d_model": 512,
+  "decoder_attention_heads": 8,
+  "decoder_ffn_dim": 2048,
+  "decoder_layerdrop": 0.0,
+  "decoder_layers": 6,
+  "decoder_start_token_id": 32000,
+  "decoder_vocab_size": 32001,
+  "dropout": 0.1,
+  "dtype": "float16",
+  "encoder_attention_heads": 8,
+  "encoder_ffn_dim": 2048,
+  "encoder_layerdrop": 0.0,
+  "encoder_layers": 6,
+  "eos_token_id": 0,
+  "forced_eos_token_id": 0,
+  "init_std": 0.02,
+  "is_encoder_decoder": true,
+  "max_length": null,
+  "max_position_embeddings": 512,
+  "model_type": "marian",
+  "normalize_embedding": false,
+  "num_beams": null,
+  "num_hidden_layers": 6,
+  "pad_token_id": 32000,
+  "scale_embedding": true,
+  "share_encoder_decoder_embeddings": true,
+  "static_position_embeddings": true,
+  "transformers_version": "4.57.3",
+  "use_cache": true,
+  "vocab_size": 32001
+}

generation_config.json

@@ -0,0 +1,16 @@
+{
+  "_from_model_config": true,
+  "bad_words_ids": [
+    [
+      32000
+    ]
+  ],
+  "bos_token_id": 0,
+  "decoder_start_token_id": 32000,
+  "eos_token_id": 0,
+  "forced_eos_token_id": 0,
+  "max_length": 512,
+  "num_beams": 8,
+  "pad_token_id": 32000,
+  "transformers_version": "4.57.3"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38b4b78e7bb9206834d231ae41785165b17f8deae57a5c286f1b80b982a9d7fa
+size 153908130

source.spm

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d561cdf0fc7ad693c1bf1fe21732c6434650623ec69dc712aceb36483587914d
+size 805644

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

target.spm

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d561cdf0fc7ad693c1bf1fe21732c6434650623ec69dc712aceb36483587914d
+size 805644

tokenizer_config.json

@@ -0,0 +1,48 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32000": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32001": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "model_max_length": 512,
+  "pad_token": "<pad>",
+  "separate_vocabs": false,
+  "source_lang": null,
+  "sp_model_kwargs": {},
+  "target_lang": null,
+  "tokenizer_class": "MarianTokenizer",
+  "unk_token": "<unk>"
+}