lang model & tokenizer

bpe_tokenizer/bpe-tokenizer.json

@@ -0,0 +1,415 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "[PAD]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "[BLANK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "[SOS]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 3,
+      "content": "[EOS]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": {
+    "type": "Sequence",
+    "normalizers": [
+      {
+        "type": "NFD"
+      },
+      {
+        "type": "Lowercase"
+      },
+      {
+        "type": "StripAccents"
+      },
+      {
+        "type": "Replace",
+        "pattern": {
+          "String": " ' s"
+        },
+        "content": "'s"
+      }
+    ]
+  },
+  "pre_tokenizer": {
+    "type": "BertPreTokenizer"
+  },
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "SpecialToken": {
+          "id": "[SOS]",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "[EOS]",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {
+      "[EOS]": {
+        "id": "[EOS]",
+        "ids": [
+          3
+        ],
+        "tokens": [
+          "[EOS]"
+        ]
+      },
+      "[SOS]": {
+        "id": "[SOS]",
+        "ids": [
+          2
+        ],
+        "tokens": [
+          "[SOS]"
+        ]
+      }
+    }
+  },
+  "decoder": {
+    "type": "WordPiece",
+    "prefix": "##",
+    "cleanup": true
+  },
+  "model": {
+    "type": "BPE",
+    "dropout": null,
+    "unk_token": "[UNK]",
+    "continuing_subword_prefix": "##",
+    "end_of_word_suffix": null,
+    "fuse_unk": false,
+    "byte_fallback": false,
+    "ignore_merges": false,
+    "vocab": {
+      "[PAD]": 0,
+      "[BLANK]": 1,
+      "[SOS]": 2,
+      "[EOS]": 3,
+      "'": 4,
+      "a": 5,
+      "b": 6,
+      "c": 7,
+      "d": 8,
+      "e": 9,
+      "f": 10,
+      "g": 11,
+      "h": 12,
+      "i": 13,
+      "j": 14,
+      "k": 15,
+      "l": 16,
+      "m": 17,
+      "n": 18,
+      "o": 19,
+      "p": 20,
+      "q": 21,
+      "r": 22,
+      "s": 23,
+      "t": 24,
+      "u": 25,
+      "v": 26,
+      "w": 27,
+      "x": 28,
+      "y": 29,
+      "z": 30,
+      "##r": 31,
+      "##u": 32,
+      "##c": 33,
+      "##h": 34,
+      "##t": 35,
+      "##e": 36,
+      "##o": 37,
+      "##s": 38,
+      "##v": 39,
+      "##i": 40,
+      "##d": 41,
+      "##a": 42,
+      "##l": 43,
+      "##b": 44,
+      "##n": 45,
+      "##p": 46,
+      "##y": 47,
+      "##g": 48,
+      "##f": 49,
+      "##k": 50,
+      "##w": 51,
+      "##z": 52,
+      "##m": 53,
+      "##j": 54,
+      "##x": 55,
+      "##q": 56,
+      "th": 57,
+      "the": 58,
+      "##er": 59,
+      "##nd": 60,
+      "##in": 61,
+      "##ed": 62,
+      "##ou": 63,
+      "##at": 64,
+      "##en": 65,
+      "and": 66,
+      "##or": 67,
+      "of": 68,
+      "##es": 69,
+      "##on": 70,
+      "to": 71,
+      "##is": 72,
+      "##ing": 73,
+      "##ar": 74,
+      "##an": 75,
+      "##it": 76,
+      "##as": 77,
+      "in": 78,
+      "##ll": 79,
+      "##re": 80,
+      "wh": 81,
+      "##om": 82,
+      "he": 83,
+      "ha": 84,
+      "be": 85,
+      "##le": 86,
+      "##ic": 87,
+      "##ot": 88,
+      "##ow": 89,
+      "##ut": 90,
+      "it": 91,
+      "##ly": 92,
+      "##ld": 93,
+      "that": 94,
+      "##gh": 95,
+      "sh": 96,
+      "was": 97,
+      "##ve": 98,
+      "on": 99
+    },
+    "merges": [
+      [
+        "t",
+        "##h"
+      ],
+      [
+        "th",
+        "##e"
+      ],
+      [
+        "##e",
+        "##r"
+      ],
+      [
+        "##n",
+        "##d"
+      ],
+      [
+        "##i",
+        "##n"
+      ],
+      [
+        "##e",
+        "##d"
+      ],
+      [
+        "##o",
+        "##u"
+      ],
+      [
+        "##a",
+        "##t"
+      ],
+      [
+        "##e",
+        "##n"
+      ],
+      [
+        "a",
+        "##nd"
+      ],
+      [
+        "##o",
+        "##r"
+      ],
+      [
+        "o",
+        "##f"
+      ],
+      [
+        "##e",
+        "##s"
+      ],
+      [
+        "##o",
+        "##n"
+      ],
+      [
+        "t",
+        "##o"
+      ],
+      [
+        "##i",
+        "##s"
+      ],
+      [
+        "##in",
+        "##g"
+      ],
+      [
+        "##a",
+        "##r"
+      ],
+      [
+        "##a",
+        "##n"
+      ],
+      [
+        "##i",
+        "##t"
+      ],
+      [
+        "##a",
+        "##s"
+      ],
+      [
+        "i",
+        "##n"
+      ],
+      [
+        "##l",
+        "##l"
+      ],
+      [
+        "##r",
+        "##e"
+      ],
+      [
+        "w",
+        "##h"
+      ],
+      [
+        "##o",
+        "##m"
+      ],
+      [
+        "h",
+        "##e"
+      ],
+      [
+        "h",
+        "##a"
+      ],
+      [
+        "b",
+        "##e"
+      ],
+      [
+        "##l",
+        "##e"
+      ],
+      [
+        "##i",
+        "##c"
+      ],
+      [
+        "##o",
+        "##t"
+      ],
+      [
+        "##o",
+        "##w"
+      ],
+      [
+        "##u",
+        "##t"
+      ],
+      [
+        "i",
+        "##t"
+      ],
+      [
+        "##l",
+        "##y"
+      ],
+      [
+        "##l",
+        "##d"
+      ],
+      [
+        "th",
+        "##at"
+      ],
+      [
+        "##g",
+        "##h"
+      ],
+      [
+        "s",
+        "##h"
+      ],
+      [
+        "w",
+        "##as"
+      ],
+      [
+        "##v",
+        "##e"
+      ],
+      [
+        "o",
+        "##n"
+      ]
+    ]
+  }
+}

bpe_tokenizer/bpe-tokenizer.meta.json

@@ -0,0 +1 @@
+{"name": "bpe-tokenizer", "min_frequency": 0, "vocab_size": 100, "data_dir": "/media/atem/Data/HSE_videos/4_DLA/hw_1_ASR/data/text_datasets", "dataset_name": "librispeech-lm-norm"}