Add files using upload-large-folder tool

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizers/german_phoneme.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,35 @@
+---
+library_name: onnx
+pipeline_tag: text-to-speech
+tags:
+- onnx
+- onnxruntime
+- text-to-speech
+- magpie-tts
+- nemo
+license: other
+---
+
+# ONNX Magpie Runtime Bundle
+
+This repository contains ONNX Runtime artifacts for a local MagpieTTS conversion.
+It is intended for use with the `onnx-magpie` runtime and does not require NeMo
+or PyTorch at inference time.
+
+## Files
+
+- `magpie.pipeline.json`: runtime pipeline manifest.
+- `magpie_text_context.onnx`: text/speaker context graph.
+- `magpie_decoder_prefix.onnx`: autoregressive decoder prefix graph.
+- `magpie_codec_decoder.onnx`: codec-token to waveform graph.
+- `tokenizers/`: dependency-free tokenizer artifacts.
+- `onnx-magpie-hub.json`: download manifest consumed by the runtime.
+
+## Runtime
+
+```powershell
+python scripts/run_onnx_magpie.py --hf-repo mweinbach1/onnx-magpie --text "Hello world." --wav-output speech.wav
+```
+
+The original MagpieTTS checkpoint is governed by NVIDIA's model license. Verify
+that your use of these converted artifacts complies with the upstream terms.

magpie.export.json

@@ -0,0 +1,47 @@
+{
+  "format": "onnx-magpie-export-summary-v1",
+  "nemo_path": "magpie_tts_multilingual_357m\\magpie_tts_multilingual_357m.nemo",
+  "opset": 18,
+  "graphs": [
+    {
+      "path": "models\\magpie_text_context.onnx",
+      "inputs": [
+        "text",
+        "text_lens",
+        "speaker_indices"
+      ],
+      "outputs": [
+        "cond",
+        "cond_mask",
+        "additional_decoder_input",
+        "additional_decoder_mask"
+      ]
+    },
+    {
+      "path": "models\\magpie_decoder_prefix.onnx",
+      "inputs": [
+        "audio_tokens",
+        "audio_tokens_lens",
+        "cond",
+        "cond_mask",
+        "additional_decoder_input",
+        "additional_decoder_mask"
+      ],
+      "outputs": [
+        "next_logits",
+        "decoder_state"
+      ]
+    },
+    {
+      "path": "models\\magpie_codec_decoder.onnx",
+      "inputs": [
+        "codes",
+        "codes_lens"
+      ],
+      "outputs": [
+        "audio",
+        "audio_lens"
+      ]
+    }
+  ]
+}

magpie.pipeline.json

@@ -0,0 +1,23 @@
+{
+  "format": "onnx-magpie-autoregressive-v1",
+  "sample_rate": 22050,
+  "tokenizer_path": "tokenizers\\magpie.tokenizers.json",
+  "graphs": {
+    "text_context": "magpie_text_context.onnx",
+    "decoder_prefix": "magpie_decoder_prefix.onnx",
+    "codec_decoder": "magpie_codec_decoder.onnx"
+  },
+  "generation": {
+    "max_decoder_steps": 500,
+    "min_generated_frames": 4,
+    "temperature": 0.6,
+    "topk": 80,
+    "num_audio_codebooks": 8,
+    "num_all_tokens_per_codebook": 2024,
+    "frame_stacking_factor": 1,
+    "audio_bos_id": 2016,
+    "audio_eos_id": 2017,
+    "sampling_mode": "argmax"
+  },
+  "waveform_output": "audio"
+}

magpie_codec_decoder.manifest.json

@@ -0,0 +1,47 @@
+{
+  "model_path": "models\\magpie_codec_decoder.onnx",
+  "ir_version": 8,
+  "producer_name": "pytorch",
+  "producer_version": "2.10.0",
+  "opsets": [
+    {
+      "domain": "ai.onnx",
+      "version": 18
+    }
+  ],
+  "inputs": [
+    {
+      "name": "codes",
+      "elem_type": "INT64",
+      "shape": [
+        1,
+        8,
+        "codec_time"
+      ]
+    },
+    {
+      "name": "codes_lens",
+      "elem_type": "INT64",
+      "shape": [
+        1
+      ]
+    }
+  ],
+  "outputs": [
+    {
+      "name": "audio",
+      "elem_type": "FLOAT",
+      "shape": [
+        "Reshapeaudio_dim_0",
+        "sample_time"
+      ]
+    },
+    {
+      "name": "audio_lens",
+      "elem_type": "INT64",
+      "shape": [
+        1
+      ]
+    }
+  ]
+}

magpie_codec_decoder.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2d12c6267d626f65cd76559c0cb420b26144bea9aeb8c3f33f530df0450ddce
+size 128635008

magpie_decoder_prefix.manifest.json

@@ -0,0 +1,82 @@
+{
+  "model_path": "models\\magpie_decoder_prefix.onnx",
+  "ir_version": 8,
+  "producer_name": "pytorch",
+  "producer_version": "2.10.0",
+  "opsets": [
+    {
+      "domain": "ai.onnx",
+      "version": 18
+    }
+  ],
+  "inputs": [
+    {
+      "name": "audio_tokens",
+      "elem_type": "INT64",
+      "shape": [
+        1,
+        8,
+        "audio_time"
+      ]
+    },
+    {
+      "name": "audio_tokens_lens",
+      "elem_type": "INT64",
+      "shape": [
+        1
+      ]
+    },
+    {
+      "name": "cond",
+      "elem_type": "FLOAT",
+      "shape": [
+        1,
+        "text_time",
+        768
+      ]
+    },
+    {
+      "name": "cond_mask",
+      "elem_type": "BOOL",
+      "shape": [
+        1,
+        "text_time"
+      ]
+    },
+    {
+      "name": "additional_decoder_input",
+      "elem_type": "FLOAT",
+      "shape": [
+        1,
+        110,
+        768
+      ]
+    },
+    {
+      "name": "additional_decoder_mask",
+      "elem_type": "BOOL",
+      "shape": [
+        1,
+        110
+      ]
+    }
+  ],
+  "outputs": [
+    {
+      "name": "next_logits",
+      "elem_type": "FLOAT",
+      "shape": [
+        "Gathernext_logits_dim_0",
+        16192
+      ]
+    },
+    {
+      "name": "decoder_state",
+      "elem_type": "FLOAT",
+      "shape": [
+        "Gathernext_logits_dim_0",
+        768
+      ]
+    }
+  ]
+}

magpie_decoder_prefix.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:642ec9330eea5b3df3a4cbf13956d0a45cca1cca26e73fa9d35310b3362f114d
+size 482141691

magpie_text_context.manifest.json

@@ -0,0 +1,72 @@
+{
+  "model_path": "models\\magpie_text_context.onnx",
+  "ir_version": 8,
+  "producer_name": "pytorch",
+  "producer_version": "2.10.0",
+  "opsets": [
+    {
+      "domain": "ai.onnx",
+      "version": 18
+    }
+  ],
+  "inputs": [
+    {
+      "name": "text",
+      "elem_type": "INT64",
+      "shape": [
+        1,
+        "text_time"
+      ]
+    },
+    {
+      "name": "text_lens",
+      "elem_type": "INT64",
+      "shape": [
+        1
+      ]
+    },
+    {
+      "name": "speaker_indices",
+      "elem_type": "INT64",
+      "shape": [
+        1
+      ]
+    }
+  ],
+  "outputs": [
+    {
+      "name": "cond",
+      "elem_type": "FLOAT",
+      "shape": [
+        "LayerNormalizationcond_dim_0",
+        "text_time",
+        768
+      ]
+    },
+    {
+      "name": "cond_mask",
+      "elem_type": "BOOL",
+      "shape": [
+        1,
+        "text_time"
+      ]
+    },
+    {
+      "name": "additional_decoder_input",
+      "elem_type": "FLOAT",
+      "shape": [
+        1,
+        110,
+        768
+      ]
+    },
+    {
+      "name": "additional_decoder_mask",
+      "elem_type": "BOOL",
+      "shape": [
+        1,
+        "Lessadditional_decoder_mask_dim_1"
+      ]
+    }
+  ]
+}

magpie_text_context.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:561208fd9e46fd595e13ff7715bd6f8c7158589f8551f02f4e5fc2f239e9ddc8
+size 428673443

onnx-magpie-hub.json

@@ -0,0 +1,90 @@
+{
+  "format": "onnx-magpie-hub-v1",
+  "pipeline": "magpie.pipeline.json",
+  "files": [
+    {
+      "path": "README.md",
+      "size": 1071
+    },
+    {
+      "path": "magpie.export.json",
+      "size": 1017
+    },
+    {
+      "path": "magpie.pipeline.json",
+      "size": 660
+    },
+    {
+      "path": "magpie_codec_decoder.manifest.json",
+      "size": 780
+    },
+    {
+      "path": "magpie_codec_decoder.onnx",
+      "size": 128635008
+    },
+    {
+      "path": "magpie_decoder_prefix.manifest.json",
+      "size": 1401
+    },
+    {
+      "path": "magpie_decoder_prefix.onnx",
+      "size": 482141691
+    },
+    {
+      "path": "magpie_text_context.manifest.json",
+      "size": 1243
+    },
+    {
+      "path": "magpie_text_context.onnx",
+      "size": 428673443
+    },
+    {
+      "path": "onnx-magpie-hub.json",
+      "size": 1855
+    },
+    {
+      "path": "tokenizers/english_phoneme.json",
+      "size": 7111300
+    },
+    {
+      "path": "tokenizers/french_chartokenizer.json",
+      "size": 219
+    },
+    {
+      "path": "tokenizers/german_phoneme.json",
+      "size": 17462337
+    },
+    {
+      "path": "tokenizers/hindi_chartokenizer.json",
+      "size": 2583
+    },
+    {
+      "path": "tokenizers/italian_phoneme.json",
+      "size": 222
+    },
+    {
+      "path": "tokenizers/japanese_phoneme.json",
+      "size": 5743
+    },
+    {
+      "path": "tokenizers/magpie.tokenizers.json",
+      "size": 7493
+    },
+    {
+      "path": "tokenizers/mandarin_phoneme.json",
+      "size": 494008
+    },
+    {
+      "path": "tokenizers/spanish_phoneme.json",
+      "size": 4736224
+    },
+    {
+      "path": "tokenizers/text_ce_tokenizer.json",
+      "size": 222
+    },
+    {
+      "path": "tokenizers/vietnamese_phoneme.json",
+      "size": 222
+    }
+  ]
+}

tokenizers/french_chartokenizer.json

@@ -0,0 +1,11 @@
+{
+  "kind": "byt5",
+  "class_name": "ByT5Tokenizer",
+  "offset": 633,
+  "pad_id": 633,
+  "num_tokens": 384,
+  "aggregate_offset": 633,
+  "byte_offset": 3,
+  "eos_token_id": 1,
+  "global_eos_token_id": 2361
+}

tokenizers/german_phoneme.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e22f249116339544bea0f5c7a60476867a0306cd6aa762e83f0fdf595bab3b37
+size 17462337

tokenizers/hindi_chartokenizer.json

@@ -0,0 +1,219 @@
+{
+  "kind": "symbol_table",
+  "class_name": "HindiCharsTokenizer",
+  "offset": 1017,
+  "pad_id": 1206,
+  "num_tokens": 191,
+  "tokens": [
+    " ",
+    "अ",
+    "आ",
+    "इ",
+    "ई",
+    "उ",
+    "ऊ",
+    "ऋ",
+    "ॠ",
+    "ए",
+    "ऐ",
+    "ओ",
+    "औ",
+    "ऍ",
+    "ऑ",
+    "क",
+    "ख",
+    "ग",
+    "घ",
+    "ङ",
+    "च",
+    "छ",
+    "ज",
+    "झ",
+    "ञ",
+    "ट",
+    "ठ",
+    "ड",
+    "ढ",
+    "ण",
+    "त",
+    "थ",
+    "द",
+    "ध",
+    "न",
+    "प",
+    "फ",
+    "ब",
+    "भ",
+    "म",
+    "य",
+    "र",
+    "ल",
+    "व",
+    "श",
+    "ष",
+    "स",
+    "ह",
+    "ळ",
+    "ऩ",
+    "ऱ",
+    "ा",
+    "ि",
+    "ी",
+    "ु",
+    "ू",
+    "ृ",
+    "ॄ",
+    "े",
+    "ै",
+    "ो",
+    "ौ",
+    "ॅ",
+    "ॉ",
+    "ँ",
+    "ं",
+    "ः",
+    "्",
+    "़",
+    "ॊ",
+    "ॢ",
+    "ॣ",
+    "ॆ",
+    "।",
+    "अ",
+    "आ",
+    "इ",
+    "ई",
+    "उ",
+    "ऊ",
+    "ऋ",
+    "ॠ",
+    "ए",
+    "ऐ",
+    "ओ",
+    "औ",
+    "ऍ",
+    "ऑ",
+    "क",
+    "ख",
+    "ग",
+    "घ",
+    "ङ",
+    "च",
+    "छ",
+    "ज",
+    "झ",
+    "ञ",
+    "ट",
+    "ठ",
+    "ड",
+    "ढ",
+    "ण",
+    "त",
+    "थ",
+    "द",
+    "ध",
+    "न",
+    "प",
+    "फ",
+    "ब",
+    "भ",
+    "म",
+    "य",
+    "र",
+    "ल",
+    "व",
+    "श",
+    "ष",
+    "स",
+    "ह",
+    "ळ",
+    "ऩ",
+    "ऱ",
+    "ा",
+    "ि",
+    "ी",
+    "ु",
+    "ू",
+    "ृ",
+    "ॄ",
+    "े",
+    "ै",
+    "ो",
+    "ौ",
+    "ॅ",
+    "ॉ",
+    "ँ",
+    "ं",
+    "ः",
+    "्",
+    "़",
+    "ॊ",
+    "ॢ",
+    "ॣ",
+    "ॆ",
+    "।",
+    "a",
+    "b",
+    "c",
+    "d",
+    "e",
+    "f",
+    "g",
+    "h",
+    "i",
+    "j",
+    "k",
+    "l",
+    "m",
+    "n",
+    "o",
+    "p",
+    "q",
+    "r",
+    "s",
+    "t",
+    "u",
+    "v",
+    "w",
+    "x",
+    "y",
+    "z",
+    "'",
+    "!",
+    "\"",
+    "(",
+    ")",
+    ",",
+    "-",
+    ".",
+    "/",
+    ":",
+    ";",
+    "?",
+    "[",
+    "]",
+    "{",
+    "}",
+    "<pad>",
+    "<oov>"
+  ],
+  "pad_with_space": true,
+  "punct": true,
+  "punct_list": [
+    "!",
+    "\"",
+    "(",
+    ")",
+    ",",
+    "-",
+    ".",
+    "/",
+    ":",
+    ";",
+    "?",
+    "[",
+    "]",
+    "{",
+    "}"
+  ]
+}

tokenizers/italian_phoneme.json

@@ -0,0 +1,11 @@
+{
+  "kind": "byt5",
+  "class_name": "ByT5Tokenizer",
+  "offset": 1208,
+  "pad_id": 1208,
+  "num_tokens": 384,
+  "aggregate_offset": 1208,
+  "byte_offset": 3,
+  "eos_token_id": 1,
+  "global_eos_token_id": 2361
+}

tokenizers/japanese_phoneme.json

@@ -0,0 +1,435 @@
+{
+  "kind": "symbol_table",
+  "class_name": "JapanesePhonemeTokenizer",
+  "offset": 458,
+  "pad_id": 631,
+  "num_tokens": 175,
+  "tokens": [
+    " ",
+    "0",
+    "1",
+    "ァ",
+    "ア",
+    "ィ",
+    "イ",
+    "ゥ",
+    "ウ",
+    "ェ",
+    "エ",
+    "ォ",
+    "オ",
+    "カ",
+    "ガ",
+    "キ",
+    "ギ",
+    "ク",
+    "グ",
+    "ケ",
+    "ゲ",
+    "コ",
+    "ゴ",
+    "サ",
+    "ザ",
+    "シ",
+    "ジ",
+    "ス",
+    "ズ",
+    "セ",
+    "ゼ",
+    "ソ",
+    "ゾ",
+    "タ",
+    "ダ",
+    "チ",
+    "ヂ",
+    "ッ",
+    "ツ",
+    "ヅ",
+    "テ",
+    "デ",
+    "ト",
+    "ド",
+    "ナ",
+    "ニ",
+    "ヌ",
+    "ネ",
+    "ノ",
+    "ハ",
+    "バ",
+    "パ",
+    "ヒ",
+    "ビ",
+    "ピ",
+    "フ",
+    "ブ",
+    "プ",
+    "ヘ",
+    "ベ",
+    "ペ",
+    "ホ",
+    "ボ",
+    "ポ",
+    "マ",
+    "ミ",
+    "ム",
+    "メ",
+    "モ",
+    "ャ",
+    "ヤ",
+    "ュ",
+    "ユ",
+    "ョ",
+    "ヨ",
+    "ラ",
+    "リ",
+    "ル",
+    "レ",
+    "ロ",
+    "ヮ",
+    "ワ",
+    "ヲ",
+    "ン",
+    "ヴ",
+    "ヵ",
+    "ヶ",
+    "ー",
+    "A",
+    "B",
+    "C",
+    "D",
+    "E",
+    "F",
+    "G",
+    "H",
+    "I",
+    "J",
+    "K",
+    "L",
+    "M",
+    "N",
+    "O",
+    "P",
+    "Q",
+    "R",
+    "S",
+    "T",
+    "U",
+    "V",
+    "W",
+    "X",
+    "Y",
+    "Z",
+    "!",
+    "\"",
+    "(",
+    ")",
+    ",",
+    "-",
+    ".",
+    "/",
+    ":",
+    ";",
+    "?",
+    "[",
+    "]",
+    "{",
+    "}",
+    "«",
+    "»",
+    "•",
+    "‥",
+    "…",
+    "‹",
+    "›",
+    "※",
+    "◦",
+    "、",
+    "。",
+    "〃",
+    "〈",
+    "〉",
+    "《",
+    "》",
+    "「",
+    "」",
+    "『",
+    "』",
+    "【",
+    "】",
+    "〒",
+    "〓",
+    "〔",
+    "〕",
+    "〖",
+    "〗",
+    "〘",
+    "〙",
+    "〚",
+    "〛",
+    "〜",
+    "〽",
+    "・",
+    "・・・",
+    "ー",
+    "﹅",
+    "﹆",
+    "！",
+    "＊",
+    "？",
+    "｟",
+    "｠",
+    "<pad>",
+    "<oov>"
+  ],
+  "japanese": {
+    "pad_with_space": true,
+    "punct": true,
+    "punct_list": [
+      "!",
+      "\"",
+      "(",
+      ")",
+      ",",
+      "-",
+      ".",
+      "/",
+      ":",
+      ";",
+      "?",
+      "[",
+      "]",
+      "{",
+      "}",
+      "«",
+      "»",
+      "•",
+      "‥",
+      "…",
+      "‹",
+      "›",
+      "※",
+      "◦",
+      "、",
+      "。",
+      "〃",
+      "〈",
+      "〉",
+      "《",
+      "》",
+      "「",
+      "」",
+      "『",
+      "』",
+      "【",
+      "】",
+      "〒",
+      "〓",
+      "〔",
+      "〕",
+      "〖",
+      "〗",
+      "〘",
+      "〙",
+      "〚",
+      "〛",
+      "〜",
+      "〽",
+      "・",
+      "・・・",
+      "ー",
+      "﹅",
+      "﹆",
+      "！",
+      "＊",
+      "？",
+      "｟",
+      "｠"
+    ],
+    "phoneme_list": [
+      "0",
+      "1",
+      "ァ",
+      "ア",
+      "ィ",
+      "イ",
+      "ゥ",
+      "ウ",
+      "ェ",
+      "エ",
+      "ォ",
+      "オ",
+      "カ",
+      "ガ",
+      "キ",
+      "ギ",
+      "ク",
+      "グ",
+      "ケ",
+      "ゲ",
+      "コ",
+      "ゴ",
+      "サ",
+      "ザ",
+      "シ",
+      "ジ",
+      "ス",
+      "ズ",
+      "セ",
+      "ゼ",
+      "ソ",
+      "ゾ",
+      "タ",
+      "ダ",
+      "チ",
+      "ヂ",
+      "ッ",
+      "ツ",
+      "ヅ",
+      "テ",
+      "デ",
+      "ト",
+      "ド",
+      "ナ",
+      "ニ",
+      "ヌ",
+      "ネ",
+      "ノ",
+      "ハ",
+      "バ",
+      "パ",
+      "ヒ",
+      "ビ",
+      "ピ",
+      "フ",
+      "ブ",
+      "プ",
+      "ヘ",
+      "ベ",
+      "ペ",
+      "ホ",
+      "ボ",
+      "ポ",
+      "マ",
+      "ミ",
+      "ム",
+      "メ",
+      "モ",
+      "ャ",
+      "ヤ",
+      "ュ",
+      "ユ",
+      "ョ",
+      "ヨ",
+      "ラ",
+      "リ",
+      "ル",
+      "レ",
+      "ロ",
+      "ヮ",
+      "ワ",
+      "ヲ",
+      "ン",
+      "ヴ",
+      "ヵ",
+      "ヶ",
+      "ー"
+    ],
+    "ascii_letter_list": [
+      "A",
+      "B",
+      "C",
+      "D",
+      "E",
+      "F",
+      "G",
+      "H",
+      "I",
+      "J",
+      "K",
+      "L",
+      "M",
+      "N",
+      "O",
+      "P",
+      "Q",
+      "R",
+      "S",
+      "T",
+      "U",
+      "V",
+      "W",
+      "X",
+      "Y",
+      "Z"
+    ],
+    "g2p_cache": {
+      "こんにちは。": [
+        "0",
+        "コ",
+        "1",
+        "ン",
+        "1",
+        "ニ",
+        "1",
+        "チ",
+        "1",
+        "ワ",
+        "。"
+      ],
+      "コンニチハ。": [
+        "0",
+        "コ",
+        "1",
+        "ン",
+        "1",
+        "ニ",
+        "0",
+        "チ",
+        "0",
+        "ハ",
+        "。"
+      ],
+      "世界。": [
+        "1",
+        "セ",
+        "0",
+        "カ",
+        "0",
+        "イ",
+        "。"
+      ],
+      "こんにちはONNX Magpieの世界。": [
+        "0",
+        "コ",
+        "1",
+        "ン",
+        "1",
+        "ニ",
+        "1",
+        "チ",
+        "1",
+        "ワ",
+        "O",
+        "N",
+        "N",
+        "X",
+        " ",
+        "M",
+        "A",
+        "G",
+        "P",
+        "I",
+        "E",
+        "0",
+        "ノ",
+        "1",
+        "セ",
+        "0",
+        "カ",
+        "0",
+        "イ",
+        "。"
+      ]
+    }
+  }
+}

tokenizers/magpie.tokenizers.json

@@ -0,0 +1,407 @@
+{
+  "format": "onnx-magpie-tokenizers-v1",
+  "source": {
+    "nemo_path": "magpie_tts_multilingual_357m\\magpie_tts_multilingual_357m.nemo",
+    "nemo_commit": "9fc8b801177515117be811f6f77f89dbb080ae41",
+    "eos_token_id": 2361
+  },
+  "language_to_tokenizer": {
+    "en": "english_phoneme",
+    "es": "spanish_phoneme",
+    "de": "german_phoneme",
+    "fr": "french_chartokenizer",
+    "it": "italian_phoneme",
+    "vi": "vietnamese_phoneme",
+    "zh": "mandarin_phoneme",
+    "hi": "hindi_chartokenizer",
+    "ja": "japanese_phoneme"
+  },
+  "tokenizer_files": {
+    "english_phoneme": "english_phoneme.json",
+    "spanish_phoneme": "spanish_phoneme.json",
+    "german_phoneme": "german_phoneme.json",
+    "mandarin_phoneme": "mandarin_phoneme.json",
+    "japanese_phoneme": "japanese_phoneme.json",
+    "french_chartokenizer": "french_chartokenizer.json",
+    "hindi_chartokenizer": "hindi_chartokenizer.json",
+    "italian_phoneme": "italian_phoneme.json",
+    "vietnamese_phoneme": "vietnamese_phoneme.json",
+    "text_ce_tokenizer": "text_ce_tokenizer.json"
+  },
+  "fixtures": [
+    {
+      "language": "en",
+      "text": "Hello world from ONNX Magpie.",
+      "sample_text": "Hello world from ONNX Magpie.",
+      "chunk_index": 0,
+      "tokenizer": "english_phoneme",
+      "ids": [
+        55,
+        79,
+        90,
+        59,
+        62,
+        87,
+        93,
+        90,
+        68,
+        82,
+        59,
+        52,
+        93,
+        27,
+        39,
+        36,
+        34,
+        93,
+        36,
+        35,
+        35,
+        45,
+        93,
+        34,
+        22,
+        28,
+        37,
+        30,
+        26,
+        7,
+        2361
+      ]
+    },
+    {
+      "language": "es",
+      "text": "Hola mundo desde ONNX Magpie.",
+      "sample_text": "Hola mundo desde ONNX Magpie.",
+      "chunk_index": 0,
+      "tokenizer": "spanish_phoneme",
+      "ids": [
+        196,
+        187,
+        148,
+        145,
+        136,
+        196,
+        120,
+        128,
+        121,
+        111,
+        122,
+        196,
+        138,
+        187,
+        139,
+        151,
+        173,
+        139,
+        196,
+        122,
+        121,
+        121,
+        131,
+        196,
+        120,
+        108,
+        114,
+        123,
+        116,
+        112,
+        103,
+        196,
+        2361
+      ]
+    },
+    {
+      "language": "de",
+      "text": "Hallo Welt von ONNX Magpie.",
+      "sample_text": "Hallo Welt von ONNX Magpie.",
+      "chunk_index": 0,
+      "tokenizer": "german_phoneme",
+      "ids": [
+        346,
+        290,
+        332,
+        285,
+        294,
+        297,
+        334,
+        346,
+        303,
+        332,
+        322,
+        294,
+        301,
+        346,
+        248,
+        241,
+        240,
+        346,
+        215,
+        214,
+        214,
+        224,
+        346,
+        213,
+        227,
+        233,
+        242,
+        235,
+        231,
+        277,
+        346,
+        2361
+      ]
+    },
+    {
+      "language": "fr",
+      "text": "Bonjour le monde depuis ONNX Magpie.",
+      "sample_text": "Bonjour le monde depuis ONNX Magpie.",
+      "chunk_index": 0,
+      "tokenizer": "french_chartokenizer",
+      "ids": [
+        702,
+        747,
+        746,
+        742,
+        747,
+        753,
+        750,
+        668,
+        744,
+        737,
+        668,
+        745,
+        747,
+        746,
+        736,
+        737,
+        668,
+        736,
+        737,
+        748,
+        753,
+        741,
+        751,
+        668,
+        715,
+        714,
+        714,
+        724,
+        668,
+        713,
+        733,
+        739,
+        748,
+        741,
+        737,
+        682,
+        634,
+        2361
+      ]
+    },
+    {
+      "language": "it",
+      "text": "Ciao mondo da ONNX Magpie.",
+      "sample_text": "Ciao mondo da ONNX Magpie.",
+      "chunk_index": 0,
+      "tokenizer": "italian_phoneme",
+      "ids": [
+        1278,
+        1316,
+        1308,
+        1322,
+        1243,
+        1320,
+        1322,
+        1321,
+        1311,
+        1322,
+        1243,
+        1311,
+        1308,
+        1243,
+        1290,
+        1289,
+        1289,
+        1299,
+        1243,
+        1288,
+        1308,
+        1314,
+        1323,
+        1316,
+        1312,
+        1257,
+        1209,
+        2361
+      ]
+    },
+    {
+      "language": "vi",
+      "text": "Xin chào thế giới từ ONNX Magpie.",
+      "sample_text": "Xin chào thế giới từ ONNX Magpie.",
+      "chunk_index": 0,
+      "tokenizer": "vietnamese_phoneme",
+      "ids": [
+        1683,
+        1700,
+        1705,
+        1627,
+        1694,
+        1699,
+        1790,
+        1755,
+        1706,
+        1627,
+        1711,
+        1699,
+        1820,
+        1781,
+        1786,
+        1627,
+        1698,
+        1700,
+        1820,
+        1782,
+        1750,
+        1700,
+        1627,
+        1711,
+        1820,
+        1782,
+        1766,
+        1627,
+        1674,
+        1673,
+        1673,
+        1683,
+        1627,
+        1672,
+        1692,
+        1698,
+        1707,
+        1700,
+        1696,
+        1641,
+        1593,
+        2361
+      ]
+    },
+    {
+      "language": "zh",
+      "text": "你好，ONNX Magpie 世界。",
+      "sample_text": "你好，ONNX Magpie 世界。",
+      "chunk_index": 0,
+      "tokenizer": "mandarin_phoneme",
+      "ids": [
+        349,
+        362,
+        356,
+        392,
+        376,
+        352,
+        392,
+        422,
+        409,
+        408,
+        408,
+        418,
+        349,
+        407,
+        395,
+        401,
+        410,
+        403,
+        399,
+        349,
+        385,
+        356,
+        393,
+        371,
+        357,
+        353,
+        393,
+        423,
+        349,
+        2361
+      ]
+    },
+    {
+      "language": "hi",
+      "text": "नमस्ते दुनिया ONNX Magpie से।",
+      "sample_text": "नमस्ते दुनिया ONNX Magpie से।",
+      "chunk_index": 0,
+      "tokenizer": "hindi_chartokenizer",
+      "ids": [
+        1017,
+        1124,
+        1129,
+        1136,
+        1157,
+        1120,
+        1148,
+        1017,
+        1122,
+        1144,
+        1124,
+        1142,
+        1130,
+        1141,
+        1017,
+        1164,
+        1170,
+        1179,
+        1172,
+        1168,
+        1017,
+        1136,
+        1148,
+        1163,
+        1017,
+        2361
+      ]
+    },
+    {
+      "language": "ja",
+      "text": "こんにちはONNX Magpieの世界。",
+      "sample_text": "こんにちはONNX Magpieの世界。",
+      "chunk_index": 0,
+      "tokenizer": "japanese_phoneme",
+      "ids": [
+        458,
+        459,
+        479,
+        460,
+        541,
+        460,
+        503,
+        460,
+        493,
+        460,
+        539,
+        560,
+        559,
+        559,
+        569,
+        458,
+        558,
+        546,
+        552,
+        561,
+        554,
+        550,
+        459,
+        506,
+        460,
+        487,
+        459,
+        471,
+        459,
+        464,
+        597,
+        458,
+        2361
+      ]
+    }
+  ]
+}

tokenizers/text_ce_tokenizer.json

@@ -0,0 +1,11 @@
+{
+  "kind": "byt5",
+  "class_name": "ByT5Tokenizer",
+  "offset": 1976,
+  "pad_id": 1976,
+  "num_tokens": 384,
+  "aggregate_offset": 1976,
+  "byte_offset": 3,
+  "eos_token_id": 1,
+  "global_eos_token_id": 2361
+}

tokenizers/vietnamese_phoneme.json

@@ -0,0 +1,11 @@
+{
+  "kind": "byt5",
+  "class_name": "ByT5Tokenizer",
+  "offset": 1592,
+  "pad_id": 1592,
+  "num_tokens": 384,
+  "aggregate_offset": 1592,
+  "byte_offset": 3,
+  "eos_token_id": 1,
+  "global_eos_token_id": 2361
+}