HebTTS (code, models, paper)

.gitattributes

@@ -33,3 +33,12 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+A[[:space:]]Language[[:space:]]Modeling[[:space:]]Approach[[:space:]]to[[:space:]]Diacritic-Free[[:space:]]Hebrew[[:space:]]TTS.pdf filter=lfs diff=lfs merge=lfs -text
+Beyond[[:space:]]RoboShaul.[[:space:]]Zero-shot[[:space:]]speech[[:space:]]synthesis[[:space:]]in[[:space:]]Hebrew[[:space:]](Medium.com).pdf filter=lfs diff=lfs merge=lfs -text
+Beyond[[:space:]]RoboShaul.[[:space:]]Zero-shot[[:space:]]speech[[:space:]]synthesis[[:space:]]in[[:space:]]Hebrew.pdf filter=lfs diff=lfs merge=lfs -text
+dataset/HebDB.[[:space:]]A[[:space:]]Weakly[[:space:]]Supervised[[:space:]]Dataset[[:space:]]for[[:space:]]Hebrew[[:space:]]Speech[[:space:]]Processing.pdf filter=lfs diff=lfs merge=lfs -text
+dataset/SASPEECH.[[:space:]]A[[:space:]]Hebrew[[:space:]]Single[[:space:]]Speaker[[:space:]]Dataset[[:space:]]for[[:space:]]Text[[:space:]]To[[:space:]]Speech[[:space:]]and[[:space:]]Voice[[:space:]]Conversion.pdf filter=lfs diff=lfs merge=lfs -text
+Enhancing[[:space:]]TTS[[:space:]]Stability[[:space:]]in[[:space:]]Hebrew[[:space:]]using[[:space:]]Discrete[[:space:]]Semantic[[:space:]]Units.pdf filter=lfs diff=lfs merge=lfs -text
+models/HebTTS_implementation/additional_info.pdf filter=lfs diff=lfs merge=lfs -text
+models/HebTTS2.0/unsloth.Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text
+models/HebTTS3.0/unsloth.Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text

A Language Modeling Approach to Diacritic-Free Hebrew TTS.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e527faf34344d36fa00b4a9e27f25a3efb521b02f00d3323d9e253a2f3479fa6
+size 413504

Beyond RoboShaul. Zero-shot speech synthesis in Hebrew (Medium.com).pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fdc465c95edffc8af74e97bac7052044ad2d5886510aeb3f218f0100484f54c4
+size 200287

Beyond RoboShaul. Zero-shot speech synthesis in Hebrew.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3005edd88a8b9197ed78d3ac8eaee2702485ada7444e3548efb39f3a6157bff
+size 509426

Enhancing TTS Stability in Hebrew using Discrete Semantic Units.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7577639df7592f2965011dc70211a081f633ee5b1b261ceab70ea2c8587c617
+size 323945

code/HebTTS [D4niel0s] +513 -5.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3d9e7089d8b88aa89d6ef3f072a5474585eaa9f2020a306165e04cfe4cce3bc
+size 19705007

code/HebTTS [adarcook] + 2 INFERENCE.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a400cb13150d3c7347edfb866c44a5ab27291e603a6623e1191bb3cda783fe1
+size 4797191

code/HebTTS [kinneretm] +2 -5.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d4c76ec87b7fcc352f6d70f13106a51bf258d83797f0fcff2e8c7173a41886fc
+size 4348150

code/HebTTS [moojnick] +3 -5.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ca8625d70e1873d8bc88803eaec2d9675d7db2a9727fcfa2c34eac85ab6eb77
+size 12039697

code/HebTTS.ipynb

@@ -0,0 +1,173 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "source": [
+        "# HebTTS Quickstart\n",
+        "We created a jupyter notebook in order to easily generate your desired samples!\n",
+        "\n",
+        "\n",
+        "  <a href='https://arxiv.org/abs/2407.12206'><img src='https://img.shields.io/badge/ArXiv-PDF-red'></a> &nbsp;\n",
+        "   <a href='https://pages.cs.huji.ac.il/adiyoss-lab/HebTTS/'><img src='https://img.shields.io/badge/Project-Page-Green'></a> &nbsp;\n",
+        "  [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1f3-6Dqbna9_hI5C9V4qTIG05dixW-r72?usp=sharing) &nbsp;\n",
+        "  [![Open In Colab](https://badges.aleen42.com/src/github.svg)](https://github.com/slp-rl/HebTTS) &nbsp;\n",
+        "\n",
+        "\n",
+        "\n",
+        "\n",
+        "---\n",
+        "\n",
+        "\n"
+      ],
+      "metadata": {
+        "id": "3wmzNy2fDSsr"
+      }
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "56kkbeJahIe1"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Install ‍💻\n",
+        "\n",
+        "%%capture\n",
+        "! git clone https://github.com/slp-rl/HebTTS.git\n",
+        "\n",
+        "! pip install torch torchaudio\n",
+        "! pip install torchmetrics\n",
+        "! pip install omegaconf\n",
+        "! pip install git+https://github.com/lhotse-speech/lhotse\n",
+        "! pip install librosa\n",
+        "! pip install encodec\n",
+        "! pip install phonemizer\n",
+        "! pip install audiocraft  # optional\n",
+        "! gdown https://drive.google.com/uc?id=11NoOJzMLRX9q1C_Q4sX0w2b9miiDjGrv\n",
+        "\n",
+        "from pathlib import Path\n",
+        "import glob\n",
+        "import os\n",
+        "from IPython.display import Audio, display\n",
+        "from pathlib import Path\n",
+        "\n",
+        "\n",
+        "def display_audio(prompt_file):\n",
+        "  l=100\n",
+        "  speaker = os.path.basename(os.path.dirname((os.path.dirname(prompt_file))))\n",
+        "  audio_files = list(Path(os.path.join(os.path.dirname(prompt_file))).rglob(\"*.wav\"))\n",
+        "  print(f\"Prompt: {open(prompt_file).read()}\")\n",
+        "  print(f\"Speaker: {speaker}\")\n",
+        "  print(\"-\"*100)\n",
+        "\n",
+        "  for audio_file in audio_files:\n",
+        "    display(Audio(audio_file, autoplay=False))\n",
+        "    print(\"-\"*100)\n",
+        "    print()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "uIBF-IFwxLjL",
+        "cellView": "form"
+      },
+      "outputs": [],
+      "source": [
+        "# @title Enter Hebrew text for generation\n",
+        "# @markdown You can concatenate prompts with '|' to generate few samples at once\n",
+        "hebrew_text_to_generate = \"היי מה קורה | ובשביל להבין למה מחיר הדלק כל כך עלה צריך לחזור שנתיים אחרונית\" # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "RFrYpBJsxQW3",
+        "cellView": "form"
+      },
+      "outputs": [],
+      "source": [
+        "\n",
+        "# @title Choose speaker\n",
+        "speaker = \"shaul\" # @param [\"\\\"osim\\\"\", \"\\\"geek\\\"\", \"\\\"shaul\\\"\"] {type:\"raw\", allow-input: true}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "**Note:** inference may be slower based on the allocated resources by google colab. It is recomended to choose a gpu in `Edit` -> `Notebook settings` -> `Hardware accelerator`."
+      ],
+      "metadata": {
+        "id": "NMvd3VytEJEZ"
+      }
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "U8Hszx6axgns"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Generate!\n",
+        "from pathlib import Path\n",
+        "\n",
+        "speaker_dir = Path(f\"./out/{speaker}\")\n",
+        "speaker_dir.mkdir(exist_ok=True, parents=True)\n",
+        "i=0\n",
+        "if any(speaker_dir.iterdir()):\n",
+        "    max(map(lambda x:int(x.name),speaker_dir.glob(\"*\")))+1\n",
+        "output_dir = speaker_dir / f\"{i}\"\n",
+        "output_dir.mkdir(exist_ok=True, parents=True)\n",
+        "prompt_file = output_dir / \"prompt.txt\"\n",
+        "with open(prompt_file, \"w\") as f:\n",
+        "    f.write(hebrew_text_to_generate)\n",
+        "\n",
+        "! python HebTTS/infer.py \\\n",
+        "    --checkpoint checkpoint.pt \\\n",
+        "    --output-dir $output_dir                                             \\\n",
+        "    --text \"$hebrew_text_to_generate\" \\\n",
+        "    --speaker \"$speaker\" \\\n",
+        "    --speaker-yaml HebTTS/speakers/speakers.yaml \\\n",
+        "    --tokens-file HebTTS/tokenizer/unique_words_tokens_all.k2symbols \\\n",
+        "    --vocab-file HebTTS/tokenizer/vocab.txt \\\n",
+        "    --mbd True\n",
+        "\n",
+        "display_audio(prompt_file)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "#@title Listen\n",
+        "for prompt_file in glob.glob(\"./out/*/*/prompt.txt\"):\n",
+        "  display_audio(prompt_file)"
+      ],
+      "metadata": {
+        "cellView": "form",
+        "id": "CXiMWmNxJzNZ"
+      },
+      "execution_count": null,
+      "outputs": []
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "provenance": [],
+      "gpuType": "T4"
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "name": "python"
+    },
+    "accelerator": "GPU"
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

code/HebTTS.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0161f47fe07f2ce1ccb677f64bb7af575ba29d3ce5250cdc00e5b6dbd93facb6
+size 3877260

dataset/HebDB. A Weakly Supervised Dataset for Hebrew Speech Processing.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97febbefdb88fd63147d9eec9bc60138480a78a44eb0d40061bdce2e26a6cfa8
+size 278562

dataset/SASPEECH. A Hebrew Single Speaker Dataset for Text To Speech and Voice Conversion.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b16fac282d398cbf0a263f3eb81530b4f31745548d38493edc3b4896b7035c2e
+size 694511

models/HebTTS2.0/.gitattributes

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+unsloth.Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text

models/HebTTS2.0/README.md

@@ -0,0 +1,22 @@
+---
+base_model: unsloth/orpheus-3b-0.1-ft-unsloth-bnb-4bit
+tags:
+- text-generation-inference
+- transformers
+- unsloth
+- llama
+- gguf
+license: apache-2.0
+language:
+- en
+---
+
+# Uploaded  model
+
+- **Developed by:** Trojanssafsdg
+- **License:** apache-2.0
+- **Finetuned from model :** unsloth/orpheus-3b-0.1-ft-unsloth-bnb-4bit
+
+This llama model was trained 2x faster with [Unsloth](https://github.com/unslothai/unsloth) and Huggingface's TRL library.
+
+[<img src="https://raw.githubusercontent.com/unslothai/unsloth/main/images/unsloth%20made%20with%20love.png" width="200"/>](https://github.com/unslothai/unsloth)

models/HebTTS2.0/config.json

@@ -0,0 +1,3 @@
+{
+    "model_type": "llama"
+}

models/HebTTS2.0/source.txt

@@ -0,0 +1 @@
+https://huggingface.co/Trojanssafsdg/hebTTS2.0

models/HebTTS2.0/unsloth.Q4_K_M.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efc23a134a66cf9e67520abec46254afcfa6055774a88c269f57ef0226907338
+size 2092567584

models/HebTTS3.0/.gitattributes

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+unsloth.Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text

models/HebTTS3.0/README.md

@@ -0,0 +1,22 @@
+---
+base_model: unsloth/orpheus-3b-0.1-ft-unsloth-bnb-4bit
+tags:
+- text-generation-inference
+- transformers
+- unsloth
+- llama
+- gguf
+license: apache-2.0
+language:
+- en
+---
+
+# Uploaded  model
+
+- **Developed by:** Trojanssafsdg
+- **License:** apache-2.0
+- **Finetuned from model :** unsloth/orpheus-3b-0.1-ft-unsloth-bnb-4bit
+
+This llama model was trained 2x faster with [Unsloth](https://github.com/unslothai/unsloth) and Huggingface's TRL library.
+
+[<img src="https://raw.githubusercontent.com/unslothai/unsloth/main/images/unsloth%20made%20with%20love.png" width="200"/>](https://github.com/unslothai/unsloth)

models/HebTTS3.0/config.json

@@ -0,0 +1,3 @@
+{
+    "model_type": "llama"
+}

models/HebTTS3.0/source.txt

@@ -0,0 +1 @@
+https://huggingface.co/Trojanssafsdg/hebTTS3.0

models/HebTTS3.0/unsloth.Q4_K_M.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08a34931b65f2eee1c7bb9cf5546d94bd8dc17133be42d70de9f0a8c33b2bf11
+size 2092567584

models/HebTTS_implementation/.gitattributes

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+additional_info.pdf filter=lfs diff=lfs merge=lfs -text

models/HebTTS_implementation/README.md

@@ -0,0 +1,19 @@
+---
+datasets:
+- SLPRL-HUJI/HebDB
+language:
+- he
+metrics:
+- wer
+- cer
+pipeline_tag: text-to-speech
+---
+
+
+# Details
+
+This model is an implementation of the vall-e architecture, with the AlephBert text tokenizer.
+This model was trained as a final project in the "DSP & audio processing using Deep Learning" course at Tel-Aviv University.
+
+Implementation details and references can be found in the included 'paper' PDF. \
+Code can be found on [our GitHub Repo](https://github.com/D4niel0s/HebTTS)

models/HebTTS_implementation/additional_info.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89b412730083c73688f540f0358bc1808ef038baf33f27df7e729e0ec7e8f9ec
+size 509426

models/HebTTS_implementation/checkpoint-150000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1fecc21e683dd103a7bfed57f986c14c84213f93aec815af73947e3dfec63e81
+size 2551217450

models/HebTTS_implementation/source.txt

@@ -0,0 +1 @@
+https://huggingface.co/D4niel0s/HebTTS_implementation