updated pattern

README.md

@@ -1,4 +1,15 @@
-# BAT NLP Campaign Audio Data
+# Arabic NLP 
+HuggingFace: https://huggingface.co/rakib72642/Arabic_NLP
+
+sudo apt install iproute2 && sudo apt install wget && sudo apt install unzip && sudo apt install nvtop && sudo apt-get install git-lfs && sudo apt-get update && sudo apt-get install libgl1 && curl -s https://ngrok-agent.s3.amazonaws.com/ngrok.asc | sudo tee /etc/apt/trusted.gpg.d/ngrok.asc >/dev/null && echo "deb https://ngrok-agent.s3.amazonaws.com buster main" | sudo tee /etc/apt/sources.list.d/ngrok.list && sudo apt update && sudo apt install ngrok && ngrok config add-authtoken 2Qm8hS1zPhVXiLjEdlI4738tLzF_2QJwGJMK5oTbQD33QSVXS && sudo apt update && sudo apt upgrade && ngrok http --domain=hawkeyes.ngrok.app 8000
+
+git clone https://huggingface.co/rakib72642/Arabic_NLP && cd Arabic_NLP && sudo apt update && sudo apt upgrade && python updated_api.py
+
+cd Arabic_NLP && python updated_api.py
+
+hypercorn updated_api:app --bind 127.0.0.1:8020 --workers 4
+
+
 config the ngrok auth: ngrok config add-authtoken 2Qm8hS1zPhVXiLjEdlI4738tLzF_2QJwGJMK5oTbQD33QSVXS
 
 ngrok http --domain=batnlp.ngrok.app 1111

get_text.ipynb

@@ -2,26 +2,9 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 1,
    "metadata": {},
-   "outputs": [
-    {
-     "ename": "ModuleNotFoundError",
-     "evalue": "No module named 'certifi'",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
-      "\u001b[1;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
-      "\u001b[1;32md:\\Projects\\BAT\\BAT_NLP_Campaign\\get_text.ipynb Cell 1\u001b[0m line \u001b[0;36m2\n\u001b[0;32m      <a href='vscode-notebook-cell:/d%3A/Projects/BAT/BAT_NLP_Campaign/get_text.ipynb#W0sZmlsZQ%3D%3D?line=0'>1</a>\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39mre\u001b[39;00m\n\u001b[1;32m----> <a href='vscode-notebook-cell:/d%3A/Projects/BAT/BAT_NLP_Campaign/get_text.ipynb#W0sZmlsZQ%3D%3D?line=1'>2</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mupdated_api\u001b[39;00m \u001b[39mimport\u001b[39;00m \u001b[39m*\u001b[39m\n\u001b[0;32m      <a href='vscode-notebook-cell:/d%3A/Projects/BAT/BAT_NLP_Campaign/get_text.ipynb#W0sZmlsZQ%3D%3D?line=2'>3</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mtyping_extensions\u001b[39;00m \u001b[39mimport\u001b[39;00m Annotated\n\u001b[0;32m      <a href='vscode-notebook-cell:/d%3A/Projects/BAT/BAT_NLP_Campaign/get_text.ipynb#W0sZmlsZQ%3D%3D?line=3'>4</a>\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39mnltk\u001b[39;00m\n",
-      "File \u001b[1;32md:\\Projects\\BAT\\BAT_NLP_Campaign\\updated_api.py:9\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39muvicorn\u001b[39;00m\n\u001b[0;32m      8\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39mnltk\u001b[39;00m\n\u001b[1;32m----> 9\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39mhttpx\u001b[39;00m\n\u001b[0;32m     10\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mfastapi\u001b[39;00m \u001b[39mimport\u001b[39;00m FastAPI\n\u001b[0;32m     11\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mpydantic\u001b[39;00m \u001b[39mimport\u001b[39;00m BaseModel\n",
-      "File \u001b[1;32mc:\\Users\\naymm\\miniconda3\\envs\\nlpBat\\lib\\site-packages\\httpx\\__init__.py:2\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39m.\u001b[39;00m\u001b[39m__version__\u001b[39;00m \u001b[39mimport\u001b[39;00m __description__, __title__, __version__\n\u001b[1;32m----> 2\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39m.\u001b[39;00m\u001b[39m_api\u001b[39;00m \u001b[39mimport\u001b[39;00m delete, get, head, options, patch, post, put, request, stream\n\u001b[0;32m      3\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39m.\u001b[39;00m\u001b[39m_auth\u001b[39;00m \u001b[39mimport\u001b[39;00m Auth, BasicAuth, DigestAuth, NetRCAuth\n\u001b[0;32m      4\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39m.\u001b[39;00m\u001b[39m_client\u001b[39;00m \u001b[39mimport\u001b[39;00m USE_CLIENT_DEFAULT, AsyncClient, Client\n",
-      "File \u001b[1;32mc:\\Users\\naymm\\miniconda3\\envs\\nlpBat\\lib\\site-packages\\httpx\\_api.py:4\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39mtyping\u001b[39;00m\n\u001b[0;32m      2\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mcontextlib\u001b[39;00m \u001b[39mimport\u001b[39;00m contextmanager\n\u001b[1;32m----> 4\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39m.\u001b[39;00m\u001b[39m_client\u001b[39;00m \u001b[39mimport\u001b[39;00m Client\n\u001b[0;32m      5\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39m.\u001b[39;00m\u001b[39m_config\u001b[39;00m \u001b[39mimport\u001b[39;00m DEFAULT_TIMEOUT_CONFIG\n\u001b[0;32m      6\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39m.\u001b[39;00m\u001b[39m_models\u001b[39;00m \u001b[39mimport\u001b[39;00m Response\n",
-      "File \u001b[1;32mc:\\Users\\naymm\\miniconda3\\envs\\nlpBat\\lib\\site-packages\\httpx\\_client.py:11\u001b[0m\n\u001b[0;32m      9\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39m.\u001b[39;00m\u001b[39m__version__\u001b[39;00m \u001b[39mimport\u001b[39;00m __version__\n\u001b[0;32m     10\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39m.\u001b[39;00m\u001b[39m_auth\u001b[39;00m \u001b[39mimport\u001b[39;00m Auth, BasicAuth, FunctionAuth\n\u001b[1;32m---> 11\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39m.\u001b[39;00m\u001b[39m_config\u001b[39;00m \u001b[39mimport\u001b[39;00m (\n\u001b[0;32m     12\u001b[0m     DEFAULT_LIMITS,\n\u001b[0;32m     13\u001b[0m     DEFAULT_MAX_REDIRECTS,\n\u001b[0;32m     14\u001b[0m     DEFAULT_TIMEOUT_CONFIG,\n\u001b[0;32m     15\u001b[0m     Limits,\n\u001b[0;32m     16\u001b[0m     Proxy,\n\u001b[0;32m     17\u001b[0m     Timeout,\n\u001b[0;32m     18\u001b[0m )\n\u001b[0;32m     19\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39m.\u001b[39;00m\u001b[39m_decoders\u001b[39;00m \u001b[39mimport\u001b[39;00m SUPPORTED_DECODERS\n\u001b[0;32m     20\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39m.\u001b[39;00m\u001b[39m_exceptions\u001b[39;00m \u001b[39mimport\u001b[39;00m (\n\u001b[0;32m     21\u001b[0m     InvalidURL,\n\u001b[0;32m     22\u001b[0m     RemoteProtocolError,\n\u001b[0;32m     23\u001b[0m     TooManyRedirects,\n\u001b[0;32m     24\u001b[0m     request_context,\n\u001b[0;32m     25\u001b[0m )\n",
-      "File \u001b[1;32mc:\\Users\\naymm\\miniconda3\\envs\\nlpBat\\lib\\site-packages\\httpx\\_config.py:7\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39mtyping\u001b[39;00m\n\u001b[0;32m      5\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mpathlib\u001b[39;00m \u001b[39mimport\u001b[39;00m Path\n\u001b[1;32m----> 7\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39mcertifi\u001b[39;00m\n\u001b[0;32m      9\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39m.\u001b[39;00m\u001b[39m_compat\u001b[39;00m \u001b[39mimport\u001b[39;00m set_minimum_tls_version_1_2\n\u001b[0;32m     10\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39m.\u001b[39;00m\u001b[39m_models\u001b[39;00m \u001b[39mimport\u001b[39;00m Headers\n",
-      "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'certifi'"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "import re\n",
     "from updated_api import *\n",
@@ -45,7 +28,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -58,7 +41,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -80,20 +63,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "<coroutine object detect_audio at 0x00000255D1384900>"
-      ]
-     },
-     "execution_count": 17,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "filename = input(\"Give Audio Name: \")\n",
     "audio_url = upload(filename)\n",
@@ -103,7 +75,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -116,7 +88,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -129,7 +101,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -149,7 +121,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -171,29 +143,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "{'Unique Capsule': ['unique capsul'], 'Refreshing Taste and Smell': ['refreshing taste smell'], 'Benson & Hadges Breeze': [('banson', 'b', 'a', 'hages niyashe ekti unique capsule offer panson hages', 'br'), ('panson', 'p', 'a', 'hages', 'br')]}\n"
-     ]
-    },
-    {
-     "data": {
-      "text/plain": [
-       "{'Unique Capsule': 1,\n",
-       " 'Refreshing Taste and Smell': 1,\n",
-       " 'Benson & Hadges Breeze': 2}"
-      ]
-     },
-     "execution_count": 10,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "text = \"Clean text :  apnea janet kushihaban banson hages niyashe ekti unique capsule offer panson hages bridge panson hages breeze air capsule atom agnoton tharna refreshing taste smell darn offer tea trial cora jonu apnea ekti trial kit nitaparin thunobat\"\n",
     "\n",
@@ -202,7 +154,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -215,7 +167,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -237,17 +189,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "<assemblyai.transcriber.Transcript object at 0x0000029377EFD480>\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "import assemblyai as aai\n",
     "from updated_api import *\n",
@@ -266,12 +210,92 @@
     "print(transcript)\n"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 57,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processing Audio\n",
+      "town saw revival new business opening\n",
+      "{'Unique Capsule': [], 'Refreshing Taste and Smell': [], 'Benson & Hadges Breeze': []}\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "{'Unique Capsule': 0,\n",
+       " 'Refreshing Taste and Smell': 0,\n",
+       " 'Benson & Hadges Breeze': 0}"
+      ]
+     },
+     "execution_count": 57,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import re\n",
+    "from nlp_api import *\n",
+    "from typing_extensions import Annotated\n",
+    "import string\n",
+    "\n",
+    "patterns = {\n",
+    "    'Unique Capsule': r\"\\b(((u(?:nit|niq).*?)\\s+(?:capsul))|(?:.*?uni.*?capsul))\",\n",
+    "    'Refreshing Taste and Smell': r\"\\b((((ref|rif|rip|rep|ep|pre).*?)\\s+t(?:a|e|i|y)s(.*?)\\s+(sm|(?:.*?(sm|m)))(?:el|il|al|ol|.*?))|((?:in.*?)\\s+t(?:a|e|i|y)s.*?\\s+(.*?)(sm|m)(?:el|il|al|ol|ail|eal)))\",\n",
+    "    'Benson & Hadges Breeze':r\"\\b((b|p|v|f)(?:(an|en|a|e)(?:s|ch|t)(?:on|an|en).*?)\\s+h(?:.*?)\\s+(b|p|v|f)(?:re|ee|e|ri))\",\n",
+    "}\n",
+    "\n",
+    "\n",
+    "    # Find and count matches for each pattern\n",
+    "def nlp_bat(text):\n",
+    "    results = {}\n",
+    "    all_match = {}\n",
+    "    for name, pattern in patterns.items():\n",
+    "        matches = re.findall(pattern, text, re.IGNORECASE)\n",
+    "        m = {name:matches}\n",
+    "        all_match.update(m)\n",
+    "        count = len(matches)\n",
+    "        results[name] = count\n",
+    "    \n",
+    "    \n",
+    "    print(all_match)    \n",
+    "\n",
+    "    return results\n",
+    "\n",
+    "async def lemmatize_and_clean(text):\n",
+    "    words = nltk.word_tokenize(text.lower())\n",
+    "    words = [word for word in words if word.isalpha() and word not in set(stopwords.words('english'))]\n",
+    "    lemmatizer = WordNetLemmatizer()\n",
+    "    words = [await asyncio.to_thread(lemmatizer.lemmatize, word) for word in words]\n",
+    "    return ' '.join(words)\n",
+    "\n",
+    "\n",
+    "# # input\n",
+    "filename = input(\"Give Audio Name: \")\n",
+    "audio_url = upload(filename)\n",
+    "\n",
+    "\n",
+    "# # transcribe\n",
+    "detect_audio(audio_url, 'file_title')"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
    "outputs": [],
-   "source": []
+   "source": [
+    "patterns = {\n",
+    "    \"Clear\":r\"\\b((c|k)(:?l..r))\",\n",
+    "    \"Confidence\":r\"\\b(((f|t|th)(u|i))(?:(|c|q|k|ck)(?:(a|e|o))))|(((f|t|th)(u|i))(?:(|c|q|k|ck)(?:a|.a)))\",\n",
+    "    \"Revival\":r\"((a)(?:(sh|yush|rch)))\",\n",
+    "    \"Anti-Dandruff\":r\"((al)(?:.*?(k|q)(?:a|i|o|u)(?:(s|sh))))|((k|q)(?:a|i|o|u)(?:(s|sh)((?:a|o))))|((k|q)(?:a|i|o|u)(?:(s|sh)(r(?:a|o|u))))\",\n",
+    "}"
+   ]
   }
  ],
  "metadata": {
@@ -290,7 +314,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.13"
+   "version": "3.12.2"
   }
  },
  "nbformat": 4,

textData.md

@@ -0,0 +1,49 @@
+1. clear fikoh ash al qasra
+2. clear fika ash al qaeda
+3. clear fikah ash al kishra
+4. ash alokosra vika clear
+5. clear al kashara
+6. clear fuqah ayush vidal kashakti
+7. clear fuqah ayush vidal kashakti
+8. ash alokosra vika clear
+9. clear fik india loch
+10. al koshro clear fikah asha
+11. clear vika asha al koshuru
+12. clear fik ash vid al khushra
+13. asha al ghoshuru clear fikah
+14. asha al kosharo clear
+15. lehrer fik ash dudul kusha
+16. clear fika taj vit al khashoggi
+17. fikah clear asha al kosru
+18. clear fikar ash vid al khusho
+19. clear fika inash bidal kushu may
+20. clear fika ash mid al kushu
+21. clear fico bush
+22. clear ar vital kosho
+23. clear fico vidal kusho
+24. clear fikah ash al kishra
+25. clear thickah ash al kishra
+26. fika vidal kishra ash fika
+27. clear fik ayesh vidal kashir
+28. player fika faith al kish ash
+30. fid alakishra ash fika clear ash fika alakashra
+31. clear fikach ash vidal kosher
+32. clear fica arch alcohol
+33. clear fikach ash vidal kosher
+34. clear fika ash al kosher
+35. clear thicker arch alcohol
+# ###################################################
+1. yunkenulil belsamil mudadil il kashrati yuhadiya farwata rasi waemna al jafa
+2. yusaidu shampul mudadul il kushrati ala muharrabatil kushrati wal hakati
+4. intarshal edabul kilesi kiyo fishabiyetihi
+6. alistair mul mustamirula jatil mudodati lil kushrati yunkinu yuaziza sahata faruatiratsi
+7. hatidanal fashali kafur satin lita lumiyua zizuthiqata
+8. artini zojajatan mina sham ko shafi
+10. alistair mul muntadimul muntajatil mudadati lil kashrati yumkinu yukala minta yuji faruatirat
+11. tahdida ehdefin kabilatin litikata
+12. tahdida ehdefin kabilatin litikata
+13. ihatota nafsibi afraid
+14. shahid al mokaotari khiyu inti ashan ladazuari badata tashdidi
+15. shahid al fari kuriyad yuntyashan mahmuda ribin jadedan
+16. town saw revival new business opening
+ 

updated_api.py

@@ -62,12 +62,12 @@ async def lemmatize_and_clean(text):
 
 
 patterns = {
-    'Unique Capsule': r"\b(((u(?:nit|niq).*?)\s+(?:capsul))|(?:.*?uni.*?capsul))",
-    'Refreshing Taste and Smell': r"\b((((ref|rif|rip|rep|ep|pre).*?)\s+t(?:a|e|i|y)s(.*?)\s+(sm|(?:.*?(sm|m)))(?:el|il|al|ol|.*?))|((?:in.*?)\s+t(?:a|e|i|y)s.*?\s+(.*?)(sm|m)(?:el|il|al|ol|ail|eal)))",
-    'Benson & Hadges Breeze':r"\b((b|p|v|f)(?:(an|en|a|e)(?:s|ch|t)(?:on|an|en).*?)\s+h(?:.*?)\s+(b|p|v|f)(?:re|ee|e|ri))",
+    "Clear":r"\b(((c|k)(?:(l..r|lir|lar|il))))",
+    "Confidence":r"\b((((f|t|th)(u|i))(?:(|c|q|k|ck)(?:(a|e|o))))|(((f|t|th)(u|i))(?:(|c|q|k|ck)(?:a|.a))))",
+    "Revival":r"\b(((a)(?:(sh|yush|rch))))",
+    "Anti-Dandruff":r"\b(((al)(?:.*?(k|q|kh)(?:a|i|o|u)(?:(s|sh))))|((k|q|kh)(?:a|i|o|u)(?:(s|sh)((?:a|o))))|((k|q|kh)(?:a|i|o|u)(?:(s|sh)(r(?:a|o|u)))))",
 }
 
-
 async def nlp_bat(text): 
     results = {}
     all_match = {}