Spaces:
Sleeping
Sleeping
File size: 4,652 Bytes
2aabb82 5937b4b 2aabb82 f686d1b 2aabb82 5937b4b 2aabb82 5937b4b 2aabb82 5937b4b 2aabb82 5937b4b 2aabb82 5937b4b 2aabb82 5937b4b 2aabb82 5937b4b 2aabb82 5937b4b 2aabb82 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 | import pandas as pd
from transformers import pipeline
from wordcloud import WordCloud
import matplotlib.pyplot as plt
from collections import Counter
import nltk
from nltk.corpus import stopwords
import random
from datetime import datetime, timedelta
from langdetect import detect
from deep_translator import GoogleTranslator
nltk.download('stopwords', quiet=True)
class MCACommentAnalyzerLight:
def __init__(self):
self.sentiment_model = pipeline(
"sentiment-analysis",
model="cardiffnlp/twitter-roberta-base-sentiment",
device=-1
)
self.summarizer = pipeline(
"summarization",
model="sshleifer/distilbart-cnn-6-6",
device=-1
)
self.stop_words = set(stopwords.words('english'))
def translate_to_english(self, text):
try:
lang = detect(text)
if lang != "en":
return GoogleTranslator(source='auto', target='en').translate(text)
return text
except:
return text
def map_sentiment(self, pred, text):
text_lower = text.lower()
violation_keywords = ["violation", "violates", "illegal", "non-compliant", "breach", "unlawful", "risk", "penalty"]
suggestion_keywords = ["should", "recommend", "suggest", "advise", "better if", "could", "need to"]
positive_keywords = ["clear", "helpful", "good", "appreciate", "support"]
negative_keywords = ["confusing", "unclear", "bad", "problem", "needs clarification"]
if any(w in text_lower for w in violation_keywords):
return "Violation"
if any(w in text_lower for w in suggestion_keywords):
return "Suggestion"
if any(w in text_lower for w in positive_keywords):
return "Positive"
if any(w in text_lower for w in negative_keywords):
return "Negative"
label = pred['label'].upper()
if label in ["POSITIVE", "LABEL_2"]:
return "Positive"
elif label in ["NEGATIVE", "LABEL_0"]:
return "Negative"
else:
return "Neutral"
def process_comment(self, comment):
translated_comment = self.translate_to_english(comment)
pred = self.sentiment_model(translated_comment)[0]
sentiment = self.map_sentiment(pred, translated_comment)
# Summary
if len(translated_comment.split()) < 10:
summary_text = " ".join(translated_comment.split()[:10])
else:
try:
summary_text = self.summarizer(
translated_comment,
max_length=20,
min_length=5,
do_sample=False
)[0]['summary_text']
except:
summary_text = translated_comment
words = [w for w in translated_comment.lower().split() if w.isalpha() and w not in self.stop_words]
keywords = list(Counter(words).keys())
top_keywords = ", ".join(keywords[:3])
return sentiment, summary_text, keywords, top_keywords
def process_comments(self, comments_list):
sentiments, summaries, all_keywords, top_keywords_list, timestamps = [], [], [], [], []
start_date = datetime.now() - timedelta(days=30)
for comment in comments_list:
sentiment, summary, keywords, top_kw = self.process_comment(comment)
sentiments.append(sentiment)
summaries.append(summary)
all_keywords.extend(keywords)
top_keywords_list.append(top_kw)
timestamps.append(start_date + timedelta(days=random.randint(0, 30)))
df = pd.DataFrame({
"Timestamp": timestamps,
"Comment": comments_list,
"Summary": summaries,
"Sentiment": sentiments,
"Top Keywords": top_keywords_list
})
df.sort_values(by='Timestamp', inplace=True, ascending=True)
keyword_freq = pd.DataFrame(
Counter(all_keywords).items(),
columns=['Keyword', 'Frequency']
).sort_values(by='Frequency', ascending=False)
return df, keyword_freq
def generate_wordcloud(self, keyword_freq, filename=None):
wc_dict = dict(zip(keyword_freq['Keyword'], keyword_freq['Frequency']))
wc = WordCloud(width=600, height=300, background_color="white").generate_from_frequencies(wc_dict)
plt.figure(figsize=(8,4))
plt.imshow(wc, interpolation="bilinear")
plt.axis("off")
if filename:
plt.savefig(filename, bbox_inches='tight')
return plt
|