Spaces:
Sleeping
Sleeping
File size: 4,592 Bytes
2aabb82 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 | import pandas as pd
from transformers import pipeline
from wordcloud import WordCloud
import matplotlib.pyplot as plt
from collections import Counter
import nltk
from nltk.corpus import stopwords
import random
from datetime import datetime, timedelta
from langdetect import detect
from deep_translator import GoogleTranslator
nltk.download('stopwords')
class MCACommentAnalyzer:
def __init__(self):
self.sentiment_model = pipeline(
"sentiment-analysis",
model="distilbert-base-uncased-finetuned-sst-2-english"
)
self.summarizer = pipeline(
"summarization",
model="sshleifer/distilbart-cnn-12-6"
)
self.stop_words = set(stopwords.words('english'))
def translate_to_english(self, text):
try:
lang = detect(text)
if lang != "en":
return GoogleTranslator(source='auto', target='en').translate(text)
return text
except:
return text
def map_sentiment(self, pred, text):
text_lower = text.lower()
violation_keywords = ["violation", "violates", "illegal", "non-compliant", "breach", "unlawful", "risk", "penalty"]
if any(w in text_lower for w in violation_keywords):
return "Violation"
suggestion_keywords = ["should", "recommend", "suggest", "advise", "better if", "could", "need to"]
if any(w in text_lower for w in suggestion_keywords):
return "Suggestion"
positive_keywords = ["clear", "helpful", "good", "appreciate", "support"]
if any(w in text_lower for w in positive_keywords):
return "Positive"
negative_keywords = ["confusing", "unclear", "bad", "problem", "needs clarification"]
if any(w in text_lower for w in negative_keywords):
return "Negative"
label = pred['label'].upper()
if label == "POSITIVE":
return "Positive"
elif label == "NEGATIVE":
return "Negative"
else:
return "Neutral"
def process_comment(self, comment):
translated_comment = self.translate_to_english(comment)
pred = self.sentiment_model(translated_comment)[0]
sentiment = self.map_sentiment(pred, translated_comment)
# Summary
if len(translated_comment.split()) < 10:
summary_text = " ".join(translated_comment.split()[:10])
else:
try:
summary_text = self.summarizer(
translated_comment,
max_length=30,
min_length=5,
do_sample=False
)[0]['summary_text']
except:
summary_text = translated_comment
# Keywords
words = [w for w in translated_comment.lower().split() if w.isalpha() and w not in self.stop_words]
keywords = list(Counter(words).keys())
top_keywords = ", ".join(keywords[:3])
return sentiment, summary_text, keywords, top_keywords
def process_comments(self, comments_list):
sentiments, summaries, all_keywords, top_keywords_list, timestamps = [], [], [], [], []
start_date = datetime.now() - timedelta(days=30)
for comment in comments_list:
sentiment, summary, keywords, top_kw = self.process_comment(comment)
sentiments.append(sentiment)
summaries.append(summary)
all_keywords.extend(keywords)
top_keywords_list.append(top_kw)
timestamps.append(start_date + timedelta(days=random.randint(0, 30)))
df = pd.DataFrame({
"Timestamp": timestamps,
"Comment": comments_list,
"Summary": summaries,
"Sentiment": sentiments,
"Top Keywords": top_keywords_list
})
df.sort_values(by='Timestamp', inplace=True, ascending=True)
keyword_freq = pd.DataFrame(
Counter(all_keywords).items(),
columns=['Keyword', 'Frequency']
).sort_values(by='Frequency', ascending=False)
return df, keyword_freq
def generate_wordcloud(self, keyword_freq, filename=None):
wc_dict = dict(zip(keyword_freq['Keyword'], keyword_freq['Frequency']))
wc = WordCloud(width=800, height=400, background_color="white").generate_from_frequencies(wc_dict)
plt.figure(figsize=(10,5))
plt.imshow(wc, interpolation="bilinear")
plt.axis("off")
if filename:
plt.savefig(filename, bbox_inches='tight')
return plt
|