| from nltk.corpus import stopwords |
| from nltk.stem import WordNetLemmatizer |
| from bs4 import BeautifulSoup |
| import re |
| def clean(df): |
| stop_words = set(stopwords.words('english')) |
| lemmatizer = WordNetLemmatizer() |
| cleaned_headlines = [] |
|
|
| for headline in df['title']: |
| headline = BeautifulSoup(headline, 'html.parser').get_text() |
| headline = re.sub(r'[^a-zA-Z0-9\s]', '', headline) |
| headline = re.sub(r'\s+', ' ', headline).strip() |
| headline = headline.lower() |
|
|
| words = headline.split() |
| words = [lemmatizer.lemmatize(word) for word in words if word not in stop_words] |
|
|
| cleaned_headline = ' '.join(words) |
| cleaned_headlines.append(cleaned_headline) |
|
|
| df['title'] = cleaned_headlines |
| df.drop_duplicates(subset=['title'], inplace=True) |
|
|
| return df |
|
|
