| """ |
| Generate UPDATED IEEE-format PDF paper with: |
| - Real TweetEval results |
| - 5 embedded figures |
| - Honest domain gap analysis |
| - Proper IEEE formatting |
| """ |
|
|
| from fpdf import FPDF |
| import os, json |
|
|
| class IEEEPaper(FPDF): |
| def __init__(self): |
| super().__init__('P', 'mm', 'Letter') |
| self.set_auto_page_break(auto=True, margin=25) |
| |
| def header(self): |
| if self.page_no() > 1: |
| self.set_font('Helvetica', 'I', 8) |
| self.set_text_color(128, 128, 128) |
| self.cell(0, 5, 'IEEE Conference Paper - Transformer-Based Social Media Sentiment Analysis', 0, 0, 'C') |
| self.ln(8) |
| self.set_text_color(0, 0, 0) |
| |
| def footer(self): |
| self.set_y(-15) |
| self.set_font('Helvetica', 'I', 8) |
| self.set_text_color(128, 128, 128) |
| self.cell(0, 10, f'{self.page_no()}', 0, 0, 'C') |
| self.set_text_color(0, 0, 0) |
|
|
| def rx(self): self.set_x(self.l_margin) |
|
|
| def section(self, num, title): |
| self.rx(); self.ln(4) |
| self.set_font('Helvetica', 'B', 11) |
| t = f'{num}. {title.upper()}' if num else title.upper() |
| self.cell(0, 7, t, 0, 1, 'C') |
| self.ln(1); self.rx() |
|
|
| def subsection(self, title): |
| self.rx(); self.ln(2) |
| self.set_font('Helvetica', 'B', 10) |
| self.multi_cell(0, 6, title, 0, 'L') |
| self.ln(1); self.rx() |
|
|
| def text(self, content): |
| self.rx() |
| self.set_font('Helvetica', '', 9) |
| self.multi_cell(0, 4.5, content, 0, 'J') |
| self.ln(1); self.rx() |
|
|
| def bullets(self, items): |
| self.rx() |
| self.set_font('Helvetica', '', 9) |
| for item in items: |
| self.set_x(self.l_margin) |
| self.multi_cell(0, 4.5, f' - {item}', 0, 'J') |
| self.ln(1); self.rx() |
|
|
| def numbers(self, items): |
| self.rx() |
| self.set_font('Helvetica', '', 9) |
| for i, item in enumerate(items, 1): |
| self.set_x(self.l_margin) |
| self.multi_cell(0, 4.5, f' {i}) {item}', 0, 'J') |
| self.ln(1); self.rx() |
|
|
| def table(self, caption, headers, rows, widths=None): |
| self.rx(); self.ln(2) |
| self.set_font('Helvetica', 'B', 9) |
| self.cell(0, 5, caption, 0, 1, 'C') |
| self.ln(1) |
| n = len(headers) |
| if widths is None: widths = [170/n]*n |
| tw = sum(widths) |
| x0 = (self.w - tw) / 2 |
| self.set_x(x0) |
| self.set_font('Helvetica', 'B', 7.5) |
| self.set_fill_color(220, 220, 220) |
| for i, h in enumerate(headers): |
| self.cell(widths[i], 6, h, 1, 0, 'C', True) |
| self.ln() |
| self.set_font('Helvetica', '', 7.5) |
| for row in rows: |
| self.set_x(x0) |
| bold = row[0].startswith('**') |
| if bold: |
| row = [r.strip('*') for r in row] |
| self.set_font('Helvetica', 'B', 7.5) |
| for i, cell in enumerate(row): |
| self.cell(widths[i], 5, str(cell), 1, 0, 'L' if i==0 else 'C') |
| self.ln() |
| if bold: self.set_font('Helvetica', '', 7.5) |
| self.ln(2); self.rx() |
|
|
| def figure(self, path, caption, width=140): |
| self.rx(); self.ln(3) |
| x = (self.w - width) / 2 |
| self.image(path, x=x, w=width) |
| self.ln(2) |
| self.set_font('Helvetica', 'I', 8) |
| self.multi_cell(0, 4, caption, 0, 'C') |
| self.ln(2); self.rx() |
|
|
| def code_block(self, code): |
| self.rx() |
| self.set_font('Courier', '', 7) |
| self.set_fill_color(245, 245, 245) |
| for line in code.strip().split('\n'): |
| self.set_x(self.l_margin + 5) |
| self.cell(0, 3.8, line, 0, 1, 'L', True) |
| self.set_font('Helvetica', '', 9) |
| self.ln(2); self.rx() |
|
|
| def references(self, refs): |
| self.rx(); self.ln(3) |
| self.set_font('Helvetica', 'B', 9) |
| self.cell(0, 5, 'REFERENCES', 0, 1, 'C') |
| self.ln(1) |
| self.set_font('Helvetica', '', 7) |
| for i, ref in enumerate(refs, 1): |
| self.set_x(self.l_margin) |
| self.multi_cell(0, 3.5, f'[{i}] {ref}', 0, 'J') |
| self.ln(0.5) |
|
|
|
|
| |
| |
| |
|
|
| pdf = IEEEPaper() |
| pdf.add_page() |
|
|
| |
| pdf.set_font('Helvetica', 'B', 17) |
| pdf.multi_cell(0, 8, 'Transformer-Based Social Media Sentiment Analysis:\nA Comprehensive Evaluation of DeBERTa-v3, RoBERTa,\nand BERT on Real Tweet Data', 0, 'C') |
| pdf.ln(4) |
| pdf.set_font('Helvetica', '', 12) |
| pdf.cell(0, 6, 'Raj Vivan', 0, 1, 'C') |
| pdf.set_font('Helvetica', 'I', 10) |
| pdf.cell(0, 5, 'Department of Computer Science, Independent Researcher', 0, 1, 'C') |
| pdf.set_font('Helvetica', '', 9) |
| pdf.cell(0, 5, 'rajvivan@huggingface.co', 0, 1, 'C') |
| pdf.ln(5) |
|
|
| |
| pdf.set_font('Helvetica', 'B', 10) |
| pdf.cell(0, 6, 'Abstract', 0, 1, 'L') |
| pdf.set_font('Helvetica', 'I', 9) |
| pdf.multi_cell(0, 4.5, ( |
| "We present a comprehensive evaluation of four transformer models for social media sentiment analysis, " |
| "tested on real tweet data from the TweetEval benchmark (12,284 tweets, 3-class). DeBERTa-v3-base achieves " |
| "96.44% accuracy on SST-2 (movie reviews), surpassing the 95% target. However, on real social media data, " |
| "Twitter-RoBERTa (pre-trained on 124M tweets) significantly outperforms all general-domain models, achieving " |
| "72.40% macro-F1 on 3-class tweet sentiment versus 40.64% for DeBERTa-v3. This reveals a critical domain gap: " |
| "models trained on formal text fail to classify neutral tweets, a class comprising 48% of real social media " |
| "data. We provide confusion matrices, per-class F1 analysis, and dataset distribution visualizations. " |
| "Our findings demonstrate that domain-specific pre-training is essential for social media NLP, and that " |
| "benchmark performance on SST-2 does not transfer to real-world tweet classification." |
| ), 0, 'J') |
| pdf.ln(2) |
| pdf.set_font('Helvetica', 'B', 9) |
| pdf.cell(18, 5, 'Keywords:', 0, 0) |
| pdf.set_font('Helvetica', 'I', 9) |
| pdf.cell(0, 5, 'sentiment analysis, social media, NLP, DeBERTa, RoBERTa, transformer, TweetEval, Twitter', 0, 1) |
| pdf.ln(3) |
|
|
| |
| pdf.section('I', 'Introduction') |
| pdf.text( |
| "Social media platforms generate vast amounts of user-generated content daily, making automated sentiment " |
| "analysis essential for brand monitoring, public opinion tracking, and social trend analysis [1]. However, " |
| "social media text presents unique challenges: informal language, slang, sarcasm, emoticons, and a dominant " |
| "neutral class that does not exist in traditional sentiment benchmarks [2]." |
| ) |
| pdf.text( |
| "Pre-trained transformer models have achieved remarkable accuracy on standard benchmarks. DeBERTa-v3-base [7] " |
| "achieves 96.44% on SST-2 [8], and BERT-base [4] reaches 92.43%. However, these benchmarks use formal text " |
| "(movie reviews), raising questions about transferability to real social media. The TweetEval benchmark [9] " |
| "provides standardized evaluation on actual tweets, where the best models achieve only ~73% macro-recall [9]." |
| ) |
| pdf.text("This paper makes the following contributions:") |
| pdf.numbers([ |
| "Comprehensive evaluation of 4 transformer models (DeBERTa-v3, Twitter-RoBERTa, BERT, DistilBERT) on both SST-2 and TweetEval benchmarks using identical evaluation protocols", |
| "Quantification of the domain gap: 96.44% SST-2 accuracy does NOT transfer to tweets (40.64% macro-F1 on 3-class TweetEval for the same DeBERTa model)", |
| "Evidence that domain-specific pre-training (Twitter-RoBERTa) is essential, achieving 72.40% macro-F1 on real tweets vs. <41% for all general-domain models", |
| "Per-class analysis revealing the neutral class as the primary failure mode for general-domain models", |
| "Open-source release of all code, figures, and results on Hugging Face Hub", |
| ]) |
|
|
| |
| pdf.section('II', 'Related Work') |
| pdf.subsection('A. Transformer-Based Sentiment Analysis') |
| pdf.text( |
| "BERT [4] established the foundation for transfer learning in NLP. RoBERTa [5] optimized pre-training with " |
| "larger batches and dynamic masking. DeBERTa-v3 [7] introduced disentangled attention and ELECTRA-style RTD " |
| "pre-training, achieving 95.6% on SST-2 and state-of-the-art on the GLUE benchmark [12]." |
| ) |
| pdf.subsection('B. Social Media NLP') |
| pdf.text( |
| "Barbieri et al. [9] introduced TweetEval, a unified benchmark for tweet understanding with 7 tasks. On " |
| "sentiment (3-class), RoBERTa-base achieves 71.3% and Twitter-retrained RoBERTa achieves 72.6% macro-recall. " |
| "BERTweet [16], pre-trained on 850M tweets, achieves competitive results. Burnham et al. [13] showed that " |
| "fine-tuned small models beat GPT-4 on tweet classification (94% vs 87%)." |
| ) |
|
|
| |
| pdf.section('III', 'Methodology') |
| pdf.subsection('A. Models Evaluated') |
| pdf.text( |
| "We evaluate four models spanning different sizes and pre-training domains:" |
| ) |
| pdf.table( |
| "TABLE I: Models Evaluated", |
| ["Model", "Parameters", "Pre-training Domain", "Output Classes"], |
| [ |
| ["DistilBERT-SST2", "66M", "Wikipedia + BookCorpus", "Binary (pos/neg)"], |
| ["BERT-base-SST2", "110M", "Wikipedia + BookCorpus", "Binary (pos/neg)"], |
| ["Twitter-RoBERTa", "125M", "124M tweets (2018-2021)", "3-class (neg/neu/pos)"], |
| ["DeBERTa-v3-base-SST2", "184M", "Wikipedia + BookCorpus", "Binary (pos/neg)"], |
| ], |
| [35, 22, 45, 38] |
| ) |
|
|
| pdf.subsection('B. Datasets') |
| pdf.text( |
| "TweetEval Sentiment [9]: 45,615 training, 2,000 validation, and 12,284 test real tweets labeled as " |
| "negative (0), neutral (1), or positive (2). Class distribution is imbalanced: 48.3% neutral, 32.3% negative, " |
| "19.3% positive. SST-2 [8]: 67,349 training and 872 validation movie review sentences (binary: positive/negative)." |
| ) |
|
|
| |
| pdf.figure('/app/figures/fig4_data_distribution.png', |
| 'Fig. 1. Dataset class distributions. TweetEval (left) shows heavy neutral dominance (48.3%). ' |
| 'SST-2 (right) has no neutral class, explaining the domain mismatch.', width=155) |
|
|
| pdf.subsection('C. Twitter Preprocessing') |
| pdf.text( |
| "Following TimeLM [9]: @mentions replaced with @user, URLs replaced with http, hashtags and emojis preserved. " |
| "Maximum sequence length: 128 tokens for all models." |
| ) |
|
|
| pdf.subsection('D. Evaluation Protocol') |
| pdf.text( |
| "All models evaluated on the same test sets with identical preprocessing. Metrics: Accuracy, Macro-F1, " |
| "Weighted Precision, Weighted Recall. For binary models on 3-class TweetEval, positive maps to class 2 " |
| "and negative to class 0 (neutral tweets cannot be predicted). We report both 3-class and binary " |
| "(neutral excluded) evaluations for completeness." |
| ) |
|
|
| |
| pdf.section('IV', 'Experimental Results') |
|
|
| pdf.subsection('A. Main Results') |
| pdf.table( |
| "TABLE II: Comprehensive Results on Both Benchmarks", |
| ["Model", "SST-2 Acc", "TweetEval Acc", "TweetEval Macro-F1"], |
| [ |
| ["DistilBERT (66M)", "91.06%", "42.01%", "36.53%"], |
| ["BERT-base (110M)", "92.43%", "44.21%", "38.53%"], |
| ["**Twitter-RoBERTa (125M)**", "**86.12%**", "**72.18%**", "**72.40%**"], |
| ["DeBERTa-v3 (184M)", "96.44%", "46.90%", "40.64%"], |
| ], |
| [40, 30, 30, 35] |
| ) |
|
|
| pdf.text( |
| "Key finding: DeBERTa-v3-base achieves the highest SST-2 accuracy (96.44%) but the LOWEST real social media " |
| "performance among evaluated models. Twitter-RoBERTa, despite lower SST-2 accuracy (86.12%), achieves " |
| "72.40% macro-F1 on TweetEval -- nearly 2x better than DeBERTa-v3 on real tweets. This demonstrates that " |
| "SST-2 performance is NOT predictive of social media sentiment analysis capability." |
| ) |
|
|
| |
| pdf.figure('/app/figures/fig3_model_comparison.png', |
| 'Fig. 2. Model comparison: SST-2 accuracy vs TweetEval macro-F1. The red dashed line marks the 95% ' |
| 'target. DeBERTa-v3 exceeds it on SST-2 but fails on real tweets. Twitter-RoBERTa shows the opposite pattern.', |
| width=155) |
|
|
| pdf.subsection('B. The Domain Gap Problem') |
| pdf.text( |
| "The 55.8 percentage-point gap between DeBERTa-v3's SST-2 accuracy (96.44%) and TweetEval 3-class accuracy " |
| "(40.64% macro-F1) is striking. This gap arises because: (1) SST-2 has no neutral class, while 48.3% of " |
| "TweetEval is neutral; (2) Movie reviews use formal sentiment markers, while tweets use slang and sarcasm; " |
| "(3) Binary models assign ALL neutral tweets to either positive or negative, causing massive misclassification." |
| ) |
|
|
| pdf.text( |
| "When neutral tweets are excluded (binary evaluation), DeBERTa-v3 recovers to 90.77% accuracy on tweets, " |
| "confirming that the neutral class -- not sentiment detection itself -- is the primary failure mode." |
| ) |
|
|
| pdf.subsection('C. Confusion Matrix Analysis') |
| |
| pdf.figure('/app/figures/fig1_confusion_roberta.png', |
| 'Fig. 3. Confusion matrix for Twitter-RoBERTa on TweetEval. The model correctly identifies all three ' |
| 'classes with reasonable accuracy, though neutral tweets show some confusion with both positive and negative.', |
| width=110) |
|
|
| pdf.figure('/app/figures/fig2_confusion_deberta.png', |
| 'Fig. 4. Confusion matrix for DeBERTa-v3-base on TweetEval. The binary model cannot predict neutral ' |
| '(middle row), assigning all neutral tweets to either negative or positive.', |
| width=110) |
|
|
| pdf.subsection('D. Per-Class Performance') |
| pdf.figure('/app/figures/fig5_per_class_f1.png', |
| 'Fig. 5. Per-class F1 scores on TweetEval. Twitter-RoBERTa achieves balanced performance across all ' |
| 'three classes. DeBERTa-v3 scores 0% F1 on neutral (cannot predict this class) but competitive on pos/neg.', |
| width=130) |
|
|
| pdf.text( |
| "Fig. 5 reveals that DeBERTa-v3 achieves competitive negative and positive F1 when those classes are present, " |
| "but its complete inability to predict neutral tweets (0.0% F1) devastates overall macro-F1. Twitter-RoBERTa's " |
| "balanced per-class performance demonstrates the value of domain-matched pre-training." |
| ) |
|
|
| |
| pdf.section('V', 'Analysis and Discussion') |
|
|
| pdf.subsection('A. Why Domain-Specific Pre-training Matters') |
| pdf.text( |
| "Twitter-RoBERTa was pre-trained on 124M tweets from 2018-2021, exposing it to informal language patterns, " |
| "abbreviations (@, #, RT), emoticons, and the neutral-leaning distribution of social media. General-domain " |
| "models (BERT, DeBERTa) were pre-trained on Wikipedia and BookCorpus, which contain no tweets and no neutral " |
| "sentiment class. This pre-training domain mismatch is the dominant factor -- model architecture and size " |
| "matter far less than pre-training data for this task." |
| ) |
|
|
| pdf.subsection('B. The Neutral Class Challenge') |
| pdf.text( |
| "The neutral class represents 48.3% of TweetEval test data. Most tweets are informational, factual, or " |
| "mixed-sentiment -- not clearly positive or negative. Binary models fundamentally cannot address this. " |
| "Even Twitter-RoBERTa's 72.40% macro-F1 shows room for improvement on neutral detection, which remains " |
| "the hardest class in social media sentiment analysis." |
| ) |
|
|
| pdf.subsection('C. Practical Recommendations') |
| pdf.bullets([ |
| "For general sentiment (formal text): DeBERTa-v3-base at 96.44% SST-2 accuracy", |
| "For social media sentiment: Twitter-RoBERTa at 72.40% TweetEval macro-F1", |
| "For production systems: Consider ensemble of both models or fine-tune DeBERTa-v3 on TweetEval training data", |
| "Always evaluate on in-domain data -- SST-2 results do NOT predict social media performance", |
| ]) |
|
|
| pdf.subsection('D. Limitations') |
| pdf.bullets([ |
| "Binary models evaluated on 3-class task (inherent disadvantage on neutral class)", |
| "Single seed evaluation (no confidence intervals)", |
| "English only -- results may not generalize to other languages", |
| "No fine-tuning of DeBERTa on TweetEval (future work)", |
| "No ablation on pre-training data size effects", |
| ]) |
|
|
| |
| pdf.section('VI', 'Conclusion and Future Work') |
| pdf.text( |
| "We presented a comprehensive evaluation of four transformer models on real social media sentiment data. " |
| "Our key finding is that SST-2 accuracy (the standard benchmark) does NOT predict social media performance. " |
| "DeBERTa-v3-base achieves 96.44% on SST-2 but only 40.64% macro-F1 on real tweets, while Twitter-RoBERTa " |
| "(86.12% SST-2) achieves 72.40% on tweets -- nearly 2x better. The neutral class, comprising 48.3% of real " |
| "tweets, is the primary failure mode for general-domain models." |
| ) |
| pdf.text("Future work includes:") |
| pdf.numbers([ |
| "Fine-tuning DeBERTa-v3 on TweetEval training data (expected to significantly close the domain gap)", |
| "Ensemble approaches combining DeBERTa-v3 and Twitter-RoBERTa", |
| "Multi-seed evaluation with statistical significance tests", |
| "Aspect-based sentiment analysis for social media", |
| "Multilingual tweet sentiment analysis", |
| ]) |
|
|
| |
| pdf.section(None, 'Reproducibility') |
| pdf.text("All code, data, figures, and results:") |
| pdf.bullets([ |
| "Repository: https://huggingface.co/rajvivan/social-media-sentiment-analysis-paper", |
| "Datasets: stanfordnlp/sst2, cardiffnlp/tweet_eval (sentiment config)", |
| "Models: cliang1453/deberta-v3-base-sst2, cardiffnlp/twitter-roberta-base-sentiment-latest, textattack/bert-base-uncased-SST-2, distilbert/distilbert-base-uncased-finetuned-sst-2-english", |
| ]) |
|
|
| |
| pdf.references([ |
| 'B. Liu, "Sentiment analysis and opinion mining," Synthesis Lectures on HLT, vol. 5, no. 1, 2012.', |
| 'A. Giachanou and F. Crestani, "Like it or not: A survey of Twitter sentiment analysis," ACM Comp. Surveys, vol. 49, no. 2, 2016.', |
| 'A. Vaswani et al., "Attention is all you need," NeurIPS, vol. 30, 2017.', |
| 'J. Devlin et al., "BERT: Pre-training of deep bidirectional transformers," Proc. NAACL-HLT, 2019.', |
| 'Y. Liu et al., "RoBERTa: A robustly optimized BERT pretraining approach," arXiv:1907.11692, 2019.', |
| 'P. He et al., "DeBERTa: Decoding-enhanced BERT with disentangled attention," Proc. ICLR, 2021.', |
| 'P. He et al., "DeBERTaV3: Improving DeBERTa using ELECTRA-style pre-training with GDES," Proc. ICLR, 2023.', |
| 'R. Socher et al., "Recursive deep models for semantic compositionality," Proc. EMNLP, 2013.', |
| 'F. Barbieri et al., "TweetEval: Unified benchmark and comparative evaluation for tweet classification," Findings of EMNLP, 2020.', |
| 'J. Wei and K. Zou, "EDA: Easy data augmentation for text classification," Proc. EMNLP-IJCNLP, 2019.', |
| 'T. Vu et al., "Stacking ensemble methods for sentiment analysis," arXiv:2009.12357, 2020.', |
| 'A. Wang et al., "GLUE: A multi-task benchmark for NLU," Proc. EMNLP BlackboxNLP, 2018.', |
| 'M. J. Burnham et al., "Fine-tuned small LLMs outperform zero-shot generative AI," arXiv:2406.08660, 2024.', |
| 'S. Ruder, "Neural transfer learning for NLP," Ph.D. thesis, NUI Galway, 2019.', |
| '"Tweet sentiment extraction," Kaggle, 2020. https://www.kaggle.com/c/tweet-sentiment-extraction', |
| 'D. Q. Nguyen et al., "BERTweet: A pre-trained language model for English tweets," Proc. EMNLP Demos, 2020.', |
| ]) |
|
|
| os.makedirs("/app/paper", exist_ok=True) |
| pdf.output("/app/paper/Social_Media_Sentiment_Analysis_IEEE_v2.pdf") |
| print(f"Updated PDF generated: /app/paper/Social_Media_Sentiment_Analysis_IEEE_v2.pdf") |
| print(f"Pages: {pdf.page_no()}") |
|
|
