Refactor: Clean up code structure and improve payload handling in SEO and Social Media analysis

classes/response_off.py

@@ -32,6 +32,7 @@ class SeoOffPageAnalyst:
 
         # AGENT NAME
         #st.header(self.analyst_name)
+    
     def request_model(self, payload_txt, headers):
         response = requests.post(self.model_url, json=payload_txt, headers=headers)
         response.raise_for_status()

classes/response_on_page.py

@@ -60,7 +60,13 @@ class SeoOn:
         myclient = MongoClient(mongodb_uri)
         mydb = myclient.get_database()
         mycol = mydb["df_data"]
-        x = mycol.find_one({"data_field": data_field})
+        
+        # Sort by timestamp field in descending order
+        x = mycol.find_one(
+            {"data_field": data_field},
+            sort=[("timestamp", -1)]  
+        )
+        
         x = x["result"]
         return x
 
@@ -68,55 +74,19 @@ class SeoOn:
                 with st.spinner('SEO On Page...', show_time=True):
                         st.write('')
                         # OUTPUT FOR SEO ANALYST
-                        headers = {"Content-Type": "application/json", "x-api-key": f"{os.getenv('x-api-key')}"}
-                        payload = ""
-                        try:
-                             payload += self.fetch_data("First Meaningful Paint")
-                        except Exception as e:
-                            pass
-                        try:
-                             payload += self.fetch_data("Crawl File")
-                        except Exception as e:
-                            pass
-
-                        try:
-                            session_first_meaningful_paint = st.session_state['first_meaningful_paint']
-                            session_crawl_file = st.session_state['crawl_file']
-                            if session_first_meaningful_paint or session_crawl_file == 'uploaded':
-                                payload_txt = {"input_value": payload, "output_type": "text", "input_type": "chat"}
+                        headers = {"Content-Type": "application/json", "x-api-key": f"{os.getenv('x-api-key')}"}                                  
+                        try:                          
+                                payload_txt = {"input_value": self.payload, "output_type": "text", "input_type": "chat"}
                                 payload_txt_model = self.request_model(payload_txt, headers)
                                 debug_info = {'data_field' : 'On Page Analyst', 'result': payload_txt_model}
                                 upload_response(debug_info)
 
                                 st.session_state['first_meaningful_paint'] = ''
                                 st.session_state['crawl_file'] = ''
+                                count = 0
                         except Exception as e:
                              pass
 
-                        #end_time = time.time()
-                        #time_lapsed = end_time - start_time
-
-                        #debug_info = {'data_field' : 'GT Metrix', 'result': result}
-                        
-                        
-                        '''
-                        debug_info = {#'analyst': self.analyst_name,
-                                      'url_uuid': self.model_url.split("-")[-1],
-                                      'time_lapsed' : time_lapsed, 
-                                    'crawl_file': [file.name for file in self.uploaded_files] if self.uploaded_files else ['Not available'],
-                                      'gt_metrix': [file.name for file in self.gtmetrix] if self.gtmetrix else ['Not available'],
-                                      'payload': payload_txt, 
-                                      'result': result}
-                        
-                        if self.gtmetrix:
-                            collect_telemetry(debug_info)
-                        '''
- 
-                            
-                        #with st.expander("Debug information", icon="⚙"):
-                        #    st.write(debug_info)
-                        
-
                         st.session_state['analyzing'] = False
                         try:
                             self.file_dict.popitem()
@@ -124,13 +94,25 @@ class SeoOn:
                             pass
                         
     def row1(self):
-           
-            #st.write("") # FOR THE HIDE BUTTON
-            #st.write("") # FOR THE HIDE BUTTON
-            #st.write("AI Analyst Output: ")
             st.session_state['analyzing'] = False
-            #st.write("") # FOR THE HIDE BUTTON
-            self.process()
+            self.payload = ""
+            count = 0
+            try:
+                session_first_meaningful_paint = st.session_state['first_meaningful_paint']
+                if session_first_meaningful_paint == 'uploaded':
+                    count += 1
+                    self.payload += self.fetch_data("First Meaningful Paint")
+            except Exception as e:
+                pass
+            try:
+                session_crawl_file = st.session_state['crawl_file']
+                if session_crawl_file == 'uploaded':
+                    count += 1
+                    self.payload += self.fetch_data("Crawl File")
+            except Exception as e:
+                pass
+            if count >= 1:
+                self.process()
                  
 
 if __name__ == "__main__":

classes/response_seo.py

@@ -48,6 +48,7 @@ class Seo:
         response = requests.post(self.model_url, json=payload_txt, headers=headers)
         response.raise_for_status()
         output = response.json()
+        #st.write(output)
         text = output["outputs"][0]["outputs"][0]["results"]["text"]["data"]["text"]
         text = json.loads(text)
         #st.write(text)
@@ -67,7 +68,13 @@ class Seo:
         myclient = MongoClient(mongodb_uri)
         mydb = myclient.get_database()
         mycol = mydb["df_data"]
-        x = mycol.find_one({"data_field": data_field})
+        
+        # Sort by timestamp field in descending order
+        x = mycol.find_one(
+            {"data_field": data_field},
+            sort=[("timestamp", -1)]  
+        )
+        
         x = x["result"]
         return x
     
@@ -143,47 +150,9 @@ class Seo:
     def process (self):    
         with st.spinner('Seo Analyst...', show_time=True):
                         st.write('')
-                        headers = {"Content-Type": "application/json", "x-api-key": f"{os.getenv('x-api-key')}"}
-                        payload = ""
-                        count = 0
-                        try:
-                            session_traffic_aqcuisition = st.session_state['df_seo']
-                            payload += self.fetch_data("SEO Keywords")
-                            count += 1
-                        except Exception as e:
-                            pass
-                        try:
-                            session_traffic_channels = st.session_state['df_traffic']
-                            payload += self.fetch_data("Traffic Channels")
-                            count += 1
-                        except Exception as e:
-                            pass
-                        try:
-                            session_others = st.session_state['others']
-                            payload += self.fetch_data("Traffic Acquisition")
-                            count += 1
-                        except Exception as e:
-                            pass
-                        try:
-                            session_page_index = st.session_state['pages_index']
-                            payload += self.fetch_data("Pages Indexed")
-                            count += 1
-                        except Exception as e:
-                            pass
-                        try:
-                            session_bounce_rate = st.session_state['bounce_rate']
-                            payload += self.fetch_data("Bounce Rate")
-                            count += 1
-                        except Exception as e:
-                            pass
-                        try:
-                            payload += self.fetch_backlinks("Backlinks")
-                            count += 1
-                        except Exception as e:
-                            pass
+                        headers = {"Content-Type": "application/json", "x-api-key": f"{os.getenv('x-api-key')}"}         
                         try:
-                            if count >= 1:
-                                payload_txt = {"input_value": payload, "output_type": "text", "input_type": "chat"}
+                                payload_txt = {"input_value": self.payload, "output_type": "text", "input_type": "chat"}
                                 payload_txt_model = self.request_model(payload_txt, headers)
                                 debug_info = {'data_field' : 'SEO Analyst', 'result': payload_txt_model}
                                 upload_response(debug_info)
@@ -202,8 +171,57 @@ class Seo:
             st.session_state['analyzing'] = False
             #st.write("") # FOR THE HIDE BUTTON
             #analyze_button = st.button("Analyze", disabled=initialize_analyze_session())
+            self.payload = ""  
+            count = 0
+            try:
+                session_traffic_aqcuisition = st.session_state['df_seo']
+                if session_traffic_aqcuisition == 'uploaded':
+                    count += 1
+                    self.payload += self.fetch_data("SEO Keywords")
+                
+            except Exception as e:
+                pass
+            try:
+                session_traffic_channels = st.session_state['df_traffic']
+                if session_traffic_channels == 'uploaded':
+                    count += 1
+                    self.payload += self.fetch_data("Traffic Channels")
+            except Exception as e:
+                pass
+            try:
+                session_others = st.session_state['others']
+                if session_others == 'uploaded':
+                    count += 1
+                    self.payload += self.fetch_data("Traffic Acquisition")
+                
+            except Exception as e:
+                pass
+            try:
+                session_page_index = st.session_state['pages_index']
+                if session_page_index == 'uploaded':
+                    count += 1
+                    self.payload += self.fetch_data("Pages Indexed")            
+            except Exception as e:
+                pass
+            try:
+                session_bounce_rate = st.session_state['bounce_rate']
+                if session_bounce_rate == 'uploaded':
+                    print("run")
+                    count += 1
+                    self.payload += self.fetch_data("Bounce Rate")                 
+            except Exception as e:
+                pass
+            try:
+                session_backlinks = st.session_state["off_page_file_uploaded"] 
+                if session_backlinks == 'uploaded':
+                    count += 1
+                    self.payload += self.fetch_backlinks("Backlinks") 
+            except Exception as e:
+                pass
+
+            if count >= 1:
+                self.process()
             
-            self.process()
 
 if __name__ == "__main__":
     st.set_page_config(layout="wide")

classes/response_social_media.py

@@ -71,7 +71,7 @@ class SocialMedia:
         # Sort by timestamp field in descending order
         x = mycol.find_one(
             {"data_field": data_field},
-            sort=[("timestamp", -1)]  # -1 means descending order
+            sort=[("timestamp", -1)]  
         )
         
         x = x["result"]
@@ -81,55 +81,9 @@ class SocialMedia:
                     with st.spinner('Social Media Analyst...', show_time=True):
                             st.write('')
 
-                            headers = {"Content-Type": "application/json", "x-api-key": f"{os.getenv('x-api-key')}"}
-                            payload = ""
-                            
+                            headers = {"Content-Type": "application/json", "x-api-key": f"{os.getenv('x-api-key')}"}              
                             try:
-                                session_fb = st.session_state['fb_upload']
-                                if session_fb == 'uploaded':
-                                    payload += self.fetch_data("Facebook")
-         
-                            except Exception as e:
-                                pass
-                            try:
-                                session_ig = st.session_state['ig_upload']
-                                if session_ig == 'uploaded':
-                                    print("running ig")
-                                    payload += self.fetch_data("Instagram")
-
-                            except Exception as e:
-                                pass
-                            try:
-                                session_twitter = st.session_state['twitter_upload']
-                                if session_twitter == 'uploaded':
-                                    payload += self.fetch_data("Twitter")
-
-                            except Exception as e:
-                                pass
-                            try:
-                                session_yt = st.session_state['youtube_upload']
-                                if session_yt == 'uploaded':
-                                    payload += self.fetch_data("YouTube")
-
-                            except Exception as e:
-                                pass
-                            try:
-                                session_linkedin = st.session_state['linkedin_upload']
-                                if session_linkedin == 'uploaded':
-                                    payload += self.fetch_data("Linkedin")
-                        
-                            except Exception as e:
-                                pass
-                            try:
-                                session_tiktok = st.session_state['tiktok_upload']
-                                if session_tiktok == 'uploaded':
-                                    payload += self.fetch_data("Tiktok")
-
-                            except Exception as e:
-                                pass
-                            try:
-                                 if session_fb or session_ig or session_twitter or session_yt or session_linkedin or session_tiktok == 'uploaded':
-                                        payload_txt = {"input_value": payload, "output_type": "text", "input_type": "chat"}
+                                        payload_txt = {"input_value": self.payload, "output_type": "text", "input_type": "chat"}
                                         payload_txt_model = self.request_model(payload_txt, headers)
                                         debug_info = {'data_field' : 'Social Media Analyst', 'result': payload_txt_model}
                                         upload_response(debug_info)
@@ -140,13 +94,58 @@ class SocialMedia:
                                         st.session_state['youtube_upload'] = ''
                                         st.session_state['linkedin_upload'] = ''
                                         st.session_state['tiktok_upload'] = ''
+                                        count = 0
                                         
                             except Exception as e:
                                  pass
-                                      
-                           
+                                                         
     def row1(self):
-            self.process()
+            self.payload = ""
+            count = 0                           
+            try:
+                session_fb = st.session_state['fb_upload']
+                if session_fb == 'uploaded':
+                    count += 1
+                    self.payload += self.fetch_data("Facebook")        
+            except Exception as e:
+                pass
+            try:
+                session_ig = st.session_state['ig_upload']
+                if session_ig == 'uploaded':
+                    count += 1
+                    self.payload += self.fetch_data("Instagram")
+            except Exception as e:
+                pass
+            try:
+                session_twitter = st.session_state['twitter_upload']
+                if session_twitter == 'uploaded':
+                    count += 1
+                    self.payload += self.fetch_data("Twitter")
+            except Exception as e:
+                pass
+            try:
+                session_yt = st.session_state['youtube_upload']
+                if session_yt == 'uploaded':
+                    count += 1
+                    self.payload += self.fetch_data("YouTube")
+            except Exception as e:
+                pass
+            try:
+                session_linkedin = st.session_state['linkedin_upload']
+                if session_linkedin == 'uploaded':
+                    count += 1
+                    self.payload += self.fetch_data("Linkedin")           
+            except Exception as e:
+                pass
+            try:
+                session_tiktok = st.session_state['tiktok_upload']
+                if session_tiktok == 'uploaded':
+                    count += 1
+                    self.payload += self.fetch_data("Tiktok")
+            except Exception as e:
+                pass
+            if count >= 1:
+                self.process()
             
 
 if __name__ == "__main__":

classes/response_website_and_tools.py

@@ -59,118 +59,23 @@ class WebsiteAndTools:
         myclient = MongoClient(mongodb_uri)
         mydb = myclient.get_database()
         mycol = mydb["df_data"]
-        x = mycol.find_one({"data_field": data_field})
+        
+        # Sort by timestamp field in descending order
+        x = mycol.find_one(
+            {"data_field": data_field},
+            sort=[("timestamp", -1)]  
+        )
+        
         x = x["result"]
         return x
 
     def process(self):
                 with st.spinner('Website and Tools...', show_time=True):
-                        st.write('')
-                        '''
-                        # OUTPUT FOR WEBSITE RESPONSIVENESS
-                        payload_txt_website_responsiveness = {"result": website_responsiveness}
-                        result_website_responsiveness = self.request_model(payload_txt_website_responsiveness)
-
-                        # OUTPUT FOR CONTENT MANAGEMENT SYSTEM
-                        payload_txt_content_management_system = {"question": content_management_system}
-                        #result_content_management_system = self.request_model(content_management_system)
-
-                        # OUTPUT FOR SSL CERTIFICATE
-                        payload_txt_SSL_certificate = {"question": SSL_certificate}
-                        #result_SSL_certificate = self.request_model(SSL_certificate)
-
-                        # OUTPUT FOR WEB ANALYTICS
-                        payload_txt_web_analytics = {"question": web_analytics}
-                        #result_web_analytics = self.request_model(web_analytics)
-
-                        # OUTPUT FOR CLIENT RELATIONS MANAGEMENT SYSTEM
-                        payload_txt_client_relations_management_system = {"question": client_relations_management_system}
-                        #result_client_relations_management_system = self.request_model(client_relations_management_system)
-                        
-                        # OUTPUT FOR LEAD GENERATION MECHANISM
-                        payload_txt_lead_generation_mechanism = {"question": lead_generation_mechanism}
-                        #result_lead_generation_mechanism = self.request_model(lead_generation_mechanism)
-                        '''
-                        # OUTPUT FOR SEO ANALYST
-                      
-                        #print(x)
+                        st.write('')       
                         headers = {"Content-Type": "application/json", "x-api-key": f"{os.getenv('x-api-key')}"}
+                                         
                         try:
-                            payload = ""
-                            session_website_responsiveness = st.session_state['website_responsiveness']
-                            if session_website_responsiveness == 'uploaded':
-                                payload += self.fetch_data("Website Responsiveness")
-                        except Exception as e:
-                             pass
-                        try:
-                            session_content_management_system = st.session_state['content_management_system']
-                            if session_content_management_system == 'uploaded':
-                                payload += self.fetch_data("Content Management System")
-                        except Exception as e:
-                             pass
-                        try:
-                            session_SSL_certificate = st.session_state['SSL_certificate']
-                            if session_SSL_certificate == 'uploaded':
-                                payload += self.fetch_data("SSL Certificate")
-                                
-                        except Exception as e:
-                             pass
-                        try:
-                            session_mobile_responsiveness = st.session_state['mobile_responsiveness']
-                            if session_mobile_responsiveness == 'uploaded':
-                                payload += self.fetch_data("Mobile Responsiveness")
-                        except Exception as e:
-                             pass
-                        try:
-                            session_desktop_loading_speed = st.session_state['desktop_loading_speed']
-                            if session_desktop_loading_speed == 'uploaded':
-                                payload += self.fetch_data("Desktop Loading Speed")
-                        except Exception as e:
-                             pass
-                        try:
-                            session_mobile_loading_speed = st.session_state['mobile_loading_speed']
-                            if session_mobile_loading_speed == 'uploaded':
-                                payload += self.fetch_data("Mobile Loading Speed")
-                        except Exception as e:
-                             pass
-                        try:
-                            session_first_meaningful_paint = st.session_state['first_meaningful_paint']
-                            if session_first_meaningful_paint == 'uploaded':
-                                payload += self.fetch_data("First Meaningful Paint")
-                        except Exception as e:
-                             pass
-                        try:
-                            session_web_analytics = st.session_state['web_analytics']
-                            if session_web_analytics == 'uploaded':
-                                payload += self.fetch_data("Web Analytics")
-                        except Exception as e:
-                             pass
-                        try:
-                            session_client_relations_management_system = st.session_state['client_relations_management_system']
-                            if session_client_relations_management_system == 'uploaded':
-                                payload += self.fetch_data("Client Relations Management System")
-                        except Exception as e:
-                             pass
-                        try:
-                            session_lead_generation_mechanism = st.session_state['lead_generation_mechanism']
-                            if session_lead_generation_mechanism == 'uploaded':
-                                payload += self.fetch_data("Lead Generation Mechanism")
-                        except Exception as e:
-                             pass
-                        
-                        try:
-                            session_website_responsiveness = st.session_state['website_responsiveness']
-                            session_client_relations_management_system = st.session_state['client_relations_management_system']
-                            session_lead_generation_mechanism = st.session_state['lead_generation_mechanism']
-                            session_web_analytics = st.session_state['web_analytics']
-                            session_mobile_responsiveness = st.session_state['mobile_responsiveness']
-                            session_desktop_loading_speed = st.session_state['desktop_loading_speed']
-                            session_mobile_loading_speed = st.session_state['mobile_loading_speed']
-                            session_SSL_certificate = st.session_state['SSL_certificate']
-                            session_content_management_system = st.session_state['content_management_system']
-
-                            if session_website_responsiveness or session_client_relations_management_system or session_lead_generation_mechanism or session_web_analytics or session_mobile_responsiveness or session_desktop_loading_speed or session_mobile_loading_speed or session_content_management_system or session_SSL_certificate == 'uploaded':
-                                payload_txt = {"input_value": payload, "output_type": "text", "input_type": "chat"}
+                                payload_txt = {"input_value": self.payload, "output_type": "text", "input_type": "chat"}
                                 payload_txt_model = self.request_model(payload_txt, headers)
                                 debug_info = {'data_field' : 'Website and Tools Analyst', 'result': payload_txt_model}
                                 upload_response(debug_info)
@@ -184,6 +89,7 @@ class WebsiteAndTools:
                                 st.session_state['web_analytics'] = ''
                                 st.session_state['client_relations_management_system'] = ''
                                 st.session_state['lead_generation_mechanism'] = ''
+                                count = 0
                         except Exception as e:
                              pass
                         #end_time = time.time()
@@ -217,15 +123,84 @@ class WebsiteAndTools:
                             pass
                         
     def row1(self):
-           
-            #st.write("") # FOR THE HIDE BUTTON
-            #st.write("") # FOR THE HIDE BUTTON
-            #st.write("AI Analyst Output: ")
             st.session_state['analyzing'] = False
-            #st.write("") # FOR THE HIDE BUTTON
-            self.process()
+            self.payload = ""
+            count = 0
+            try:
+                payload = ""
+                session_website_responsiveness = st.session_state['website_responsiveness']
+                if session_website_responsiveness == 'uploaded':
+                    count += 1
+                    self.payload += self.fetch_data("Website Responsiveness")
+            except Exception as e:
+                pass
+            try:
+                session_content_management_system = st.session_state['content_management_system']
+                if session_content_management_system == 'uploaded':
+                    count += 1
+                    self.payload += self.fetch_data("Content Management System")
+            except Exception as e:
+                pass
+            try:
+                session_SSL_certificate = st.session_state['SSL_certificate']
+                if session_SSL_certificate == 'uploaded':
+                    count += 1
+                    self.payload += self.fetch_data("SSL Certificate")
+                                
+            except Exception as e:
+                pass
+            try:
+                session_mobile_responsiveness = st.session_state['mobile_responsiveness']
+                if session_mobile_responsiveness == 'uploaded':
+                    count += 1
+                    self.payload += self.fetch_data("Mobile Responsiveness")
+            except Exception as e:
+                pass
+            try:
+                session_desktop_loading_speed = st.session_state['desktop_loading_speed']
+                if session_desktop_loading_speed == 'uploaded':
+                    count += 1
+                    self.payload += self.fetch_data("Desktop Loading Speed")
+            except Exception as e:
+                pass
+            try:
+                session_mobile_loading_speed = st.session_state['mobile_loading_speed']
+                if session_mobile_loading_speed == 'uploaded':
+                    count += 1
+                    self.payload += self.fetch_data("Mobile Loading Speed")
+            except Exception as e:
+                pass
+            try:
+                session_first_meaningful_paint = st.session_state['first_meaningful_paint']
+                if session_first_meaningful_paint == 'uploaded':
+                    count += 1
+                    self.payload += self.fetch_data("First Meaningful Paint")
+            except Exception as e:
+                pass
+            try:
+                session_web_analytics = st.session_state['web_analytics']
+                if session_web_analytics == 'uploaded':
+                    count += 1
+                    self.payload += self.fetch_data("Web Analytics")
+            except Exception as e:
+                pass
+            try:
+                session_client_relations_management_system = st.session_state['client_relations_management_system']
+                if session_client_relations_management_system == 'uploaded':
+                    count += 1
+                    self.payload += self.fetch_data("Client Relations Management System")
+            except Exception as e:
+                pass
+            try:
+                session_lead_generation_mechanism = st.session_state['lead_generation_mechanism']
+                if session_lead_generation_mechanism == 'uploaded':
+                    count += 1
+                    self.payload += self.fetch_data("Lead Generation Mechanism")
+            except Exception as e:
+                pass
+            if count >= 1:
+                self.process()
                  
-
 if __name__ == "__main__":
     st.set_page_config(layout="wide")
 

pages/analyzing_page.py

@@ -9,7 +9,6 @@ from classes.response_seo import Seo
 from classes.response_social_media import SocialMedia
 
 def run_analysis():
-    
     # Placeholders for status updates
     off_page_status = st.empty()
     on_page_status = st.empty()