Added deleted posts update

app.py

@@ -42,6 +42,7 @@ TOP_K = 75
 HISTORY_WINDOW = timedelta(days=1000)
 TIMEZONE = ZoneInfo("UTC")
 UPDATE_INTERVAL = 300  # In seconds (5 minutes)
+START_TIME = datetime.now(TIMEZONE)
 
 # Global variables
 supabase_client = None
@@ -119,6 +120,43 @@ class Recommender:
         self.reply_weight = reply_weight
         self.reply_like_weight = reply_like_weight
 
+    async def fetch_existing_post_ids() -> set:
+        supabase = get_supabase_client()
+        page_size = 1000
+        page = 0
+        post_ids = set()
+
+        while True:
+            response = await asyncio.to_thread(
+                supabase.table('posts')
+                .select('id')
+                .range(page * page_size, (page + 1) * page_size - 1)
+                .execute
+            )
+
+            if not response.data:
+                break
+
+            for row in response.data:
+                post_ids.add(row['id'])
+
+            page += 1
+
+        return post_ids
+
+    def clean_deleted_posts(self, existing_post_ids: set):
+        # Determine which posts are missing
+        all_cached_ids = set(post_metadata.keys())
+        deleted_ids = all_cached_ids - existing_post_ids
+
+        for post_id in deleted_ids:
+            post_metadata.pop(post_id, None)
+            post_features.pop(post_id, None)
+            self.post_popularity.pop(post_id, None)
+
+        for user_id in user_interactions:
+            user_interactions[user_id] -= deleted_ids  # remove any deleted post_ids
+
     async def fetch_all_rows(self, table_name: str, columns: str, last_update: datetime, post_id_not_null: bool):
         """Fetch all rows from a table using pagination."""
         supabase = get_supabase_client()
@@ -259,6 +297,9 @@ class Recommender:
             for post_id, embedding in zip(post_ids, embeddings):
                 post_features[post_id] = embedding / np.linalg.norm(embedding)
 
+        existing_post_ids = await self.fetch_existing_post_ids()
+        self.clean_deleted_posts(existing_post_ids)
+
         self.last_update = datetime.now(TIMEZONE)
         total_interactions = len(likes) + len(comments) + \
             len(commentlikes) + len(replies) + len(replylikes)