Added a fix for 1000 batch reqs

app.py

@@ -38,7 +38,7 @@ random.seed(SEED)
 np.random.seed(SEED)
 
 TOP_K = 75
-HISTORY_WINDOW = timedelta(days=365)
+HISTORY_WINDOW = timedelta(days=1000)
 TIMEZONE = ZoneInfo("UTC")
 UPDATE_INTERVAL = 300  # In seconds (5 minutes)
 
@@ -83,14 +83,36 @@ class Recommender:
         self.post_popularity = defaultdict(float)
         self.last_update = datetime.now(TIMEZONE) - HISTORY_WINDOW
 
-    async def update_data(self):
+    async def fetch_all_rows(self, table_name: str, columns: str, last_update: datetime):
+        """Fetch all rows from a table using pagination."""
         supabase = get_supabase_client()
+        page_size = 1000
+        page = 0
+        all_data = []
+
+        while True:
+            response = await asyncio.to_thread(
+                supabase.table(table_name)
+                .select(columns)
+                .gt('created_at', last_update.isoformat())
+                .range(page * page_size, (page + 1) * page_size - 1)
+                .execute
+            )
+            
+            if not response.data:
+                break
+            
+            all_data.extend(response.data)
+            page += 1
+
+        return all_data
 
-        # Fetch likes and comments
-        likes = await asyncio.to_thread(supabase.table('likes').select('user_id, post_id, created_at').gt('created_at', self.last_update.isoformat()).execute)
-        comments = await asyncio.to_thread(supabase.table('comments').select('user_id, post_id, created_at').gt('created_at', self.last_update.isoformat()).execute)
+    async def update_data(self):
+        # Fetch all likes and comments with pagination
+        likes = await recommender.fetch_all_rows('likes', 'user_id, post_id, created_at', self.last_update)
+        comments = await recommender.fetch_all_rows('comments', 'user_id, post_id, created_at', self.last_update)
 
-        for interaction in likes.data + comments.data:
+        for interaction in likes + comments:
             user_id = interaction['user_id']
             post_id = interaction['post_id']
             user_interactions[user_id].add(post_id)
@@ -99,10 +121,10 @@ class Recommender:
                 self.user_profiles[user_id] += post_features[post_id]
 
         # Fetch posts
-        posts = await asyncio.to_thread(supabase.table('posts').select('*').gt('created_at', self.last_update.isoformat()).execute)
+        posts = await recommender.fetch_all_rows('posts', '*', self.last_update)
 
         post_texts, post_ids = [], []
-        for post in posts.data:
+        for post in posts:
             post_id = post['id']
             text = f"{post.get('movie_name', '')} {post.get('content', '')}".strip()
             post_texts.append(text)
@@ -117,7 +139,7 @@ class Recommender:
                 post_features[post_id] = embedding / np.linalg.norm(embedding)
 
         self.last_update = datetime.now(TIMEZONE)
-        logger.info("Data updated: {} posts, {} interactions".format(len(posts.data), len(likes.data) + len(comments.data)))
+        logger.info("Data updated: {} posts, {} interactions".format(len(posts), len(likes) + len(comments)))
 
     def get_recommendations(self, user_id: str) -> List[Dict]:
         user_profile = self.user_profiles[user_id]
@@ -183,3 +205,7 @@ async def shutdown_event():
 @app.get("/")
 async def health_check():
     return {"status": "success", "message": "Service operational"}
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=8000)