feat: update robots.txt to allow /llms.txt + add AI crawler rules

src/app/robots.ts

@@ -28,12 +28,14 @@ export default function robots(): MetadataRoute.Robots {
                     '/about',
                     '/forum',
                     '/categories',
+                    '/category/',
                     '/extension',
                     '/docs',
-                    '/api/og',   // Allow OG image generation for crawlers
+                    '/llms.txt',
+                    '/api/og',
                 ],
                 disallow: [
-                    '/api/',          // Block all API routes (except /api/og below)
+                    '/api/',
                     '/dashboard',
                     '/settings',
                     '/create',
@@ -45,7 +47,7 @@ export default function robots(): MetadataRoute.Robots {
                     '/admin/',
                 ],
             },
-            // Allow specific bots to access OG images
+            // Allow social bots to access OG images
             {
                 userAgent: 'Twitterbot',
                 allow: ['/api/og'],
@@ -58,30 +60,40 @@ export default function robots(): MetadataRoute.Robots {
                 userAgent: 'LinkedInBot',
                 allow: ['/api/og'],
             },
-            // AI crawlers — allow full indexing to appear in AI search results
+            // AI crawlers — allow full indexing for AI search results
             {
                 userAgent: 'GPTBot',
-                allow: ['/'],
+                allow: ['/', '/llms.txt'],
                 disallow: ['/api/', '/dashboard', '/settings', '/sign-in', '/sign-up'],
             },
             {
                 userAgent: 'Google-Extended',
-                allow: ['/'],
+                allow: ['/', '/llms.txt'],
                 disallow: ['/api/', '/dashboard', '/settings', '/sign-in', '/sign-up'],
             },
             {
                 userAgent: 'anthropic-ai',
-                allow: ['/'],
+                allow: ['/', '/llms.txt'],
+                disallow: ['/api/', '/dashboard', '/settings', '/sign-in', '/sign-up'],
+            },
+            {
+                userAgent: 'ClaudeBot',
+                allow: ['/', '/llms.txt'],
                 disallow: ['/api/', '/dashboard', '/settings', '/sign-in', '/sign-up'],
             },
             {
                 userAgent: 'PerplexityBot',
-                allow: ['/'],
+                allow: ['/', '/llms.txt'],
                 disallow: ['/api/', '/dashboard', '/settings', '/sign-in', '/sign-up'],
             },
             {
                 userAgent: 'Applebot',
-                allow: ['/'],
+                allow: ['/', '/llms.txt'],
+                disallow: ['/api/', '/dashboard', '/settings', '/sign-in', '/sign-up'],
+            },
+            {
+                userAgent: 'Bytespider',
+                allow: ['/', '/llms.txt'],
                 disallow: ['/api/', '/dashboard', '/settings', '/sign-in', '/sign-up'],
             },
         ],