src/app/robots.ts

import type { MetadataRoute } from 'next'

const BASE_URL = process.env.NEXT_PUBLIC_APP_URL || 'https://open-prompt.netlify.app'

export default function robots(): MetadataRoute.Robots {
    return {
        rules: [
            {
                userAgent: '*',
                allow: [
                    '/',
                    '/explore',
                    '/tools',
                    '/tools/',
                    '/p/',
                    '/creator/',
                    '/creators',
                    '/collections',
                    '/characters',
                    '/image-prompts',
                    '/workflows',
                    '/leaderboard',
                    '/frameworks',
                    '/thunderdome',
                    '/pricing',
                    '/blog',
                    '/guides',
                    '/about',
                    '/forum',
                    '/categories',
                    '/category/',
                    '/extension',
                    '/docs',
                    '/llms.txt',
                    '/api/og',
                ],
                disallow: [
                    '/api/',
                    '/dashboard',
                    '/settings',
                    '/create',
                    '/edit/',
                    '/sign-in',
                    '/sign-up',
                    '/handler/',
                    '/_next/',
                    '/admin/',
                ],
            },
            // Allow social bots to access OG images
            {
                userAgent: 'Twitterbot',
                allow: ['/api/og'],
            },
            {
                userAgent: 'facebookexternalhit',
                allow: ['/api/og'],
            },
            {
                userAgent: 'LinkedInBot',
                allow: ['/api/og'],
            },
            // AI crawlers — allow full indexing for AI search results
            {
                userAgent: 'GPTBot',
                allow: ['/', '/llms.txt'],
                disallow: ['/api/', '/dashboard', '/settings', '/sign-in', '/sign-up'],
            },
            {
                userAgent: 'Google-Extended',
                allow: ['/', '/llms.txt'],
                disallow: ['/api/', '/dashboard', '/settings', '/sign-in', '/sign-up'],
            },
            {
                userAgent: 'anthropic-ai',
                allow: ['/', '/llms.txt'],
                disallow: ['/api/', '/dashboard', '/settings', '/sign-in', '/sign-up'],
            },
            {
                userAgent: 'ClaudeBot',
                allow: ['/', '/llms.txt'],
                disallow: ['/api/', '/dashboard', '/settings', '/sign-in', '/sign-up'],
            },
            {
                userAgent: 'PerplexityBot',
                allow: ['/', '/llms.txt'],
                disallow: ['/api/', '/dashboard', '/settings', '/sign-in', '/sign-up'],
            },
            {
                userAgent: 'Applebot',
                allow: ['/', '/llms.txt'],
                disallow: ['/api/', '/dashboard', '/settings', '/sign-in', '/sign-up'],
            },
            {
                userAgent: 'Bytespider',
                allow: ['/', '/llms.txt'],
                disallow: ['/api/', '/dashboard', '/settings', '/sign-in', '/sign-up'],
            },
        ],
        sitemap: `${BASE_URL}/sitemap.xml`,
        host: BASE_URL,
    }
}