Spaces:
Configuration error
Configuration error
File size: 3,445 Bytes
bcce530 e8b2c80 bcce530 e8b2c80 bcce530 e8b2c80 bcce530 e8b2c80 bcce530 e8b2c80 bcce530 e8b2c80 bcce530 e8b2c80 bcce530 e8b2c80 bcce530 e8b2c80 bcce530 e8b2c80 bcce530 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 | import type { MetadataRoute } from 'next'
const BASE_URL = process.env.NEXT_PUBLIC_APP_URL || 'https://open-prompt.netlify.app'
export default function robots(): MetadataRoute.Robots {
return {
rules: [
{
userAgent: '*',
allow: [
'/',
'/explore',
'/tools',
'/tools/',
'/p/',
'/creator/',
'/creators',
'/collections',
'/characters',
'/image-prompts',
'/workflows',
'/leaderboard',
'/frameworks',
'/thunderdome',
'/pricing',
'/blog',
'/guides',
'/about',
'/forum',
'/categories',
'/category/',
'/extension',
'/docs',
'/llms.txt',
'/api/og',
],
disallow: [
'/api/',
'/dashboard',
'/settings',
'/create',
'/edit/',
'/sign-in',
'/sign-up',
'/handler/',
'/_next/',
'/admin/',
],
},
// Allow social bots to access OG images
{
userAgent: 'Twitterbot',
allow: ['/api/og'],
},
{
userAgent: 'facebookexternalhit',
allow: ['/api/og'],
},
{
userAgent: 'LinkedInBot',
allow: ['/api/og'],
},
// AI crawlers — allow full indexing for AI search results
{
userAgent: 'GPTBot',
allow: ['/', '/llms.txt'],
disallow: ['/api/', '/dashboard', '/settings', '/sign-in', '/sign-up'],
},
{
userAgent: 'Google-Extended',
allow: ['/', '/llms.txt'],
disallow: ['/api/', '/dashboard', '/settings', '/sign-in', '/sign-up'],
},
{
userAgent: 'anthropic-ai',
allow: ['/', '/llms.txt'],
disallow: ['/api/', '/dashboard', '/settings', '/sign-in', '/sign-up'],
},
{
userAgent: 'ClaudeBot',
allow: ['/', '/llms.txt'],
disallow: ['/api/', '/dashboard', '/settings', '/sign-in', '/sign-up'],
},
{
userAgent: 'PerplexityBot',
allow: ['/', '/llms.txt'],
disallow: ['/api/', '/dashboard', '/settings', '/sign-in', '/sign-up'],
},
{
userAgent: 'Applebot',
allow: ['/', '/llms.txt'],
disallow: ['/api/', '/dashboard', '/settings', '/sign-in', '/sign-up'],
},
{
userAgent: 'Bytespider',
allow: ['/', '/llms.txt'],
disallow: ['/api/', '/dashboard', '/settings', '/sign-in', '/sign-up'],
},
],
sitemap: `${BASE_URL}/sitemap.xml`,
host: BASE_URL,
}
}
|