# Robots.txt for Fenestra.app
# Allow all crawlers for public content, block sensitive areas

User-agent: *

# Allow public pages
Allow: /
Allow: /blog
Allow: /blog/*
Allow: /login
Allow: /pricing
Allow: /documentation
Allow: /documentation/*
Allow: /solutions
Allow: /solutions/*
Allow: /changelog
Allow: /about
Allow: /contact
Allow: /privacy
Allow: /termsandconditions
Allow: /cookiespolicy

# Block API routes
Disallow: /api/

# Block user-specific content
Disallow: /profile/
Disallow: /spaces/
Disallow: /shared/
Disallow: /add-credits/

# Block internal/legacy pages
Disallow: /slice-simulator/
Disallow: /blob/
Disallow: /rarefounders/

# Block Vercel internals (Next.js static assets must remain crawlable so Googlebot can render pages)
Disallow: /_vercel/

# Block auth and callback URLs (defensive)
Disallow: /auth/
Disallow: /callback/

# Block demo pages (defensive)
Disallow: /demo/
Disallow: /demo-2/

# Block sensitive query parameters
Disallow: /*?*token*
Disallow: /*?*api_key*
Disallow: /*?*session*

# Crawl delays for search engines
User-agent: Googlebot
Crawl-delay: 1

User-agent: Bingbot
Crawl-delay: 1

# Control aggressive SEO crawlers
User-agent: SemrushBot
Crawl-delay: 10
Disallow: /api/
Disallow: /profile/
Disallow: /spaces/

User-agent: AhrefsBot
Crawl-delay: 10
Disallow: /api/
Disallow: /profile/
Disallow: /spaces/

User-agent: MJ12bot
Crawl-delay: 10
Disallow: /api/
Disallow: /profile/
Disallow: /spaces/

# AI Search Engine Crawlers (Allow for GEO visibility)
# These bots power AI search engines like ChatGPT, Perplexity, Claude, etc.

User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /profile/
Disallow: /spaces/
Disallow: /shared/
Disallow: /add-credits/

User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /profile/
Disallow: /spaces/
Disallow: /shared/
Disallow: /add-credits/

User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /profile/
Disallow: /spaces/
Disallow: /shared/
Disallow: /add-credits/

User-agent: ClaudeBot
Allow: /
Disallow: /api/
Disallow: /profile/
Disallow: /spaces/
Disallow: /shared/
Disallow: /add-credits/

User-agent: anthropic-ai
Allow: /
Disallow: /api/
Disallow: /profile/
Disallow: /spaces/
Disallow: /shared/
Disallow: /add-credits/

User-agent: Google-Extended
Allow: /
Disallow: /api/
Disallow: /profile/
Disallow: /spaces/
Disallow: /shared/
Disallow: /add-credits/

User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /profile/
Disallow: /spaces/
Disallow: /shared/
Disallow: /add-credits/

User-agent: Bytespider
Allow: /
Disallow: /api/
Disallow: /profile/
Disallow: /spaces/
Disallow: /shared/
Disallow: /add-credits/

User-agent: Applebot-Extended
Allow: /
Disallow: /api/
Disallow: /profile/
Disallow: /spaces/
Disallow: /shared/
Disallow: /add-credits/

# Sitemap location
Sitemap: https://www.fenestra.app/sitemap.xml

# LLM context file (llms.txt spec)
# https://www.fenestra.app/llms.txt