# robots.txt for https://textgenerator.net — SEO + AI-friendly (2025-09-09)

Sitemap: https://textgenerator.net/sitemap.xml

# Core crawl policy: render assets, block private areas only
User-agent: *
Allow: /$
Allow: /assets/
Allow: /uploads/
Allow: /*.css$
Allow: /*.js$
Allow: /favicon.ico
Disallow: /admin/
Disallow: /cp/
Disallow: /dashboard/
Disallow: /login
Disallow: /register
Disallow: /cgi-bin/
Disallow: /api/
Disallow: /vendor/
Disallow: /storage/
# Note: don’t blanket-block URL parameters here; rely on canonicals/internal links.

# AI / research crawlers — explicitly allowed for discovery & citations
User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: CCBot
Allow: /

User-agent: Amazonbot
Allow: /