# robots.txt file for full open access
# This setup allows all well-behaved search engines and AI bots to crawl the entire website.

User-agent: *
Allow: /
Disallow: /admin/
Disallow: /studio/
Disallow: /edit/

# Explicitly allow major search and AI crawlers
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

# AI model crawlers
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: PerplexityBot
Allow: /

# Sitemaps and LLM context
Sitemap: https://anurag.edu.in/sitemap.xml
LLMs: https://anurag.edu.in/llms.txt