User-agent: *
Disallow: /dev/
Disallow: /api/component-usage

# Google Search
User-agent: Googlebot
Allow: /

# Microsoft Bing (powers Copilot)
User-agent: Bingbot
Allow: /

# OpenAI (ChatGPT, GPT-4)
User-agent: GPTBot
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Anthropic Claude
User-agent: Claude-Web
Allow: /

# Google's LLM crawler (Bard/Gemini)
User-agent: Google-Extended
Allow: /

# Meta AI (Facebook/Instagram AI)
User-agent: FacebookBot
Allow: /

# Apple Intelligence
User-agent: Applebot
Allow: /

# Common Crawl (used by many AI companies)
User-agent: CCBot
Allow: /

Sitemap: https://www-qwt6.onrender.com/sitemap/index.xml