# ================================
# Robots.txt for AI/LLM Crawlers
# Updated: 2025-09-18
# ================================

# --- OpenAI ---
User-agent: GPTBot
Disallow: /private/

User-agent: ChatGPT-User
Disallow: /private/

# --- Anthropic ---
User-agent: ClaudeBot
Disallow: /private/

# --- Google AI (Gemini / Bard) ---
User-agent: Google-Extended
Disallow: /private/

# --- Perplexity ---
User-agent: PerplexityBot
Disallow: /private/

# --- Common Crawl (used by many LLMs) ---
User-agent: CCBot
Disallow: /private/

# --- Scale AI ---
User-agent: ScaleSerpBot
Disallow: /private/

# --- Facebook / Meta (LLaMA training) ---
User-agent: FacebookBot
Disallow: /private/

User-agent: MetaAI
Disallow: /private/

# --- Amazon (training + Alexa AI) ---
User-agent: Amazonbot
Disallow: /private/

# --- Apple (Applebot + AI) ---
User-agent: Applebot
Disallow: /private/

# --- Microsoft (Copilot / Bing) ---
User-agent: Bingbot
Disallow: /private/

User-agent: BingPreview
Disallow: /private/

# --- You.com (AI search) ---
User-agent: YouBot
Disallow: /private/

# --- DuckDuckGo (AI training) ---
User-agent: DuckDuckBot
Disallow: /private/

# --- Neeva AI (search + LLM, now acquired by Snowflake) ---
User-agent: NeevaBot
Disallow: /private/

# --- Other AI/LLM-related crawlers ---
User-agent: cohere-ai
Disallow: /private/

User-agent: ai21
Disallow: /private/

User-agent: WriterBot
Disallow: /private/

User-agent: JasperBot
Disallow: /private/


User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

Disallow: */feed/

#below is added by dushyant 
User-agent: *
Disallow: /*?filter_
Disallow: /*?stock_status=
Disallow: /*?sale_status=
Disallow: /*?orderby=
Disallow: /*?min_price=
Disallow: /*?max_price=

Sitemap: https://mycareindia.co.in/sitemap.xml