# ============================================================
# robots.txt for ergospace.ae
# Last updated: 30 April 2026
# ============================================================

# ------------------------------------------------------------
# Cloudflare Content Signals (2026 proposed standard)
# Declares your intent for how AI may use your content.
# Honored by Anthropic, OpenAI, Perplexity, Google.
# ------------------------------------------------------------


# ------------------------------------------------------------
# Global rules — apply to every crawler unless overridden
# ------------------------------------------------------------
User-agent: *

# Account & checkout (private, no SEO/AI value)
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /register
Disallow: /account/
Disallow: /my-account/
Disallow: /password-reset
Disallow: /forgot-password
Disallow: /cart
Disallow: /cart/
Disallow: /checkout
Disallow: /checkout/
Disallow: /order/
Disallow: /order-confirmation
Disallow: /payment/
Disallow: /thank-you

# Internal site search
Disallow: /search
Disallow: /*?q=
Disallow: /*?search=

# Filter / faceted nav (protects crawl budget on 5000+ SKUs)
Disallow: /*?color=
Disallow: /*?size=
Disallow: /*?material=
Disallow: /*?finish=
Disallow: /*?frame=
Disallow: /*?handle=
Disallow: /*?pedestal=
Disallow: /*?wheel=
Disallow: /*?shelf=
Disallow: /*?tabletop=
Disallow: /*?socket=
Disallow: /*?option=
Disallow: /*?price_min=
Disallow: /*?price_max=
Disallow: /*?sort=
Disallow: /*?orderby=
Disallow: /*?per_page=

# Tracking parameters (prevent duplicate URLs from ad campaigns)
Disallow: /*?utm_
Disallow: /*?gclid=
Disallow: /*?fbclid=
Disallow: /*?msclkid=

# System / staging
Disallow: /admin/
Disallow: /staging/
Disallow: /dev/
Disallow: /test/
Disallow: /private/

# Allow render-essential assets (CRITICAL for AI page comprehension)
Allow: /*.css$
Allow: /*.js$
Allow: /*.svg$
Allow: /*.webp$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.gif$
Allow: /*.woff$
Allow: /*.woff2$

# ------------------------------------------------------------
# AI Bots — ALL ALLOWED for maximum visibility in
# ChatGPT, Claude, Perplexity, Gemini, Copilot, Apple, etc.
# (Listed explicitly to signal intent + cover all 3 tiers:
#  training / search-index / user-fetcher)
# ------------------------------------------------------------

# OpenAI (training, search, user-fetch)
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
Allow: /

# Anthropic (training, search, user-fetch — anthropic-ai is deprecated)
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
Allow: /

# Google AI (Gemini training + AI Overviews + Vertex)
User-agent: Google-Extended
User-agent: Google-CloudVertexBot
Allow: /

# Perplexity (index + user-fetch)
User-agent: PerplexityBot
User-agent: Perplexity-User
Allow: /

# Apple Intelligence
User-agent: Applebot
User-agent: Applebot-Extended
Allow: /

# Microsoft / Bing AI
User-agent: Bingbot
User-agent: BingPreview
Allow: /

# Meta AI (Llama training)
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: FacebookBot
Allow: /

# Other major AI engines
User-agent: Amazonbot
User-agent: DuckAssistBot
User-agent: MistralAI-User
User-agent: cohere-ai
User-agent: CCBot
Allow: /

# ------------------------------------------------------------
# SEO / Scraping tools — throttled (10s between requests)
# Allows self-monitoring tools to work, prevents server abuse
# ------------------------------------------------------------

User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Crawl-delay: 30

User-agent: DotBot
Crawl-delay: 30

User-agent: BLEXBot
Crawl-delay: 30

User-agent: PetalBot
Crawl-delay: 30

User-agent: SeznamBot
Crawl-delay: 30

User-agent: DataForSeoBot
Crawl-delay: 30

# ------------------------------------------------------------
# Sitemaps
# ------------------------------------------------------------
Sitemap: https://westernfurniture.ae/sitemap.xml