# ==============================================================
# K Raheja Realty — robots.txt
# Website: https://www.krahejarealty.com
# Last updated: June 2026
# ==============================================================


# --------------------------------------------------------------
# STANDARD SEARCH ENGINE CRAWLERS
# Allow all search engines full access by default
# --------------------------------------------------------------
User-agent: *
Disallow: /cgi-bin/
Disallow: /brochure/interface-heights/compliance-certificate.pdf
Disallow: /brochure/interface-heights/environment-clerance-certificate.pdf
Disallow: /brochure/raheja-residency/compliance-certificate.pdf
Disallow: /brochure/raheja-residency/environment-clerance-certificate.pdf


# --------------------------------------------------------------
# AI CRAWLERS — EXPLICITLY ALLOWED
# These bots power ChatGPT, Claude, Perplexity, Google AI,
# and other AI answer engines. We explicitly welcome them
# so K Raheja Realty appears in AI-generated answers.
# --------------------------------------------------------------

# OpenAI / ChatGPT
User-agent: GPTBot
Allow: /

# Anthropic / Claude
User-agent: ClaudeBot
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Google AI (Gemini, AI Overviews, Search Generative Experience)
User-agent: Google-Extended
Allow: /

# Common Crawl (used to train many AI/LLM models)
User-agent: CCBot
Allow: /

# Meta AI
User-agent: FacebookBot
Allow: /

# Apple (used for Apple Intelligence / Siri)
User-agent: Applebot
Allow: /

# Microsoft / Bing Copilot
User-agent: bingbot
Allow: /

# Amazon Alexa
User-agent: Amazonbot
Allow: /


# --------------------------------------------------------------
# SITEMAP & LLM REFERENCE
# --------------------------------------------------------------
Sitemap: https://www.krahejarealty.com/sitemap.xml

# LLM reference file for AI crawlers (AEO)
# LLM-Reference: https://www.krahejarealty.com/llm.txt