User-agent: *
Sitemap: http://agiloncables.com/sitemap.xml


##############
#   custom   #
##############

# ==============================================================
#  robots.txt — Agilon Cables India Private Limited
#  Website : https://agiloncables.com
#  Updated : 2026-06-23
# ==============================================================


# --------------------------------------------------------------
#  SECTION 1 — DEFAULT RULE FOR ALL CRAWLERS
#  Allow full public access; block only sensitive/private paths.
# --------------------------------------------------------------

User-agent: *
Allow: /

# Admin & backend
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-json/
Disallow: /xmlrpc.php

# E-commerce & account flows
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /account/
Disallow: /login/
Disallow: /register/
Disallow: /logout/

# Temporary, duplicate & internal paths
Disallow: /private/
Disallow: /temp/
Disallow: /tmp/
Disallow: /cache/
Disallow: /cgi-bin/
Disallow: /includes/
Disallow: /vendor/
Disallow: /node_modules/
Disallow: /uploads/private/

# Raw data & API endpoints
Disallow: /api/
Disallow: /*.json$
Disallow: /*.xml$
Disallow: /*?preview=true
Disallow: /*?s=
Disallow: /*?replytocom=

Crawl-delay: 10


# --------------------------------------------------------------
#  SECTION 2 — MAJOR SEARCH ENGINES (faster crawl allowed)
# --------------------------------------------------------------

User-agent: Googlebot
Allow: /
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Crawl-delay: 2

User-agent: Googlebot-Image
Allow: /
Crawl-delay: 5

User-agent: Googlebot-News
Allow: /blog/
Allow: /press-releases/
Allow: /events/
Crawl-delay: 5

User-agent: Bingbot
Allow: /
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /cart/
Disallow: /checkout/
Crawl-delay: 3

User-agent: DuckDuckBot
Allow: /
Crawl-delay: 5

User-agent: YandexBot
Allow: /
Crawl-delay: 10

User-agent: Baiduspider
Allow: /
Crawl-delay: 10

User-agent: Slurp
Allow: /
Crawl-delay: 10


# --------------------------------------------------------------
#  SECTION 3 — LLM & AI CRAWLERS (explicitly allowed)
#
#  Agilon Cables welcomes AI crawlers for indexing, training,
#  retrieval-augmented generation (RAG), and AI assistant use.
#  See /llms.txt for structured product and contact information.
# --------------------------------------------------------------

# ------- OpenAI / ChatGPT-------------------------------
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# ----- Anthropic / Claude -----------------------------------
User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

# --- Google Gemini / Bard / Vertex AI -----------------------
User-agent: Google-Extended
Allow: /

# -- Meta AI / LLaMA -----------------------------------------
User-agent: FacebookBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

# --- Amazon Alexa / Bedrock / Titan ------------------------
User-agent: Amazonbot
Allow: /

# --- Apple Intelligence / Siri -----------------------------
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# --- Perplexity AI -----------------------------------------
User-agent: PerplexityBot
Allow: /

# ---- Cohere -----------------------------------------------
User-agent: cohere-ai
Allow: /

# --- Mistral AI --------------------------------------------
User-agent: MistralBot
Allow: /

# --- xAI / Grok --------------------------------------------
User-agent: xAI-Bot
Allow: /

# --- ByteDance / Doubao / Coze -----------------------------
User-agent: Bytespider
Allow: /

# --- Common Crawl (used by many open-source LLMs) ----------
User-agent: CCBot
Allow: /

# --- Diffbot (structured AI data extraction) ---------------
User-agent: Diffbot
Allow: /

# --- You.com -----------------------------------------------
User-agent: YouBot
Allow: /

# --- Brave Search / Leo AI ---------------------------------
User-agent: BraveBot
Allow: /

# --- Timpi (AI-native search) ------------------------------
User-agent: Timpibot
Allow: /

# --- DataForSeo (AI content intelligence) ------------------
User-agent: DataForSeoBot
Allow: /

# --- Webz.io / Omgili (news & data LLM feeds) --------------
User-agent: omgili
Allow: /

User-agent: omgilibot
Allow: /


# --------------------------------------------------------------
#  SECTION 4 — SECURITY & SPAM BOTS (blocked)
# --------------------------------------------------------------

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: 360Spider
Disallow: /

User-agent: ia_archiver
Disallow: /


# --------------------------------------------------------------
#  SECTION 5 — SITEMAP & LLM MANIFEST
# --------------------------------------------------------------

Sitemap: https://agiloncables.com/sitemap.xml

# AI-readable structured content (llms.txt standard):
# https://agiloncables.com/llms.txt