# 3DCarParts.org robots.txt
#
# Policy:
#   • Legitimate search engines (Googlebot, Bingbot, DuckDuckBot, Yandex, etc.)
#     are welcome and are governed by the `User-agent: *` group below — this is
#     how we keep our organic search ranking. They are deliberately NOT given
#     their own groups: a crawler obeys only its single most-specific matching
#     group, so a per-bot group would silently exempt that bot from every
#     Disallow in the `*` group.
#   • AI-training / LLM data crawlers and aggressive commercial scrapers are
#     blocked in full via dedicated `Disallow: /` groups. A whole-site block is
#     the one case where a per-bot group is correct — there are no shared rules
#     for those bots to inherit, so nothing is lost by isolating them.
#
# robots.txt is advisory, NOT a security control. It tells well-behaved bots to
# stay out and records our stated no-scraping / no-AI-training position (see our
# Terms of Service, "Automated Access, Scraping & AI Training"). Bots that ignore
# it are unauthorized under those Terms and subject to IP blocking and legal
# remedies. /api and /admin are protected by server-side auth + Supabase RLS
# regardless of anything in this file.

User-agent: *
Allow: /
Allow: /feeds/
Allow: /catalog/
Allow: /.well-known/ucp
Allow: /ucp/
Allow: /checkout_sessions
Allow: /checkout
Allow: /make-part
Disallow: /admin
Disallow: /admin-panel
Disallow: /aggregation-console
Disallow: /api/
Disallow: /account
Disallow: /auth/
Disallow: /edit-profile
Disallow: /email-preview
Disallow: /my-purchases
Disallow: /orders
Disallow: /part-review
Disallow: /security
Disallow: /test-
Disallow: /*.json$
Crawl-delay: 1

# ---------------------------------------------------------------------------
# AI training / LLM data-collection crawlers — full block.
# Blocking these does NOT affect search ranking: the "-Extended" variants
# (Google-Extended, Applebot-Extended) gate AI training only and are separate
# from the search crawlers (Googlebot, Applebot) which remain allowed above.
# ---------------------------------------------------------------------------
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: Google-Extended
User-agent: anthropic-ai
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: CCBot
User-agent: Bytespider
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Amazonbot
User-agent: Applebot-Extended
User-agent: Meta-ExternalAgent
User-agent: meta-externalagent
User-agent: FacebookBot
User-agent: Diffbot
User-agent: Omgilibot
User-agent: Omgili
User-agent: ImagesiftBot
User-agent: YouBot
User-agent: AI2Bot
User-agent: cohere-ai
User-agent: Timpibot
User-agent: Webzio-Extended
User-agent: PanguBot
User-agent: DuckAssistBot
Disallow: /

# ---------------------------------------------------------------------------
# Aggressive commercial scrapers / SEO data miners — full block.
# These crawl heavily to resell catalog, backlink, and content data.
# ---------------------------------------------------------------------------
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: DataForSeoBot
User-agent: BLEXBot
User-agent: PetalBot
User-agent: rogerbot
User-agent: Barkrowler
User-agent: MegaIndex
User-agent: SerpstatBot
User-agent: magpie-crawler
User-agent: Scrapy
User-agent: python-requests
Disallow: /

Sitemap: https://3dcarparts.org/sitemap.xml