# =========================================================
#      1) BONS BOTS : autorisation générale du site,
#         sauf les parties sensibles de Magento, 
#         paramétrages et URLs spécifiques à bloquer
# =========================================================

User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-News
User-agent: Googlebot-Video
User-agent: Google-InspectionTool
User-agent: Bingbot
User-agent: BingPreview
User-agent: Slurp
User-agent: DuckDuckBot
User-agent: Baiduspider
User-agent: YandexBot
User-agent: Applebot
User-agent: Facebookbot
User-agent: Twitterbot
User-agent: Pinterestbot
User-agent: LinkedInBot
User-agent: WhatsApp
User-agent: TelegramBot
User-agent: Discordbot
User-agent: Redditbot
User-agent: PetalBot
User-agent: CCBot
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: PerplexityBot
User-agent: MojeekBot
User-agent: SistrixBot
User-agent: AhrefsSiteAudit
User-agent: SiteAuditBot

# -- En général, on “autorise” la racine
Allow: /

# ---------------------------------------------------------
#    BLOQUER LES RÉPERTOIRES SENSIBLES DE MAGENTO
# ---------------------------------------------------------
Disallow: /404/
Disallow: /app/
Disallow: /cgi-bin/
Disallow: /downloader/
Disallow: /errors/
Disallow: /includes/
Disallow: /lib/
Disallow: /magento/
Disallow: /pkginfo/
Disallow: /report/
Disallow: /scripts/
Disallow: /shell/
Disallow: /stats/
Disallow: /var/

# ---------------------------------------------------------
#    BLOQUER CERTAINES PAGES / FONCTIONNALITÉS
#    (versions propres des URL : /catalogsearch/, /checkout/, etc.)
# ---------------------------------------------------------
Disallow: */catalogsearch/result/
Disallow: */catalog/product_compare/
Disallow: */catalog/category/view/
Disallow: */catalog/product/view/
Disallow: */catalogsearch/
Disallow: */checkout/
Disallow: */control/
Disallow: */contacts/
Disallow: */customer/
Disallow: */customize/
Disallow: */newsletter/
Disallow: */poll/
Disallow: */review/
Disallow: */sales/
Disallow: */maillog/
Disallow: */sendfriend/
Disallow: */tag/
Disallow: */wishlist/
Disallow: */kyrena/
Disallow: */downloadable/
Disallow: */hipay/
Disallow: */payzen/
Disallow: */paypal/
Disallow: */shippingmax/
Disallow: */rewards/
Disallow: */onestepcheckout/
Disallow: */rss/
Disallow: */zendesk/
Disallow: */pslogin/
Disallow: */urlcheckout/

# ---------------------------------------------------------
#    BLOQUER CERTAINS FICHIERS SENSIBLES (cron, install…)
# ---------------------------------------------------------
Disallow: /cron.php
Disallow: /cron.sh
Disallow: /error_log
Disallow: /index.php
Disallow: /install.php
Disallow: /LICENSE.html
Disallow: /LICENSE.txt
Disallow: /LICENSE_AFL.txt
Disallow: /STATUS.txt
Disallow: /vimeo.php

# ---------------------------------------------------------
#    BLOQUER VIEUX FICHIERS PDF DÉJÀ INDEXÉS
# ---------------------------------------------------------

Disallow: /cb-guide.pdf
Disallow: /cb-guide-cz.pdf
Disallow: /cb-guide-pl.pdf
Disallow: /cb-guide-de.pdf

# ---------------------------------------------------------
#    BLOQUER LES URLS AVEC PARAMÈTRES SPÉCIFIQUES (utm_, coupon…)
# ---------------------------------------------------------
Disallow: /.php
Disallow: /?p=&
Disallow: /?SID=
Disallow: /?limit=all
Disallow: /?___from_store=
Disallow: /?koupon=
Disallow: /&koupon=
Disallow: /?coupon=
Disallow: /&coupon=
Disallow: /?utm_source=
Disallow: /*&utm_source=

# =========================================================
#      2) MAUVAIS BOTS CONNUS : blocage total
# =========================================================

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: dotbot
Disallow: /

User-agent: BlexBot
Disallow: /

User-agent: MauiBot
Disallow: /

User-agent: SEOkicks-Robot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Sogou
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: AspiegelBot
Disallow: /

User-agent: Exabot
Disallow: /

User-agent: VoilaBot
Disallow: /

User-agent: spbot
Disallow: /

User-agent: Cliqzbot
Disallow: /

User-agent: Cocolyzebot
Disallow: /

User-agent: Yeti
Disallow: /

User-agent: TurnitinBot
Disallow: /

User-agent: magpie-crawler
Disallow: /

User-agent: Scrapy
Disallow: /

User-agent: curl
Disallow: /

User-agent: wget
Disallow: /

User-agent: python-requests
Disallow: /

User-agent: python-urllib
Disallow: /

User-agent: Java
Disallow: /

# =========================================================
#      3) TOUS LES AUTRES BOTS (INCONNUS) : blocage total
# =========================================================
User-agent: *
Disallow: /