# robots.txt – koeder-laden.de
# Stand: 2026-09-22
# Policy: Suchmaschinen + KI-Systeme (Suche, Antworten, Training) erlaubt,
#         Session-/Warenkorb-/Konto-/Filter-URLs gesperrt,
#         SEO-Tools und aggressive Scraper komplett gesperrt.

# ── 1. Standard für alle Crawler ─────────────────────────────────

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /admin/
Disallow: /Core/
Disallow: /tmp/
Disallow: /views/
Disallow: /Setup/
Disallow: /log/
Disallow: /EXCEPTION_LOG.txt
#
Disallow: /newsletter/
Disallow: /en/newsletter/
Disallow: /index.php?cl=newsletter
#
Disallow: /AGB/
Disallow: /en/Terms-and-Conditions/
#
Disallow: /warenkorb/
Disallow: /en/cart/
Disallow: /index.php?cl=basket
#
Disallow: /mein-konto/
Disallow: /en/my-account/
Disallow: /index.php?cl=account
#
Disallow: /mein-merkzettel/
Disallow: /en/my-wishlist/
Disallow: /index.php?cl=account_noticelist
#
Disallow: /mein-wunschzettel/
Disallow: /en/my-gift-registry/
Disallow: /index.php?cl=account_wishlist
#
Disallow: /konto-eroeffnen/
Disallow: /en/open-account/
Disallow: /index.php?cl=register
#
Disallow: /passwort-vergessen/
Disallow: /en/forgot-password/
Disallow: /index.php?cl=forgotpwd
#
Disallow: /index.php?cl=moredetails
Disallow: /index.php?cl=review
Disallow: /index.php?cl=search
#
# Wildcards am Ende, weil manche Crawler sie sonst als Fehler werten
Disallow: /*?cl=newsletter
Disallow: /*&cl=newsletter
Disallow: /*?cl=basket
Disallow: /*&cl=basket
Disallow: /*?cl=account
Disallow: /*&cl=account
Disallow: /*?cl=register
Disallow: /*&cl=register
Disallow: /*?cl=forgotpwd
Disallow: /*&cl=forgotpwd
Disallow: /*?cl=moredetails
Disallow: /*&cl=moredetails
Disallow: /*?cl=review
Disallow: /*&cl=review
Disallow: /*?cl=search
Disallow: /*&cl=search
#
Disallow: /*&fnc=tobasket
Disallow: /*&fnc=tocomparelist
Disallow: /*&addcompare=
#
# Session-IDs und Währungsparameter nie crawlen (löst force_sid-Redirect-Problem)
Disallow: /*/sid/
Disallow: /*?sid=
Disallow: /*&sid=
Disallow: /*?force_sid=
Disallow: /*&force_sid=
Disallow: /*?cur=
Disallow: /*&cur=

Disallow: /*cl=oetagstagcontroller

# ── 2. KI-Crawler: Suche, Nutzeranfragen und Training erlaubt ────
# Eigene Gruppe, weil explizite AI-Einträge gefordert sind.
# Achtung: Diese Gruppe ERSETZT für die genannten Bots die *-Gruppe,
# deshalb stehen alle relevanten Sperren hier noch einmal.
# Kein Crawl-delay: wird von OpenAI/Google ignoriert, und Such-/User-Bots
# sollen aktuelle Preise und Verfügbarkeit ohne Verzögerung abrufen.
# pgNr= (Kategorie-Paginierung) ist bewusst ERLAUBT.

# Amazon
User-agent: Amzn-SearchBot
User-agent: Amzn-User
# OpenAI
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
# Anthropic
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: Claude-Web
User-agent: anthropic-ai
# Perplexity
User-agent: PerplexityBot
User-agent: Perplexity-User
# Google (Gemini / Vertex AI – Steuerungs-Token, kein eigener Crawler)
User-agent: Google-Extended
# Apple (Siri/Spotlight + Apple Intelligence)
User-agent: Applebot
User-agent: Applebot-Extended
# Common Crawl (Trainingsdatenbasis vieler Modelle)
User-agent: CCBot
# Weitere KI-Assistenten
User-agent: DuckAssistBot
User-agent: MistralAI-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /admin/
Disallow: /Core/
Disallow: /tmp/
Disallow: /views/
Disallow: /Setup/
Disallow: /log/
Disallow: /EXCEPTION_LOG.txt
#
Disallow: /warenkorb/
Disallow: /en/cart/
Disallow: /mein-konto/
Disallow: /en/my-account/
Disallow: /mein-merkzettel/
Disallow: /en/my-wishlist/
Disallow: /mein-wunschzettel/
Disallow: /en/my-gift-registry/
Disallow: /konto-eroeffnen/
Disallow: /en/open-account/
Disallow: /passwort-vergessen/
Disallow: /en/forgot-password/
Disallow: /newsletter/
Disallow: /en/newsletter/
#
Disallow: /*cl=basket
Disallow: /*cl=account
Disallow: /*cl=register
Disallow: /*cl=forgotpwd
Disallow: /*cl=newsletter
Disallow: /*cl=search
Disallow: /*cl=moredetails
Disallow: /*cl=review
Disallow: /*fnc=tobasket
Disallow: /*fnc=tocomparelist
Disallow: /*addcompare=
#
# Filter-/Sortier-Varianten (Duplicate Content, hohe Last)
Disallow: /*attrfilter
Disallow: /*fnc=executefilter
Disallow: /*listorderby=
Disallow: /*listorder=
Disallow: /*_artperpage=
Disallow: /*ldtype=
Disallow: /*searchparam=
#
Disallow: /*/sid/
Disallow: /*?sid=
Disallow: /*&sid=
Disallow: /*?force_sid=
Disallow: /*&force_sid=
Disallow: /*?cur=
Disallow: /*&cur=

Disallow: /*cl=oetagstagcontroller

User-agent: ClaudeBot
Crawl-delay: 2
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /admin/
Disallow: /Core/
Disallow: /tmp/
Disallow: /views/
Disallow: /Setup/
Disallow: /log/
Disallow: /EXCEPTION_LOG.txt
#
Disallow: /warenkorb/
Disallow: /en/cart/
Disallow: /mein-konto/
Disallow: /en/my-account/
Disallow: /mein-merkzettel/
Disallow: /en/my-wishlist/
Disallow: /mein-wunschzettel/
Disallow: /en/my-gift-registry/
Disallow: /konto-eroeffnen/
Disallow: /en/open-account/
Disallow: /passwort-vergessen/
Disallow: /en/forgot-password/
Disallow: /newsletter/
Disallow: /en/newsletter/
#
Disallow: /*cl=basket
Disallow: /*cl=account
Disallow: /*cl=register
Disallow: /*cl=forgotpwd
Disallow: /*cl=newsletter
Disallow: /*cl=search
Disallow: /*cl=moredetails
Disallow: /*cl=review
Disallow: /*fnc=tobasket
Disallow: /*fnc=tocomparelist
Disallow: /*addcompare=
#
# Filter-/Sortier-Varianten (Duplicate Content, hohe Last)
Disallow: /*attrfilter
Disallow: /*fnc=executefilter
Disallow: /*listorderby=
Disallow: /*listorder=
Disallow: /*_artperpage=
Disallow: /*ldtype=
Disallow: /*searchparam=
#
Disallow: /*/sid/
Disallow: /*?sid=
Disallow: /*&sid=
Disallow: /*?force_sid=
Disallow: /*&force_sid=
Disallow: /*?cur=
Disallow: /*&cur=

Disallow: /*cl=oetagstagcontroller


User-agent: GPTBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /admin/
Disallow: /Core/
Disallow: /tmp/
Disallow: /views/
Disallow: /Setup/
Disallow: /log/
Disallow: /EXCEPTION_LOG.txt
#
Disallow: /warenkorb/
Disallow: /en/cart/
Disallow: /mein-konto/
Disallow: /en/my-account/
Disallow: /mein-merkzettel/
Disallow: /en/my-wishlist/
Disallow: /mein-wunschzettel/
Disallow: /en/my-gift-registry/
Disallow: /konto-eroeffnen/
Disallow: /en/open-account/
Disallow: /passwort-vergessen/
Disallow: /en/forgot-password/
Disallow: /newsletter/
Disallow: /en/newsletter/
#
Disallow: /*cl=basket
Disallow: /*cl=account
Disallow: /*cl=register
Disallow: /*cl=forgotpwd
Disallow: /*cl=newsletter
Disallow: /*cl=search
Disallow: /*cl=moredetails
Disallow: /*cl=review
Disallow: /*fnc=tobasket
Disallow: /*fnc=tocomparelist
Disallow: /*addcompare=
#
# Filter-/Sortier-Varianten (Duplicate Content, hohe Last)
Disallow: /*attrfilter
Disallow: /*fnc=executefilter
Disallow: /*listorderby=
Disallow: /*listorder=
Disallow: /*_artperpage=
Disallow: /*ldtype=
Disallow: /*searchparam=
#
Disallow: /*/sid/
Disallow: /*?sid=
Disallow: /*&sid=
Disallow: /*?force_sid=
Disallow: /*&force_sid=
Disallow: /*?cur=
Disallow: /*&cur=
Disallow: /*searchtag=

Disallow: /*cl=oetagstagcontroller

# ── 3. SEO-Tools: gedrosselt ─────────────────────────────────────

User-agent: AhrefsBot
Crawl-delay: 30
Disallow: /admin/
Disallow: /warenkorb/
Disallow: /en/cart/
Disallow: /mein-konto/
Disallow: /en/my-account/
Disallow: /*cl=basket
Disallow: /*cl=account
Disallow: /*cl=search
Disallow: /*cl=register
Disallow: /*cl=forgotpwd
Disallow: /*cl=moredetails
Disallow: /*cl=review
Disallow: /*fnc=tobasket
Disallow: /*fnc=tocomparelist
Disallow: /*addcompare=
Disallow: /*attrfilter
Disallow: /*fnc=executefilter
Disallow: /*listorderby=
Disallow: /*listorder=
Disallow: /*_artperpage=
Disallow: /*ldtype=
Disallow: /*pgNr=
Disallow: /*searchparam=
Disallow: /*/sid/
Disallow: /*?sid=
Disallow: /*&sid=
Disallow: /*?force_sid=
Disallow: /*&force_sid=
Disallow: /*?cur=
Disallow: /*&cur=

Disallow: /*cl=oetagstagcontroller

User-agent: IbouBot
User-agent: AwarioBot
Crawl-delay: 60
Disallow: /admin/
Disallow: /warenkorb/
Disallow: /en/cart/
Disallow: /mein-konto/
Disallow: /en/my-account/
Disallow: /*cl=basket
Disallow: /*cl=account
Disallow: /*cl=search
Disallow: /*cl=register
Disallow: /*cl=forgotpwd
Disallow: /*cl=moredetails
Disallow: /*cl=review
Disallow: /*fnc=tobasket
Disallow: /*fnc=tocomparelist
Disallow: /*addcompare=
Disallow: /*attrfilter
Disallow: /*fnc=executefilter
Disallow: /*listorderby=
Disallow: /*listorder=
Disallow: /*_artperpage=
Disallow: /*ldtype=
Disallow: /*pgNr=
Disallow: /*searchparam=
Disallow: /*/sid/
Disallow: /*?sid=
Disallow: /*&sid=
Disallow: /*?force_sid=
Disallow: /*&force_sid=
Disallow: /*?cur=
Disallow: /*&cur=

Disallow: /*cl=oetagstagcontroller

# ── 4. Komplett gesperrt ─────────────────────────────────────────
# meta-externalagent: ~4 Mio. Requests/30 Tage, zusätzlich per Cloudflare geblockt
# (Link-Vorschauen laufen über facebookexternalhit und bleiben erlaubt)

User-agent: SemrushBot
User-agent: DataForSeoBot
User-agent: DotBot
User-agent: PetalBot
User-agent: Bytedance
User-agent: Bytespider
User-agent: meta-externalagent
User-agent: meta-webindexer
# Amazon (Alexa / Rufus)
User-agent: Amazonbot
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

# ── Sitemaps ─────────────────────────────────────────────────────

Sitemap: https://www.koeder-laden.de/sitemap_de.xml
Sitemap: https://www.koeder-laden.de/sitemap_en.xml