# cleanfoodcertified.org — updated 2026-08-13
# Search engines (Google, Bing, DuckDuckGo) remain fully allowed.

User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-cron.php
Disallow: /?s=
Disallow: /search/

# AI training / dataset crawlers. Remove any line here you WANT to be
# included in that product's index or training data.
User-agent: GPTBot
Disallow: /

User-agent: OAI-SearchBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: meta-externalagent
Disallow: /

# Commercial SEO backlink crawlers — no benefit to this site, high bandwidth cost.
# These are also hard-blocked at the web server; see cleanfoodcertified.conf.
User-agent: SERankingBacklinksBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /