# infotrust.com robots.txt
# Source of truth: InfoTrust Website repo, robots/robots.txt. Deploy to the web root. Staging first.

User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /?s=
Disallow: /search/
Disallow: /*?replytocom=
Disallow: /*/feed/
Disallow: /wp-content/uploads/*.mp4$
Disallow: /wp-content/uploads/*.mov$

# AI crawlers: allowed on purpose (GEO), so InfoTrust content can be cited by AI answer engines and
# is represented in AI models. A crawler that matches its own group ignores the "*" group, so the
# same exclusions are repeated here. To opt out of AI training only, move the "training" user agents
# into a group with "Disallow: /".
# Answer and search engines (fetch pages to answer questions and cite sources):
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Claude-SearchBot
User-agent: Claude-User
# Model training:
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: CCBot
User-agent: meta-externalagent
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /?s=
Disallow: /search/
Disallow: /*?replytocom=
Disallow: /*/feed/
Disallow: /wp-content/uploads/*.mp4$
Disallow: /wp-content/uploads/*.mov$

# Crawlers with no search value for InfoTrust's markets that crawl aggressively.
User-agent: Sogou web spider
User-agent: Sogou inst spider
User-agent: Bytespider
User-agent: PetalBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: BLEXBot
User-agent: DataForSeoBot
User-agent: serpstatbot
Disallow: /

Sitemap: https://infotrust.com/sitemap.xml