# robots.txt for https://www.saasu.com
#
# Policy:
#   - Search engines, AI search crawlers and user-triggered AI agents: allowed
#   - AI training crawlers and some SEO scrapers: blocked
#
# A crawler obeys only the ONE most specific group below. Rules in the "*" group
# do not carry over to a named group, so the shared WordPress rules are repeated
# in every group that allows crawling.

# --- 1. AI training crawlers and scrapers: blocked ---
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: CCBot
User-agent: Applebot-Extended
User-agent: meta-externalagent
User-agent: Bytespider
User-agent: Diffbot
User-agent: DotBot
User-agent: BLEXBot
Disallow: /

# --- 2. SEO tool crawlers: allowed, but slowed down ---
User-agent: AhrefsBot
User-agent: MJ12bot
Crawl-delay: 10
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /xmlrpc.php
Disallow: /cgi-bin/
Disallow: /trackback/
Disallow: /feed/
Disallow: /*/feed/
Disallow: /comments/feed/
Disallow: /*?s=
Disallow: /*&s=
Disallow: /*?replytocom=
Disallow: /*&replytocom=
Disallow: /*?preview=
Disallow: /*&preview=

# --- 3. Search engines, AI search crawlers and user-triggered agents: allowed ---
User-agent: Googlebot
User-agent: Bingbot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: FacebookBot
User-agent: LinkedInBot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /xmlrpc.php
Disallow: /cgi-bin/
Disallow: /trackback/
Disallow: /feed/
Disallow: /*/feed/
Disallow: /comments/feed/
Disallow: /*?s=
Disallow: /*&s=
Disallow: /*?replytocom=
Disallow: /*&replytocom=
Disallow: /*?preview=
Disallow: /*&preview=

# --- 4. Google-Extended is a control token for Gemini, not a crawler ---
User-agent: Google-Extended
Allow: /

# --- 5. Everyone else ---
User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /xmlrpc.php
Disallow: /cgi-bin/
Disallow: /trackback/
Disallow: /feed/
Disallow: /*/feed/
Disallow: /comments/feed/
Disallow: /*?s=
Disallow: /*&s=
Disallow: /*?replytocom=
Disallow: /*&replytocom=
Disallow: /*?preview=
Disallow: /*&preview=

Sitemap: https://www.saasu.com/sitemap_index.xml