# Cloudflare AI Crawl Control prepends the Content Signals block and the
# training-crawler Disallow list (Amazonbot, Applebot-Extended, Bytespider,
# CCBot, ClaudeBot, CloudflareBrowserRenderingCrawler, Google-Extended, GPTBot,
# meta-externalagent). Do not restate those agents here - duplicate groups for
# the same user-agent resolve inconsistently across crawlers.
#
# This file covers only the agents Cloudflare does not emit: the user-prompted
# assistants, the AI search indexers, and the classic search engines.

User-agent: *
Allow: /
Disallow: /profile/
Disallow: /signup
Disallow: /login
Disallow: /reset-password
Disallow: /forgot-password
Disallow: /email-verification
Disallow: /auth/
Disallow: /widget/iframe
Disallow: /dev/
Disallow: /test/
Disallow: /staging/
Disallow: /go/

# AI assistants - user-prompted fetches, allowed for public content
User-agent: ChatGPT-User
Allow: /
Disallow: /profile/
Disallow: /signup
Disallow: /login
Disallow: /reset-password
Disallow: /forgot-password
Disallow: /email-verification
Disallow: /auth/
Disallow: /widget/iframe
Disallow: /dev/
Disallow: /test/
Disallow: /staging/
Disallow: /go/

User-agent: Claude-User
Allow: /
Disallow: /profile/
Disallow: /signup
Disallow: /login
Disallow: /reset-password
Disallow: /forgot-password
Disallow: /email-verification
Disallow: /auth/
Disallow: /widget/iframe
Disallow: /dev/
Disallow: /test/
Disallow: /staging/
Disallow: /go/

User-agent: Perplexity-User
Allow: /
Disallow: /profile/
Disallow: /signup
Disallow: /login
Disallow: /reset-password
Disallow: /forgot-password
Disallow: /email-verification
Disallow: /auth/
Disallow: /widget/iframe
Disallow: /dev/
Disallow: /test/
Disallow: /staging/
Disallow: /go/

User-agent: DuckAssistBot
Allow: /
Disallow: /profile/
Disallow: /signup
Disallow: /login
Disallow: /reset-password
Disallow: /forgot-password
Disallow: /email-verification
Disallow: /auth/
Disallow: /widget/iframe
Disallow: /dev/
Disallow: /test/
Disallow: /staging/
Disallow: /go/

User-agent: MistralAI-User
Allow: /
Disallow: /profile/
Disallow: /signup
Disallow: /login
Disallow: /reset-password
Disallow: /forgot-password
Disallow: /email-verification
Disallow: /auth/
Disallow: /widget/iframe
Disallow: /dev/
Disallow: /test/
Disallow: /staging/
Disallow: /go/

# AI search indexers - citation traffic, allowed for public content
User-agent: OAI-SearchBot
Allow: /
Disallow: /profile/
Disallow: /signup
Disallow: /login
Disallow: /reset-password
Disallow: /forgot-password
Disallow: /email-verification
Disallow: /auth/
Disallow: /widget/iframe
Disallow: /dev/
Disallow: /test/
Disallow: /staging/
Disallow: /go/

User-agent: Claude-SearchBot
Allow: /
Disallow: /profile/
Disallow: /signup
Disallow: /login
Disallow: /reset-password
Disallow: /forgot-password
Disallow: /email-verification
Disallow: /auth/
Disallow: /widget/iframe
Disallow: /dev/
Disallow: /test/
Disallow: /staging/
Disallow: /go/

User-agent: PerplexityBot
Allow: /
Disallow: /profile/
Disallow: /signup
Disallow: /login
Disallow: /reset-password
Disallow: /forgot-password
Disallow: /email-verification
Disallow: /auth/
Disallow: /widget/iframe
Disallow: /dev/
Disallow: /test/
Disallow: /staging/
Disallow: /go/

# Search engines
User-agent: Googlebot
Allow: /
Disallow: /go/

User-agent: Bingbot
Allow: /
Disallow: /go/

User-agent: Applebot
Allow: /
Disallow: /go/

Sitemap: https://wirelesschoices.com.au/sitemap.xml
Sitemap: https://wirelesschoices.com.au/sitemap-outages.xml
Sitemap: https://wirelesschoices.com.au/sitemap-plans.xml
Sitemap: https://wirelesschoices.com.au/sitemap-providers.xml
Sitemap: https://wirelesschoices.com.au/sitemap-energy.xml
Sitemap: https://wirelesschoices.com.au/sitemap-fuel.xml
Sitemap: https://wirelesschoices.com.au/sitemap-blog.xml
Sitemap: https://wirelesschoices.com.au/sitemap-devices.xml