# TaxReclaim - robots.txt
# https://taxreclaim.eu

# Default rules for all crawlers
User-agent: *
Allow: /
Allow: /contact
Allow: /popular-dividend-shares
Allow: /popular-wht-funds
Allow: /supported-forms
Allow: /ticker-directory
Allow: /tax-guide
Allow: /tax-glossary
Allow: /forms
Allow: /dividend-stocks
Disallow: /login
Disallow: /register
Disallow: /dashboard
Disallow: /claims
Disallow: /settings
Disallow: /tax-credits
Disallow: /recent-interest-bonds
Disallow: /auth/
Disallow: /api/

# Google Search Bot
User-agent: Googlebot
Allow: /
Disallow: /login
Disallow: /register
Disallow: /dashboard
Disallow: /claims
Disallow: /settings
Disallow: /tax-credits
Disallow: /recent-interest-bonds
Disallow: /auth/

# Google Image Bot
User-agent: Googlebot-Image
Allow: /og-image.png
Allow: /favicon.svg
Allow: /favicon-512x512.png
Allow: /apple-touch-icon.png

# Google-Extended controls Gemini training and use, not Google Search or AI Overviews
User-agent: Google-Extended
Allow: /
Disallow: /dashboard
Disallow: /claims
Disallow: /settings
Disallow: /auth/

# ChatGPT search & citation bot — allow so ChatGPT can cite your content
User-agent: GPTBot
Allow: /
Disallow: /dashboard
Disallow: /claims
Disallow: /settings
Disallow: /auth/

# ChatGPT browsing (real-time) — allow so ChatGPT with browsing can cite you
User-agent: ChatGPT-User
Allow: /

# OpenAI search crawler — allow search indexing and citations
User-agent: OAI-SearchBot
Allow: /
Disallow: /dashboard
Disallow: /claims
Disallow: /settings
Disallow: /auth/

# Claude (Anthropic) — allow citation
User-agent: anthropic-ai
Allow: /
Disallow: /dashboard
Disallow: /claims
Disallow: /settings
Disallow: /auth/

# ClaudeBot — allow crawling and citation
User-agent: ClaudeBot
Allow: /
Disallow: /dashboard
Disallow: /claims
Disallow: /settings
Disallow: /auth/

# Perplexity — allow indexing and citation
User-agent: PerplexityBot
Allow: /
Disallow: /dashboard
Disallow: /claims
Disallow: /settings
Disallow: /auth/

# Bing Search Bot (also powers Microsoft Copilot)
User-agent: Bingbot
Allow: /
Disallow: /dashboard
Disallow: /claims
Disallow: /settings
Disallow: /auth/

# Social Media Bots
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Slackbot
Allow: /

User-agent: WhatsApp
Allow: /

# Block AI training-only crawlers (not search/citation bots)
# CCBot is used by Common Crawl for training data only — not for search or citation
User-agent: CCBot
Disallow: /

# Block aggressive scrapers
User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

# Sitemap location
Sitemap: https://taxreclaim.eu/sitemap.xml