Disallow: /email-addresses/



# --- Allow Google ---
User-agent: Googlebot
Allow: /
User-agent: Googlebot-Image
Disallow: /
User-agent: Googlebot-News
Allow: /
User-agent: Googlebot-Video
Allow: /
User-agent: AdsBot-Google
Disallow: /
User-agent: AdsBot-Google-Mobile
Allow: /
# LLM/training control for Google
User-agent: Google-Extended
Allow: /

# --- Allow Bing ---
User-agent: Bingbot
Allow: /
User-agent: BingPreview
Allow: /

# --- Allow Facebook / Meta link scrapers ---
User-agent: facebookexternalhit
Allow: /
User-agent: Meta-ExternalAgent
Allow: /

# --- Allow common LLM crawlers ---
# OpenAI
User-agent: GPTBot
Allow: /
User-agent: ChatGPT-User
Allow: /
# (Optional) OpenAI Search indexing
User-agent: OAI-SearchBot
Allow: /
# Anthropic
User-agent: ClaudeBot
Allow: /
User-agent: Claude-User
Allow: /
User-agent: Claude-SearchBot
Allow: /
User-agent: Claude-Web
Allow: /
# Perplexity
User-agent: PerplexityBot
Allow: /
User-agent: DeepSeekBot
Allow: /
User-agent: XAI-Bot
Allow: /
User-agent: MistralBot
Allow: /
User-agent: Anthropic-AI
Allow: /
User-agent: Z-Bot
Allow: /
User-agent: QwenBot
Allow: /


User-agent: SemrushBot
Disallow: /

User-agent: SiteAuditBot
Disallow: /

User-agent: SemrushBot-BA
Disallow: /

User-agent: SemrushBot-SI
Disallow: /

User-agent: SemrushBot-SWA
Disallow: /

User-agent: SplitSignalBot
Disallow: /

User-agent: SemrushBot-OCOB
Disallow: /
T
User-agent: SemrushBot-FT
Disallow: /

User-agent: RyteBot
Disallow: /

User-agent: SemrushBot-ESI
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: Buck
Disallow: /

# --- (Redundant but explicit) Block known Russian & Chinese crawlers ---
# These are already blocked by "User-agent: *  Disallow: /"
User-agent: Yandex
Disallow: /
User-agent: YandexBot
Disallow: /
User-agent: YandexImages
Disallow: /
User-agent: Baiduspider
Disallow: /
User-agent: Baiduspider-image
Disallow: /
User-agent: Baiduspider-video
Disallow: /
User-agent: 360Spider
Disallow: /
User-agent: Sogou web spider
Disallow: /
User-agent: PetalBot
Disallow: /
User-agent: Bytespider
Disallow: /
User-agent: YisouSpider
Disallow: /
User-agent: HaoSouSpider
Disallow: /


User-agent: NaverBot
User-agent: Yeti
Disallow: /


User-agent: sogou spider
Disallow: /

User-agent: YoudaoBot
Disallow: /

User-agent: moget
User-agent: ichiro
Disallow: /


# Block everyone by default
User-agent: *
Disallow: /
# (Optional) add your sitemap(s)
Sitemap: https://yourdomain.com/sitemap.xml