# ============================================
# robots.txt for yabupushelberg.com
# Optimized for International SEO/GEO
# Last Updated: December 2024
# ============================================

# ============================================
# MAJOR SEARCH ENGINES - ALLOWED
# Essential for international visibility
# ============================================

# Google (Global - Primary)
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-News
User-agent: Googlebot-Video
Allow: /

# Bing/Microsoft (Global - US/Europe)
User-agent: Bingbot
User-agent: msnbot
User-agent: BingPreview
Allow: /

# Baidu (CRITICAL for China market)
User-agent: Baiduspider
User-agent: Baiduspider-image
User-agent: Baiduspider-video
User-agent: Baiduspider-news
Allow: /

# Sogou (China - Secondary)
User-agent: Sogou
User-agent: sogou spider
Allow: /

# 360 Search (China)
User-agent: 360Spider
User-agent: HaoSouSpider
Allow: /

# Yandex (Russia/Eastern Europe)
User-agent: Yandex
User-agent: YandexBot
User-agent: YandexImages
Allow: /

# Naver (South Korea)
User-agent: Yeti
User-agent: NaverBot
Allow: /

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /

# Yahoo/Japan (Important for Japanese market)
User-agent: Slurp
Allow: /

# Qwant (European privacy-focused)
User-agent: Qwantify
Allow: /

# Apple
User-agent: Applebot
Allow: /

# ============================================
# SOCIAL MEDIA CRAWLERS - ALLOWED
# Essential for link previews when sharing
# ============================================

User-agent: facebookexternalhit
User-agent: Facebot
User-agent: Twitterbot
User-agent: LinkedInBot
User-agent: Pinterest
User-agent: Pinterestbot
User-agent: Slackbot
User-agent: Slackbot-LinkExpanding
User-agent: WhatsApp
User-agent: TelegramBot
Allow: /

# ============================================
# SEO & ANALYTICS TOOLS - ALLOWED
# Helpful for monitoring site performance
# ============================================

User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: rogerbot
User-agent: MJ12bot
User-agent: DotBot
User-agent: BLEXBot
User-agent: DataForSeoBot
User-agent: serpstatbot
User-agent: SEOkicks
User-agent: Screaming Frog SEO Spider
Allow: /

# ============================================
# AI TRAINING CRAWLERS - BLOCKED
# Prevent content from training AI models
# ============================================

User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: FacebookBot
Disallow: /

# ============================================
# EMAIL HARVESTERS - BLOCKED
# ============================================

User-agent: EmailCollector
Disallow: /

User-agent: EmailSiphon
Disallow: /

User-agent: EmailWolf
Disallow: /

User-agent: ExtractorPro
Disallow: /

# ============================================
# SITE COPIERS & SCRAPERS - BLOCKED
# ============================================

User-agent: Wget
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebZip
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: TurnitinBot
Disallow: /

User-agent: magpie-crawler
Disallow: /

User-agent: Nimbostratus-Bot
Disallow: /

# ============================================
# ARCHIVE BOTS - BLOCKED (optional)
# Remove if you want site archived
# ============================================

User-agent: ia_archiver
Disallow: /

User-agent: archive.org_bot
Disallow: /

# ============================================
# DEFAULT RULE - All other bots
# ============================================

User-agent: *
# Block admin/private areas
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /login/
Disallow: /private/
Disallow: /staging/
Disallow: /dev/
Disallow: /test/
Disallow: /tmp/
Disallow: /cgi-bin/
Disallow: /api/
Disallow: /search/
Disallow: /search?
Disallow: /*?*

# Allow everything else
Allow: /

# Crawl delay (seconds between requests)
Crawl-delay: 10

# ============================================
# SITEMAPS
# ============================================

Sitemap: https://yabupushelberg.com/sitemap.xml

# Future localized sitemaps (uncomment when live):
# Sitemap: https://yabupushelberg.com/zh-cn/sitemap.xml
# Sitemap: https://yabupushelberg.com/ar/sitemap.xml
# Sitemap: https://yabupushelberg.com/ja/sitemap.xml
# Sitemap: https://yabupushelberg.com/fr/sitemap.xml
# Sitemap: https://yabupushelberg.com/es/sitemap.xml

# ============================================
# SUMMARY FOR YABU PUSHELBERG
# ============================================
#
# ✅ ALLOWED:
#    - Google, Bing, Baidu, Sogou, 360 (search engines)
#    - Yandex, Naver, Yahoo Japan (regional search)
#    - Facebook, LinkedIn, Twitter, Pinterest (social)
#    - Ahrefs, SEMrush, Moz, etc. (SEO tools)
#
# ❌ BLOCKED:
#    - GPTBot, CCBot, etc. (AI training)
#    - Email harvesters
#    - Site copiers (HTTrack, WebZip)
#    - Archive.org (remove if you want archiving)
#
# TO DO:
#    1. Upload to root of yabupushelberg.com
#    2. Verify sitemap URL is correct
#    3. Add locale sitemaps when launched
# ============================================

Sitemap: https://www.yabupushelberg.com/sitemap.xml