User-agent: *
Disallow: /api
Disallow: /2fa
Disallow: /confirm_signup
Disallow: /example.txt
Disallow: /signup


# Allow major search engines and AI bots
User-agent: Googlebot
User-agent: Bingbot
User-agent: DuckDuckGo
User-agent: Yandex
User-agent: Baiduspider
User-agent: Applebot
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: PerplexityBot
User-agent: CCBot
User-agent: ChatGPT-User
User-agent: YouBot
User-agent: NeevaBot
User-agent: AI-Searcher
User-agent: Google-Extended
User-agent: OpenAI
Allow: /


# Block known bad bots and scrapers
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: DataForSeoBot
User-agent: Seomozbot
User-agent: UptimeRobot
Disallow: /


# Sitemap location
Sitemap: https://streamingmafia.com/sitemap.xml