User-agent: *
Disallow: /admin/
Disallow: /api/
Disallow: /storage/
Disallow: /vendor/
Disallow: /config/
Disallow: /bootstrap/
Disallow: /database/
Disallow: /resources/
Disallow: /routes/
Disallow: /tests/
Disallow: /app/
Disallow: /cart
Disallow: /checkout
Disallow: /login
Disallow: /register
Disallow: /dashboard
Disallow: /profile
Disallow: /search?
Disallow: /filter?
Disallow: /*?sort=
Disallow: /*?page=
Disallow: /*?filter=
Crawl-delay: 10

# Allow specific search engines
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 2

User-agent: YandexBot
Allow: /
Crawl-delay: 3

# SEO tools with restrictions
User-agent: AhrefsBot
Disallow: /admin/
Disallow: /api/
Disallow: /storage/
Disallow: /cart
Disallow: /checkout
Disallow: /login
Disallow: /register
Disallow: /dashboard
Disallow: /profile
Crawl-delay: 30

# Block bad bots
User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

# OpenAI (ChatGPT)
User-Agent: GPTBot
Crawl-delay: 10

User-Agent: ChatGPT-User
Crawl-delay: 10

# Anthropic (Claude)
User-Agent: ClaudeBot
Crawl-delay: 10

User-Agent: Claude-User
Crawl-delay: 10

# Common Crawl (Used by many AI models)
User-Agent: CCBot
Crawl-delay: 10

# Google AI
User-Agent: Google-Extended
Crawl-delay: 10

# Perplexity AI
User-Agent: PerplexityBot
Crawl-delay: 10

# Meta/Facebook
User-Agent: Facebookbot
Crawl-delay: 10

# Amazon
User-Agent: Amazonbot
Crawl-delay: 10

# ByteDance/TikTok
User-Agent: Bytespider
Crawl-delay: 10

Sitemap: https://intop.lv/sitemap.xml