# Konecteaze.com - Internet Provider Comparison Platform
# Optimized for maximum SEO visibility and crawler efficiency

# Allow all major search engines FULL access to valuable content
User-agent: Googlebot
Allow: /
Allow: /providers/
Allow: /blog/
Allow: /locations/
Allow: /city/
Allow: /zip/
Allow: /compare
Allow: /speed-test
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Allow: /providers/
Allow: /blog/
Allow: /locations/
Allow: /city/
Allow: /zip/
Allow: /compare
Allow: /speed-test
Crawl-delay: 1

User-agent: Slurp
Allow: /
Crawl-delay: 1

User-agent: DuckDuckBot
Allow: /
Crawl-delay: 1

# AI Search Crawlers (ChatGPT, Claude, etc.) - Critical for modern SEO
User-agent: GPTBot
Allow: /
Allow: /providers/
Allow: /blog/
Allow: /locations/
Allow: /city/
Allow: /compare
Allow: /speed-test
Disallow: /admin/
Disallow: /api/
Crawl-delay: 2

User-agent: ChatGPT-User
Allow: /
Allow: /providers/
Allow: /blog/
Allow: /locations/
Allow: /city/
Allow: /compare
Allow: /speed-test
Disallow: /admin/
Disallow: /api/
Crawl-delay: 2

User-agent: CCBot
Allow: /
Allow: /providers/
Allow: /blog/
Allow: /locations/
Allow: /city/
Allow: /compare
Allow: /speed-test
Disallow: /admin/
Disallow: /api/
Crawl-delay: 2

User-agent: Claude-Web
Allow: /
Allow: /providers/
Allow: /blog/
Allow: /locations/
Allow: /city/
Allow: /compare
Allow: /speed-test
Disallow: /admin/
Disallow: /api/
Crawl-delay: 2

# General crawlers - Allow valuable content, block admin/API
User-agent: *
Allow: /
Allow: /providers/
Allow: /blog/
Allow: /locations/
Allow: /city/
Allow: /compare
Allow: /speed-test
Allow: /about
Allow: /contact
Allow: /privacy-policy
Allow: /terms
Allow: /sitemap.xml

# BLOCK administrative and sensitive areas
Disallow: /admin/
Disallow: /secured-management/
Disallow: /api/
Disallow: /_next/
Disallow: /auth/
Disallow: /login
Disallow: /dashboard

# BLOCK low-value or duplicate content pages
Disallow: /*?*
Disallow: /search?*
Disallow: /filter?*
Disallow: /*utm_*
Disallow: /*fbclid*
Disallow: /*gclid*
Disallow: /*ref=*

# BLOCK development and system files
Disallow: /.well-known/
Disallow: /temp/
Disallow: /tmp/
Disallow: /backup/
Disallow: /*~
Disallow: /*.bak
Disallow: /*.tmp
Disallow: /*.log

# SEO Tools - Allow with rate limiting
User-agent: AhrefsBot
Allow: /
Disallow: /admin/
Disallow: /api/
Crawl-delay: 10

User-agent: SemrushBot
Allow: /
Disallow: /admin/
Disallow: /api/
Crawl-delay: 10

User-agent: MajesticSEO
Allow: /
Disallow: /admin/
Disallow: /api/
Crawl-delay: 5

# Block aggressive/malicious crawlers
User-agent: MJ12bot
Crawl-delay: 30

User-agent: DotBot
Crawl-delay: 30

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Crawl-delay: 60

User-agent: SiteAuditBot
Crawl-delay: 30

# COMPREHENSIVE SITEMAP INDEX for maximum discoverability
Sitemap: https://www.konecteaze.com/sitemap.xml
Sitemap: https://www.konecteaze.com/sitemap-index.xml
Sitemap: https://www.konecteaze.com/sitemap-providers.xml
Sitemap: https://www.konecteaze.com/sitemap-blog.xml
Sitemap: https://www.konecteaze.com/sitemap-locations.xml
Sitemap: https://www.konecteaze.com/sitemap-cities.xml
Sitemap: https://www.konecteaze.com/sitemap-states.xml

# Host directive (Google preference)
Host: https://www.konecteaze.com