# Konecteaze.com - Internet Provider Comparison Service
# Optimized robots.txt for SEO and crawler efficiency

# Allow all major search engines full access to content
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: Slurp
Allow: /
Crawl-delay: 1

User-agent: DuckDuckBot
Allow: /
Crawl-delay: 1

User-agent: Baiduspider
Allow: /
Crawl-delay: 2

User-agent: YandexBot
Allow: /
Crawl-delay: 2

# Allow other crawlers with rate limiting
User-agent: *
Allow: /
Crawl-delay: 3

# Disallow administrative and sensitive areas
Disallow: /admin/
Disallow: /api/admin/
Disallow: /secured-management/
Disallow: /_next/
Disallow: /static/

# Disallow API endpoints (except sitemap APIs)
Disallow: /api/
Allow: /api/sitemap*

# Disallow development and build files
Disallow: /.next/
Disallow: /node_modules/
Disallow: /.git/
Disallow: /.env*
Disallow: /package*.json
Disallow: /tsconfig*.json
Disallow: /next.config*
Disallow: /tailwind.config*
Disallow: /postcss.config*

# Disallow temporary and backup files
Disallow: /*~
Disallow: /*.bak
Disallow: /*.tmp
Disallow: /*_backup*
Disallow: /backup/
Disallow: /temp/
Disallow: /tmp/

# Allow important content areas
Allow: /providers/
Allow: /provider/
Allow: /blog/
Allow: /speed-test
Allow: /compare
Allow: /about
Allow: /contact
Allow: /privacy-policy
Allow: /terms
Allow: /cookie-policy
Allow: /provider-support

# Allow state and city pages for local SEO
Allow: /*/
Allow: /*/*/

# Allow AI search crawlers for maximum SEO traffic
User-agent: CCBot
Allow: /
Disallow: /api/
Disallow: /admin/
Crawl-delay: 2

User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /admin/
Crawl-delay: 2

User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /admin/
Crawl-delay: 2

User-agent: Google-Extended
Allow: /
Disallow: /api/
Disallow: /admin/
Crawl-delay: 2

User-agent: AhrefsBot
Crawl-delay: 10
Disallow: /api/

User-agent: SemrushBot
Crawl-delay: 10
Disallow: /api/

User-agent: MJ12bot
Crawl-delay: 30

User-agent: DotBot
Crawl-delay: 10

# Block malicious bots
User-agent: BLEXBot
User-agent: DataForSeoBot
Disallow: /api/
Disallow: /admin/
Crawl-delay: 30

# Comprehensive sitemap index
Sitemap: https://konecteaze.com/sitemap.xml
Sitemap: https://konecteaze.com/sitemap-index.xml
Sitemap: https://konecteaze.com/sitemap-static.xml
Sitemap: https://konecteaze.com/sitemap-providers.xml
Sitemap: https://konecteaze.com/sitemap-cities.xml
Sitemap: https://konecteaze.com/sitemap-states.xml
Sitemap: https://konecteaze.com/sitemap-blog.xml

# Host directive for production
Host: https://konecteaze.com