# robots.txt for DigiMirchi Infotech - SEO & AI Search Optimized
# This file tells search engine crawlers which pages or files they can or can't request from your site

User-agent: *

# Allow all crawlers to access the main pages
Allow: /
Allow: /index.html
Allow: /about.html
Allow: /service.html
Allow: /portfolio.html
Allow: /technology.html
Allow: /blog.html
Allow: /career.html
Allow: /contact.html

# Allow crawlers to access assets for proper rendering
Allow: /assets/
Allow: /Services/
Allow: /Portfolio/
Allow: /Policy/
Allow: /blog/

# Disallow crawlers from accessing backend files
Disallow: /php/
Disallow: /*.php
Disallow: /assets/scss/
Disallow: /assets/maps/

# Disallow crawlers from accessing temporary or system files
Disallow: /tmp/
Disallow: /.git/
Disallow: /.env
Disallow: /node_modules/

# Disallow crawlers from accessing error pages
Disallow: /error.html
Disallow: /coming-soon.html

# Allow access to sitemap
Allow: /sitemap.xml
Allow: /sitemap.html

# Specify the location of the sitemap
Sitemap: https://digimirchiinfotech.com/sitemap.xml

# AI Search Engine Bots (2025 Optimization - Correct User Agents)
User-agent: GPTBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: bingbot
Allow: /

# Social Media Crawlers
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

# Crawl delay to be respectful to server resources
Crawl-delay: 1