# Smart BDM - AI Sales Platform Robots.txt Configuration
# Generated: 2026-04-11T11:15:24.747Z

# Allow all crawlers to access the site
User-agent: *
Allow: /

# Allow public marketing pages
Allow: /pricing
Allow: /contact
Allow: /affiliate
Allow: /privacy-policy
Allow: /terms
Allow: /industry/real-estate
Allow: /industry/education
Allow: /industry/insurance
Allow: /industry/healthcare
Allow: /industry/automotive
Allow: /industry/financial-services
Allow: /industry/travel-hospitality

# Block all authentication routes
Disallow: /login
Disallow: /login-verify-otp
Disallow: /password-login
Disallow: /forgot-password
Disallow: /register

# Block all dashboard and functionality
Disallow: /dashboard
Disallow: /ai-buddy/
Disallow: /notifications
Disallow: /pipelines
Disallow: /tasks
Disallow: /inbox
Disallow: /calls
Disallow: /max-attention
Disallow: /search-results

# Block all analytics routes
Disallow: /analytics/

# Block all dynamic routes with parameters
Disallow: /stage/
Disallow: /pipeline/
Disallow: /uncontacted-leads
Disallow: /followup-leads
Disallow: /lead/
Disallow: /profile/
Disallow: /call/
Disallow: /inbox/

# Block all admin routes
Disallow: /admin/

# Block Next.js internal routes
Disallow: /_next/
Disallow: /api/

# Block temporary and cache files
Disallow: /*.json$
Disallow: /temp/

# Rate limiting for crawlers to prevent server overload
Crawl-delay: 1

# Specific rules for major search engines
User-agent: Googlebot
Allow: /
Allow: /pricing
Allow: /contact
Allow: /affiliate
Allow: /privacy-policy
Allow: /terms
Allow: /industry/real-estate
Allow: /industry/education
Allow: /industry/insurance
Allow: /industry/healthcare
Allow: /industry/automotive
Allow: /industry/financial-services
Allow: /industry/travel-hospitality
Disallow: /login
Disallow: /login-verify-otp
Disallow: /password-login
Disallow: /forgot-password
Disallow: /register
Disallow: /dashboard
Disallow: /ai-buddy/
Disallow: /notifications
Disallow: /pipelines
Disallow: /tasks
Disallow: /inbox
Disallow: /calls
Disallow: /max-attention
Disallow: /analytics/
Disallow: /stage/
Disallow: /pipeline/
Disallow: /uncontacted-leads
Disallow: /followup-leads
Disallow: /lead/
Disallow: /profile/
Disallow: /call/
Disallow: /admin/

User-agent: Bingbot
Allow: /
Allow: /pricing
Allow: /contact
Allow: /affiliate
Allow: /privacy-policy
Allow: /terms
Allow: /industry/real-estate
Allow: /industry/education
Allow: /industry/insurance
Allow: /industry/healthcare
Allow: /industry/automotive
Allow: /industry/financial-services
Allow: /industry/travel-hospitality
Disallow: /login
Disallow: /login-verify-otp
Disallow: /password-login
Disallow: /forgot-password
Disallow: /register
Disallow: /dashboard
Disallow: /ai-buddy/
Disallow: /notifications
Disallow: /pipelines
Disallow: /tasks
Disallow: /inbox
Disallow: /calls
Disallow: /max-attention
Disallow: /analytics/
Disallow: /stage/
Disallow: /pipeline/
Disallow: /uncontacted-leads
Disallow: /followup-leads
Disallow: /lead/
Disallow: /profile/
Disallow: /call/
Disallow: /admin/

# Block aggressive SEO crawlers that may impact performance
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

# Static pages sitemap
Sitemap: https://rd-dev.smartbdm.ai/sitemap.xml

# Dynamic blog content sitemap
Sitemap: https://rd-dev.smartbdm.ai/sitemap-dynamic.xml

# Host directive for canonical domain
Host: https://rd-dev.smartbdm.ai