# BhauBox - Dog Subscription Box Service
# Robots.txt for SEO and Crawler Management
# Last Updated: 2024-12-19
# CRAWLER-FRIENDLY CONFIGURATION

# Global Rules for All Crawlers - MAXIMUM ACCESS
User-agent: *
Allow: /
Allow: /login
Allow: /dashboard
Allow: /admin
Allow: /api/vetchat
Allow: /sitemap.xml
Allow: /robots.txt
Allow: /llms.txt

# Explicitly allow important pages
Allow: /login
Allow: /dashboard
Allow: /admin
Allow: /api/vetchat

# Allow all static assets and images
Allow: /images/
Allow: /public/
Allow: /*.webp
Allow: /*.png
Allow: /*.jpg
Allow: /*.jpeg
Allow: /*.gif
Allow: /*.svg
Allow: /*.ico
Allow: /*.css
Allow: /*.js
Allow: /*.mp4

# Disallow sensitive admin API endpoints (but allow public ones)
Disallow: /api/admin/
Disallow: /api/admin/analytics
Disallow: /api/admin/box-previews
Disallow: /api/admin/boxes
Disallow: /api/admin/orders
Disallow: /api/admin/stats
Disallow: /api/admin/subscriptions
Disallow: /api/admin/users

# Disallow development and build files
Disallow: /.next/
Disallow: /node_modules/
Disallow: /.git/
Disallow: /.env
Disallow: /.env.local
Disallow: /.env.production
Disallow: /package.json
Disallow: /package-lock.json
Disallow: /pnpm-lock.yaml
Disallow: /tsconfig.json
Disallow: /tailwind.config.ts
Disallow: /postcss.config.mjs
Disallow: /next.config.mjs
Disallow: /middleware.ts
Disallow: /SUPABASE_SETUP.md

# Disallow temporary and cache files
Disallow: /tmp/
Disallow: /cache/
Disallow: /.cache/
Disallow: /build/
Disallow: /dist/

# Disallow user-specific content (if any)
Disallow: /user/
Disallow: /profile/
Disallow: /account/

# Disallow form submissions and dynamic content
Disallow: /submit/
Disallow: /form/
Disallow: /checkout/process
Disallow: /payment/
Disallow: /billing/

# Specific Rules for Major Search Engines

# Google Bot - Most permissive for SEO
User-agent: Googlebot
Allow: /
Allow: /login
Allow: /dashboard
Allow: /api/vetchat
Allow: /images/
Allow: /*.webp
Allow: /*.png
Allow: /*.jpg
Allow: /*.jpeg
Allow: /*.svg
Allow: /*.ico
Allow: /*.css
Allow: /*.js
Disallow: /api/admin/
Disallow: /.next/
Disallow: /node_modules/
Disallow: /.git/

# Bing Bot
User-agent: Bingbot
Allow: /
Allow: /login
Allow: /dashboard
Allow: /api/vetchat
Allow: /images/
Allow: /*.webp
Allow: /*.png
Allow: /*.jpg
Allow: /*.jpeg
Allow: /*.svg
Allow: /*.ico
Allow: /*.css
Allow: /*.js
Disallow: /api/admin/
Disallow: /.next/
Disallow: /node_modules/
Disallow: /.git/

# Yahoo Bot
User-agent: Slurp
Allow: /
Allow: /login
Allow: /dashboard
Allow: /api/vetchat
Allow: /images/
Allow: /*.webp
Allow: /*.png
Allow: /*.jpg
Allow: /*.jpeg
Allow: /*.svg
Allow: /*.ico
Allow: /*.css
Allow: /*.js
Disallow: /api/admin/
Disallow: /.next/
Disallow: /node_modules/
Disallow: /.git/

# DuckDuckGo Bot
User-agent: DuckDuckBot
Allow: /
Allow: /login
Allow: /dashboard
Allow: /api/vetchat
Allow: /images/
Allow: /*.webp
Allow: /*.png
Allow: /*.jpg
Allow: /*.jpeg
Allow: /*.svg
Allow: /*.ico
Allow: /*.css
Allow: /*.js
Disallow: /api/admin/
Disallow: /.next/
Disallow: /node_modules/
Disallow: /.git/

# Facebook Bot (for social media sharing)
User-agent: facebookexternalhit
Allow: /
Allow: /login
Allow: /dashboard
Allow: /api/vetchat
Allow: /images/
Allow: /*.webp
Allow: /*.png
Allow: /*.jpg
Allow: /*.jpeg
Allow: /*.svg
Allow: /*.ico
Disallow: /api/admin/
Disallow: /.next/
Disallow: /node_modules/
Disallow: /.git/

# Twitter Bot (for social media sharing)
User-agent: Twitterbot
Allow: /
Allow: /login
Allow: /dashboard
Allow: /api/vetchat
Allow: /images/
Allow: /*.webp
Allow: /*.png
Allow: /*.jpg
Allow: /*.jpeg
Allow: /*.svg
Allow: /*.ico
Disallow: /api/admin/
Disallow: /.next/
Disallow: /node_modules/
Disallow: /.git/

# LinkedIn Bot (for professional sharing)
User-agent: LinkedInBot
Allow: /
Allow: /login
Allow: /dashboard
Allow: /api/vetchat
Allow: /images/
Allow: /*.webp
Allow: /*.png
Allow: /*.jpg
Allow: /*.jpeg
Allow: /*.svg
Allow: /*.ico
Disallow: /api/admin/
Disallow: /.next/
Disallow: /node_modules/
Disallow: /.git/

# WhatsApp Bot (for messaging sharing)
User-agent: WhatsApp
Allow: /
Allow: /login
Allow: /dashboard
Allow: /api/vetchat
Allow: /images/
Allow: /*.webp
Allow: /*.png
Allow: /*.jpg
Allow: /*.jpeg
Allow: /*.svg
Allow: /*.ico
Disallow: /api/admin/
Disallow: /.next/
Disallow: /node_modules/
Disallow: /.git/

# AI and LLM Crawlers (for AI training and indexing)
User-agent: GPTBot
Allow: /
Allow: /login
Allow: /dashboard
Allow: /api/vetchat
Allow: /images/
Allow: /*.webp
Allow: /*.png
Allow: /*.jpg
Allow: /*.jpeg
Allow: /*.svg
Allow: /*.ico
Allow: /*.css
Allow: /*.js
Disallow: /api/admin/
Disallow: /.next/
Disallow: /node_modules/
Disallow: /.git/

User-agent: ChatGPT-User
Allow: /
Allow: /login
Allow: /dashboard
Allow: /api/vetchat
Allow: /images/
Allow: /*.webp
Allow: /*.png
Allow: /*.jpg
Allow: /*.jpeg
Allow: /*.svg
Allow: /*.ico
Allow: /*.css
Allow: /*.js
Disallow: /api/admin/
Disallow: /.next/
Disallow: /node_modules/
Disallow: /.git/

User-agent: Claude-Web
Allow: /
Allow: /login
Allow: /dashboard
Allow: /api/vetchat
Allow: /images/
Allow: /*.webp
Allow: /*.png
Allow: /*.jpg
Allow: /*.jpeg
Allow: /*.svg
Allow: /*.ico
Allow: /*.css
Allow: /*.js
Disallow: /api/admin/
Disallow: /.next/
Disallow: /node_modules/
Disallow: /.git/

# Block aggressive or unwanted crawlers
User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

# Crawl Delay (optimized for performance)
Crawl-delay: 0.5

# Additional Crawler-Friendly Directives
Request-rate: 1/1s
Visit-time: 0600-2300

# Sitemap Location
Sitemap: https://bhaubox.com/sitemap.xml

# Additional Sitemaps (if you have multiple)
# Sitemap: https://bhaubox.com/sitemap-images.xml
# Sitemap: https://bhaubox.com/sitemap-videos.xml

# Host Declaration (preferred domain)
Host: https://bhaubox.com

# Contact Information (optional)
# Contact: webmaster@bhaubox.com