# Zongo Shipping - Robots.txt
# This file guides search engine crawlers on how to index our website

# Global rules for all crawlers
User-agent: *
Allow: /

# Crawl delay to be respectful to our server
Crawl-delay: 1

# Sitemap location
Sitemap: https://zongoshipping.com/sitemap.xml

# Disallow private/admin areas
Disallow: /staff-dashboard.html
Disallow: /staff-login.html
Disallow: /staff-dashboard.js
Disallow: /staff-login.js
Disallow: /supabase-config.js
Disallow: /create-staff-user.js
Disallow: /create-staff-user.sql
Disallow: /diagnostic.js

# Disallow test files
Disallow: /test-*.html
Disallow: /test-*.js

# Disallow documentation files (optional - you may want these indexed)
Disallow: /README.md
Disallow: /EMAIL_SETUP.md
Disallow: /FIX-CURRENT-ERRORS.md
Disallow: /FIX-ERROR-34.md
Disallow: /TROUBLESHOOTING.md
Disallow: /QUICK_SETUP.md
Disallow: /STAFF_SETUP.md
Disallow: /SUPABASE_SETUP.md
Disallow: /SEO_OPTIMIZATION.md
Disallow: /supabase-schema.sql

# Disallow Excel files (business data)
Disallow: /*.xlsx
Disallow: /*.xls

# Allow important public resources
Allow: /styles.css
Allow: /script.js
Allow: /favicon.ico
Allow: /favicon.svg
Allow: /site.webmanifest

# Allow images
Allow: /*.png
Allow: /*.jpg
Allow: /*.jpeg
Allow: /*.gif
Allow: /*.svg
Allow: /*.webp

# Specific rules for Googlebot
User-agent: Googlebot
Allow: /
Crawl-delay: 1

# Specific rules for Bingbot
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Block AI training crawlers (optional)
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: Omgilibot
Disallow: /