# robots.txt for https://primaconsulting.org
# Last updated: October 2025
# Purpose: Maximize visibility for search engines & AI crawlers while protecting sensitive areas

User-agent: *
Allow: /
Crawl-delay: 10

# Block backend and private directories
Disallow: /wp-admin/
Disallow: /cgi-bin/
Disallow: /tmp/
Disallow: /drafts/
Disallow: /internal/
Disallow: /private/
Disallow: /backup/
Disallow: /reports/
Disallow: /old/
Disallow: /wordfence/

# Block search pages and REST API endpoints (WordPress)
Disallow: /?s=
Disallow: /page/*/?s=
Disallow: /search/
Disallow: /wp-json/
Disallow: /?rest_route=

# Block query parameters that create duplicate or low-value URLs
Disallow: /*?
Disallow: /*?v=
Disallow: /*?remove_item=
Disallow: /*?filter_color=
Disallow: /*?wordfence_lh=
Disallow: /*?utm_
Disallow: /*?fbclid

# Allow essential rendering files
Allow: /*.js$
Allow: /*.css$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.svg$
Allow: /*.gif$
Allow: /wp-content/uploads/

# Sitemap
Sitemap: https://primaconsulting.org/sitemap_index.xml

# -----------------------------------------------
# AI crawler permissions
# -----------------------------------------------
User-agent: ChatGPT-User
Allow: /

User-agent: GPTBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Gemini
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Grok
Allow: /

User-agent: DeepSeekBot
Allow: /

User-agent: KimiBot
Allow: /

User-agent: ClaudeBot
Allow: /

# -----------------------------------------------
# Search crawler optimization
# -----------------------------------------------
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-News
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: Yandex
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Applebot
Allow: /

User-agent: LinkedInBot
Allow: /

# Optional crawl-delay for non-Google bots to prevent server overload
User-agent: Bingbot
Crawl-delay: 10

User-agent: Baiduspider
Crawl-delay: 10

# -----------------------------------------------
# End of file
# -----------------------------------------------