User-agent: *
# Block access to sensitive directories and files
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /account/
Disallow: /search/
Disallow: /wishlist/
Disallow: /login/
Disallow: /register/
Disallow: /reset-password/
Disallow: /customer/
Disallow: /orders/
Disallow: /private/
Disallow: /debug/
Disallow: /test/
Disallow: /wp-content/uploads/wc-logs/
Disallow: /wp-content/uploads/woocommerce_transient_files/
Disallow: /wp-content/uploads/woocommerce_uploads/

# Block unnecessary query parameters to prevent duplicate content
Disallow: /*?*
Disallow: /*add-to-cart=
Disallow: /*orderby=
Disallow: /*filter=

# Disallow directories not relevant for search engines
Disallow: /cgi-bin/
Disallow: /tmp/
Disallow: /logs/
Disallow: /cache/

# Allow public assets and media for SEO
Allow: /wp-content/uploads/

# Specify sitemap location for efficient crawling
Sitemap: https://www.verstolo.com/sitemap_index.xml

# Block aggressive or unnecessary bots
User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: Sogou spider
Disallow: /

User-agent: Sogou Spider
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: Sosospider
Disallow: /

# Rules for specific search engine bots
User-agent: Bingbot
Disallow:  # Allow Bingbot full access.

User-agent: Yahoobot
Disallow:  # Allow Yahoobot full access.

User-agent: Yandex
Disallow: /  # Block Yandex completely if you don't target Russian markets.

# Optional: Add crawl delay if server experiences strain
# Crawl-delay: 10

# Manage Google crawlers
User-agent: Googlebot
Disallow:  # This allows Googlebot full access.

User-agent: Googlebot-image
Disallow:  # This allows Googlebot-image full access. (Do not disallow if you want images to rank.)

# End of robots.txt