# ===========================================
# Robots.txt for https://www.vixen-escorts.com/
# Last updated: 2025-08-17
# Purpose: Improve SEO crawling and block bad bots
# ===========================================

# ----------------------------
# General rules for all GOOD bots
# (Googlebot, Bingbot, etc.)
# ----------------------------
User-agent: *
Disallow: /cgi-bin/
Allow: /

# ----------------------------
# Block known unwanted bots
# ----------------------------
User-agent: AboutUsBot
User-agent: ia_archiver
User-agent: HypeStat
User-agent: CuteStat
User-agent: URLmetrics
User-agent: ClearWebStats
Disallow: /

# ----------------------------
# (Optional) Example: block private sections
# Uncomment if you want to hide certain folders
# User-agent: *
# Disallow: /private/
# Disallow: /tmp/
# ----------------------------

# ----------------------------
# Notes:
# - Legitimate search engines are allowed full access (except /cgi-bin/).
# - Spammy bots are explicitly blocked.
# - Sensitive or temporary pages should also use <meta name="robots" content="noindex">
#   because some bad bots ignore robots.txt.
# ----------------------------