diff --git a/robots.txt b/robots.txt index 040124231..72158267b 100644 --- a/robots.txt +++ b/robots.txt @@ -1,14 +1 @@ -# Added due to slurping up everything as fast as it could -User-agent: sistrix -Disallow: / - -# I watched this bot for 4 consecutive days, it indexed the whole site 4 times -# each day (using GETs) -User-agent: AhrefsBot -Disallow: / - -# Repeated indexing -User-agent: Ezooms -Disallow: / - User-Agent: *