update robots.txt allowing GitHub to manage robots

2023-11-24 20:30:25 -08:00
1 parent e3c5232404
commit 8aaaebb4dc
@@ -1,14 +1 @@
-# Added due to slurping up everything as fast as it could
-User-agent: sistrix
-Disallow: /
-
-# I watched this bot for 4 consecutive days, it indexed the whole site 4 times
-# each day (using GETs)
-User-agent: AhrefsBot
-Disallow: /
-
-# Repeated indexing
-User-agent: Ezooms
-Disallow: /
-
 User-Agent: *