User-agent: *
Disallow: /agave-usage/ # This is an infinite virtual URL space
Disallow: /caseta-usage/ # This is an infinite virtual URL space
Disallow: /webmail/ # This is an infinite virtual URL space
Disallow: /cgi-bin/ # This is an infinite virtual URL space
Disallow: /tmp/ # these will soon disappear
Disallow: /missing.html
Disallow: /~* # Do not crawl user directories