# Disallows identified webcrawlers from scanning site.
# A "Disallow: /" would block all scans.
 
User-agent: *
Disallow: /spider
Disallow: /crawl
Disallow: /WISENutbot
Disallow: /cgi-bin/ 
Disallow: /cgi_html/ 
Disallow: /ycdb/