# /robots.txt file for http://webcrawler.com/
#http://www.robotstxt.org/wc/robots.html
#  1. Allow our google search appliance to crawl everything.
#	 2. Disable all other enginges from searching cap_data_docs.
#	 3. Prevent "slurp" from excessive crawling
#
User-agent: basf-crawler
Allow: /cap_data_docs

User-agent: Slurp
Disallow: /cap_data_docs
Crawl-delay: 10

User-agent: *
Disallow: /cap_data_docs