# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file # robots.txt for http://www.hirethings.co.nz User-agent: * Crawl-delay: 3 # yahoo specific for the Slurp crawler Disallow: /myprofile/* Disallow: /contact/list/* Disallow: /loan/* Disallow: /booking/* Disallow: /use_request/* Disallow: /picker/* Disallow: /search/* Disallow: /javascripts/* Disallow: /home/* Disallow: /category/select_list/* Disallow: /payment/*