# robots.txt for metacrawler.com User-agent: * Disallow: /pics/ Disallow: /cgi-bin/