# robots.txt for http://www.mnot.net/ # # Too greedy User-agent: http://www.almaden.ibm.com/cs/crawler Disallow: / User-agent: BrowserShots Disallow: User-agent: ia_archiver Disallow: / User-agent: msnbot Crawl-delay: 120 User-agent: * # just a working area, nothing to see Disallow: /test/ # # probably best not to index these Disallow: /photo/ # # nice try, referer spammers! Disallow: /stats/