User-agent: *Disallow: /cgi-bin/User-agent: msnbotCrawl-delay: 10User-agent: TeomaCrawl-delay: 10User-agent: SlurpCrawl-delay: 10User-agent: aipbotDisallow: /User-agent: BecomeBotDisallow: /User-agent: psbotDisallow: /User-agent: Xenu Link Sleuth 1.2gDisallow: /User-agent: XenuDisallow: /