# Most crawlers will need to keep off:
User-agent: *
Disallow: /sitecore/
Disallow: /Global/Pages/Search.aspx
Disallow: /sitesearch/

# Google seems to need this:
User-agent: Googlebot
Allow: /