User-agent: Slurp
Crawl-delay: 10
Disallow: /error/
Disallow: /exhibits/
Disallow: /images/
Disallow: /includes/
Disallow: /offers/
Disallow: /order/
Disallow: /pdf/
Disallow: /rss/
Disallow: /site_graphics/
Disallow: /statistics/
Disallow: /tmp/
Disallow: /redesign/
Disallow: /catalog/
Disallow: /catalog.php?recid=
Sitemap: https://www.hup.harvard.edu/sitemap.xml

User-agent: MSNBot
Crawl-delay: 10
Disallow: /error/
Disallow: /exhibits/
Disallow: /images/
Disallow: /includes/
Disallow: /offers/
Disallow: /order/
Disallow: /pdf/
Disallow: /rss/
Disallow: /site_graphics/
Disallow: /statistics/
Disallow: /tmp/
Disallow: /redesign/
Disallow: /catalog/
Disallow: /catalog.php?recid=
Sitemap: https://www.hup.harvard.edu/sitemap.xml

User-agent: hul-wax
Disallow: 

User-agent: *
Crawl-delay: 10
Disallow: /error/
Disallow: /exhibits/
Disallow: /images/
Disallow: /includes/
Disallow: /offers/
Disallow: /order/
Disallow: /pdf/
Disallow: /rss/
Disallow: /site_graphics/
Disallow: /statistics/
Disallow: /tmp/
Disallow: /redesign/
Disallow: /catalog/
Disallow: /catalog.php?recid=
Sitemap: https://www.hup.harvard.edu/sitemap.xml

# Certain social media sites are whitelisted to allow images in sharing previews. -ACF 11/30/18
User-agent: Twitterbot
Allow: /images/
Disallow: /images/badges/
Disallow: /images/graphics/

# FB image sharing is actually working OK so don’t need this ATM. -ACF 11/30/18
# User-agent: facebookexternalhit
# Allow: /images/