# 
# Agent-specific settings for indexing the site.
#

#  Allow Google for full text crawling 
User-agent: GoogleBot
# Google ignores Crawl-delay
# Crawl-delay: 2
# URL prefixes
# URL wildcards
Disallow: /admin/
Disallow: /alerts/
Disallow: /cache/
Disallow: /carnet/
Disallow: /cart
Disallow: /cart/
Disallow: /cartregistration/
Disallow: /cc/
Disallow: /commenting/
Disallow: /counterstats/
Disallow: /deliver/
Disallow: /editcontent/
Disallow: /email
Disallow: /email/
Disallow: /error/
Disallow: /export/
Disallow: /librarians/uploadbranding
Disallow: /licence-admin/
Disallow: /managesavedsearches
Disallow: /markedlist/
Disallow: /mostviewed/
Disallow: /offer/
Disallow: /openurl/profile
Disallow: /orders
Disallow: /orders/
Disallow: /pricing/
Disallow: /profile
Disallow: /profile/
Disallow: /pubstats/
Disallow: /recommendto
Disallow: /reference/
Disallow: /regactivation
Disallow: /regactivation/
Disallow: /registration
Disallow: /registration/
Disallow: /resourcelookup
Disallow: /runsavedsearch
Disallow: /savedsearches
Disallow: /search
Disallow: /search/
Disallow: /selfservice/
Disallow: /session
Disallow: /session/
Disallow: /statslogredirect
Disallow: /statslogwrapper
Disallow: /subscribedtitles
Disallow: /uploadfile
Disallow: /user/
Disallow: /workflow/
Disallow: /content/contributor
Disallow: /content/contributor/
Disallow: /content/institution
Disallow: /content/institution/

# 
# Yandex have two bots so they can work twice as slowly
#
User-agent: YandexBot
Crawl-delay: 4
Disallow: /admin/
Disallow: /alerts/
Disallow: /cache/
Disallow: /carnet/
Disallow: /cart
Disallow: /cart/
Disallow: /cartregistration/
Disallow: /cc/
Disallow: /commenting/
Disallow: /counterstats/
Disallow: /deliver/
Disallow: /editcontent/
Disallow: /email
Disallow: /email/
Disallow: /error/
Disallow: /export/
Disallow: /librarians/uploadbranding
Disallow: /licence-admin/
Disallow: /managesavedsearches
Disallow: /markedlist/
Disallow: /mostviewed/
Disallow: /offer/
Disallow: /openurl/profile
Disallow: /orders
Disallow: /orders/
Disallow: /pricing/
Disallow: /profile
Disallow: /profile/
Disallow: /pubstats/
Disallow: /recommendto
Disallow: /reference/
Disallow: /regactivation
Disallow: /regactivation/
Disallow: /registration
Disallow: /registration/
Disallow: /resourcelookup
Disallow: /runsavedsearch
Disallow: /savedsearches
Disallow: /search
Disallow: /search/
Disallow: /selfservice/
Disallow: /session
Disallow: /session/
Disallow: /statslogredirect
Disallow: /statslogwrapper
Disallow: /subscribedtitles
Disallow: /uploadfile
Disallow: /user/
Disallow: /workflow/
Disallow: /content/contributor
Disallow: /content/contributor/
Disallow: /content/institution
Disallow: /content/institution/

# 
# Yandex have two bots so they can work twice as slowly
#
User-agent: YandexImages
Crawl-delay: 4
Disallow: /admin/
Disallow: /alerts/
Disallow: /cache/
Disallow: /carnet/
Disallow: /cart
Disallow: /cart/
Disallow: /cartregistration/
Disallow: /cc/
Disallow: /commenting/
Disallow: /counterstats/
Disallow: /deliver/
Disallow: /editcontent/
Disallow: /email
Disallow: /email/
Disallow: /error/
Disallow: /export/
Disallow: /librarians/uploadbranding
Disallow: /licence-admin/
Disallow: /managesavedsearches
Disallow: /markedlist/
Disallow: /mostviewed/
Disallow: /offer/
Disallow: /openurl/profile
Disallow: /orders
Disallow: /orders/
Disallow: /pricing/
Disallow: /profile
Disallow: /profile/
Disallow: /pubstats/
Disallow: /recommendto
Disallow: /reference/
Disallow: /regactivation
Disallow: /regactivation/
Disallow: /registration
Disallow: /registration/
Disallow: /resourcelookup
Disallow: /runsavedsearch
Disallow: /savedsearches
Disallow: /search
Disallow: /search/
Disallow: /selfservice/
Disallow: /session
Disallow: /session/
Disallow: /statslogredirect
Disallow: /statslogwrapper
Disallow: /subscribedtitles
Disallow: /uploadfile
Disallow: /user/
Disallow: /workflow/
Disallow: /content/contributor
Disallow: /content/contributor/
Disallow: /content/institution
Disallow: /content/institution/

# 
# Bing specific configuration
#
User-agent: bingbot
Crawl-delay: 2
Disallow: /admin/
Disallow: /alerts/
Disallow: /cache/
Disallow: /carnet/
Disallow: /cart
Disallow: /cart/
Disallow: /cartregistration/
Disallow: /cc/
Disallow: /commenting/
Disallow: /counterstats/
Disallow: /deliver/
Disallow: /editcontent/
Disallow: /email
Disallow: /email/
Disallow: /error/
Disallow: /export/
Disallow: /librarians/uploadbranding
Disallow: /licence-admin/
Disallow: /managesavedsearches
Disallow: /markedlist/
Disallow: /mostviewed/
Disallow: /offer/
Disallow: /openurl/profile
Disallow: /orders
Disallow: /orders/
Disallow: /pricing/
Disallow: /profile
Disallow: /profile/
Disallow: /pubstats/
Disallow: /recommendto
Disallow: /reference/
Disallow: /regactivation
Disallow: /regactivation/
Disallow: /registration
Disallow: /registration/
Disallow: /resourcelookup
Disallow: /runsavedsearch
Disallow: /savedsearches
Disallow: /search
Disallow: /search/
Disallow: /selfservice/
Disallow: /session
Disallow: /session/
Disallow: /statslogredirect
Disallow: /statslogwrapper
Disallow: /subscribedtitles
Disallow: /uploadfile
Disallow: /user/
Disallow: /workflow/
Disallow: /content/contributor
Disallow: /content/contributor/
Disallow: /content/institution
Disallow: /content/institution/

# 
# Baidu specific configuration
#
User-agent: Baiduspider
Crawl-delay: 2
Disallow: /admin/
Disallow: /alerts/
Disallow: /cache/
Disallow: /carnet/
Disallow: /cart
Disallow: /cart/
Disallow: /cartregistration/
Disallow: /cc/
Disallow: /commenting/
Disallow: /counterstats/
Disallow: /deliver/
Disallow: /editcontent/
Disallow: /email
Disallow: /email/
Disallow: /error/
Disallow: /export/
Disallow: /librarians/uploadbranding
Disallow: /licence-admin/
Disallow: /managesavedsearches
Disallow: /markedlist/
Disallow: /mostviewed/
Disallow: /offer/
Disallow: /openurl/profile
Disallow: /orders
Disallow: /orders/
Disallow: /pricing/
Disallow: /profile
Disallow: /profile/
Disallow: /pubstats/
Disallow: /recommendto
Disallow: /reference/
Disallow: /regactivation
Disallow: /regactivation/
Disallow: /registration
Disallow: /registration/
Disallow: /resourcelookup
Disallow: /runsavedsearch
Disallow: /savedsearches
Disallow: /search
Disallow: /search/
Disallow: /selfservice/
Disallow: /session
Disallow: /session/
Disallow: /statslogredirect
Disallow: /statslogwrapper
Disallow: /subscribedtitles
Disallow: /uploadfile
Disallow: /user/
Disallow: /workflow/
Disallow: /content/contributor
Disallow: /content/contributor/
Disallow: /content/institution
Disallow: /content/institution/

User-agent: facebookexternalhit
User-agent: LinkedInBot
User-agent: Twitterbot
Allow: /

User-agent: GPTBot
Disallow: /

# 
# Generic settings for indexing the site.
#
User-agent: *
Crawl-delay: 2
Disallow: /admin/
Disallow: /alerts/
Disallow: /cache/
Disallow: /carnet/
Disallow: /cart
Disallow: /cart/
Disallow: /cartregistration/
Disallow: /cc/
Disallow: /commenting/
Disallow: /counterstats/
Disallow: /deliver/
Disallow: /editcontent/
Disallow: /email
Disallow: /email/
Disallow: /error/
Disallow: /export/
Disallow: /librarians/uploadbranding
Disallow: /licence-admin/
Disallow: /managesavedsearches
Disallow: /markedlist/
Disallow: /mostviewed/
Disallow: /offer/
Disallow: /openurl/profile
Disallow: /orders
Disallow: /orders/
Disallow: /pricing/
Disallow: /profile
Disallow: /profile/
Disallow: /pubstats/
Disallow: /recommendto
Disallow: /reference/
Disallow: /regactivation
Disallow: /regactivation/
Disallow: /registration
Disallow: /registration/
Disallow: /resourcelookup
Disallow: /runsavedsearch
Disallow: /savedsearches
Disallow: /search
Disallow: /search/
Disallow: /selfservice/
Disallow: /session
Disallow: /session/
Disallow: /statslogredirect
Disallow: /statslogwrapper
Disallow: /subscribedtitles
Disallow: /uploadfile
Disallow: /user/
Disallow: /workflow/
Disallow: /content/contributor
Disallow: /content/contributor/
Disallow: /content/institution
Disallow: /content/institution/