# Veridian robots.txt file #=========================================================# # Block all web crawlers from accessing the site #=========================================================# User-agent: * Disallow: / # Applebot needs to be specifically told to stop; otherwise, it follows Googlebot User-agent: Applebot Disallow: / # Explicitly state that it is not for Google’s machine learning purposes User-agent: Google-Extended Disallow: / #=========================================================# # Only allow the following bots #=========================================================# User-agent: archive.org_bot Sitemap: https://idnc.library.illinois.edu/sitemap-illinois-index.xml Crawl-delay: 1 # Wait one second between requests Disallow: /cgi-bin/imageserver.pl # Block imageserver requests Disallow: /?a=a # Block authentication Disallow: /?a=cb # Block content blocking Disallow: /?a=gs # Block graphserver requests Disallow: /?a=is # Block imageserver requests Disallow: /?a=pme # Block publication metadata editor Disallow: /?a=q # Block queries Disallow: /?a=tc # Block text correction Disallow: /?a=ua # Block user account Disallow: /?a=ur # Block user registration Disallow: /*iiif* # Block all IIIF requests Disallow: /?*&e=* # Block all internal links # ILLINOIS CUSTOMISATION: Additional blocks Disallow: /?a=d&d=ITH # Block ITH documents Disallow: /?a=d&d=DIL19690521.2.66 Disallow: /?a=d&d=AUE19770128.2.4 User-agent: bingbot Sitemap: https://idnc.library.illinois.edu/sitemap-illinois-index.xml Crawl-delay: 1 # Wait one second between requests Disallow: /cgi-bin/imageserver.pl # Block imageserver requests Disallow: /?a=a # Block authentication Disallow: /?a=cb # Block content blocking Disallow: /?a=gs # Block graphserver requests Disallow: /?a=is # Block imageserver requests Disallow: /?a=pme # Block publication metadata editor Disallow: /?a=q # Block queries Disallow: /?a=tc # Block text correction Disallow: /?a=ua # Block user account Disallow: /?a=ur # Block user registration Disallow: /*iiif* # Block all IIIF requests Disallow: /?*&e=* # Block all internal links # ILLINOIS CUSTOMISATION: Additional blocks Disallow: /?a=d&d=ITH # Block ITH documents Disallow: /?a=d&d=DIL19690521.2.66 Disallow: /?a=d&d=AUE19770128.2.4 User-agent: Googlebot Sitemap: https://idnc.library.illinois.edu/sitemap-illinois-index.xml Crawl-delay: 1 # Wait one second between requests Disallow: /cgi-bin/imageserver.pl # Block imageserver requests Disallow: /?a=a # Block authentication Disallow: /?a=cb # Block content blocking Disallow: /?a=gs # Block graphserver requests Disallow: /?a=is # Block imageserver requests Disallow: /?a=pme # Block publication metadata editor Disallow: /?a=q # Block queries Disallow: /?a=tc # Block text correction Disallow: /?a=ua # Block user account Disallow: /?a=ur # Block user registration Disallow: /*iiif* # Block all IIIF requests Disallow: /?*&e=* # Block all internal links # ILLINOIS CUSTOMISATION: Additional blocks Disallow: /?a=d&d=ITH # Block ITH documents Disallow: /?a=d&d=DIL19690521.2.66 Disallow: /?a=d&d=AUE19770128.2.4 User-agent: Yahoo Sitemap: https://idnc.library.illinois.edu/sitemap-illinois-index.xml Crawl-delay: 1 # Wait one second between requests Disallow: /cgi-bin/imageserver.pl # Block imageserver requests Disallow: /?a=a # Block authentication Disallow: /?a=cb # Block content blocking Disallow: /?a=gs # Block graphserver requests Disallow: /?a=is # Block imageserver requests Disallow: /?a=pme # Block publication metadata editor Disallow: /?a=q # Block queries Disallow: /?a=tc # Block text correction Disallow: /?a=ua # Block user account Disallow: /?a=ur # Block user registration Disallow: /*iiif* # Block all IIIF requests Disallow: /?*&e=* # Block all internal links # ILLINOIS CUSTOMISATION: Additional blocks Disallow: /?a=d&d=ITH # Block ITH documents Disallow: /?a=d&d=DIL19690521.2.66 Disallow: /?a=d&d=AUE19770128.2.4