# robots.txt für https://pub.uni-bielefeld.de

User-agent: *
Allow: /record/
Allow: /person/
Allow: /siteindex.xml
Allow: /sitemap
Disallow: /record/*.mods
Disallow: /record/*.datacite
Disallow: /record/*.rtf
Disallow: /record/*.dc
Disallow: /record/*.json
Disallow: /record/*.jsonl
Disallow: /record/*.bibtex
Disallow: /record/*.yaml
Disallow: /record/*.csl_json
Disallow: /record/*.dc_json
Disallow: /record/*.aref
Disallow: /record/*.ris
Disallow: /record?
Disallow: /
Crawl-delay: 1.5

User-agent: SemrushBot
Disallow: /

Sitemap: https://pub.uni-bielefeld.de/siteindex.xml