# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file

User-agent: *
Disallow: /observations?
Disallow: /observations/?
Disallow: /observations.csv
Disallow: /observations.csv?
Disallow: /taxa/search
Disallow: /taxa/search?
Disallow: /*?

User-agent: Googlebot
Disallow: /taxa/*/description$
Disallow: *.csv*
Disallow: *page=*
Disallow: /*?

User-agent: ia_archiver
Disallow: