# /robots.txt for http://uspest.org
# comments to coopl@science.oregonstate.edu

# These user agents hit us 10-100 times as much as google.
# IMO, that's grossly wasteful and perhaps abusive.

User-agent: The Knowledge AI
Disallow: /
Disallow: *

User-agent: Seekport Crawler
Disallow: /
Disallow: *

User-agent: SemrushBot
Disallow: /
Disallow: *

User-agent: MegaIndex.ru
Disallow: /
Disallow: *

User-agent: YandexBot
Disallow: /
Disallow: *

User-agent: DotBot
Disallow: /
Disallow: *

User-agent: AhrefsBot
Disallow: /
Disallow: *

User-agent: BLEXBot
Disallow: /
Disallow: *

User-agent: MJ12bot
Disallow: /
Disallow: *

User-agent: GPTBot
Disallow: /
Disallow: *

User-agent: Gecko/20100115
Disallow: /
Disallow: *

User-agent: (Windows; U; Windows NT 6.1; fr; rv:1.9.2) Gecko/20100115 Firefox/3.6
Disallow: /
Disallow: *

# Added by Len to get Google Analytics to work better:
User-agent: Googlebot
Allow: /wea/
Allow: /risk/
Allow: /dd/
Allow: /cgi-bin/
Disallow: /data/
Disallow: /histor_data/
Disallow: /histor_data2/
Disallow: /Java/


# Everybody else 
User-agent: *
Disallow: /Java/
Disallow: /aw/
Disallow: /calc/
Disallow: /current/
Disallow: /data/
Disallow: /fcst/
Disallow: /grasslinks/
Disallow: /histor_data/
Disallow: /histor_data2/
Disallow: /lyrics/
Disallow: /manual/
Disallow: /mpl/
Disallow: /pnw/
Disallow: /virt/
Disallow: /sdb
Disallow: /tmp
Disallow: /kengray
Disallow: /wchannel
Disallow: /ipmp
Disallow: /wea_dd
Disallow: /dd/do_model
Disallow: /risk/models