#State Google appliance, disallowing past sessions
User-agent: gsa-crawler
Disallow: /bills.aspx
Disallow: /print.aspx
Disallow: /billcentral.aspx
Disallow: /billtracking/bills95/
Disallow: /content.aspx?info=/bills95/
Disallow: /billtracking/bills96/
Disallow: /content.aspx?info=/bills96/
Disallow: /billtracking/bills97/
Disallow: /content.aspx?info=/bills97/
Disallow: /billtracking/bills98/
Disallow: /content.aspx?info=/bills98/
Disallow: /billtracking/bills99/
Disallow: /content.aspx?info=/bills99/
Disallow: /billtracking/bills00/
Disallow: /content.aspx?info=/bills00/
Disallow: /billtracking/bills01/
Disallow: /content.aspx?info=/bills01/
Disallow: /billtracking/spec01/
Disallow: /content.aspx?info=/spec01/
Disallow: /billtracking/bills02/
Disallow: /content.aspx?info=/bills02/
Disallow: /billtracking/bills03/
Disallow: /content.aspx?info=/bills03/
Disallow: /billtracking/bills033/
Disallow: /content.aspx?info=/bills033/
Disallow: /billtracking/bills034/
Disallow: /content.aspx?info=/bills034/
Disallow: /billtracking/bills041/
Disallow: /content.aspx?info=/bills041/
Disallow: /billtracking/bills051/
Disallow: /content.aspx?info=/bills051/
Disallow: /billtracking/bills053/
Disallow: /content.aspx?info=/bills053/
Disallow: /billtracking/bills061/
Disallow: /content.aspx?info=/bills061/
Disallow: /billtracking/bills071/
Disallow: /content.aspx?info=/bills071/
Disallow: /billtracking/bills073/
Disallow: /content.aspx?info=/bills073/
Disallow: /billtracking/bills081/
Disallow: /content.aspx?info=/bills081/

User-agent: *
Disallow: /*.axd$
Disallow: /DXR.axd
Disallow: /bills.aspx
Disallow: /print.aspx
Disallow: /billcentral.aspx
Disallow: /javascript:false
Disallow: /*?
Crawl-delay: 120


#Yahoo. No indication what the delay frequency is, max listed at 5. *Supports delay
User-agent: Slurp
Disallow: /bills.aspx
Disallow: /print.aspx
Disallow: /billcentral.aspx
Crawl-delay: 5

#MSN. *Supports delay
User-agent: MSNBot
Disallow: /bills.aspx
Disallow: /print.aspx
Disallow: /billcentral.aspx
Crawl-delay: 120

#Google. *Does not support crawl delay.
User-agent: Googlebot
Disallow: /bills.aspx
Disallow: /print.aspx
Disallow: /billcentral.aspx

#Google Images.*Does not support crawl delay.
User-agent: Googlebot-Image
Disallow: /bills.aspx
Disallow: /print.aspx
Disallow: /billcentral.aspx

#MSN Images. 
User-agent: psbot
Disallow: /bills.aspx
Disallow: /print.aspx
Disallow: /billcentral.aspx
Crawl-delay: 120

#SingingFish. 
User-agent: asterias
Disallow: /bills.aspx
Disallow: /print.aspx
Disallow: /billcentral.aspx
Crawl-delay: 120

#Yahoo Multimedia.
User-agent: yahoo-mmcrawler
Disallow: /bills.aspx
Disallow: /print.aspx
Disallow: /billcentral.aspx
Crawl-delay: 120

#Ask.com. *Supports delay
User-agent: Teoma
Disallow: /bills.aspx
Disallow: /print.aspx
Disallow: /billcentral.aspx
Crawl-delay: 120

#Searchme.com.
User-agent: Charlotte
Disallow: /bills.aspx
Disallow: /print.aspx
Disallow: /billcentral.aspx
Crawl-delay: 120

#Lexis-Nexis
User-agent: lnbot
Disallow: /bills.aspx
Disallow: /print.aspx
Disallow: /billcentral.aspx
Crawl-delay: 120

#Zibb.com. Has preset crawl delay of 30 seconds. Appears to be following a 60 second delay instead.
User-agent: zibber-v0.1(www.zibb.com/crawler/)
Disallow: /bills.aspx
Disallow: /print.aspx
Disallow: /billcentral.aspx

#Others.
User-agent: Gigabot
Disallow: /bills.aspx
Disallow: /print.aspx
Disallow: /billcentral.aspx
Crawl-delay: 120

User-agent: Scrubby
Disallow: /bills.aspx
Disallow: /print.aspx
Disallow: /billcentral.aspx
Crawl-delay: 120

User-agent: Robozilla
Disallow: /bills.aspx
Disallow: /print.aspx
Disallow: /billcentral.aspx
Crawl-delay: 120

User-agent: Nutch
Disallow: /bills.aspx
Disallow: /print.aspx
Disallow: /billcentral.aspx
Crawl-delay: 120

User-agent: ia_archiver
Disallow: /bills.aspx
Disallow: /print.aspx
Disallow: /billcentral.aspx
Crawl-delay: 120

User-agent: baiduspider
Disallow: /bills.aspx
Disallow: /print.aspx
Disallow: /billcentral.aspx
Crawl-delay: 120

User-agent: FiscalNote/1.0
Disallow: /bills.aspx
Disallow: /print.aspx
Disallow: /billcentral.aspx
Crawl-delay: 120

#If you are trying to get legislative data, use https://house.mo.gov/xmlexports.html instead of crawling the site.