#Allow state google engine to search cgi-bin/news
User-Agent: gsa-crawler
Allow: /cgi-bin/news/
# This is for every spider!
User-Agent: *
# stay away from this
Disallow: /cgi-bin/ #and everything in it
Disallow: /mdcdev/ #and everything in it
Disallow: /masternaturalist/ #and everything in it
Disallow: /hmtr.htm #clues for cybersurfari
Disallow:/chrisd/
Disallow:/library/
Disallow:/test/   #directory for testing
Disallow: state.mo.us
Disallow: *_new.htm #Should disallow google search of new pages
Disallow: *_old.htm*