#robots.txt file for http://www.pedestrians.org and http://www.pedestrian.org
User-agent: *
Disallow: /cgi-bin
Disallow: /htbin
Disallow: /klwg
Disallow: /autoresponders
Disallow: /_vti_bin
Disallow: /_vti-cnf
Disallow: /_vti_log
Disallow: /_vti_pvt
Disallow: /_vti_txt
Disallow: /_private
Disallow: /_fpclass

Hit-rate: 12
# wait 12 seconds before starting a new URL request
Visiting-hours: 20:00EDT-05:00EDT
# index this site ONLY between  8PM - 5AM EDT 
Concurrent-hits: 2
# limit concurrent active URLS to 2 for each index server