############################### # Created by the webmaster[at]histosoft.com # robots.txt file for www.myhistosoft.com # Last modified Saturday, June 23, 2007 # # addresses all robots by using wild card * # User-agent: * # # list folders robots are not allowed to index # #Disallow: /*.htaccess$ #Disallow: /*.htpasswd$ Disallow: /.htaccess Disallow: /.htpasswd Disallow: /_testing/ Disallow: /_tmpdata/ Disallow: /cgi-bin/ Disallow: /images/ Disallow: /js/ Disallow: /main/exclude/ Disallow: /main/includes/ Disallow: /main/member/ #Disallow: /main/member2b/ Disallow: /search/cgi-bin/ Disallow: /style/ #User-agent: googlebot #User-agent: * #Disallow: *.cgi #Disallow: *.con #Disallow: *.dat #Disallow: *.log #Disallow: *.php #Disallow: *.pl #Disallow: *.txt # # list specific files robots are not allowed to index # Disallow: /tutorials/custom_error_page.html # # forbidden robots #User-agent: TurnitinBot #Disallow: / #User-agent: http://www.almaden.ibm.com/cs/crawler #Disallow: / # # End of robots.txt file # ###############################