User-agent: * # list folders robots are not allowed to index #Disallow: /Maildir/ #Disallow: /logs/ #Disallow: /cgi-bin/ #Disallow: /public_html/rates/ # # list specific files robots are not allowed to index # # # End of robots.txt file