# $Id: robots.txt 92365 2007-09-23 14:04:27Z oden $ # $HeadURL: svn+ssh://svn.mandriva.com/svn/packages/cooker/apache-conf/current/SOURCES/robots.txt $ # exclude help system from robots User-agent: * Disallow: /char_move/ Disallow: /compressed/ Disallow: /css/ Disallow: /eqatlas/ Disallow: /images/ Disallow: /include/ Disallow: /login_backup/ Disallow: /login_stats/ Disallow: /old/ Disallow: /temp/ Disallow: /update/ Disallow: /test/ Disallow: /launcher/ # the next line is a spam bot trap, for grepping the logs. you should _really_ change this to something else... Disallow: /all_our_e-mail_addresses # same idea here... Disallow: /admin/ # but allow htdig to index our doc-tree #User-agent: htdig #Disallow: # disallow stress test user-agent: stress-agent Disallow: /