# $MirOS: www/files/robots.txt,v 1.2 2007/10/02 18:55:00 tg Exp $

# Allow downloads

User-Agent: wget
Disallow:

# Disallow the W3C Linkchecker
# XXX to be revisited

User-Agent: W3C-checklink
# dynamically generated content
Disallow: /htman
Disallow: /man
Disallow: /man1
Disallow: /man2
Disallow: /man3
Disallow: /man4
Disallow: /man5
Disallow: /man6
Disallow: /man7
Disallow: /man8
Disallow: /man9
Disallow: /cman/
Disallow: /r9man/
# content which is out of date
Disallow: /?wlog-1
Disallow: /?wlog-2
Disallow: /?wlog-3
Disallow: /?wlog-4
Disallow: /?wlog-5
Disallow: /?wlog-6
# other not-to-be-listed content
Disallow: /GBG/
Disallow: /MirOS/
Disallow: /current/

# Disallow other bots

User-Agent: *
Disallow: /GBG/
# search machines shouldn't index dups
Disallow: /man1
Disallow: /man2
Disallow: /man3
Disallow: /man4
Disallow: /man5
Disallow: /man6
Disallow: /man7
Disallow: /man8
Disallow: /man9
# nor the download area
Disallow: /MirOS/
Disallow: /current/