User-agent: *
# HJH, 05/16/08, these are subdirectories accessed through view
Disallow: /twiki/bin/view/TWiki/
Disallow: /twiki/bin/view/Sandbox/
Disallow: /twiki/bin/view/Main/
# HJH, 05/16/08, crawlers should not crawl attachments at all
Disallow: /twiki/pub/
Disallow: /twiki/pub/
Disallow: /twiki/pub/
# HJH, 05/16/08, came with default twiki robots.txt file
Disallow: /twiki/list/
# HJH, 05/16/08, nothing in our repository should be crawled
Disallow: /svn/
# HJH, 05/16/08, kwiki stuff is all out of date
Disallow: /~csrm/kwiki/
Disallow: /~linuxug/kwiki/
Disallow: /~csrr/kwiki/
# HJH, 05/16/08, these are all specific programs or files in bin
Disallow: /twiki/bin/LocalLib.cfg
Disallow: /twiki/bin/attach
Disallow: /twiki/bin/bibsearch
Disallow: /twiki/bin/changes
Disallow: /twiki/bin/configure
Disallow: /twiki/bin/edit
Disallow: /twiki/bin/extender
Disallow: /twiki/bin/geturl
Disallow: /twiki/bin/html2tml
Disallow: /twiki/bin/installpasswd
Disallow: /twiki/bin/login
Disallow: /twiki/bin/logon
Disallow: /twiki/bin/logos
Disallow: /twiki/bin/mailnotify
Disallow: /twiki/bin/manage
Disallow: /twiki/bin/minifycss
Disallow: /twiki/bin/minifyjs
Disallow: /twiki/bin/mod_perl_startup
Disallow: /twiki/bin/natsearch
Disallow: /twiki/bin/oops
Disallow: /twiki/bin/passwd
Disallow: /twiki/bin/preview
Disallow: /twiki/bin/rdiff
Disallow: /twiki/bin/rdiffauth
Disallow: /twiki/bin/register
Disallow: /twiki/bin/rename
Disallow: /twiki/bin/render.pl
Disallow: /twiki/bin/render.sh
Disallow: /twiki/bin/resetpasswd
Disallow: /twiki/bin/rest
Disallow: /twiki/bin/rewriteshbang
Disallow: /twiki/bin/save
Disallow: /twiki/bin/savemulti
Disallow: /twiki/bin/search
Disallow: /twiki/bin/searchmulti
Disallow: /twiki/bin/setlib.cfg
Disallow: /twiki/bin/statistics
Disallow: /twiki/bin/testenv
Disallow: /twiki/bin/tick_twiki
Disallow: /twiki/bin/tml2html
Disallow: /twiki/bin/upgrade_emails
Disallow: /twiki/bin/upload
Disallow: /twiki/bin/viewauth
Disallow: /twiki/bin/viewfile