# wwwd/robots.txt   2009-9-22   AK.
# $Id: robots.txt 28477 2009-09-22 05:07:45Z akenning $
# => http://www.topology.org/robots.txt

# asterias is the spider of singingfish.com. topology.org has no multimedia!
# Altavista's Scooter indexes once per day. That's too often!!
# Zao (Kototoi) downloads full copies every 48 hours or so.
# Zao does _not_ respect this robots.txt file! Use the firewall for kototoi.
# Inktomi uses "slurp". So does Yahoo. But they give me almost no hits.
# Picsearch.com (psbot) is wasting its time looking for pictures here.
# The alexa.com ia_archiver downloads once a day! This is the wayback machine!
# Zyborg may be the string for wisenutbot used by looksmart.
# Zyborg as used by WISEnutbot.com ignores the Zyborg string.
# Speedy is the label for entireweb.com which indexes every 48 hours. Grrr....
# Fast is for webcrawler: http://www.alltheweb.com/help/webmaster/crawler
# 2004-6-5: Maybe Teoma is too rampant these days.
# Krugle: http://www.krugle.com/crawler/info.html
User-agent: asterias
User-agent: DepSpid
User-agent: envolk
User-agent: fast
User-agent: gigabot
User-agent: http://www.almaden.ibm.com/cs/crawler
User-agent: ia_archiver
User-agent: InfoNavirobot
User-agent: Krugle
User-agent: msnbot
User-agent: Nutch
User-agent: panscient.com
User-agent: psbot
User-agent: Scooter
User-agent: slurp
User-agent: speedy
User-agent: szukacz
User-agent: Teoma
User-agent: turnitinbot
User-agent: TutorGig
User-agent: Yandex
User-agent: Zao
User-agent: ZyBorg
Disallow: /

# Googlebot is a sophisticated and polite robot which understands everything!
# Full marks: 20/20 with a gold star!
# 2006-4-24: Googlebot has been getting steadily worse in the last 12 months.
# But that might be just the China-based googlebot which downloads too fast.
User-agent: Googlebot
Disallow: /bluetooth.html
Disallow: /bwshare-
Disallow: /sim.html
Disallow: /ak/
Disallow: /attacks/
Disallow: /ddc/
Disallow: /dia/
Disallow: /extra/
Disallow: /fb/
Disallow: /human/
Disallow: /ideas/trag
Disallow: /images/
Disallow: /iso/
Disallow: /java/
Disallow: /midi/ratpc/
Disallow: /midi/songs/
Disallow: /php/
Disallow: /php3/
Disallow: /phpvnconv/
Disallow: /ps/
Disallow: /soc/fbc.html
Disallow: /src/akpref/
Disallow: /status/
Disallow: /tex/conc/ps/
Disallow: /tex/lilypond/
Disallow: /tex/vs/
Disallow: /x/
Disallow: /*.akm$
Disallow: /*.asc$
Disallow: /*.bz2$
Disallow: /*.c$
Disallow: /*.css$
Disallow: /*.csv$
Disallow: /*.dat$
Disallow: /*.doc$
Disallow: /*.gif$
Disallow: /*.gz$
Disallow: /*.h$
Disallow: /*.java$
Disallow: /*.jpeg$
Disallow: /*.jpg$
Disallow: /*.JPG$
Disallow: /*.ly$
Disallow: /*.m4$
Disallow: /*.mid$
Disallow: /*.mp3$
Disallow: /*.ogg$
Disallow: /*.pdf$
Disallow: /*.pdf.gz$
Disallow: /*.phpmod$
Disallow: /*.pl$
Disallow: /*.pm$
Disallow: /*.png$
Disallow: /*.ps$
Disallow: /*.ps.gz$
Disallow: /*.ps.zip$
Disallow: /*.sys$
Disallow: /*.tex$
Disallow: /*.tgz$
Disallow: /*.tsv$
Disallow: /*.xml$
Disallow: /*.zip$
Disallow: /*.Z$
Disallow: /*/makefile
Disallow: /*/LICENCE
Disallow: /*/CHANGES

User-agent: *
Disallow: /