# This file lists local URLs that well-behaved robots should ignore

User-agent: *
Disallow: /cgi-bin		# includes some large virtual spaces
Disallow: /urchin		# skip usage reports
Disallow: /logs			# skip logs

User-agent: linkwalker
Disallow: