# -*- Mode: Makefile -*-   (not really, but about right for xemacs modes)
#
# See http://info.webcrawler.com/mak/projects/robots/robots.html
# for details 
#
# There's nothing to see here. Move along
#
# Actually, this is almost a complete list of places the robot can access
# It would be simplier to just do
# Disallow:/
# But in case you feel like letting the robot roam your search server
# (or certain parts of it) here's a list you can comment out as needed.
#
User-Agent: *
#Disallow:/		# Uncomment for NO ACCESS at all
Disallow:/cgi-bin/	# scripts
Disallow:/compass/	# search UI
Disallow:/docs/gv/    	# gVine stuff
Disallow:/images/	# pretty pictures
Disallow:/java/	        # Don't bother
Disallow:/mc-icons/	# pretty pictures
Disallow:/ns-icons/	# pretty pictures (same)
Disallow:/queries.html	# list of all queries
Disallow:/rdm/		# server guts
Disallow:/reports/	# old site,query reports
Disallow:/search-ui/	# Enterprise-compatible SearchUI
Disallow:/tests/	# robot and RDM test GUIs
Disallow:/ug/		# User Guide (nethelp)
Disallow:/control_help.html	# random stuff in the default docroot
Disallow:/forbidden.html	# custom server error codes
Disallow:/notfound.html		# custom server error codes
Disallow:/robot_cmd.html
Disallow:/server_error.html	# custom server error codes
Disallow:/test_robot.html
Disallow:/unauthorized.html	# custom server error codes
# useful robot food. 'Allow'
#Disallow:/sites.html	# list of all sites cataloged