# robots.txt file (ALL LOWERCASE) # This file should go in your web site's ROOT directory # apply settings to ALL search engine spiders/crawlers User-agent: * # These settings will keep spiders from indexing your unwanted pages Disallow: /catalog/admin Disallow: /catalog/account.php # Disallow: /catalog/advanced_search.php Disallow: /catalog/checkout_shipping.php Disallow: /catalog/checkout_success.php Disallow: /catalog/create_account.php Disallow: /catalog/login.php Disallow: /catalog/password_forgotten.php # Disallow: /catalog/popup_image.php Disallow: /catalog/shopping_cart.php # IF YOU DO NOT WISH TO HAVE THE GOOGLE IMAGE BOT SCAN YOUR DOMAIN FOR IMAGES # THEN YOU CAN INCLUDE THE FOLLOWING IN YOUR ROBOTS FILE. # User-agent: Googlebot-Image # Disallow: /