########################## ## FS robots.txt: 20/10/17 ########################## #User-agent: archive.org_bot #User-agent: bingbot #User-agent: Googlebot #User-agent: msnbot #User-agent: psbot #User-agent: Slurp #Disallow: /catalog/images/ User-agent: * Disallow: /catalog/ext/ Disallow: /catalog/includes/ Disallow: /catalog/mts/ Disallow: /catalog/account.php Disallow: /catalog/account_edit.php Disallow: /catalog/account_history.php Disallow: /catalog/account_history_info.php Disallow: /catalog/account_newsletters.php Disallow: /catalog/account_notifications.php Disallow: /catalog/account_password.php Disallow: /catalog/address_book.php Disallow: /catalog/address_book_process.php Disallow: /catalog/advanced_search.php Disallow: /catalog/advanced_search_result.php Disallow: /catalog/checkout_confirmation.php Disallow: /catalog/checkout_payment.php Disallow: /catalog/checkout_payment_address.php Disallow: /catalog/checkout_paypalipn.php Disallow: /catalog/checkout_process.php Disallow: /catalog/checkout_shipping.php Disallow: /catalog/checkout_shipping_address.php Disallow: /catalog/checkout_success.php Disallow: /catalog/conditions.php Disallow: /catalog/contact_us.php Disallow: /catalog/cookie_usage.php Disallow: /catalog/create_account.php Disallow: /catalog/create_account_success.php Disallow: /catalog/download.php Disallow: /catalog/info_shopping_cart.php Disallow: /catalog/login.php Disallow: /catalog/logoff.php Disallow: /catalog/mailhive.php Disallow: /catalog/opensearch.php Disallow: /catalog/password_forgotten.php Disallow: /catalog/password_reset.php Disallow: /catalog/popup_image.php Disallow: /catalog/popup_search_help.php Disallow: /catalog/product_reviews_write.php Disallow: /catalog/redirect.php Disallow: /catalog/shopping_cart.php Disallow: /catalog/tell_a_friend.php # User-agent: Googlebot-Image User-agent: msnbot-media User-agent: 007ac9 Crawler User-agent: Abonti User-agent: AdnormCrawler User-agent: AdvBot User-agent: AhrefsBot User-agent: BaiduSpider User-agent: BaiduSpider-image User-agent: betaBot User-agent: bitlybot User-agent: BLEXBot User-agent: BPImageWalker User-agent: BUbiNG User-agent: ca-crawler User-agent: centurybot9 User-agent: cmscrawler User-agent: CMS crawler User-agent: Cliqzbot User-agent: coccoc User-agent: datagnionbot User-agent: deadlinkchecker User-agent: DeuSu User-agent: DomainAppender User-agent: DomainCrawler User-agent: DotBot User-agent: EasouSpider User-agent: electricmonk User-agent: Exabot User-agent: Experibot User-agent: Findxbot User-agent: gimme60bot User-agent: gimmeUSAbot User-agent: Gowikibot User-agent: GrapeshotCrawler User-agent: GWPImages User-agent: HaosouSpider User-agent: ia_archiver User-agent: IErachnid User-agent: imagescoccoc User-agent: ips-agent User-agent: KickFire User-agent: Linkdexbot User-agent: LinkParser User-agent: LinkSaver User-agent: Lipperhey User-agent: LSSRocketCrawler User-agent: ltx71 User-agent: Mail.RU_Bot User-agent: meanpathbot User-agent: memoryBot User-agent: MJ12bot User-agent: MojeekBot User-agent: NetcraftSurveyAgent User-agent: netEstate NE Crawler User-agent: NetSeer crawler User-agent: nbot User-agent: Nutch User-agent: OrangeBot User-agent: PagesInventory User-agent: PageAnalyser User-agent: panscient User-agent: Pinterest User-agent: plukkie User-agent: PrivacyAwareBot User-agent: publiclibraryarchive.org/1.0 User-agent: Quantify User-agent: Riddler User-agent: rogerbot User-agent: SafeDNSbot User-agent: ScoutJet User-agent: Screaming Frog User-agent: SearchmetricsBot User-agent: SemrushBot User-agent: SeznamBot User-agent: SISTRIX Crawler User-agent: SiteExplorer User-agent: SkimBot User-agent: SMTBot User-agent: SocialRankIOBot User-agent: spbot User-agent: ssearch_bot User-agent: TestCrawler User-agent: TurnitinBot User-agent: uMBot User-agent: Uptimebot User-agent: Vagabondo User-agent: VoilaBot User-agent: WBSearchBot User-agent: webcollage User-agent: webcrawler User-agent: WeSEE User-agent: woobot User-agent: wonderbot User-agent: worldwebheritage User-agent: worldwebheritage.org User-agent: Wotbox User-agent: Yacybot User-agent: YandexBot User-agent: YandexImages User-agent: Yeti User-agent: YisouSpider User-agent: yoozBot Disallow: / ################# ## robots.txt-END