# Disallow indexing of specific file extensions User-agent: * Disallow: /*.png$ User-agent: * Disallow: /*.gif$ User-agent: * Disallow: /*.jpg$ User-agent: * Disallow: /*.zip$ User-agent: * Disallow: /*.doc$ User-agent: * Disallow: /*.exe$ User-agent: * Disallow: /*.pdf$ User-agent: * Disallow: /*.php # Added 2010-08-08 Prevent Russian bots from indexing the pages User-agent: Yandex Disallow: / User-agent: spbot Disallow: / # Added 2009-02-23 Prevent Fasterfox from prefetching pages User-agent: Fasterfox Disallow: / # Disallow Collectors and Spam # Added 2008-12-14 User-agent: NextGenSearchBot Disallow: / User-agent: panscient.com Disallow: / User-agent: atSpider Disallow: / User-agent: cherrypicker Disallow: / User-agent: DSurf Disallow: / User-agent: EliteSys Entry Disallow: / User-agent: EmailCollector Disallow: / User-agent: EmailSiphon Disallow: / User-agent: EmailWolf Disallow: / User-agent: Mail Sweeper Disallow: / User-agent: munky Disallow: / User-agent: Roverbot Disallow: / User-agent: WebEmailExtrac Disallow: / # Disallow Offline Browsers User-agent: xGet Disallow: / User-agent: wGet Disallow: / User-agent: WebWalk Disallow: / User-agent: webvac Disallow: / User-agent: WebReaper Disallow: / User-agent: WebMirror Disallow: / User-agent: WebFetcher Disallow: / User-agent: WebCopy Disallow: / User-agent: webcopier Disallow: / User-agent: WebCatcher Disallow: / User-agent: WebBandit Disallow: / User-agent: w3mir Disallow: / User-agent: vobsub Disallow: / User-agent: Templeton Disallow: / User-agent: ssearcher100 Disallow: / User-agent: SpiderBot Disallow: / User-agent: PBWF Disallow: / User-agent: LightningDownload Disallow: / User-agent: KDD Exploror Disallow: / User-agent: Jeeves Disallow: / User-agent: Internet Explore Disallow: / User-agent: InfoSpiders Disallow: / User-agent: httrack Disallow: / User-agent: HavIndex Disallow: / User-agent: GetUrl Disallow: / User-agent: GetBot Disallow: / User-agent: ESIRover Disallow: / User-agent: Download Wonder Disallow: / User-agent: Collage Disallow: / # BEGIN XML-SITEMAP-PLUGIN Sitemap: http://www.reviewsbycomparisonmatrix.com/sitemap.xml.gz # END XML-SITEMAP-PLUGIN