User-agent: * # Default ############ Disallow: /CustomErrors/ Disallow: /WebPortal/* Disallow: /*.xml$ Disallow: /id/* Disallow: */basket.htm$ Disallow: */checkout.htm$ Disallow: */favorite.htm$ Disallow: */compare.htm$ Allow: /WebPortal/$ Allow: /sitemap.xml$ Allow: /CatStruct_*.xml$ Allow: /CatItem_*.xml$ Allow: /RSSGoogleShopping_*.xml$ Allow: /RSSFacebook_*.xml$ User-agent: AhrefsBot Crawl-Delay: 10 User-agent: SemrushBot Crawl-Delay: 10 User-agent: bingbot Crawl-Delay: 10 User-agent: MJ12bot Crawl-Delay: 10 User-agent: DataForSeoBot Crawl-Delay: 10 # Project