# robots.txt - homeessentialsdirect.com
  # Optimized 2026-02-08
  # ============================================================

  # Sitemaps
  Sitemap: https://www.homeessentialsdirect.com/content/sitemap-new-updated.xml

  # =================================
  # Google crawlers (Merchant/Shopping)
  # =================================
  User-agent: Googlebot
  Allow: /
  # Keep transactional pages blocked
  Disallow: /account.php
  Disallow: /login.php
  Disallow: /logout.php
  Disallow: /cart.php
  Disallow: /checkout
  Disallow: /checkout.php
  Disallow: /finishorder.php
  Disallow: /orderstatus.php
  Disallow: /wishlist.php
  Disallow: /wishlist
  Disallow: /compare
  Disallow: /giftcertificates.php
  Disallow: /postreview.php
  Disallow: /admin/
  Disallow: /search.php
  Disallow: /amp/
  Disallow: /xmlconnect/
  # IMPORTANT for Merchant rendering/images
  Allow: /productimage.php
  Allow: /cdn-cgi/image/
  # --- Block parameterized/faceted URLs (prevents 7K+ junk pages) ---
  Disallow: /*?_bc_fsnf=*
  Disallow: /*&_bc_fsnf=*
  Disallow: /*?*in_stock=*
  Disallow: /*&*in_stock=*
  Disallow: /*?*price*=*
  Disallow: /*&*price*=*
  Disallow: /*?*sort=*
  Disallow: /*&*sort=*
  Disallow: /*?*limit=*
  Disallow: /*&*limit=*
  Disallow: /*?*brand=*
  Disallow: /*&*brand=*
  Disallow: /*?*category=*
  Disallow: /*&*category=*
  Disallow: /*?*has_free_shipping=*
  Disallow: /*&*has_free_shipping=*
  # --- Block non-Google tracking parameters ---
  Disallow: /*?*fbclid=*
  Disallow: /*&*fbclid=*
  Disallow: /*?*msclkid=*
  Disallow: /*&*msclkid=*
  # --- Search tracking params ---
  Disallow: /*?*searchid=*
  Disallow: /*&*searchid=*
  Disallow: /*?*search_query=*
  Disallow: /*&*search_query=*

  User-agent: Googlebot-Image
  Allow: /
  Allow: /productimage.php
  Allow: /cdn-cgi/image/

  # Merchant Center / Shopping (Storebot needs product pages)
  User-agent: Storebot-Google
  Allow: /
  Disallow: /account.php
  Disallow: /login.php
  Disallow: /logout.php
  Disallow: /cart.php
  Disallow: /checkout
  Disallow: /checkout.php
  Disallow: /finishorder.php
  Disallow: /orderstatus.php
  Disallow: /wishlist.php
  Disallow: /wishlist
  Disallow: /admin/
  Disallow: /search.php
  Disallow: /amp/
  Disallow: /xmlconnect/
  Allow: /productimage.php
  Allow: /cdn-cgi/image/

  # AdsBot checks landing pages (keep permissive)
  User-agent: AdsBot-Google
  Allow: /
  Allow: /productimage.php
  Allow: /cdn-cgi/image/

  # =================================
  # Bing / Microsoft
  # =================================
  User-agent: bingbot
  Allow: /
  Disallow: /account.php
  Disallow: /login.php
  Disallow: /logout.php
  Disallow: /cart.php
  Disallow: /checkout
  Disallow: /checkout.php
  Disallow: /finishorder.php
  Disallow: /orderstatus.php
  Disallow: /wishlist.php
  Disallow: /wishlist
  Disallow: /compare
  Disallow: /giftcertificates.php
  Disallow: /postreview.php
  Disallow: /admin/
  Disallow: /search.php
  Disallow: /amp/
  Disallow: /xmlconnect/
  Disallow: /*?_bc_fsnf=*
  Disallow: /*&_bc_fsnf=*
  Disallow: /*?*in_stock=*
  Disallow: /*&*in_stock=*
  Disallow: /*?*price*=*
  Disallow: /*&*price*=*
  Disallow: /*?*sort=*
  Disallow: /*&*sort=*
  Disallow: /*?*limit=*
  Disallow: /*&*limit=*
  Disallow: /*?*brand=*
  Disallow: /*&*brand=*
  Disallow: /*?*category=*
  Disallow: /*&*category=*
  Disallow: /*?*has_free_shipping=*
  Disallow: /*&*has_free_shipping=*
  Disallow: /*?*fbclid=*
  Disallow: /*&*fbclid=*
  Disallow: /*?*msclkid=*
  Disallow: /*&*msclkid=*
  Disallow: /*?*searchid=*
  Disallow: /*&*searchid=*
  Disallow: /*?*search_query=*
  Disallow: /*&*search_query=*

  # ----------------------------
  # AI / LLM crawlers (allowed for visibility, transactional blocked)
  # ----------------------------
  User-agent: ChatGPT-User
  User-agent: GPTBot
  User-agent: OAI-SearchBot
  User-agent: Claude-Web
  User-agent: ClaudeBot
  User-agent: anthropic-ai
  User-agent: CCBot
  User-agent: PerplexityBot
  Allow: /
  Disallow: /account.php
  Disallow: /login.php
  Disallow: /logout.php
  Disallow: /cart.php
  Disallow: /checkout
  Disallow: /checkout.php
  Disallow: /finishorder.php
  Disallow: /orderstatus.php
  Disallow: /wishlist.php
  Disallow: /wishlist
  Disallow: /admin/
  Disallow: /search.php
  Disallow: /xmlconnect/

  # ----------------------------
  # Everyone else (catch-all)
  # ----------------------------
  User-agent: *

  # --- Account / transactional ---
  Disallow: /account.php
  Disallow: /login.php
  Disallow: /logout.php
  Disallow: /cart.php
  Disallow: /checkout
  Disallow: /checkout.php
  Disallow: /finishorder.php
  Disallow: /orderstatus.php
  Disallow: /wishlist.php
  Disallow: /wishlist
  Disallow: /compare
  Disallow: /giftcertificates.php
  Disallow: /postreview.php

  # --- Utility / internal endpoints ---
  Disallow: /remote.php
  Disallow: /viewfile.php
  Disallow: /admin/
  Disallow: /__socialshop/
  Disallow: /amp/
  Disallow: /xmlconnect/
  Disallow: /productimage.php
  Disallow: /cdn-cgi/

  # --- Internal site search ---
  Disallow: /search.php

  # --- BigCommerce faceted/filter crawl traps ---
  Disallow: /*?_bc_fsnf=*
  Disallow: /*&_bc_fsnf=*
  Disallow: /*?*in_stock=*
  Disallow: /*&*in_stock=*
  Disallow: /*?*price*=*
  Disallow: /*&*price*=*
  Disallow: /*?*sort=*
  Disallow: /*&*sort=*
  Disallow: /*?*limit=*
  Disallow: /*&*limit=*
  Disallow: /*?*brand=*
  Disallow: /*&*brand=*
  Disallow: /*?*category=*
  Disallow: /*&*category=*
  Disallow: /*?*has_free_shipping=*
  Disallow: /*&*has_free_shipping=*

  # --- Non-Google tracking parameters ---
  Disallow: /*?*fbclid=*
  Disallow: /*&*fbclid=*
  Disallow: /*?*msclkid=*
  Disallow: /*&*msclkid=*

  # --- Search tracking params ---
  Disallow: /*?*searchid=*
  Disallow: /*&*searchid=*
  Disallow: /*?*search_query=*
  Disallow: /*&*search_query=*

  # ----------------------------
  # Rate-limited / scraper bots
  # ----------------------------
  User-agent: AI2Bot
  User-agent: Ai2Bot-Dolma
  User-agent: Amazonbot
  User-agent: Applebot
  User-agent: Applebot-Extended
  User-agent: Bytespider
  User-agent: Diffbot
  User-agent: FacebookBot
  User-agent: FriendlyCrawler
  User-agent: Google-Extended
  User-agent: GoogleOther
  User-agent: GoogleOther-Image
  User-agent: GoogleOther-Video
  User-agent: ICC-Crawler
  User-agent: ISSCyberRiskCrawler
  User-agent: ImagesiftBot
  User-agent: Kangaroo Bot
  User-agent: Meta-ExternalAgent
  User-agent: Meta-ExternalFetcher
  User-agent: PetalBot
  User-agent: Scrapy
  User-agent: Sidetrade indexer bot
  User-agent: Timpibot
  User-agent: VelenPublicWebCrawler
  User-agent: Webzio-Extended
  User-agent: YouBot
  User-agent: cohere-ai
  User-agent: facebookexternalhit
  User-agent: iaskspider/2.0
  User-agent: img2dataset
  User-agent: omgili
  User-agent: omgilibot
  Disallow: /account.php
  Disallow: /login.php
  Disallow: /logout.php
  Disallow: /cart.php
  Disallow: /checkout
  Disallow: /checkout.php
  Disallow: /finishorder.php
  Disallow: /orderstatus.php
  Disallow: /wishlist.php
  Disallow: /wishlist
  Disallow: /compare
  Disallow: /admin/
  Disallow: /search.php
  Disallow: /xmlconnect/
  Disallow: /*?_bc_fsnf=*
  Disallow: /*&_bc_fsnf=*
  Disallow: /*?*price*=*
  Disallow: /*&*price*=*
  Disallow: /*?*sort=*
  Disallow: /*&*sort=*
  Disallow: /*?*limit=*
  Disallow: /*&*limit=*

User-agent: *
Disallow: /*?_bc_fsnf=1*
Disallow: /*&_bc_fsnf=1*

User-agent: AI2Bot
User-agent: Ai2Bot-Dolma
User-agent: Amazonbot
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Bytespider
User-agent: CCBot
User-agent: ChatGPT-User
User-agent: Claude-Web
User-agent: ClaudeBot
User-agent: Diffbot
User-agent: FacebookBot
User-agent: FriendlyCrawler
User-agent: GPTBot
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: GoogleOther-Image
User-agent: GoogleOther-Video
User-agent: ICC-Crawler
User-agent: ISSCyberRiskCrawler
User-agent: ImagesiftBot
User-agent: Kangaroo Bot
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: OAI-SearchBot
User-agent: PerplexityBot
User-agent: PetalBot
User-agent: Scrapy
User-agent: Sidetrade indexer bot
User-agent: Timpibot
User-agent: VelenPublicWebCrawler
User-agent: Webzio-Extended
User-agent: YouBot
User-agent: anthropic-ai
User-agent: cohere-ai
User-agent: facebookexternalhit
User-agent: iaskspider/2.0
User-agent: img2dataset
User-agent: omgili
User-agent: omgilibot
Crawl-delay: 10