Sitemap: https://www.industrialproducts.com/sitemap/industrialproducts/blog_sitemap.xml
Sitemap: https://www.industrialproducts.com/sitemap/industrialproducts/sitemap.xml

# DEFAULT CRAWL RULES
User-agent: *
Crawl-delay: 5
Disallow: /rest/
Disallow: /cdn-cgi/

# CORE MAGENTO DIRECTORIES
Disallow: /app/
Disallow: /bin/
Disallow: /dev/
Disallow: /lib/
Disallow: /phpserver/
Disallow: /pkginfo/
Disallow: /setup/
Disallow: /update/
Disallow: /var/
Disallow: /vendor/
Disallow: /generated/


# ADMIN & BACKEND
Disallow: /admin/
Disallow: /downloader/

# CUSTOMER AREAS
Disallow: /customer/
Disallow: /checkout/
Disallow: /wishlist/
Disallow: /control/

# CATALOG FUNCTIONALITY (Not Product/Category Pages)
Disallow: /catalog/product_compare/
Disallow: /catalog/category/view/
Disallow: /catalog/product/view/
Disallow: /catalog/product/gallery/
Disallow: /catalogsearch/
Disallow: /*?

# USER INTERACTION PAGES
Disallow: /review/
Disallow: /sendfriend/
Disallow: /newsletter/
Disallow: /poll/
Disallow: /tag/

# SYSTEM PAGES
Disallow: /404/
Disallow: /report/
Disallow: /index.php/

# SYSTEM FILES
Disallow: /cron.php
Disallow: /cron.sh
Disallow: /error_log
Disallow: /install.php
Disallow: /LICENSE.html
Disallow: /LICENSE.txt
Disallow: /LICENSE_AFL.txt
Disallow: /STATUS.txt
Disallow: /RELEASE_NOTES.txt
Disallow: /composer.json
Disallow: /composer.lock
Disallow: /package.json

# FILE EXTENSIONS
Disallow: /*.php$

# QUERY PARAMETERS - Block filtered/sorted URLs
Disallow: /*?*SID=*
Disallow: /*&SID=*
Disallow: /*?*price=*
Disallow: /*&price=*
Disallow: /*?*dir=*
Disallow: /*&dir=*
Disallow: /*?*order=*
Disallow: /*&order=*
Disallow: /*?*limit=*
Disallow: /*&limit=*
Disallow: /*?*mode=*
Disallow: /*&mode=*
Disallow: /*?q=*
Disallow: /*?*rfsn=*
Disallow: /*&rfsn=*
Disallow: /*-all-questions.html

# BAD BOTS & SCRAPERS

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /


# ALLOW GOOGLE CRAWLERS (Explicit)
User-agent: Googlebot
Disallow:

User-agent: Googlebot-Image
Disallow:

User-agent: Googlebot-Mobile
Allow: /

User-agent: Mediapartners-Google
Allow: /

User-agent: Adsbot-Google
Allow: /