User-agent: *
# Disallow access to sensitive system folders
Disallow: /app/
Disallow: /bin/
Disallow: /dev/
Disallow: /generated/
Disallow: /lib/
Disallow: /phpserver/
Disallow: /pub/errors/
Disallow: /pub/static/
Disallow: /setup/
Disallow: /update/
Disallow: /var/
Disallow: /vendor/

# Disallow unnecessary pages
Disallow: /cgi-bin/
Disallow: /customer/
Disallow: /checkout/
Disallow: /catalogsearch/
Disallow: /search/
Disallow: /wishlist/
Disallow: /sendfriend/
Disallow: /review/
Disallow: /newsletter/

# Block specific file types from being indexed
Disallow: /*.php$
Disallow: /*.cgi$
Disallow: /*.asp$
Disallow: /*.aspx$
Disallow: /*.jsp$
Disallow: /*.htm$
Disallow: /*.html$?SID=

# Allow important resources for rendering
Allow: /pub/media/
Allow: /pub/static/frontend/
Allow: /pub/static/adminhtml/
Allow: /static/version*
Allow: /media/

# Sitemap locations (replace with your actual domain)
Sitemap: https://www.example.com/sitemap.xml
Sitemap: https://www.example.com/sitemap_products.xml