########################################################################################
##################### Optimized robots.txt for Texas.co.il (GEO Ready) #################
########################################################################################
# Last Updated: 2026-07-26
# Contact: info@texas.co.il
#
# ARCHITECTURE NOTE (fixed 2026-07-26):
# A crawler obeys ONLY its most-specific matching group and ignores all others.
# Previously Googlebot/Bingbot/GPTBot had their own "Allow: /" groups, which meant
# every Disallow under "User-agent: *" was invisible to them.
# All shared rules now live under "User-agent: *". Named groups exist ONLY where
# the behaviour must differ (crawl-delay / full block), and repeat the core blocks.
########################################################################################


# ============================================
# DEFAULT GROUP - applies to Google, Bing, all
# AI/GEO bots, social bots and everything else
# ============================================

User-agent: *
Disallow: /ajax/
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

# --- AI / GEO crawlers are welcome (default = allow) ---
# GPTBot · OAI-SearchBot · ChatGPT-User · ClaudeBot · Claude-Web · anthropic-ai
# PerplexityBot · Perplexity-User · Google-Extended · GoogleOther · CCBot
# Applebot-Extended · Meta-ExternalAgent · DuckAssistBot · YouBot
# Social: facebookexternalhit · Facebot · Twitterbot · LinkedInBot · WhatsApp · TelegramBot
# They inherit this group, so they are allowed AND respect the blocks below.

# --- Images (Google Images / Lens) ---
Allow: /media/catalog/product/
Allow: /media/wysiwyg/
Allow: /media/theme/

# --- Admin & security decoys (real admin path is obscured - never list it) ---
Disallow: /admin/
Disallow: /admin.php
Disallow: /administrator/
Disallow: /wp-admin/
Disallow: /login/
Disallow: /api/
Disallow: /install/
Disallow: /installer/
Disallow: /downloader/
Disallow: /index.php/admin/
Disallow: /rss/
Disallow: /connect/

# --- Magento core directories ---
Disallow: /404/
Disallow: /app/
Disallow: /cgi-bin/
Disallow: /errors/
Disallow: /includes/
Disallow: /lib/
Disallow: /magento/
Disallow: /pkginfo/
Disallow: /report/
Disallow: /shell/
Disallow: /stats/
Disallow: /var/
Disallow: /media/captcha/
Disallow: /media/downloadable/

# --- Magento URL paths (raw routes + infinite spaces) ---
Disallow: /index.php/
Disallow: /catalog/product_compare/
Disallow: /txcartstatus/
Disallow: /catalog/category/view/
Disallow: /catalog/product/view/
Disallow: /catalog/product/gallery/
Disallow: /catalogsearch/
Disallow: /txnav/
Disallow: /checkout
Disallow: /customer/
Disallow: /customize/
Disallow: /control/
Disallow: /newsletter/
Disallow: /paypal/
Disallow: /poll/
Disallow: /review/
Disallow: /sendfriend/
Disallow: /tag/
# */tag/ - blog tag pages sit mid-path (/מידע-על-עולם.../tag/X); /tag/ alone is
# root-anchored and misses them. 75 near-identical tag pages (712-717 words).
Disallow: */tag/
Disallow: /wishlist
Disallow: /cms/index/noCookies

# --- URL parameters (duplicate content / crawl budget) ---
# NOTE: patterns omit '?' on purpose - /*?X= only matches when X is the FIRST
# param; /*X= matches anywhere (e.g. ?p=2&limit=36). Critical now that ?p= is open.
# NOTE: pagination (?p=) intentionally NOT blocked - each page has unique
# products (cat14=839, cat5=566, cat8=553). Sort/limit params ARE duplicates.
Disallow: /*SID=
Disallow: /*gclid=
Disallow: /*fbclid=
Disallow: /*utm_
Disallow: /*___store=
Disallow: /*___from_store=
Disallow: /*limit=
Disallow: /*mode=
Disallow: /*dir=
Disallow: /*order=
Disallow: /*price=
Disallow: /*cat=

# --- Layered Navigation filters (added 04/09/2026) ---
# Cloudflare 04/09: 2,449 requests to one category via ?shirt_type= combos
# = 28% of AI crawl budget. GSC 16m verified: 0 param URLs ranking.
Disallow: /*color=
Disallow: /*cup_type=
Disallow: /*line_of_business=
Disallow: /*manufacturer=
Disallow: /*marketing_badges=
Disallow: /*shirt_type=
Disallow: /*subcat=
Disallow: /7zyf/
# 12/08/2026 - the raw catalog PDF competes with the HTML flipbook page
Disallow: /catalog-rosh-hashanah/catalog.pdf

# --- Files ---
Disallow: /cron.php
Disallow: /cron.sh
Disallow: /error_log
Disallow: /install.php
Disallow: /LICENSE.html
Disallow: /LICENSE.txt
Disallow: /LICENSE_AFL.txt
Disallow: /STATUS.txt
Disallow: /RELEASE_NOTES.txt
Disallow: /php.ini.sample
Disallow: /composer.json
Disallow: /composer.lock
Disallow: /cdn-cgi/

# --- Privacy / legal ---
Disallow: /privacy-policy-cookie-restriction-mode/
Disallow: /terms/

# --- Paid landing pages ---
# Blocked in robots - meta robots injection failed (head.phtml does not render
# layout children on CMS pages). Zero external links = URL-only indexing risk
# negligible. Revisit only if an X-Robots-Tag header is verified live first.
Disallow: /importer-direct


# ============================================
# RATE-LIMITED CRAWLERS
# (core blocks repeated - groups do not inherit)
# ============================================

User-agent: AhrefsBot
# 2s not 10s - paid tool we depend on; 3,600 URLs x 10s = 10h, Site Audit never finishes
Crawl-delay: 2
Disallow: /admin/
Disallow: /catalogsearch/
Disallow: /txnav/
Disallow: /checkout
Disallow: /customer/
Disallow: /*SID=

User-agent: SemrushBot
Crawl-delay: 10
Disallow: /admin/
Disallow: /catalogsearch/
Disallow: /txnav/
Disallow: /checkout
Disallow: /customer/
Disallow: /*SID=

User-agent: MJ12bot
Crawl-delay: 10
Disallow: /admin/
Disallow: /catalogsearch/
Disallow: /txnav/
Disallow: /checkout
Disallow: /customer/

User-agent: DotBot
Crawl-delay: 10
Disallow: /admin/
Disallow: /catalogsearch/
Disallow: /txnav/
Disallow: /checkout
Disallow: /customer/

User-agent: PetalBot
Crawl-delay: 10
Disallow: /admin/
Disallow: /catalogsearch/
Disallow: /txnav/
Disallow: /checkout
Disallow: /customer/

User-agent: Amazonbot
Crawl-delay: 10
Disallow: /admin/
Disallow: /catalogsearch/
Disallow: /txnav/
Disallow: /checkout
Disallow: /customer/

User-agent: Bytespider
Crawl-delay: 10
Disallow: /admin/
Disallow: /catalogsearch/
Disallow: /txnav/
Disallow: /checkout
Disallow: /customer/

User-agent: Screaming Frog SEO Spider
Crawl-delay: 2
Allow: /
Disallow: /*.gif$
Disallow: /*.jpg$
Disallow: /*.jpeg$
Disallow: /*.png$
Disallow: /*.bmp$
Disallow: /*.webp$
Disallow: /*.css$
Disallow: /*.js$


# ============================================
# FULLY BLOCKED
# (also enforced at Cloudflare WAF - these
#  scrapers ignore robots.txt)
# ============================================

User-agent: GeedoShopProductFinder
Disallow: /


# ============================================
# SITEMAPS (global - not bound to any group)
# ============================================

Sitemap: https://www.texas.co.il/sitemap_index.xml
Sitemap: https://www.texas.co.il/sitemap_products.xml
Sitemap: https://www.texas.co.il/sitemap_categories.xml
Sitemap: https://www.texas.co.il/sitemap_blog.xml
Sitemap: https://www.texas.co.il/sitemap_cms.xml
Sitemap: https://www.texas.co.il/sitemap-images.xml
Sitemap: https://www.texas.co.il/sitemap_photogallery.xml