# robots.txt — unsfarms.com
# Version: v2.1.0
# Fixed: Google Merchant Center product page blocking
# Updated: 2026-05
# ─────────────────────────────────────────────────────────
#
# CHANGELOG:
#  v2.1.0 — Added /pages/* and /ar/pages/* to all AI bots (ChatGPT-User, GPTBot,
#            ClaudeBot, PerplexityBot, Google-Extended) for GEO indexing of About Us,
#            B2B, Farm Management and Farm Tours pages
#          — Added /ar/pages/ + /ar/pages/* to all three AdsBot blocks for Arabic
#            landing page GMC verification
#          — Added /ar/blogs/ + /ar/pages/ directory indexes to Storebot-Google
#          — Added /blogs/ + /ar/blogs/ directory indexes to facebookexternalhit
#          — Added /blogs/, /pages/, /ar/blogs/, /ar/pages/ directory indexes to
#            facebookcatalog
#          — Added /blogs/ + /ar/pages/ directory indexes to meta-externalads;
#            fixed truncated User-agent ('meta-' → 'meta-externalads')
#          — Fixed AdsBot stacking: split AdsBot-Google, AdsBot-Google-Mobile,
#            adsbot-google into separate blocks (stacked syntax was invalid)
#          — Removed duplicate Allow lines from Googlebot and Storebot-Google
#  v2.0.0 — Added explicit Allow: /products/ + /products/* to AdsBot-Google
#          — Added Storebot-Google dedicated block (Google Shopping crawler)
#          — Added Allow: /cdn/shop/products/ to AdsBot-Google for image verification
#          — Added Allow: /recommendations/products to AdsBot-Google (Dynamic Remarketing)
#          — Added Allow: /products/ to Googlebot-Image
#          — Fixed invalid syntax: */collections/*filter* → /*/collections/*filter*
#          — Removed: Disallow: /*?*variant* from Googlebot (was blocking all variant URLs)
#          — Removed: Disallow: /*?*oseid=* from Googlebot and AdsBot-Google
#          — Removed: Disallow: /*?*sort_by* from Googlebot
#
#  v1.0.0 — Initial deployment (had GMC blocking issues)
# ─────────────────────────────────────────────────────────


# ── AI Bots ──────────────────────────────────────────────
User-agent: ChatGPT-User
Disallow: /
Allow: /products/
Allow: /collections/
Allow: /blogs/
Allow: /pages/
Allow: /ar/products/
Allow: /ar/collections/
Allow: /ar/blogs/
Allow: /ar/pages/
Allow: /products/*
Allow: /collections/*
Allow: /blogs/*
Allow: /pages/*
Allow: /ar/products/*
Allow: /ar/collections/*
Allow: /ar/blogs/*
Allow: /ar/pages/*

User-agent: GPTBot
Disallow: /
Allow: /products/
Allow: /collections/
Allow: /blogs/
Allow: /pages/
Allow: /ar/products/
Allow: /ar/collections/
Allow: /ar/blogs/
Allow: /ar/pages/
Allow: /products/*
Allow: /collections/*
Allow: /blogs/*
Allow: /pages/*
Allow: /ar/products/*
Allow: /ar/collections/*
Allow: /ar/blogs/*
Allow: /ar/pages/*


User-agent: ClaudeBot
Disallow: /
Allow: /products/
Allow: /collections/
Allow: /blogs/
Allow: /pages/
Allow: /ar/products/
Allow: /ar/collections/
Allow: /ar/blogs/
Allow: /ar/pages/
Allow: /products/*
Allow: /collections/*
Allow: /blogs/*
Allow: /pages/*
Allow: /ar/products/*
Allow: /ar/collections/*
Allow: /ar/blogs/*
Allow: /ar/pages/*


User-agent: PerplexityBot
Disallow: /
Allow: /products/
Allow: /collections/
Allow: /blogs/
Allow: /pages/
Allow: /ar/products/
Allow: /ar/collections/
Allow: /ar/blogs/
Allow: /ar/pages/
Allow: /products/*
Allow: /collections/*
Allow: /blogs/*
Allow: /pages/*
Allow: /ar/products/*
Allow: /ar/collections/*
Allow: /ar/blogs/*
Allow: /ar/pages/*



# ── Google — Core ────────────────────────────────────────
# NOTE: /*?*variant* and /*?*oseid=* have been intentionally
# removed. These were blocking Shopify variant product URLs
# which are required for Google Shopping feed verification.

User-agent: Googlebot
Allow: /products/
Allow: /products/*
Allow: /collections/
Allow: /collections/*
Allow: /blogs/
Allow: /pages/
Allow: /ar/products/
Allow: /ar/products/*
Allow: /ar/collections/
Allow: /ar/collections/*
Allow: /ar/blogs/
Allow: /ar/pages/
Allow: /blogs/*
Allow: /pages/*
Allow: /ar/blogs/*
Allow: /ar/pages/*
Disallow: /admin
Disallow: /cart
Disallow: /checkout
Disallow: /checkouts/
Disallow: /account
Disallow: /orders
Disallow: /carts
Disallow: /67990880391/checkouts
Disallow: /67990880391/orders
Disallow: /*preview_theme_id*
Disallow: /*preview_script_id*
Disallow: /sf_private_access_tokens
Disallow: /.well-known/shopify/monorail
Disallow: /cdn/wpm/*.js


# ── Google Images ─────────────────────────────────────────
User-agent: Googlebot-Image
Allow: /cdn/shop/products/
Allow: /cdn/shop/files/
Allow: /files/
Allow: /products/
Allow: /products/*
Allow: /collections/*
Allow: /blogs/
Allow: /blogs/*
Allow: /pages/*
Allow: /ar/products/*
Allow: /ar/collections/*
Allow: /ar/blogs/*
Allow: /ar/pages/*
Disallow: /admin
Disallow: /checkout
Disallow: /account


# ── Google Shopping Crawler (Storebot-Google) ────────────
# Storebot-Google is Google's dedicated Shopping product
# indexer. Must have full access to products, variants,
# and the image CDN. Missing this block = Shopping indexing
# falls through to User-agent: * which lacks explicit allows.

User-agent: Storebot-Google
Allow: /ar/blogs/
Allow: /ar/pages/
Allow: /products/
Allow: /products/*
Allow: /collections/
Allow: /collections/*
Allow: /ar/products/
Allow: /ar/products/*
Allow: /ar/collections/
Allow: /ar/collections/*
Allow: /cdn/shop/products/
Allow: /cdn/shop/files/
Allow: /pages/
Allow: /blogs/
Allow: /blogs/*
Allow: /pages/*
Allow: /ar/blogs/*
Allow: /ar/pages/*
Disallow: /admin
Disallow: /checkout
Disallow: /checkouts/
Disallow: /account
Disallow: /orders
Disallow: /carts


# ── Google Extended (Gemini Apps) ────────────────────────
User-agent: Google-Extended
Disallow: /
Allow: /products/
Allow: /collections/
Allow: /blogs/
Allow: /pages/
Allow: /ar/products/
Allow: /ar/collections/
Allow: /ar/blogs/
Allow: /ar/pages/
Allow: /products/*
Allow: /collections/*
Allow: /blogs/*
Allow: /pages/*
Allow: /ar/products/*
Allow: /ar/collections/*
Allow: /ar/blogs/*
Allow: /ar/pages/*


# ── Google Ads + Merchant Center ─────────────────────────
# AdsBot-Google is the PRIMARY Google Merchant Center
# landing page verifier. Without explicit Allow: /products/
# and Allow: /products/*, GMC cannot verify product feed
# URLs, variant URLs, or Dynamic Remarketing data.
#
# MANDATORY for GMC approval:
#   Allow: /products/         ← canonical product pages
#   Allow: /products/*        ← covers ?variant=XXXXX URLs
#   Allow: /cdn/shop/products/← product image verification
#   Allow: /recommendations/products ← Dynamic Remarketing
#
# robots.txt is case-sensitive for user-agent matching.
# All three AdsBot variants below are required.

User-agent: AdsBot-Google
Allow: /products/
Allow: /products/*
Allow: /collections/
Allow: /collections/*
Allow: /ar/pages/
Allow: /ar/pages/*
Allow: /ar/products/
Allow: /ar/products/*
Allow: /ar/collections/
Allow: /ar/collections/*
Allow: /cdn/shop/products/
Allow: /cdn/shop/files/
Allow: /pages/
Allow: /pages/*
Allow: /recommendations/products
Disallow: /checkouts/
Disallow: /checkout
Disallow: /carts
Disallow: /orders
Disallow: /67990880391/checkouts
Disallow: /67990880391/orders
Disallow: /account
Disallow: /*preview_theme_id*
Disallow: /*preview_script_id*
Disallow: /cdn/wpm/*.js
Disallow: /sf_private_access_tokens


User-agent: adsbot-google
Allow: /products/
Allow: /products/*
Allow: /collections/
Allow: /collections/*
Allow: /ar/products/
Allow: /ar/products/*
Allow: /ar/pages/
Allow: /ar/pages/*
Allow: /ar/collections/
Allow: /ar/collections/*
Allow: /cdn/shop/products/
Allow: /cdn/shop/files/
Allow: /pages/
Allow: /pages/*
Allow: /recommendations/products
Disallow: /checkouts/
Disallow: /checkout
Disallow: /carts
Disallow: /orders
Disallow: /67990880391/checkouts
Disallow: /67990880391/orders
Disallow: /account
Disallow: /*preview_theme_id*
Disallow: /*preview_script_id*
Disallow: /cdn/wpm/*.js
Disallow: /sf_private_access_tokens




User-agent: AdsBot-Google-Mobile
Allow: /products/
Allow: /products/*
Allow: /collections/
Allow: /collections/*
Allow: /ar/products/
Allow: /ar/products/*
Allow: /ar/collections/
Allow: /ar/collections/*
Allow: /cdn/shop/products/
Allow: /cdn/shop/files/
Allow: /pages/
Allow: /pages/*
Allow: /ar/pages/
Allow: /ar/pages/*
Allow: /recommendations/products
Disallow: /checkouts/
Disallow: /checkout
Disallow: /carts
Disallow: /orders
Disallow: /67990880391/checkouts
Disallow: /67990880391/orders
Disallow: /account
Disallow: /*preview_theme_id*
Disallow: /*preview_script_id*
Disallow: /cdn/wpm/*.js
Disallow: /sf_private_access_tokens


# ── Meta / Facebook ───────────────────────────────────────
User-agent: facebookexternalhit
Allow: /products/
Allow: /collections/
Allow: /pages/
Allow: /ar/products/
Allow: /ar/collections/
Allow: /ar/pages/
Allow: /ar/blogs/
Allow: /ar/blogs/*
Allow: /cdn/shop/products/
Allow: /cdn/shop/files/
Allow: /products/*
Allow: /collections/*
Allow: /blogs/
Allow: /blogs/*
Allow: /pages/*
Allow: /ar/products/*
Allow: /ar/collections/*
Allow: /ar/blogs/*
Allow: /ar/pages/*
Disallow: /admin
Disallow: /cart
Disallow: /checkout
Disallow: /account
Disallow: /orders
Disallow: /*?*SID=

User-agent: meta-externalads
Allow: /blogs/
Allow: /ar/blogs/
Allow: /ar/pages/
Allow: /products/
Allow: /collections/
Allow: /ar/products/
Allow: /ar/collections/
Allow: /cdn/shop/products/
Allow: /cdn/shop/files/
Allow: /products/*
Allow: /collections/*
Allow: /blogs/*
Allow: /pages/
Allow: /pages/*
Allow: /ar/products/*
Allow: /ar/collections/*
Allow: /ar/blogs/*
Allow: /ar/pages/*
Disallow: /admin
Disallow: /cart
Disallow: /checkout
Disallow: /account
Disallow: /orders
Disallow: /search

User-agent: facebookcatalog
Allow: /blogs/
Allow: /pages/
Allow: /ar/blogs/
Allow: /ar/pages/
Allow: /products/
Allow: /collections/
Allow: /ar/products/
Allow: /ar/collections/
Allow: /cdn/shop/products/
Allow: /cdn/shop/files/
Allow: /products/*
Allow: /collections/*
Allow: /blogs/*
Allow: /pages/*
Allow: /ar/products/*
Allow: /ar/collections/*
Allow: /ar/blogs/*
Allow: /ar/pages/*
Disallow: /admin
Disallow: /cart
Disallow: /checkout
Disallow: /account


# ── AI training crawlers: blocked ──────────────────────────────
User-agent: anthropic-ai
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /


# ── Known scrapers and data extractors: blocked ────────────────
User-agent: ApifyBot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: omgili
Disallow: /

User-agent: omgilibot
Disallow: /

User-agent: PetalBot
Disallow: /


# ── Block entirely ───────────────────────────────────────
User-agent: Nutch
Disallow: /

User-agent: MJ12bot
Crawl-delay: 10
Disallow: /


# ── SEO crawlers — slow + restrict ───────────────────────
User-agent: AhrefsBot
Crawl-delay: 10
Disallow: /

User-agent: AhrefsSiteAudit
Crawl-delay: 10
Disallow: /


# ── Pinterest ────────────────────────────────────────────
User-agent: Pinterest
Crawl-delay: 1
Disallow: /admin
Disallow: /cart
Disallow: /checkout
Disallow: /account
Disallow: /orders


# ── All other bots ───────────────────────────────────────
User-agent: *
Crawl-delay: 5
Disallow: /a/downloads/-/*
Disallow: /admin
Disallow: /cart
Disallow: /orders
Disallow: /checkouts/
Disallow: /checkout
Disallow: /67990880391/checkouts
Disallow: /67990880391/orders
Disallow: /carts
Disallow: /account
Disallow: /collections/*sort_by*
Disallow: /*/collections/*sort_by*
Disallow: /collections/*+*
Disallow: /collections/*%2B*
Disallow: /collections/*%2b*
Disallow: /*/collections/*+*
Disallow: /*/collections/*%2B*
Disallow: /*/collections/*%2b*
Disallow: /collections/*products.json
# FIXED: was */collections/ (invalid — must start with /)
Disallow: /*/collections/*filter*&*filter*
Disallow: /blogs/*+*
Disallow: /blogs/*%2B*
Disallow: /blogs/*%2b*
Disallow: /*/blogs/*+*
Disallow: /*/blogs/*%2B*
Disallow: /*/blogs/*%2b*
Disallow: /*preview_theme_id*
Disallow: /*preview_script_id*
Disallow: /*/*?*ls=*&ls=*
Disallow: /sf_private_access_tokens
Disallow: /apple-app-site-association
Disallow: /.well-known/shopify/monorail
Disallow: /cdn/wpm/*.js
Disallow: /recommendations/products
Disallow: /*/recommendations/products
Disallow: /products.json

# ─────────────────────────────────────────────────────────
Sitemap: https://www.unsfarms.com/sitemap.xml