# ----------------------------------------------------------------------- # Primary robots.txt for WordPress + large listing site # Optimized for SEO-safe crawling, image discovery, and reduced junk crawl # ----------------------------------------------------------------------- # ========================= # 1) Block AI / unwanted bots # ========================= User-agent: Amazonbot Disallow: / User-agent: Applebot-Extended Disallow: / User-agent: ClaudeBot Disallow: / User-agent: GPTBot Disallow: / User-agent: PetalBot Disallow: / User-agent: PerplexityBot Disallow: / User-agent: CCBot Disallow: / User-agent: AhrefsBot Disallow: / User-agent: SemrushBot Disallow: / User-agent: DotBot Disallow: / User-agent: Yandex Disallow: / User-agent: YaK Disallow: / User-agent: EtaoSpider Disallow: / User-agent: viberbot Disallow: / User-agent: uptimerobot Disallow: / # ========================= # 2) Google AI policy # ========================= # Allow Google Search, but block Google AI training/use User-agent: Google-Extended Disallow: / # ========================= # 3) Apple search crawler # ========================= # Allow Apple search indexing, but rate control should be handled at server/WAF level User-agent: Applebot Allow: / # ========================= # 4) Google image crawler # ========================= # Restrict image crawling to the folders/files you actually want surfaced User-agent: Googlebot-Image Disallow: / Allow: /favicon.ico Allow: /wp-content/uploads/ Allow: /images/agents/ Allow: /images/udx/ Allow: /images/listings/ # ========================= # 5) Main search crawlers # ========================= User-agent: Googlebot Allow: / User-agent: Bingbot Allow: / # ========================= # 6) General rules for all crawlers # ========================= User-agent: * Allow: / Disallow: /wp-admin/ Allow: /wp-admin/admin-ajax.php # Query cleanup / crawl waste reduction Disallow: /*?attachment_id= Disallow: /*?*q= # Faceted / sort-style URLs Disallow: /*/hga Disallow: /*/hga*?*q= Disallow: /*/lga Disallow: /*/lga*?*q= Disallow: /*/oga Disallow: /*/oga*?*q= Disallow: /*/nga Disallow: /*/nga*?*q= # Error / junk patterns Disallow: /*/A Disallow: /*/Array Disallow: /*?*afghanistan* Disallow: /*?*cheapfifa23coins* # Important crawlable pages Allow: /favicon.ico Allow: /en/d/ Allow: /en/d/* Allow: /fr/d/ Allow: /fr/d/* Allow: /nl/d/ Allow: /nl/d/* Allow: /es/d/ Allow: /es/d/* # Account: 1000281 (orig) Disallow: /en/s/ Allow: /en/d/luxury-homes-cayman-halcyon-point-1157907 Allow: /en/d/luxury-beachfront-villa-grand-cayman-1157907 Allow: /en/d/halcyon-point-cayman-islands-for-sale-1157907 Allow: /en/s/luxury-real-estate-cayman-islands Allow: /en/s/luxury-homes-cayman Allow: /en/s/luxury-condos-cayman-islands Allow: /en/s/canal-front-luxury-homes-cayman Allow: /en/s/luxury-beachfront-homes-cayman Allow: /en/s/west-bay-luxury-villas Allow: /en/s/bodden-town-breakers-luxury-homes Allow: /en/s/savannah-lower-valley-luxury-properties Allow: /en/s/spotts-prospect-luxury-villas Allow: /en/s/cayman-kai-rum-point-luxury-villas Allow: /en/s/george-town-luxury-villas Allow: /en/s/south-sound-luxury-villas Allow: /en/s/luxury-villas-grand-cayman Allow: /en/s/luxury-villas-cayman-brac Allow: /en/s/luxury-villas-little-cayman Allow: /en/s/for-sale Allow: /en/s/for-sale/condos-apartments Allow: /en/s/for-sale/condos-apartments/pool Allow: /en/s/for-sale/condos-apartments/water-front Allow: /en/s/for-sale/condos-apartments/beach-oceanfront Allow: /en/s/for-sale/condos-apartments/bedrooms-1 Allow: /en/s/for-sale/condos-apartments/bedrooms-2 Allow: /en/s/for-sale/condos-apartments/bedrooms-3 Allow: /en/s/for-sale/condos-apartments/west-bay-grand-cayman Allow: /en/s/for-sale/condos-apartments/spotts-prospect-grand-cayman Allow: /en/s/for-sale/condos-apartments/south-sound-grand-cayman Allow: /en/s/for-sale/condos-apartments/savannah-newlands-grand-cayman Allow: /en/s/for-sale/condos-apartments/seven-mile-beach-grand-cayman Allow: /en/s/for-sale/condos-apartments/north-side-grand-cayman Allow: /en/s/for-sale/condos-apartments/george-town-grand-cayman Allow: /en/s/for-sale/condos-apartments/east-end-high-rock-grand-cayman Allow: /en/s/for-sale/condos-apartments/cayman-kai-rum-point-grand-cayman Allow: /en/s/for-sale/condos-apartments/bodden-town-breakers-grand-cayman Allow: /en/s/for-sale/condos-apartments/little-cayman Allow: /en/s/for-sale/condos-apartments/cayman-brac Allow: /en/s/for-sale/single-family-homes Allow: /en/s/for-sale/single-family-homes/pool Allow: /en/s/for-sale/single-family-homes/water-front Allow: /en/s/for-sale/single-family-homes/beach-oceanfront Allow: /en/s/for-sale/single-family-homes/bedrooms-2 Allow: /en/s/for-sale/single-family-homes/bedrooms-3 Allow: /en/s/for-sale/single-family-homes/bedrooms-4 Allow: /en/s/for-sale/single-family-homes/bedrooms-5 Allow: /en/s/for-sale/single-family-homes/west-bay-grand-cayman Allow: /en/s/for-sale/single-family-homes/spotts-prospect-grand-cayman Allow: /en/s/for-sale/single-family-homes/south-sound-grand-cayman Allow: /en/s/for-sale/single-family-homes/savannah-newlands-grand-cayman Allow: /en/s/for-sale/single-family-homes/seven-mile-beach-grand-cayman Allow: /en/s/for-sale/single-family-homes/north-side-grand-cayman Allow: /en/s/for-sale/single-family-homes/george-town-grand-cayman Allow: /en/s/for-sale/single-family-homes/east-end-high-rock-grand-cayman Allow: /en/s/for-sale/single-family-homes/cayman-kai-rum-point-grand-cayman Allow: /en/s/for-sale/single-family-homes/bodden-town-breakers-grand-cayman Allow: /en/s/for-sale/single-family-homes/little-cayman Allow: /en/s/for-sale/single-family-homes/cayman-brac Allow: /en/s/for-sale/vacant-land Allow: /en/s/for-sale/vacant-land/water-front Allow: /en/s/for-rent Allow: /en/s/for-rent/condos-apartments Allow: /en/s/for-rent/condos-apartments/pool Allow: /en/s/for-rent/condos-apartments/water-front Allow: /en/s/for-rent/condos-apartments/beach-oceanfront Allow: /en/s/for-rent/condos-apartments/bedrooms-1 Allow: /en/s/for-rent/condos-apartments/bedrooms-2 Allow: /en/s/for-rent/condos-apartments/bedrooms-3 Allow: /en/s/for-rent/condos-apartments/west-bay-grand-cayman Allow: /en/s/for-rent/condos-apartments/spotts-prospect-grand-cayman Allow: /en/s/for-rent/condos-apartments/south-sound-grand-cayman Allow: /en/s/for-rent/condos-apartments/savannah-newlands-grand-cayman Allow: /en/s/for-rent/condos-apartments/seven-mile-beach-grand-cayman Allow: /en/s/for-rent/condos-apartments/north-side-grand-cayman Allow: /en/s/for-rent/condos-apartments/george-town-grand-cayman Allow: /en/s/for-rent/condos-apartments/east-end-high-rock-grand-cayman Allow: /en/s/for-rent/condos-apartments/cayman-kai-rum-point-grand-cayman Allow: /en/s/for-rent/condos-apartments/bodden-town-breakers-grand-cayman Allow: /en/s/for-rent/condos-apartments/little-cayman Allow: /en/s/for-rent/condos-apartments/cayman-brac Allow: /en/s/for-rent/single-family-homes Allow: /en/s/commercial Allow: /en/s/commercial/business Allow: /en/s/commercial/commercial-mixed-use Allow: /en/s/commercial/commercial-building-office Allow: /en/s/commercial/vacant-land-agricultural Allow: /en/s/commercial/vacant-land-commercial Allow: /en/s/commercial/water-front Allow: /en/s/commercial/grand-cayman Allow: /en/s/commercial/little-cayman Allow: /en/s/commercial/cayman-brac