# HIGHERGROUND CUSTOM ROBOTS TEST 2026-04-27
# Robots.txt for HigherGround Health and Fitness
# Website: https://higherground.ca/
# Location: Toronto, Ontario
#
# Purpose:
# - Allow major search engines to crawl public website content.
# - Allow useful AI search/discovery bots where appropriate.
# - Block admin areas, junk URLs, crawl traps, and high-volume training/scraping bots.
# - Reduce unnecessary server load.

# ------------------------------------------------------------
# Default rule for normal crawlers
# ------------------------------------------------------------

User-agent: *
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

# WordPress admin/system areas
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /xmlrpc.php

# WordPress internal / low-value paths
Disallow: /wp-json/oembed/
Disallow: /trackback/
Disallow: /*/trackback/
Disallow: /*/feed/
Disallow: /comments/feed/
Disallow: /author/
Disallow: /tag/

# Search, filter, reply, print, share, and tracking URLs
Disallow: /?s=
Disallow: /search/
Disallow: /*?s=
Disallow: /*&s=
Disallow: /*?replytocom=
Disallow: /*&replytocom=
Disallow: /*?print=
Disallow: /*&print=
Disallow: /*?share=
Disallow: /*&share=
Disallow: /*?utm_
Disallow: /*&utm_
Disallow: /*?fbclid=
Disallow: /*&fbclid=
Disallow: /*?gclid=
Disallow: /*&gclid=
Disallow: /*?msclkid=
Disallow: /*&msclkid=

# Avoid crawling non-public utility files
Disallow: /readme.html
Disallow: /license.txt

# ------------------------------------------------------------
# Major search engines: allow
# ------------------------------------------------------------

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Applebot
Allow: /

# ------------------------------------------------------------
# AI search / answer engine crawlers: allow useful discovery
# ------------------------------------------------------------
# These can help AI platforms understand and cite the business.
# They should still respect the general disallow rules above.

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

# ------------------------------------------------------------
# AI training / high-volume crawlers: block
# ------------------------------------------------------------
# These are less likely to send local business leads and more likely
# to create crawl load or train models from site content.

User-agent: GPTBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

# ------------------------------------------------------------
# Known aggressive / low-value SEO, scraper, and marketing bots
# ------------------------------------------------------------

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: SemrushBot-SA
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Yandex
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: Sogou
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: MegaIndex
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: Barkrowler
Disallow: /

User-agent: serpstatbot
Disallow: /

User-agent: SeekportBot
Disallow: /

User-agent: trendictionbot
Disallow: /

User-agent: CensysInspect
Disallow: /

User-agent: InternetMeasurement
Disallow: /

# ------------------------------------------------------------
# Sitemaps and AI discovery
# ------------------------------------------------------------

Sitemap: https://higherground.ca/sitemap.xml
Sitemap: https://higherground.ca/page-sitemap.xml
Sitemap: https://higherground.ca/post-sitemap.xml
Sitemap: https://higherground.ca/sitemap.rss

LLMS: https://higherground.ca/llms.txt