# robots.txt for Magnet Health NY
# Website: https://magnethealthny.com/

# -----------------------------------------------
# Allow all major search engine crawlers
# -----------------------------------------------
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

# -----------------------------------------------
# Block AI / LLM Training Crawlers
# (Prevent content from being used for AI model training)
# -----------------------------------------------

User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: FacebookBot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Bytespider
Allow: /

User-agent: Diffbot
Allow: /


# -----------------------------------------------
# General rules for all bots
# -----------------------------------------------
User-agent: *
Allow: /

# -----------------------------------------------
# Block WordPress admin, login & system files
# -----------------------------------------------
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /wp-json/

# Block sensitive or low-value pages
Disallow: /patient-portal/
Disallow: /patient-login/
Disallow: /checkout/
Disallow: /cart/
Disallow: /my-account/
Disallow: /thank-you/
Disallow: /order-confirmation/
Disallow: /category/*

# Block internal search & filtered/paginated utility pages
Disallow: /?s=
Disallow: /search/
Disallow: /?p=
Disallow: /?page_id=
Disallow: /page/
Disallow: /tag/
Disallow: /author/

# Block duplicate or low-value archive pages
Disallow: /date/
Disallow: /feed/
Disallow: /comments/feed/
Disallow: /trackback/
Disallow: /xmlrpc.php
Disallow: /readme.html
Disallow: /license.txt
Disallow: /wp-cron.php
Disallow: /wp-config.php

# Block URL query strings that create duplicate content
Disallow: /*?replytocom=
Disallow: /*?doing_wp_cron
Disallow: /*?utm_source=
Disallow: /*?utm_medium=
Disallow: /*?utm_campaign=
Disallow: /*?fbclid=
Disallow: /*?gclid=

# -----------------------------------------------
# Crawl delay (optional — reduce server load)
# -----------------------------------------------
Crawl-delay: 5

# -----------------------------------------------
# XML Sitemap location
# -----------------------------------------------
Sitemap: https://magnethealthny.com/sitemap.xml
Sitemap: https://magnethealthny.com/sitemap_index.xml