# =========================================================================
# AllMyMeetings — robots.txt
# https://allmymeetings.com
#
# Strategy: Allow all search engines and AI assistants to index public
# marketing & content pages. Block authenticated app routes.
# Deny known data-harvesting scrapers that offer no referral value.
# =========================================================================

# ---------------------------------------------------------------------------
# Default policy — applies to Googlebot, Bingbot, and any crawler without
# a more-specific block below (including AI search and assistant crawlers like
# OAI-SearchBot, ChatGPT-User, Claude-SearchBot, Claude-User, PerplexityBot).
# ---------------------------------------------------------------------------
User-agent: *
Allow: /
Allow: /blog
Allow: /blog/
Allow: /compare/
Allow: /for/
Allow: /terms
Allow: /privacy
Allow: /developers
Allow: /openapi.json

# Agent discovery metadata (RFC 8615 well-known URIs)
Allow: /.well-known/

# Authenticated app routes
Disallow: /dashboard
Disallow: /calendar
Disallow: /calendar-accounts
Disallow: /sync
Disallow: /scheduling-pages
Disallow: /logs
Disallow: /settings
Disallow: /teams
Disallow: /api

# OAuth & auth plumbing
Disallow: /auth/
Disallow: /login/*/redirect
Disallow: /login/*/callback

# Laravel / Livewire internals
Disallow: /livewire/
Disallow: /sanctum/
Disallow: /_ignition/
Disallow: /telescope

# Allow Googlebot to fetch assets needed for page rendering
# Blocking /build/ prevents Google from rendering pages — hurts indexing
Allow: /build/
Allow: /vendor/
Allow: /flux/

# Prevent query-string duplicate content
Disallow: /*?*ref=
Disallow: /*?*utm_

# ---------------------------------------------------------------------------
# AI Training-Only Crawlers — block bots that scrape content purely for
# model training with no user-facing search/citation benefit.
# ---------------------------------------------------------------------------

# ByteDance / TikTok training spider
User-agent: Bytespider
Disallow: /

# Common Crawl (bulk dataset builder)
User-agent: CCBot
Disallow: /

# Diffbot (structured data extraction)
User-agent: Diffbot
Disallow: /

# Apple Intelligence training crawler (Applebot handles Siri / Spotlight search)
User-agent: Applebot-Extended
Disallow: /

# Google-Extended is deliberately NOT blocked: besides training, it controls
# whether Gemini apps and Vertex AI may use these pages for grounded, cited
# answers. (Search and AI Overviews follow Googlebot either way.)

# Facebook / Meta AI training crawler
User-agent: FacebookBot
Disallow: /

# Omgili / Webz.io data miner
User-agent: omgili
Disallow: /

# Legacy Anthropic token (Anthropic's crawlers are ClaudeBot, Claude-User and Claude-SearchBot)
User-agent: anthropic-ai
Disallow: /

# Cohere training crawler
User-agent: cohere-ai
Disallow: /

# Amazon AI training scraper
User-agent: Amazonbot
Disallow: /

# Timpi data harvester
User-agent: Timpibot
Disallow: /

# ---------------------------------------------------------------------------
# Aggressive SEO / Scraping Bots — these hammer servers and provide no
# direct ranking or traffic benefit worth the load.
# ---------------------------------------------------------------------------

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

# ---------------------------------------------------------------------------
# Sitemap
# ---------------------------------------------------------------------------
Sitemap: https://allmymeetings.com/sitemap.xml