User-agent: *
Allow: /

# Authenticated app pages — no value indexing these.
Disallow: /renewal-app.html
Disallow: /renewal-policy.html
Disallow: /account.html
Disallow: /welcome.html
# Note: /state.html, /carrier.html, /insights-search.html are user-facing landing
# pages — we WANT them indexed (sitemap lists state/carrier combos explicitly).
# /insights-filing.html stays blocked: individual filings are noindex-tagged.
Disallow: /insights-filing.html

# Internal preview / legacy snapshots.
Disallow: /preview/
Disallow: /index-legacy-2026-05-01.html
Disallow: /index-old-2026-05-04.html

# API + assets — wasted crawl budget.
Disallow: /api/
Disallow: /fonts/
Disallow: /pdfs/
Disallow: /node_modules/
Disallow: /backups/

# Block known content-scraping AI bots from training on our copy.
# (We're happy to be cited; we are not happy to be eaten.)
User-agent: GPTBot
Disallow: /
User-agent: ClaudeBot
Disallow: /
User-agent: CCBot
Disallow: /
User-agent: Google-Extended
Disallow: /
User-agent: anthropic-ai
Disallow: /
User-agent: PerplexityBot
Disallow: /

Sitemap: https://ratefileai.com/sitemap-index.xml
Sitemap: https://ratefileai.com/sitemap.xml
Sitemap: https://ratefileai.com/sitemap-seo.xml