User-Agent: *
Allow: /
Disallow: /auth/
Disallow: /login
Disallow: /dashboard
Disallow: /ai-visibility
Disallow: /analytics
Disallow: /clients/
Disallow: /crawls
Disallow: /domains
Disallow: /integrations/
Disallow: /localization
Disallow: /mentions
Disallow: /policies
Disallow: /regressions
Disallow: /scan
Disallow: /settings
Disallow: /verifications
Disallow: /history
Disallow: /p/
Disallow: /portal/
Disallow: /r/
Disallow: /site/
Disallow: /md/

Host: https://crawlfix.ai
Sitemap: https://crawlfix.ai/sitemap.xml