# ML Junction robots policy.
# AI crawlers (GPTBot, ClaudeBot, PerplexityBot, etc.) are explicitly welcome —
# the public catalog, docs and pricing are meant to be discoverable.
User-agent: *
Allow: /

# Non-indexable surfaces. These MUST mirror NOINDEX_PATH_PATTERNS in
# src/config/seoRoutePolicy.js - drift here re-exposes pages meant to be hidden.
# Each of these also ships a prerendered noindex meta tag, so even if a crawler
# ignores this file (or the URL is linked externally), fetching the page still
# yields an explicit noindex directive.
Disallow: /app
Disallow: /sign-in
Disallow: /sign-up
Disallow: /sso-callback

# Polite crawl pacing for the big general crawlers.
User-agent: Googlebot
Crawl-delay: 1

User-agent: Bingbot
Crawl-delay: 1

Sitemap: https://mljunction.com/sitemap.xml