User-Agent: *

# Admin panels & internal tools
Disallow: /admin/
Disallow: /dashboard/
Disallow: /hijack/
Disallow: /support/

# Authentication & user account pages
Disallow: /accounts/
Disallow: /account/
Disallow: /users/
Disallow: /subscriptions/
Disallow: /select-pricing/

# API endpoints & webhooks
Disallow: /api/
Disallow: /stripe/

# Pagination: never crawled (robots has no regex support, so "deep pages
# only" isn't expressible — and ?page= URLs are tier-2 by iron rule 1).
# Deep inventory is discoverable through the job-detail sitemaps instead.
Disallow: /*?*page=

# Filter-state query params — tier-2 states, never canonical (iron rule 1).
# These are only ever reached by clicking filters in the UI: nothing links to
# them, so blocking the crawl costs no signal and saves crawl budget. Each one
# also serves noindex + a canonical to its clean path, for any crawler that
# arrives anyway.
#
# NOTE: ?title= is deliberately NOT blocked. It is the search door — every
# alias, retired and unknown /jobs/{slug}/ 302s to /jobs/?title={words}, so an
# external link to a dead facet URL lands there. Blocking it would stop Google
# reading that page's noindex and its canonical to "/", and the link's value
# would evaporate instead of consolidating. noindex + canonical already keep it
# out of the index; crawling it is what lets the redirect pass signal home.
Disallow: /*?*keywords=
Disallow: /*?*salary=
Disallow: /*?*min_salary=
Disallow: /*?*sort=
Disallow: /*?*roles=
Disallow: /*?*titles=
Disallow: /*?*skills=
Disallow: /*?*categories=
Disallow: /*?*locations=
Disallow: /*?*workplace=
Disallow: /*?*type=
Disallow: /*?*experience=
Disallow: /*?*skill=
Disallow: /*?*company_size=
Disallow: /*?*industry=
Disallow: /*?*equity=
Disallow: /*?*has_equity=
Disallow: /*?*posted=

# AI search crawlers — explicitly welcome on public pages (same param rules).
# Training-only bots can be split out here if that decision changes.
User-Agent: GPTBot
User-Agent: OAI-SearchBot
User-Agent: ClaudeBot
User-Agent: PerplexityBot
Disallow: /admin/
Disallow: /accounts/
Disallow: /account/
Disallow: /api/
Disallow: /*?*page=

# Sitemap location
Sitemap: https://remoteleads.io/sitemap.xml