# https://www.robotstxt.org/robotstxt.html
User-agent: *

# Block authenticated and private pages
Disallow: /login
Disallow: /signup
Disallow: /verification-code
Disallow: /logout
Disallow: /settings
Disallow: /dashboard
Disallow: /user/
Disallow: /resume-builder
Disallow: /cover-letter
Disallow: /oauth/
# The admin console moved /admin -> /hq and the B2B console lives at /talent;
# both render a 200 SPA shell to a crawler before their guard resolves, so both
# need listing. /admin stays because old links to it are still in the wild.
Disallow: /admin
Disallow: /hq
Disallow: /talent
Disallow: /saved-jobs
Disallow: /my-resumes
# /community and /job-application/ were listed here until 2026-08-13. Both now
# return 410, and a Disallow'd URL cannot be crawled — so Google would never
# see the 410 and would keep the URL indexed. Deindexing requires the crawl.

# API access:
# Block PRIVATE/authenticated API namespaces only. Do NOT blanket-block /api/
# — the client app fetches public read endpoints (jobs, map, analytics) via
# XHR during rendering, and blocking them makes Googlebot render an empty/404
# page. Public JSON is fetchable here but carries an `X-Robots-Tag: noindex`
# response header (see nginx.conf) so it is not indexed as a standalone page.
Disallow: /api/users/
Disallow: /api/saved-jobs
Disallow: /api/saved-searches
Disallow: /api/cover-letter
Disallow: /api/ai/
Disallow: /api/subscription
Disallow: /api/credits
Disallow: /api/referral
Disallow: /api/prerender/
Allow: /api/jobs/
Allow: /api/map/
Allow: /api/analytics/
Allow: /api/og/

# Unlisted internal mockups (share-by-link only)
Disallow: /mockups.html
Disallow: /mockups/

# Unlisted capabilities / sales collateral (share-by-link only)
Disallow: /ayla-capabilities-statement.html

# Allow public content
Allow: /
Allow: /government-jobs
Allow: /jobs
Allow: /jobs/map
Allow: /jobs/category/
Allow: /jobs/state/
Allow: /jobs/type/
Allow: /jobs/role/
# /jobs/salary/ was listed here until 2026-09-06. The route was removed on
# 2026-08-17 (0 impressions, 0 clicks, never crawled) and nginx now returns 410
# for every URL under it — verified with curl on 2026-09-06: /jobs/salary/ and
# /jobs/salary/gs-13 both 410. An Allow: pointing at a gone route invites the
# crawl budget we do have to spend itself on tombstones.
Allow: /jobs/employer/
Allow: /jobs/remote
# /platform, /sourcing, /talent-insights and /government-job-market-data were
# listed here until 2026-09-14; all four are 301s now (see ui/nginx.conf).
Allow: /government-hiring-report
Allow: /analytics
Allow: /compensation
Allow: /job-alerts
Allow: /about
# /contact stopped being noindex on 2026-08-17: it carries the white-glove
# recruiting form, which is the only place that offer can be reached.
Allow: /contact
Allow: /terms
Allow: /privacy
# The public API documented and the crawler/agent policy (2026-09-15).
Allow: /developers
Allow: /agent-policy

# Dynamic sitemap
Sitemap: https://aylagov.com/sitemap.xml

# AI Crawlers — explicitly allow for discoverability
# See https://aylagov.com/llms.txt for AI-specific information and
# https://aylagov.com/agent-policy for the stated policy in prose. The
# public JSON API is documented at https://aylagov.com/developers; it is
# fetchable but carries X-Robots-Tag: noindex, so it stays out of every
# group's Allow list here. Each vendor's tokens are listed by role, per its
# own documentation: the training crawler, the search-index crawler, and the
# user-initiated fetcher (which may ignore this file anyway).
#
# OpenAI: GPTBot trains, OAI-SearchBot indexes for ChatGPT search,
# ChatGPT-User fetches on a person's request.
User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /dashboard
Disallow: /user/
Disallow: /settings

User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /dashboard
Disallow: /user/
Disallow: /settings

User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /dashboard
Disallow: /user/
Disallow: /settings

User-agent: Google-Extended
Allow: /

# Anthropic: ClaudeBot trains, Claude-SearchBot indexes for search,
# Claude-User fetches on a person's request. The retired `anthropic-ai` and
# `Claude-Web` tokens were listed here until 2026-09-15; Anthropic's crawler
# documentation no longer names them and no fetch has carried either since
# the July log baseline.
User-agent: ClaudeBot
Allow: /
Disallow: /api/
Disallow: /dashboard
Disallow: /user/
Disallow: /settings

User-agent: Claude-SearchBot
Allow: /
Disallow: /api/
Disallow: /dashboard
Disallow: /user/
Disallow: /settings

User-agent: Claude-User
Allow: /
Disallow: /api/
Disallow: /dashboard
Disallow: /user/
Disallow: /settings

# Perplexity: PerplexityBot indexes, Perplexity-User fetches on request.
User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /dashboard
Disallow: /user/
Disallow: /settings

User-agent: Perplexity-User
Allow: /
Disallow: /api/
Disallow: /dashboard
Disallow: /user/
Disallow: /settings

User-agent: Applebot-Extended
Allow: /

User-agent: CCBot
Allow: /
Disallow: /api/
Disallow: /dashboard
Disallow: /user/
Disallow: /settings

# Crawl-delay for polite crawling
Crawl-delay: 1