# https://www.robotstxt.org/robotstxt.html
#
# Content-Signal declares the permitted *use* of what a crawler fetches, independently of whether
# it may fetch at all: this site may be indexed for search and used as input to an AI answer
# (grounding/RAG), but its content is not offered for model training.
# https://contentsignals.org
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=no
# Sign-in endpoints 302 to the identity provider with a new session id on every request.
# Crawling them spends budget on URLs that are never pages. Other /api paths (the published
# JSON and llms.txt documents below) stay allowed.
Disallow: /api/auth/

Sitemap: https://layer.ai/sitemap-index.xml
# /docs is a separate Astro build behind the same origin, so it emits its own sitemap. Both
# are listed here: robots.txt is the only place a crawler is told a sitemap exists, and the
# root index covers the marketing build alone.
Sitemap: https://layer.ai/docs/sitemap-index.xml

# ── AI crawlers ──
#
# robots.txt groups do not merge: a crawler that matches a named group ignores the `*` group
# entirely. Every group below therefore repeats Content-Signal, or these agents would see no
# signal at all.

# Crawlers behind an answer engine, and the user-triggered fetchers that back "open this page"
# in an assistant. Allowed: being absent from ChatGPT, Claude, and Perplexity costs Layer more
# than any crawl budget. Training remains unpermitted — that is what Content-Signal says, and it
# applies to these agents too.
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Applebot-Extended
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
# Repeated from the `*` group: a named group does not inherit it.
Disallow: /api/auth/

# Crawlers that build training corpora and surface Layer in no answer engine, so there is nothing
# to lose by declining them. Disallowed as well as signaled, because a signal is a request and a
# disallow is an instruction.
User-agent: CCBot
User-agent: Bytespider
Content-Signal: search=yes, ai-input=yes, ai-train=no
Disallow: /

# LLM-readable content indexes (https://llmstxt.org)
# https://layer.ai/llms.txt
# https://layer.ai/llms-full.txt
# https://layer.ai/docs/llms.txt

# Agent mode for the home page — the same facts as / in a dense, machine-first form
# https://layer.ai/?mode=agent                  (also /agent, /agent.md)

# Machine-readable API surface
# https://layer.ai/.well-known/api-catalog      (RFC 9727 API catalog)
# https://layer.ai/.well-known/mcp              (Layer MCP servers + tool lists)
# https://layer.ai/.well-known/ard.json        (ARD manifest: every callable Layer resource)
# https://layer.ai/.well-known/ai-catalog.json  (APIs + readable content surface, for AI clients)
# https://layer.ai/.well-known/agent-skills/index.json  (Agent Skills index: one SKILL.md per capability)
# https://layer.ai/openapi.json                 (Layer REST API, OpenAPI 3.1)
# https://layer.ai/api/index.json               (REST + MCP entry points, auth, limits, policy)
# https://layer.ai/api/onboarding.json          (self-serve signup, credentials, zero-auth URLs)

# OAuth 2.0 discovery — these paths redirect to the authoritative host, which is where RFC 8414
# and RFC 9728 require the metadata to be served from.
# https://layer.ai/.well-known/oauth-authorization-server  -> https://auth.app.layer.ai/...
# https://layer.ai/.well-known/oauth-protected-resource    -> https://mcp.app.layer.ai/...

# Developer documentation
# https://layer.ai/docs                      (Layer developer documentation)
# https://layer.ai/docs/authentication       (Layer API authentication)
# https://layer.ai/docs/rate-limits          (Layer API rate limits)
# https://layer.ai/docs/versioning           (Layer API versioning)
# https://layer.ai/docs/deprecation          (Layer API deprecation policy)

# Markdown variants: append .md to any page path, or send `Accept: text/markdown`
# (https://acceptmarkdown.com)