# PromptHive's own crawler policy.
#
# Everything above this line is injected by Cloudflare at the zone level and is
# not editable from this repository. It blocks AI *training* crawlers, which is
# the intended policy. It says nothing about the agents that produce citations,
# and this section says it explicitly rather than relying on their absence from
# a list we do not control. See src/lib/crawlers.ts and DECISIONS #25.

User-agent: *
Content-Signal: search=yes,ai-input=yes,ai-train=no,use=reference
Allow: /

# Internal moderation tool, gated at the edge and noindexed.
Disallow: /admin/
# JSON endpoints; nothing here renders in a search result.
Disallow: /api/
# Per-visitor confirmation.
Disallow: /submit/received/
# Per-visitor status lookup.
Disallow: /submit/status/
# Retired ad frames. Nothing here is content.
Disallow: /ads/

# ── Answer engines ────────────────────────────────────────────────────
#
# Allowed on purpose, and named so the permission cannot lapse quietly.
#
# Each group repeats the rules above because robots.txt groups do NOT inherit
# from "User-agent: *" — a crawler obeys only its most specific group (RFC 9309
# §2.2.1). Omitting them here would expose /admin/ and the per-visitor
# confirmation pages to exactly the crawlers being invited in.

# OpenAI's search index — the source of ChatGPT Search citations.
User-agent: OAI-SearchBot
Content-Signal: search=yes,ai-input=yes,ai-train=no,use=reference
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /submit/received/
Disallow: /submit/status/
Disallow: /ads/

# Fetches a page because a ChatGPT user asked about it.
User-agent: ChatGPT-User
Content-Signal: search=yes,ai-input=yes,ai-train=no,use=reference
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /submit/received/
Disallow: /submit/status/
Disallow: /ads/

# Anthropic's search index, used to answer with citations.
User-agent: Claude-SearchBot
Content-Signal: search=yes,ai-input=yes,ai-train=no,use=reference
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /submit/received/
Disallow: /submit/status/
Disallow: /ads/

# Fetches a page on behalf of a Claude user who asked.
User-agent: Claude-User
Content-Signal: search=yes,ai-input=yes,ai-train=no,use=reference
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /submit/received/
Disallow: /submit/status/
Disallow: /ads/

# Perplexity's index; Perplexity answers cite their sources.
User-agent: PerplexityBot
Content-Signal: search=yes,ai-input=yes,ai-train=no,use=reference
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /submit/received/
Disallow: /submit/status/
Disallow: /ads/

# Fetches a page a Perplexity user is asking about.
User-agent: Perplexity-User
Content-Signal: search=yes,ai-input=yes,ai-train=no,use=reference
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /submit/received/
Disallow: /submit/status/
Disallow: /ads/

Sitemap: https://prompthive.dpdns.org/sitemap-index.xml
Sitemap: https://prompthive.dpdns.org/sitemap-0.xml