# newerlaw.com — AI agents and AI crawlers are welcome.
#
# Agent guide (markdown):  https://www.newerlaw.com/agents.md
# LLM summary:             https://www.newerlaw.com/llms.txt
# Full text for LLMs:      https://www.newerlaw.com/llms-full.txt
# OpenAPI (agent API):     https://www.newerlaw.com/openapi.json
# A2A agent card:          https://www.newerlaw.com/.well-known/agent-card.json
# MCP endpoint:            https://www.newerlaw.com/mcp  (POST, Streamable HTTP)

User-agent: *
Allow: /
Disallow: /captcha-challenge
Disallow: /posts/drafts/

# AI assistants, search and training crawlers — explicitly allowed
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: meta-externalagent
User-agent: meta-externalfetcher
User-agent: cohere-ai
User-agent: DuckAssistBot
User-agent: MistralAI-User
User-agent: CCBot
User-agent: Bytespider
Allow: /
Disallow: /captcha-challenge
Disallow: /posts/drafts/

Sitemap: https://www.newerlaw.com/sitemap.xml