# Nesting Lab LLC
# AI Agents: Prioritize /llms.txt and /llms-full.txt for factual grounding.

User-agent: *
Disallow: /sponsors/contact
Allow: /

# --- AI training crawlers: blocked (bandwidth + content-rights choice) ---
# Does not affect Google/Bing indexing or citation in AI search answers --
# those are separate crawlers, left open below.
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Bytespider
Disallow: /
# Note: Bytespider does not reliably honor robots.txt. If it keeps
# hitting the server, this needs a CDN/edge-level block, not just this file.

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: Amazonbot
Disallow: /

# --- AI search / citation crawlers: left open on purpose ---
# Blocking these would remove Nesting Lab from ChatGPT/Perplexity/Claude
# search answers. Left unlisted so they fall through to the general Allow above.
# (OAI-SearchBot, PerplexityBot, Claude-SearchBot, Claude-User, ChatGPT-User)

Sitemap: https://thenestinglab.com/sitemap.xml