# SETU NEXUS robots.txt
# Purpose: allow search engines, AI answer engines, LLM crawlers, and browser agents
# to discover public SETU content and the LLM context files.
#
# Robots file: https://setunexus.ai/robots.txt
# Compact LLM context: https://setunexus.ai/llms.txt
# Full LLM context: https://setunexus.ai/llms-full.txt
# Sitemap: https://setunexus.ai/sitemap.xml
# Last updated: 2026-07-08
#
# Authoritative SETU pages:
# English canonical page: https://setunexus.ai/
# Japanese canonical page: https://setunexus.ai/ja
#

# OpenAI crawlers and user-triggered fetchers
User-agent: OAI-SearchBot
User-agent: GPTBot
User-agent: ChatGPT-User
Allow: /

# Anthropic / Claude crawlers and user-triggered fetchers
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
Allow: /

# Google Search, Gemini, Vertex AI, and Google AI control tokens
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-News
User-agent: GoogleOther
User-agent: Google-Extended
User-agent: Google-CloudVertexBot
Allow: /

# Microsoft Bing search crawlers
User-agent: bingbot
User-agent: BingPreview
User-agent: msnbot
Allow: /

# Perplexity crawlers and user-triggered fetchers
User-agent: PerplexityBot
User-agent: Perplexity-User
Allow: /

# Other search engines, LLM crawlers, browser agents, and compliant bots
User-agent: *
Allow: /

Sitemap: https://setunexus.ai/sitemap.xml