# llms.txt - AI Usage Control Policy



#Different bots are blocked to meet legal compliance and risk reduction as indicated in the EU.

# Amazon AI   fully blocked
# Only used for training and scraping Alexa and AWS AI tools
User-agent: Amazonbot
Disallow: /
Purpose: all

# Baidu (China)   fully blocked
# Only used for Ernie bot
User-agent: BaiduSpider
Disallow: /
Purpose: all

# ByteDance (Doubao, China)   fully blocked
# Only used for training Doubao, ByteDance s LLM and TikTok AI model, and an active crawler
User-agent: Bytespider
Disallow: /
Purpose: all

# Anthropic (Claude, USA)   not supported in robots.txt due to blocking
# Assumed use for private scraping only
User-agent: ClaudeBot
Disallow: /
Purpose: all

# Meta (LLaMA / previews)   fully blocked
# Used by  Meta LLM and no public crawler
User-agent: Meta-Agent
Disallow: /
Purpose: all

# Yandex (Russia)   fully blocked
# For training and indexing Yandex AI tools
User-agent: YandexBot
Disallow: /
Purpose: all



# General AI policy for allowed bots for embedding, indexing and summarizing, not-training

# Cohere (Canada/USA)
# Known AI model builder
User-agent: CohereBot
Allow: /
Purpose: embedding, indexing, summarization
Disallow: /
Purpose: training

# Google (Gemini, USA)
# From Google AI like Gemini
User-agent: Google-Extended
Allow: /
Purpose: embedding, indexing, summarization
Disallow: /
Purpose: training

# OpenAI (ChatGPT, USA)
# Used to gather web content that honors robots.txt
User-agent: GPTBot
Allow: /
Purpose: embedding, indexing, summarization
Disallow: /
Purpose: training

# Mistral AI (France / EU)
# Used by Mistral AI
User-agent: MistralBot
Allow: /
Purpose: embedding, indexing, summarization
Disallow: /
Purpose: training

# Perplexity AI (USA)
# An AI search crawler
User-agent: PerplexityBot
Allow: /
Purpose: embedding, indexing, summarization
Disallow: /
Purpose: training

# Other unknown AI bots
User-agent: *
Allow: /
Purpose: embedding, indexing, summarization
Disallow: /
Purpose: training

# Same policy with different syntax
User-Agent: *
Use-For-Training: no
Use-For-Embedding: yes
Use-For-Indexing: yes
Use-For-Summarizing: yes
Attribution-Required: yes