# AI Model Directory — robots.txt
# https://aimodels.directory
# This data is designed to be machine-readable. All crawlers are welcome.

# Default: all crawlers get full access (except private API key routes)
User-agent: *
Allow: /
Disallow: /api/developer/keys

# ───────────────────────────────────────────────
# SEARCH ENGINES
# ───────────────────────────────────────────────

# Google (all crawlers)
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-News
Allow: /

User-agent: Googlebot-Video
Allow: /

User-agent: Google-Extended
Allow: /

# Bing / Microsoft
User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: MSNBot
Allow: /

User-agent: MSNBot-Media
Allow: /

User-agent: AdIdxBot
Allow: /

# Yandex (Russia) — all crawler variants
User-agent: YandexBot
Allow: /
Crawl-delay: 2

User-agent: YandexImages
Allow: /

User-agent: YandexVideo
Allow: /

User-agent: YandexNews
Allow: /

User-agent: YandexMobileBot
Allow: /

User-agent: YandexMarket
Allow: /

User-agent: YandexFavicons
Allow: /

User-agent: YandexSitelinks
Allow: /

User-agent: YandexScreenshotBot
Allow: /

User-agent: YandexAccessibilityBot
Allow: /

# VKontakte / VK (Russia's largest social network — link previews)
User-agent: vkShare
Allow: /

# Odnoklassniki / OK.ru (Russia's second largest social network)
User-agent: OdklBot
Allow: /

# Mail.ru / Mailru (Russia)
User-agent: Mail.RU_Bot
Allow: /

User-agent: Mediapartners-Google
Allow: /

# Baidu (China) — all crawler variants including JS renderer
User-agent: Baiduspider
Allow: /
Crawl-delay: 2

User-agent: Baiduspider-image
Allow: /

User-agent: Baiduspider-video
Allow: /

User-agent: Baiduspider-render
Allow: /

User-agent: Baiduspider+
Allow: /

# Naver (South Korea)
User-agent: Yeti
Allow: /
Crawl-delay: 2

User-agent: NaverBot
Allow: /

# Daum / Kakao (South Korea)
User-agent: Daum
Allow: /

User-agent: Kakaotalk-Scrap
Allow: /

# Line (Japan, Taiwan, Thailand, Korea — link previews)
User-agent: LineCrawler
Allow: /

# Petal Search (Huawei — strong in Middle East, Asia, Africa)
User-agent: PetalBot
Allow: /

# Yahoo Japan
User-agent: Y!J-BRZ/YBOT
Allow: /

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /

User-agent: DuckDuckGo-Favicons-Bot
Allow: /

# Brave Search
User-agent: BraveBot
Allow: /

# Ecosia (uses Bing index but has own crawler)
User-agent: EcosiaBot
Allow: /

# Mojeek (independent search engine)
User-agent: MojeekBot
Allow: /

# Qwant (France)
User-agent: Qwantify
Allow: /

# Seznam (Czech Republic)
User-agent: SeznamBot
Allow: /

# Sogou / Tencent (China)
User-agent: Sogou web spider
Allow: /

User-agent: Sogou inst spider
Allow: /

User-agent: Sogouwebspider
Allow: /

# 360 / Haosou (China)
User-agent: 360Spider
Allow: /

User-agent: HaosouSpider
Allow: /

# Shenma / UC Web / Alibaba (China — mobile search)
User-agent: ShenmaSpider
Allow: /

User-agent: UCWEB
Allow: /

# Huawei Petal Search (China + international)
User-agent: PetalBot
Allow: /

# Tencent / WeChat link preview (China social)
User-agent: TencentTraveler
Allow: /

# WeChat / Weixin (link unfurling)
User-agent: WeChat
Allow: /

# Exalead (France)
User-agent: Exabot
Allow: /

# ───────────────────────────────────────────────
# AI / LLM CRAWLERS
# Explicitly welcome — this data is designed to be machine-readable
# ───────────────────────────────────────────────

# OpenAI
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Anthropic / Claude
User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

# Google AI / Gemini
User-agent: Google-Extended
Allow: /

User-agent: Gemini-Bot
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

# Apple
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# You.com
User-agent: YouBot
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

# Amazon / Alexa
User-agent: Amazonbot
Allow: /

# Meta AI
User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

# ByteDance / TikTok / Douyin AI (China + global)
User-agent: Bytespider
Allow: /

# DeepSeek AI (China)
User-agent: DeepSeekBot
Allow: /

# Alibaba / Qwen / Tongyi AI (China)
User-agent: Qwen-Bot
Allow: /

User-agent: AlibabaBot
Allow: /

# Baidu ERNIE / Wenxin AI (China)
User-agent: ERNIEBot
Allow: /

# Zhipu AI / ChatGLM (China)
User-agent: ZhipuAIBot
Allow: /

# Moonshot / Kimi AI (China)
User-agent: KimiBot
Allow: /

# Diffbot (data extraction AI)
User-agent: Diffbot
Allow: /

# iAsk AI
User-agent: iaskspider
Allow: /

# Exa AI
User-agent: Exa-Bot
Allow: /

# Mistral AI
User-agent: MistralBot
Allow: /

# ───────────────────────────────────────────────
# SOCIAL MEDIA LINK PREVIEW BOTS
# Needed for rich previews when URLs are shared
# ───────────────────────────────────────────────

# Facebook / Instagram / Threads (Meta)
User-agent: facebookexternalhit
Allow: /

User-agent: Facebot
Allow: /

# LinkedIn
User-agent: LinkedInBot
Allow: /

# Twitter / X
User-agent: Twitterbot
Allow: /

# Slack
User-agent: Slackbot
Allow: /

User-agent: Slackbot-LinkExpanding
Allow: /

# Discord
User-agent: Discordbot
Allow: /

# Telegram
User-agent: TelegramBot
Allow: /

# WhatsApp
User-agent: WhatsApp
Allow: /

# Pinterest
User-agent: Pinterestbot
Allow: /

# Snapchat
User-agent: Snapchat
Allow: /

# ───────────────────────────────────────────────
# ARCHIVE & RESEARCH BOTS
# ───────────────────────────────────────────────

# Internet Archive / Wayback Machine
User-agent: ia_archiver
Allow: /

User-agent: archive.org_bot
Allow: /

User-agent: Heritrix
Allow: /

# Common Crawl (used by AI training datasets)
User-agent: CCBot
Allow: /

# ───────────────────────────────────────────────
# SEO & ANALYTICS TOOLS
# Allowing these improves backlink discovery and domain authority signals
# ───────────────────────────────────────────────

User-agent: SemrushBot
Allow: /
Crawl-delay: 5

User-agent: AhrefsBot
Allow: /
Crawl-delay: 5

User-agent: MJ12bot
Allow: /
Crawl-delay: 5

User-agent: DotBot
Allow: /
Crawl-delay: 5

User-agent: BLEXBot
Allow: /
Crawl-delay: 5

User-agent: DataForSeoBot
Allow: /
Crawl-delay: 5

# ───────────────────────────────────────────────
# SITEMAPS & MACHINE-READABLE ENDPOINTS
# ───────────────────────────────────────────────

Sitemap: https://aimodels.directory/sitemap.xml

# Machine-readable data endpoints
# JSON API:    https://aimodels.directory/api/models
# RSS feed:    https://aimodels.directory/rss/changelog
# OpenAPI:     https://aimodels.directory/openapi.json
# LLM guide:  https://aimodels.directory/llms.txt
# AI plugin:  https://aimodels.directory/.well-known/ai-plugin.json