{
  "name": "AI crawler list",
  "source": "https://stellarcast.ai/ai-crawlers/",
  "checked": "2026-10-30",
  "license": "CC BY 4.0",
  "bots": [
    {
      "token": "GPTBot",
      "operator": "OpenAI",
      "type": "training",
      "used_for_training": "yes",
      "robots_txt": "optout",
      "description": "Disallowing it tells OpenAI not to use the site's content for training.",
      "documentation": "https://developers.openai.com/api/docs/bots",
      "ip_list": "https://openai.com/gptbot.json"
    },
    {
      "token": "OAI-SearchBot",
      "operator": "OpenAI",
      "type": "search",
      "used_for_training": "no",
      "robots_txt": "optout",
      "description": "Surfaces sites in ChatGPT search. Blocked sites are not shown there; changes take about a day.",
      "documentation": "https://developers.openai.com/api/docs/bots",
      "ip_list": "https://openai.com/searchbot.json"
    },
    {
      "token": "ChatGPT-User",
      "operator": "OpenAI",
      "type": "user",
      "used_for_training": "no",
      "robots_txt": "partial",
      "description": "Visits a page when a ChatGPT user's action needs it. OpenAI says robots.txt may not apply.",
      "documentation": "https://developers.openai.com/api/docs/bots",
      "ip_list": "https://openai.com/chatgpt-user.json"
    },
    {
      "token": "OAI-AdsBot",
      "operator": "OpenAI",
      "type": "other",
      "used_for_training": "na",
      "robots_txt": "unstated",
      "description": "Checks the safety of pages submitted as ads on ChatGPT, and only those pages.",
      "documentation": "https://developers.openai.com/api/docs/bots",
      "ip_list": "https://openai.com/adsbot.json"
    },
    {
      "token": "ClaudeBot",
      "operator": "Anthropic",
      "type": "training",
      "used_for_training": "yes",
      "robots_txt": "yes",
      "description": "Blocking it keeps future content out of Anthropic's training data. Crawl-delay is supported.",
      "documentation": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "ip_list": "https://claude.com/crawling/bots.json"
    },
    {
      "token": "Claude-SearchBot",
      "operator": "Anthropic",
      "type": "search",
      "used_for_training": "no",
      "robots_txt": "yes",
      "description": "Indexes pages to improve the quality of Claude's search results.",
      "documentation": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "ip_list": "https://claude.com/crawling/bots.json"
    },
    {
      "token": "Claude-User",
      "operator": "Anthropic",
      "type": "user",
      "used_for_training": "no",
      "robots_txt": "yes",
      "description": "Fetches pages when a Claude user asks. Anthropic's robots.txt commitment covers all its bots.",
      "documentation": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "ip_list": "https://claude.com/crawling/bots.json"
    },
    {
      "token": "PerplexityBot",
      "operator": "Perplexity",
      "type": "search",
      "used_for_training": "no",
      "robots_txt": "optout",
      "description": "Surfaces and links sites in Perplexity answers; Perplexity says it is not used for model training.",
      "documentation": "https://docs.perplexity.ai/guides/bots",
      "ip_list": "https://www.perplexity.ai/perplexitybot.json"
    },
    {
      "token": "Perplexity-User",
      "operator": "Perplexity",
      "type": "user",
      "used_for_training": "no",
      "robots_txt": "no",
      "description": "Fetches a page when a user asks. Perplexity says it generally ignores robots.txt.",
      "documentation": "https://docs.perplexity.ai/guides/bots",
      "ip_list": "https://www.perplexity.ai/perplexity-user.json"
    },
    {
      "token": "Googlebot",
      "operator": "Google",
      "type": "search",
      "used_for_training": "na",
      "robots_txt": "yes",
      "description": "Google Search's crawler. Google names Googlebot's robots.txt rules as the control for AI features in Search; use nosnippet or noindex to limit AI Overviews and AI Mode.",
      "documentation": "https://developers.google.com/search/docs/appearance/ai-features",
      "ip_list": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json"
    },
    {
      "token": "Google-Extended",
      "operator": "Google",
      "type": "control",
      "used_for_training": "yes",
      "robots_txt": "na",
      "description": "No crawler of its own. Decides whether content may train future Gemini models and ground Gemini apps. Google says it does not affect inclusion in Google Search.",
      "documentation": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
      "ip_list": null
    },
    {
      "token": "GoogleOther",
      "operator": "Google",
      "type": "other",
      "used_for_training": "na",
      "robots_txt": "yes",
      "description": "Generic crawler for Google product teams and research. Its rules do not affect a specific product.",
      "documentation": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
      "ip_list": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json"
    },
    {
      "token": "Google-CloudVertexBot",
      "operator": "Google",
      "type": "user",
      "used_for_training": "na",
      "robots_txt": "yes",
      "description": "Crawls sites a site owner asks it to, to build Vertex AI agents. No effect on Google Search.",
      "documentation": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
      "ip_list": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json"
    },
    {
      "token": "bingbot",
      "operator": "Microsoft",
      "type": "search",
      "used_for_training": "maybe",
      "robots_txt": "yes",
      "description": "Bing's main crawler. Microsoft's AI controls are meta tags, not a robots.txt token: NOARCHIVE keeps a page out of Bing's AI answers and training (2023 guidance, written for Bing Chat).",
      "documentation": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0",
      "ip_list": "https://www.bing.com/toolbox/verify-bingbot"
    },
    {
      "token": "Applebot",
      "operator": "Apple",
      "type": "search",
      "used_for_training": "maybe",
      "robots_txt": "yes",
      "description": "Powers Spotlight, Siri and Safari search. Its data may help train Apple models unless Applebot-Extended is disallowed. Falls back to Googlebot rules if none name Applebot.",
      "documentation": "https://support.apple.com/en-us/119829",
      "ip_list": "http://search.developer.apple.com/applebot.json"
    },
    {
      "token": "Applebot-Extended",
      "operator": "Apple",
      "type": "control",
      "used_for_training": "yes",
      "robots_txt": "na",
      "description": "Does not crawl. Disallowing it stops Apple using Applebot's data to train its models; pages stay in Apple search results.",
      "documentation": "https://support.apple.com/en-us/119829",
      "ip_list": null
    },
    {
      "token": "meta-externalagent",
      "operator": "Meta",
      "type": "training",
      "used_for_training": "yes",
      "robots_txt": "optout",
      "description": "Used for training AI models and for indexing content to improve products. One token covers both.",
      "documentation": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/",
      "ip_list": null
    },
    {
      "token": "meta-webindexer",
      "operator": "Meta",
      "type": "search",
      "used_for_training": "na",
      "robots_txt": "optout",
      "description": "Improves the quality of Meta AI search results.",
      "documentation": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/",
      "ip_list": null
    },
    {
      "token": "meta-externalfetcher",
      "operator": "Meta",
      "type": "user",
      "used_for_training": "na",
      "robots_txt": "partial",
      "description": "Fetches individual links at a user's request. Meta says it may bypass robots.txt.",
      "documentation": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/",
      "ip_list": null
    },
    {
      "token": "Amazonbot",
      "operator": "Amazon",
      "type": "training",
      "used_for_training": "maybe",
      "robots_txt": "yes",
      "description": "Improves Amazon products; its data may be used to train Amazon AI models. Amazon also honours a noarchive meta tag to exclude a page from training.",
      "documentation": "https://developer.amazon.com/amazonbot",
      "ip_list": "https://developer.amazon.com/amazonbot/ip-addresses/"
    },
    {
      "token": "Amzn-SearchBot",
      "operator": "Amazon",
      "type": "search",
      "used_for_training": "no",
      "robots_txt": "yes",
      "description": "Improves Amazon search experiences; Amazon says it does not crawl for AI training.",
      "documentation": "https://developer.amazon.com/amazonbot",
      "ip_list": "https://developer.amazon.com/amazonbot/searchbot-ip-addresses/"
    },
    {
      "token": "Amzn-User",
      "operator": "Amazon",
      "type": "user",
      "used_for_training": "no",
      "robots_txt": "partial",
      "description": "Fetches live information to answer Alexa questions. Amazon says it may not follow all robots.txt rules.",
      "documentation": "https://developer.amazon.com/amazonbot",
      "ip_list": "https://developer.amazon.com/amazonbot/live-ip-addresses/"
    },
    {
      "token": "CCBot",
      "operator": "Common Crawl",
      "type": "other",
      "used_for_training": "na",
      "robots_txt": "yes",
      "description": "Builds Common Crawl's open web archive, which anyone can reuse. Supports Crawl-delay.",
      "documentation": "https://commoncrawl.org/ccbot",
      "ip_list": "https://index.commoncrawl.org/ccbot.json"
    },
    {
      "token": "MistralAI-Training",
      "operator": "Mistral AI",
      "type": "training",
      "used_for_training": "yes",
      "robots_txt": "optout",
      "description": "Builds datasets for training Mistral's models.",
      "documentation": "https://docs.mistral.ai/robots",
      "ip_list": null
    },
    {
      "token": "MistralAI-Index",
      "operator": "Mistral AI",
      "type": "search",
      "used_for_training": "no",
      "robots_txt": "optout",
      "description": "Indexing only; Mistral says it is not used for training.",
      "documentation": "https://docs.mistral.ai/robots",
      "ip_list": "https://mistral.ai/mistralai-index-ips.json"
    },
    {
      "token": "MistralAI-User",
      "operator": "Mistral AI",
      "type": "user",
      "used_for_training": "no",
      "robots_txt": "optout",
      "description": "Fetches pages for user actions in Mistral's assistant.",
      "documentation": "https://docs.mistral.ai/robots",
      "ip_list": "https://mistral.ai/mistralai-user-ips.json"
    },
    {
      "token": "DuckAssistBot",
      "operator": "DuckDuckGo",
      "type": "user",
      "used_for_training": "no",
      "robots_txt": "optout",
      "description": "Fetches pages in real time for DuckDuckGo's AI-assisted answers. A disallow takes effect after 72 hours.",
      "documentation": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot",
      "ip_list": "https://duckduckgo.com/duckassistbot.json"
    }
  ],
  "undocumented": [
    "ByteDance (Bytespider)",
    "xAI (Grok)",
    "DeepSeek",
    "FacebookBot",
    "Cohere"
  ]
}