{
  "version": 1,
  "contentRevision": "sha256:faeb704dd7053cd9016127d260c57d4247466f5cda86d24cb7a8a66d33492b5c",
  "identityRevision": "sha256:6f3fb63ef6ed50ed",
  "organization": {
    "@id": "https://thatdeveloperguy.com/#organization"
  },
  "author": {
    "@id": "https://thatdeveloperguy.com/#person"
  },
  "language": "en-US",
  "contentUpdated": "2026-10-18",
  "catalogueUpdated": "2026-10-18T23:27:16.697Z",
  "lastCompletedScan": "2026-10-19T11:24:58.711Z",
  "counts": {
    "total": 177,
    "works": 172,
    "demos": 143,
    "liveEditions": 29,
    "signature": 5
  },
  "links": {
    "guide": "https://thatwebhostingguy.com/library-guide/",
    "answers": "https://thatwebhostingguy.com/ai-answers.json",
    "catalogue": "https://thatwebhostingguy.com/discovery/catalogue.json",
    "providers": "https://thatwebhostingguy.com/discovery/providers.json",
    "text": "https://thatwebhostingguy.com/llms-full.txt",
    "identity": "https://thatwebhostingguy.com/entity.json",
    "sitemap": "https://thatwebhostingguy.com/sitemap.xml",
    "updates": "https://thatwebhostingguy.com/atom.xml",
    "robots": "https://thatwebhostingguy.com/robots.txt"
  },
  "url": "https://thatwebhostingguy.com/discovery/providers.json",
  "format": "Site-defined reference feeds, not provider registration or a provider-required JSON protocol.",
  "providers": [
    {
      "key": "google",
      "provider": "Google",
      "product": "Google Search, AI Overviews and AI Mode",
      "url": "https://thatwebhostingguy.com/discovery/providers/google.json",
      "crawlers": [
        {
          "token": "Googlebot",
          "role": "search",
          "decision": "allow",
          "source": "node properties.enginePolicy.bots",
          "httpUserAgent": true
        },
        {
          "token": "Googlebot-Image",
          "role": "search",
          "decision": "allow",
          "source": "node properties.enginePolicy.bots",
          "httpUserAgent": true
        },
        {
          "token": "Googlebot-News",
          "role": "search",
          "decision": "allow",
          "source": "node properties.enginePolicy.bots",
          "httpUserAgent": true
        },
        {
          "token": "Google-Extended",
          "role": "training",
          "decision": "allow",
          "source": "node properties.enginePolicy.bots",
          "httpUserAgent": false
        }
      ],
      "documentation": [
        {
          "title": "AI optimization guide",
          "url": "https://developers.google.com/search/docs/fundamentals/ai-optimization-guide"
        },
        {
          "title": "AI features and your website",
          "url": "https://developers.google.com/search/docs/appearance/ai-features"
        },
        {
          "title": "Google common crawlers",
          "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers"
        },
        {
          "title": "Verify requests from Google crawlers and fetchers",
          "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/verify-google-requests"
        },
        {
          "title": "How Google interprets robots.txt",
          "url": "https://developers.google.com/crawling/docs/robots-txt/robots-txt-spec"
        },
        {
          "title": "Generative AI performance report (Search)",
          "url": "https://support.google.com/webmasters/answer/16984139"
        }
      ]
    },
    {
      "key": "microsoft",
      "provider": "Microsoft",
      "product": "Bing Search, Copilot and Bing AI answers",
      "url": "https://thatwebhostingguy.com/discovery/providers/microsoft.json",
      "crawlers": [
        {
          "token": "Bingbot",
          "role": "search",
          "decision": "allow",
          "source": "node properties.enginePolicy.bots",
          "httpUserAgent": true
        },
        {
          "token": "BingPreview",
          "role": "preview",
          "decision": "allow",
          "source": "wildcard (no explicit decision on the node yet)",
          "httpUserAgent": true
        }
      ],
      "documentation": [
        {
          "title": "Introducing AI Performance in Bing Webmaster Tools",
          "url": "https://blogs.bing.com/webmaster/February-2026/Introducing-AI-Performance-in-Bing-Webmaster-Tools-Public-Preview"
        },
        {
          "title": "New AI visibility insights: Intents, Topics, Citation Share and Compare",
          "url": "https://blogs.bing.com/search/June-2026/New-AI-Visibility-Insights-in-Bing-Webmaster-Tools-Intents-Topics-Citation-Share-Compare"
        },
        {
          "title": "IndexNow documentation",
          "url": "https://www.indexnow.org/documentation"
        }
      ]
    },
    {
      "key": "openai",
      "provider": "OpenAI",
      "product": "ChatGPT search and answers",
      "url": "https://thatwebhostingguy.com/discovery/providers/openai.json",
      "crawlers": [
        {
          "token": "OAI-SearchBot",
          "role": "search",
          "decision": "allow",
          "source": "node properties.enginePolicy.bots",
          "httpUserAgent": true
        },
        {
          "token": "GPTBot",
          "role": "training",
          "decision": "allow",
          "source": "node properties.enginePolicy.bots",
          "httpUserAgent": true
        },
        {
          "token": "ChatGPT-User",
          "role": "user-fetch",
          "decision": "allow",
          "source": "node properties.enginePolicy.bots",
          "httpUserAgent": true
        },
        {
          "token": "OAI-AdsBot",
          "role": "ads",
          "decision": "allow",
          "source": "node properties.enginePolicy.bots",
          "httpUserAgent": true
        }
      ],
      "documentation": [
        {
          "title": "Overview of OpenAI crawlers",
          "url": "https://developers.openai.com/api/docs/bots"
        }
      ]
    },
    {
      "key": "anthropic",
      "provider": "Anthropic",
      "product": "Claude search and answers",
      "url": "https://thatwebhostingguy.com/discovery/providers/anthropic.json",
      "crawlers": [
        {
          "token": "Claude-SearchBot",
          "role": "search",
          "decision": "allow",
          "source": "node properties.enginePolicy.bots",
          "httpUserAgent": true
        },
        {
          "token": "ClaudeBot",
          "role": "training",
          "decision": "allow",
          "source": "node properties.enginePolicy.bots",
          "httpUserAgent": true
        },
        {
          "token": "Claude-User",
          "role": "user-fetch",
          "decision": "allow",
          "source": "node properties.enginePolicy.bots",
          "httpUserAgent": true
        }
      ],
      "documentation": [
        {
          "title": "Does Anthropic crawl data from the web?",
          "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler"
        }
      ]
    },
    {
      "key": "perplexity",
      "provider": "Perplexity",
      "product": "Perplexity answers",
      "url": "https://thatwebhostingguy.com/discovery/providers/perplexity.json",
      "crawlers": [
        {
          "token": "PerplexityBot",
          "role": "search",
          "decision": "allow",
          "source": "node properties.enginePolicy.bots",
          "httpUserAgent": true
        },
        {
          "token": "Perplexity-User",
          "role": "user-fetch",
          "decision": "allow",
          "source": "node properties.enginePolicy.bots",
          "httpUserAgent": true
        }
      ],
      "documentation": [
        {
          "title": "Perplexity crawlers",
          "url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers"
        }
      ]
    },
    {
      "key": "apple",
      "provider": "Apple",
      "product": "Siri, Spotlight and Safari suggestions; Apple Intelligence",
      "url": "https://thatwebhostingguy.com/discovery/providers/apple.json",
      "crawlers": [
        {
          "token": "Applebot",
          "role": "search",
          "decision": "allow",
          "source": "node properties.enginePolicy.bots",
          "httpUserAgent": true
        },
        {
          "token": "Applebot-Extended",
          "role": "training",
          "decision": "allow",
          "source": "wildcard (no explicit decision on the node yet)",
          "httpUserAgent": false
        }
      ],
      "documentation": [
        {
          "title": "About Applebot",
          "url": "https://support.apple.com/en-us/119829"
        }
      ]
    },
    {
      "key": "duckduckgo",
      "provider": "DuckDuckGo",
      "product": "DuckDuckGo search and DuckAssist",
      "url": "https://thatwebhostingguy.com/discovery/providers/duckduckgo.json",
      "crawlers": [
        {
          "token": "DuckDuckBot",
          "role": "search",
          "decision": "allow",
          "source": "node properties.enginePolicy.bots",
          "httpUserAgent": true
        },
        {
          "token": "DuckAssistBot",
          "role": "assistant",
          "decision": "allow",
          "source": "wildcard (no explicit decision on the node yet)",
          "httpUserAgent": true
        }
      ],
      "documentation": [
        {
          "title": "DuckDuckBot",
          "url": "https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot/"
        },
        {
          "title": "DuckAssistBot",
          "url": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot"
        }
      ]
    },
    {
      "key": "yandex",
      "provider": "Yandex",
      "product": "Yandex Search",
      "url": "https://thatwebhostingguy.com/discovery/providers/yandex.json",
      "crawlers": [
        {
          "token": "YandexBot",
          "role": "search",
          "decision": "allow",
          "source": "wildcard (no explicit decision on the node yet)",
          "httpUserAgent": true
        }
      ],
      "documentation": [
        {
          "title": "IndexNow documentation",
          "url": "https://www.indexnow.org/documentation"
        }
      ]
    },
    {
      "key": "meta",
      "provider": "Meta",
      "product": "Meta AI and link previews",
      "url": "https://thatwebhostingguy.com/discovery/providers/meta.json",
      "crawlers": [
        {
          "token": "Meta-WebIndexer",
          "role": "search",
          "decision": "allow",
          "source": "wildcard (no explicit decision on the node yet)",
          "httpUserAgent": true
        },
        {
          "token": "Meta-ExternalFetcher",
          "role": "user-fetch",
          "decision": "allow",
          "source": "wildcard (no explicit decision on the node yet)",
          "httpUserAgent": true
        },
        {
          "token": "Meta-ExternalAgent",
          "role": "training",
          "decision": "allow",
          "source": "wildcard (no explicit decision on the node yet)",
          "httpUserAgent": true
        },
        {
          "token": "facebookexternalhit",
          "role": "preview",
          "decision": "allow",
          "source": "wildcard (no explicit decision on the node yet)",
          "httpUserAgent": true
        }
      ],
      "documentation": [
        {
          "title": "Meta web crawlers",
          "url": "https://developers.facebook.com/documentation/sharing/webmasters/web-crawlers"
        }
      ]
    },
    {
      "key": "amazon",
      "provider": "Amazon",
      "product": "Alexa and Amazon services",
      "url": "https://thatwebhostingguy.com/discovery/providers/amazon.json",
      "crawlers": [
        {
          "token": "Amzn-SearchBot",
          "role": "search",
          "decision": "allow",
          "source": "wildcard (no explicit decision on the node yet)",
          "httpUserAgent": true
        },
        {
          "token": "Amzn-User",
          "role": "user-fetch",
          "decision": "allow",
          "source": "wildcard (no explicit decision on the node yet)",
          "httpUserAgent": true
        },
        {
          "token": "Amazonbot",
          "role": "training",
          "decision": "allow",
          "source": "wildcard (no explicit decision on the node yet)",
          "httpUserAgent": true
        }
      ],
      "documentation": [
        {
          "title": "Amazonbot, Amzn-SearchBot and Amzn-User",
          "url": "https://developer.amazon.com/amazonbot"
        }
      ]
    },
    {
      "key": "common-crawl",
      "provider": "Common Crawl",
      "product": "The open web corpus many models train on",
      "url": "https://thatwebhostingguy.com/discovery/providers/common-crawl.json",
      "crawlers": [
        {
          "token": "CCBot",
          "role": "training",
          "decision": "allow",
          "source": "wildcard (no explicit decision on the node yet)",
          "httpUserAgent": true
        }
      ],
      "documentation": [
        {
          "title": "CCBot",
          "url": "https://commoncrawl.org/ccbot"
        }
      ]
    },
    {
      "key": "mistral",
      "provider": "Mistral",
      "product": "Mistral search, user-requested answers and model training",
      "url": "https://thatwebhostingguy.com/discovery/providers/mistral.json",
      "crawlers": [
        {
          "token": "MistralAI-Index",
          "role": "search",
          "decision": "allow",
          "source": "wildcard (no explicit decision on the node yet)",
          "httpUserAgent": true
        },
        {
          "token": "MistralAI-User",
          "role": "user-fetch",
          "decision": "allow",
          "source": "wildcard (no explicit decision on the node yet)",
          "httpUserAgent": true
        },
        {
          "token": "MistralAI-Training",
          "role": "training",
          "decision": "allow",
          "source": "wildcard (no explicit decision on the node yet)",
          "httpUserAgent": true
        }
      ],
      "documentation": [
        {
          "title": "Mistral crawlers",
          "url": "https://docs.mistral.ai/robots"
        }
      ]
    },
    {
      "key": "brave",
      "provider": "Brave",
      "product": "Brave Search",
      "url": "https://thatwebhostingguy.com/discovery/providers/brave.json",
      "crawlers": [],
      "documentation": [
        {
          "title": "Brave Search Crawler",
          "url": "https://search.brave.com/help/brave-search-crawler"
        }
      ]
    }
  ],
  "unlistedProviders": {
    "policy": "allow",
    "answers": "https://thatwebhostingguy.com/ai-answers.json",
    "catalogue": "https://thatwebhostingguy.com/discovery/catalogue.json",
    "note": "Other assistants can retrieve the same public sources. No undocumented crawler token is invented."
  },
  "access": {
    "authenticationRequired": false,
    "javascriptRequired": false,
    "analyticsConsentRequired": false,
    "robots": "https://thatwebhostingguy.com/robots.txt",
    "protectedPaths": [
      "/api/",
      "/admin/",
      "/private/",
      "/*?*utm_",
      "/contact/thanks/",
      "/healthz",
      "/library-data.json",
      "/library-world.json"
    ]
  },
  "freshness": "Derived from the current public catalogue. ETag supports conditional requests. Identity and crawler policy come from the central organization record loaded at service startup."
}