AI Crawler Index

{
  "name": "dev.workers.pathwren.www/ai-crawler-index",
  "_meta": {
    "io.modelcontextprotocol.registry/publisher-provided": {
      "auth": "none",
      "docs": "https://www.pathwren.workers.dev/c/mcp-registry-official/mcp.html",
      "tools": [
        {
          "name": "whoami",
          "search": "who is calling: user-agent, address, our verdict - no arguments"
        },
        {
          "name": "classify_user_agent",
          "search": "identify a bot from its user-agent string"
        },
        {
          "name": "lookup_crawler",
          "search": "look up one crawler: operator, purpose, cost of blocking it"
        },
        {
          "name": "list_crawlers",
          "search": "list or filter every AI crawler in the index"
        },
        {
          "name": "generate_robots_txt",
          "search": "generate a robots.txt that blocks AI training or scrapers"
        },
        {
          "name": "is_verified_crawler_ip",
          "search": "check whether an IP really belongs to Googlebot/GPTBot"
        },
        {
          "name": "whats_changed",
          "search": "see what changed in the index and when it was built"
        },
        {
          "name": "example",
          "search": "run the worked example — no arguments"
        }
      ],
      "example": {
        "curl": "curl -s -X POST https://www.pathwren.workers.dev/c/mcp-registry-official/mcp -H 'content-type: application/json' -H 'accept: application/json, text/event-stream' -d '{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/call\",\"params\":{\"name\":\"whoami\",\"arguments\":{}}}'",
        "note": "whoami takes no arguments at all and answers about the request you just sent - user-agent, address, what our classifier makes of it. Every tool here accepts {} and answers with its own documented worked example — nothing of yours is read, nothing is fetched from a third party. Send the request below verbatim.",
        "request": {
          "id": 1,
          "method": "tools/call",
          "params": {
            "name": "whoami",
            "arguments": {}
          },
          "jsonrpc": "2.0"
        },
        "endpoint": "https://www.pathwren.workers.dev/c/mcp-registry-official/mcp",
        "transport": "streamable-http",
        "verified_live_at": "2026-09-02T08:49:08Z"
      },
      "pricing": "free",
      "keywords": [
        "robots.txt",
        "gptbot",
        "googlebot",
        "ai crawler",
        "user agent lookup",
        "block ai scrapers",
        "bot detection",
        "crawler ip",
        "llms.txt",
        "markdown",
        "markdown docs",
        "free",
        "no api key",
        "no auth"
      ],
      "markdown": {
        "note": "Everything these tools answer from is also readable without JSON-RPC and without an MCP client: every page of the host is a plain markdown file at the same address with .md — also .mdx, <page>.html.md, <page>.html.mdx, or send Accept: text/markdown to the .html page. No key, no JavaScript, CC0. The registry caps `description` at 100 characters, which is why this sentence lives here.",
        "index": "https://www.pathwren.workers.dev/c/mcp-registry-official/sitemap.md",
        "example": "curl -s https://www.pathwren.workers.dev/c/mcp-registry-official/mcp.md",
        "canonical": "the HTML page; each mirror carries Link: rel=\"canonical\" back to it"
      },
      "publisher": "Pathwren — an independent, non-commercial automated project",
      "dataLicense": "CC0-1.0",
      "authentication": "none — no key, no account, no OAuth, no rate limit"
    }
  },
  "icons": [
    {
      "src": "https://www.pathwren.workers.dev/c/mcp-registry-official/icon.png",
      "sizes": [
        "256x256"
      ],
      "mimeType": "image/png"
    }
  ],
  "title": "AI Crawler Index",
  "$schema": "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json",
  "remotes": [
    {
      "url": "https://www.pathwren.workers.dev/c/mcp-registry-official/mcp",
      "type": "streamable-http"
    }
  ],
  "version": "1.6.0",
  "websiteUrl": "https://www.pathwren.workers.dev/c/mcp-registry-official/mcp.html",
  "description": "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/call\",\"params\":{\"name\":\"whoami\",\"arguments\":{}}} free no key"
}