AI Crawler Index

{
  "_meta": {
    "io.modelcontextprotocol.registry/official": {
      "status": "active",
      "isLatest": true,
      "updatedAt": "2026-09-07T00:44:37.088429Z",
      "publishedAt": "2026-09-07T00:44:37.088429Z",
      "statusChangedAt": "2026-09-07T00:44:37.088429Z"
    }
  },
  "server": {
    "name": "dev.workers.pathwren.www/ai-crawler-index",
    "_meta": {
      "io.modelcontextprotocol.registry/publisher-provided": {
        "auth": "none",
        "docs": "https://www.pathwren.workers.dev/c/mcp-registry-official/mcp.html",
        "tools": [
          {
            "name": "whoami",
            "search": "who is calling: user-agent, address, our verdict - no arguments"
          },
          {
            "name": "classify_user_agent",
            "search": "identify a bot from its user-agent string"
          },
          {
            "name": "lookup_crawler",
            "search": "look up one crawler: operator, purpose, cost of blocking it"
          },
          {
            "name": "list_crawlers",
            "search": "list or filter every AI crawler in the index"
          },
          {
            "name": "generate_robots_txt",
            "search": "generate a robots.txt that blocks AI training or scrapers"
          },
          {
            "name": "is_verified_crawler_ip",
            "search": "check whether an IP really belongs to Googlebot/GPTBot"
          },
          {
            "name": "whats_changed",
            "search": "see what changed in the index and when it was built"
          },
          {
            "name": "example",
            "search": "run the worked example — no arguments"
          }
        ],
        "example": {
          "curl": "curl -s -X POST https://www.pathwren.workers.dev/c/mcp-registry-official/mcp -H 'content-type: application/json' -H 'accept: application/json, text/event-stream' -d '{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/call\",\"params\":{\"name\":\"whoami\",\"arguments\":{}}}'",
          "note": "whoami takes no arguments at all and answers about the request you just sent - user-agent, address, what our classifier makes of it. Every tool here accepts {} and answers with its own documented worked example — nothing of yours is read, nothing is fetched from a third party. Send the request below verbatim.",
          "request": {
            "id": 1,
            "method": "tools/call",
            "params": {
              "name": "whoami",
              "arguments": {}
            },
            "jsonrpc": "2.0"
          },
          "endpoint": "https://www.pathwren.workers.dev/c/mcp-registry-official/mcp",
          "transport": "streamable-http",
          "verified_live_at": "2026-09-02T08:49:08Z"
        },
        "pricing": "free",
        "keywords": [
          "robots.txt",
          "gptbot",
          "googlebot",
          "ai crawler",
          "user agent lookup",
          "block ai scrapers",
          "bot detection",
          "crawler ip",
          "llms.txt",
          "markdown",
          "markdown docs",
          "free",
          "no api key",
          "no auth"
        ],
        "markdown": {
          "note": "Everything these tools answer from is also readable without JSON-RPC and without an MCP client: every page of the host is a plain markdown file at the same address with .md — also .mdx, <page>.html.md, <page>.html.mdx, or send Accept: text/markdown to the .html page. No key, no JavaScript, CC0. The registry caps `description` at 100 characters, which is why this sentence lives here.",
          "index": "https://www.pathwren.workers.dev/c/mcp-registry-official/sitemap.md",
          "example": "curl -s https://www.pathwren.workers.dev/c/mcp-registry-official/mcp.md",
          "canonical": "the HTML page; each mirror carries Link: rel=\"canonical\" back to it"
        },
        "publisher": "Pathwren — an independent, non-commercial automated project",
        "dataLicense": "CC0-1.0",
        "authentication": "none — no key, no account, no OAuth, no rate limit"
      }
    },
    "icons": [
      {
        "src": "https://www.pathwren.workers.dev/c/mcp-registry-official/icon.png",
        "sizes": [
          "256x256"
        ],
        "mimeType": "image/png"
      }
    ],
    "title": "AI Crawler Index",
    "$schema": "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json",
    "remotes": [
      {
        "url": "https://www.pathwren.workers.dev/c/mcp-registry-official/mcp",
        "type": "streamable-http"
      }
    ],
    "version": "1.6.0",
    "websiteUrl": "https://www.pathwren.workers.dev/c/mcp-registry-official/mcp.html",
    "description": "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/call\",\"params\":{\"name\":\"whoami\",\"arguments\":{}}} free no key"
  }
}