polite-crawler

{
  "_meta": {
    "io.modelcontextprotocol.registry/official": {
      "status": "active",
      "isLatest": true,
      "updatedAt": "2026-10-25T07:26:15.991495Z",
      "publishedAt": "2026-10-25T07:26:15.991495Z",
      "statusChangedAt": "2026-10-25T07:26:15.991495Z"
    }
  },
  "server": {
    "name": "io.github.mark1russell7/polite-crawler",
    "title": "polite-crawler",
    "$schema": "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json",
    "version": "0.2.4",
    "packages": [
      {
        "version": "0.2.4",
        "transport": {
          "type": "stdio"
        },
        "identifier": "polite-crawler-mcp",
        "runtimeHint": "npx",
        "registryType": "npm",
        "registryBaseUrl": "https://registry.npmjs.org",
        "environmentVariables": [
          {
            "name": "POLITE_CRAWLER_DIR",
            "format": "filepath",
            "description": "The folder of the store. The default is ~/.polite-crawler."
          },
          {
            "name": "POLITE_CRAWLER_CONTACT",
            "description": "An email address or URL for the From header of each request."
          },
          {
            "name": "POLITE_CRAWLER_ALLOW_PRIVATE",
            "format": "boolean",
            "description": "Set it to 1 to let the crawler fetch loopback, private and link-local addresses. The default refuses them."
          },
          {
            "name": "POLITE_CRAWLER_ALLOW_USER_PROVIDED",
            "format": "boolean",
            "description": "Set it to 1 to let user_provided read a page that refuses AI input when the client cannot ask the user to confirm."
          },
          {
            "name": "POLITE_CRAWLER_SIGN_KEY",
            "format": "filepath",
            "description": "The file of an Ed25519 private key (PKCS#8 PEM or JWK). The crawler signs each request with it (Web Bot Auth). It needs POLITE_CRAWLER_SIGNATURE_AGENT."
          },
          {
            "name": "POLITE_CRAWLER_SIGNATURE_AGENT",
            "description": "Where a site finds the public key of POLITE_CRAWLER_SIGN_KEY: the https origin of the key directory, or the URL of a JSON Web Key Set."
          }
        ]
      }
    ],
    "repository": {
      "url": "https://github.com/mark1russell7/crawler",
      "source": "github",
      "subfolder": "packages/polite-crawler-mcp"
    },
    "websiteUrl": "https://mark1russell7.github.io/crawler/",
    "description": "A polite crawler for agents: obeys robots.txt, fetches each page once, finds and verifies quotes."
  }
}