x402

Robots.txt Parse (x402)

Fetch and parse a website's robots.txt. Returns rules per user-agent, sitemaps, crawl delays, and whether all crawlers are blocked.

get/x402/v2/robots-txt-parse

Request

  • Base URL: https://api.strale.io
  • URL: https://api.strale.io/x402/v2/robots-txt-parse
  • Auth: none required

Query parameters

urlstring required

Response

Success

existsboolean
blocks_all_crawlersboolean

Example response

{
  "url": "https://strale.dev/robots.txt",
  "rules": [
    {
      "allow": [
        "/"
      ],
      "disallow": [],
      "user_agent": "Googlebot"
    },
    {
      "allow": [
        "/"
      ],
      "disallow": [],
      "user_agent": "Bingbot"
    },
    {
      "allow": [
        "/"
      ],
      "disallow": [],
      "user_agent": "Twitterbot"
    },
    {
      "allow": [
        "/"
      ],
      "disallow": [],
      "user_agent": "facebookexternalhit"
    },
    {
      "allow": [
        "/"
      ],
      "disallow": [],
      "user_agent": "GPTBot"
    },
    {
      "allow": [
        "/"
      ],
      "disallow": [],
      "user_agent": "Claude-Web"
    },
    {
      "allow": [
        "/"
      ],
      "disallow": [],
      "user_agent": "anthropic-ai"
    },
    {
      "allow": [
        "/"
      ],
      "disallow": [],
      "user_agent": "Google-Extended"
    },
    {
      "allow": [
        "/"
      ],
      "disallow": [],
      "user_agent": "Amazonbot"
    },
    {
      "allow": [
        "/"
      ],
      "disallow": [],
      "user_agent": "cohere-ai"
    },
    {
      "allow": [
        "/"
      ],
      "disallow": [],
      "user_agent": "PerplexityBot"
    },
    {
      "allow": [
        "/",
        "/.well-known/"
      ],
      "disallow": [],
      "user_agent": "*"
    }
  ],
  "exists": true,
  "sitemaps": [
    "https://strale.dev/sitemap.xml"
  ],
  "raw_length": 505,
  "total_rules": 12,
  "user_agents": [
    "Googlebot",
    "Bingbot",
    "Twitterbot",
    "facebookexternalhit",
    "GPTBot",
    "Claude-Web",
    "anthropic-ai",
    "Google-Extended",
    "Amazonbot",
    "cohere-ai",
    "PerplexityBot",
    "*"
  ],
  "total_allow_rules": 13,
  "blocks_all_crawlers": false,
  "total_disallow_rules": 0
}

Changes

No recorded changes to this operation across all 1 revision of this API.