{"ok":true,"result":{"slug":"robots-txt-tester","name":"Robots.txt Tester","summary":"Test whether a URL path is allowed for a crawler under a site's robots.txt (fetched, or passed in), using Google's longest-match rules.","description":"Fetch a site's robots.txt or paste your own, then test whether a URL is allowed for Googlebot, Bingbot, GPTBot or ClaudeBot using Google's longest-match rules.","method":"POST","endpoint":"/api/v1/tools/robots-txt-tester","plan":"free","mcpName":"robots_txt_tester","input":{"type":"object","additionalProperties":false,"properties":{"site":{"type":"string","description":"Site whose /robots.txt to fetch, such as example.com. Leave out to test robotsTxt instead.","maxLength":2048},"robotsTxt":{"type":"string","description":"robots.txt content to test instead of fetching one.","maxLength":512000},"path":{"type":"string","description":"The URL or path to test.","default":"/","maxLength":2048},"userAgent":{"type":"string","description":"Crawler user-agent token, such as Googlebot, Bingbot, GPTBot, ClaudeBot or *.","default":"Googlebot","maxLength":100}}},"example":{"robotsTxt":"User-agent: *\nDisallow: /admin/\nAllow: /admin/help","path":"/admin/help","userAgent":"Googlebot"},"randomOutput":false,"network":true,"docs":"https://pithlab.com/developers/tools/robots-txt-tester","page":"https://pithlab.com/tools/robots-txt-tester"}}