{"ok":true,"result":{"slug":"robots-txt-generator","name":"Robots.txt Generator","summary":"Generate a robots.txt (default rules, per-crawler groups, AI crawler blocking, sitemaps), or test whether a crawler may fetch a URL under a robots.txt.","description":"Create a robots.txt file with per-crawler rules, sitemaps and one-click blocking of AI crawlers like GPTBot and CCBot, then test any URL against the rules.","method":"POST","endpoint":"/api/v1/tools/robots-txt-generator","plan":"free","mcpName":"robots_txt_generator","input":{"type":"object","additionalProperties":false,"properties":{"mode":{"type":"string","description":"generate: build a robots.txt. test: check `url` for `userAgent` against `robotsTxt` (RFC 9309 and Google matching).","default":"generate","enum":["generate","test"],"maxLength":200000},"defaultPolicy":{"type":"string","description":"generate: allow or disallow everything for all crawlers (User-agent: *) before the path rules.","default":"allow","enum":["allow","disallow"],"maxLength":200000},"disallow":{"type":"string","description":"generate: paths to disallow for all crawlers, one per line, each starting with / or *.","maxLength":20000},"allow":{"type":"string","description":"generate: paths to allow for all crawlers, one per line.","maxLength":20000},"crawlDelay":{"type":"number","description":"generate: Crawl-delay in seconds for all crawlers (ignored by Google).","minimum":0,"maximum":86400},"blockAiCrawlers":{"type":"boolean","description":"generate: block all known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, Claude-User, Claude-SearchBot, anthropic-ai, Google-Extended, Applebot-Extended, CCBot, PerplexityBot, Perplexity-User, Bytespider, meta-externalagent, FacebookBot, Amazonbot, cohere-ai, Diffbot, omgili, Timpibot).","default":false},"groups":{"type":"string","description":"generate: extra per-crawler groups as a JSON array, e.g. [{\"userAgents\":[\"Googlebot-Image\"],\"disallow\":[\"/private/\"],\"allow\":[],\"crawlDelay\":\"\"}].","maxLength":50000},"sitemaps":{"type":"string","description":"generate: absolute sitemap URLs, one per line.","maxLength":20000},"robotsTxt":{"type":"string","description":"test: the robots.txt content to test against.","maxLength":200000},"userAgent":{"type":"string","description":"test: crawler user-agent or product token, e.g. Googlebot.","maxLength":500},"url":{"type":"string","description":"test: the URL or path to check, e.g. https://example.com/admin/login.","maxLength":2048}}},"example":{"mode":"generate","disallow":"/admin/\n/cart","sitemaps":"https://example.com/sitemap.xml"},"randomOutput":false,"network":false,"docs":"https://pithlab.com/developers/tools/robots-txt-generator","page":"https://pithlab.com/tools/robots-txt-generator"}}