User-agent: *
Disallow: /Search.html
Disallow: /Wiki/W/Special:Search.html
Disallow: /Wiki/W/Special:RecentChanges.html
Disallow: /Wiki/W/Special:
Disallow: /Wiki/index.php
Disallow: /Wiki/api.php
Disallow: /Nothing3-Union.html
Crawl-delay: 5
User-agent: MJ12bot
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: dotbot
Disallow: /
User-agent: PetalBot
Disallow: /
User-agent: AhrefsBot
Disallow: /
# https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/
# Used for AI training, extreme number of requests (45k/day)
User-agent: meta-externalagent
Disallow: /
# https://developer.amazon.com/amazonbot
# Used for Alexa replies, extreme number of requests (80k/day)
User-agent: Amazonbot
Disallow: /