public/robots.txt
author Markus Bröker<broeker.markus@googlemail.com>
Fri, 15 Sep 2017 10:01:32 +0200
changeset 50 b65e2d0ece0d
permissions -rw-r--r--
Fehlende Projekt-Dateien hinzu gefügt

User-agent: *
Disallow: /persistence/

# Unseriöse Spider
User-agent: WebReaper
User-agent: WebCopier
User-agent: Offline Explorer
User-agent: HTTrack
User-agent: Microsoft.URL.Control
User-agent: EmailCollector
User-agent: penthesilea
User-agent: MJ12bot
User-agent: AhrefsBot
Disallow: /