public/robots.txt
author Markus Bröker<broeker.markus@googlemail.com>
Tue, 19 Sep 2017 13:04:18 +0200
changeset 51 bfcfa759003a
parent 50 b65e2d0ece0d
permissions -rw-r--r--
GSV ignorieren

User-agent: *
Disallow: /persistence/

# Unseriƶse Spider
User-agent: WebReaper
User-agent: WebCopier
User-agent: Offline Explorer
User-agent: HTTrack
User-agent: Microsoft.URL.Control
User-agent: EmailCollector
User-agent: penthesilea
User-agent: MJ12bot
User-agent: AhrefsBot
Disallow: /