public/robots.txt
changeset 50 b65e2d0ece0d
new file mode 100644
--- /dev/null
+++ b/public/robots.txt
@@ -0,0 +1,14 @@
+User-agent: *
+Disallow: /persistence/
+
+# Unseriƶse Spider
+User-agent: WebReaper
+User-agent: WebCopier
+User-agent: Offline Explorer
+User-agent: HTTrack
+User-agent: Microsoft.URL.Control
+User-agent: EmailCollector
+User-agent: penthesilea
+User-agent: MJ12bot
+User-agent: AhrefsBot
+Disallow: /