diff --git a/public/robots.txt b/public/robots.txt new file mode 100644 --- /dev/null +++ b/public/robots.txt @@ -0,0 +1,14 @@ +User-agent: * +Disallow: /persistence/ + +# Unseriƶse Spider +User-agent: WebReaper +User-agent: WebCopier +User-agent: Offline Explorer +User-agent: HTTrack +User-agent: Microsoft.URL.Control +User-agent: EmailCollector +User-agent: penthesilea +User-agent: MJ12bot +User-agent: AhrefsBot +Disallow: /