diff options
Diffstat (limited to 'static')
| -rw-r--r-- | static/robots.txt | 11 |
1 files changed, 7 insertions, 4 deletions
diff --git a/static/robots.txt b/static/robots.txt index 058446d..41b7ac2 100644 --- a/static/robots.txt +++ b/static/robots.txt @@ -1,5 +1,8 @@ -User-agent: * -Disallow: /about-me/ -Disallow: /css/ -Disallow: /js/ +User-agent: Googlebot +Disallow: /media/ + +User-agent: archive.org_bot Disallow: /media/ + +User-agent: * +Disallow: / |