From cdcf7328bb9821791564c3cf5c3266685beca10c Mon Sep 17 00:00:00 2001 From: Mudkip Date: Thu, 13 Jun 2024 09:12:25 +0800 Subject: [PATCH] Update robots.txt --- robots.txt | 122 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 78 insertions(+), 44 deletions(-) diff --git a/robots.txt b/robots.txt index 425e987b76a86..8a5754a123b9c 100644 --- a/robots.txt +++ b/robots.txt @@ -1,44 +1,78 @@ -User-agent: * -Disallow: /index.php -Disallow: /index.php? -Disallow: /skins/ -Disallow: /wiki/Special: -Disallow: /zh/Special: -Disallow: /zh-hans/Special: -Disallow: /zh-hant/Special: -Disallow: /wiki/File: -Disallow: /zh/File: -Disallow: /zh-hans/File: -Disallow: /zh-hant/File: -Disallow: /wiki/User_talk: -Disallow: /zh/User_talk: -Disallow: /zh-hans/User_talk: -Disallow: /zh-hant/User_talk: -Disallow: /wiki/%E9%99%84%E5%BD%95: -Disallow: /zh/%E9%99%84%E5%BD%95: -Disallow: /zh-hans/%E9%99%84%E5%BD%95: -Disallow: /zh-hant/%E9%99%84%E5%BD%95: -Disallow: /wiki/附录: -Disallow: /zh/附录: -Disallow: /zh-hans/附录: -Disallow: /zh-hant/附录: -Disallow: /wiki/Pre: -Disallow: /zh/Pre: -Disallow: /zh-hans/Pre: -Disallow: /zh-hant/Pre: -Crawl-delay: 500 - -User-agent: Googlebot-Image -Disallow: / -User-agent: Baiduspider-image -Disallow: / -User-agent: msnbot-media -Disallow: / -User-agent: Mediapartners-Google* -Disallow: / -User-agent: Sogou Pic Spider -Disallow: / -User-agent: JikeSpider -Disallow: / -User-agent: SemrushBot -Disallow: / +User-agent: * +Disallow: /index.php +Disallow: /index.php? +Disallow: /skins/ +Disallow: /wiki/Special: +Disallow: /zh/Special: +Disallow: /zh-hans/Special: +Disallow: /zh-hant/Special: +Disallow: /wiki/File: +Disallow: /zh/File: +Disallow: /zh-hans/File: +Disallow: /zh-hant/File: +Disallow: /wiki/Talk: +Disallow: /zh/Talk: +Disallow: /zh-hans/Talk: +Disallow: /zh-hant/Talk: +Disallow: /wiki/User: +Disallow: /zh/User: +Disallow: /zh-hans/User: +Disallow: /zh-hant/User: +Disallow: /wiki/User_talk: +Disallow: /zh/User_talk: +Disallow: /zh-hans/User_talk: +Disallow: /zh-hant/User_talk: +Disallow: /wiki/%E9%99%84%E5%BD%95: +Disallow: /zh/%E9%99%84%E5%BD%95: +Disallow: /zh-hans/%E9%99%84%E5%BD%95: +Disallow: /zh-hant/%E9%99%84%E5%BD%95: +Disallow: /wiki/附录: +Disallow: /zh/附录: +Disallow: /zh-hans/附录: +Disallow: /zh-hant/附录: +Disallow: /wiki/Pre: +Disallow: /zh/Pre: +Disallow: /zh-hans/Pre: +Disallow: /zh-hant/Pre: +Disallow: /wiki/%E7%A5%9E%E5%A5%87%E5%AE%9D%E8%B4%9D%E7%99%BE%E7%A7%91_talk: +Disallow: /zh/%E7%A5%9E%E5%A5%87%E5%AE%9D%E8%B4%9D%E7%99%BE%E7%A7%91_talk: +Disallow: /zh-hans/%E7%A5%9E%E5%A5%87%E5%AE%9D%E8%B4%9D%E7%99%BE%E7%A7%91_talk: +Disallow: /zh-hant/%E7%A5%9E%E5%A5%87%E5%AE%9D%E8%B4%9D%E7%99%BE%E7%A7%91_talk: +Disallow: /wiki/神奇宝贝百科_talk: +Disallow: /zh/神奇宝贝百科_talk: +Disallow: /zh-hans/神奇宝贝百科_talk: +Disallow: /zh-hant/神奇宝贝百科_talk: +Crawl-delay: 500 + +User-agent: Googlebot-Image +Disallow: / +User-agent: Baiduspider-image +Disallow: / +User-agent: msnbot-media +Disallow: / +User-agent: Mediapartners-Google* +Disallow: / +User-agent: Sogou Pic Spider +Disallow: / +User-agent: JikeSpider +Disallow: / +User-agent: SemrushBot +Disallow: / +User-agent: GPTBot +Disallow: / +User-agent: Google-Extended +Disallow: / +User-agent: Applebot-Extended +Disallow: / +User-agent: anthropic-ai +Disallow: / +User-agent: ClaudeBot +Disallow: / +User-agent: Diffbot +Disallow: / +User-agent: ImagesiftBot +Disallow: / +User-agent: PerplexityBot +Disallow: / +User-agent: cohere-ai +Disallow: /