Skip to content

Commit

Permalink
Update robots.txt
Browse files Browse the repository at this point in the history
  • Loading branch information
mudkipme committed Jun 13, 2024
1 parent 3c8cce0 commit cdcf732
Showing 1 changed file with 78 additions and 44 deletions.
122 changes: 78 additions & 44 deletions robots.txt
Original file line number Diff line number Diff line change
@@ -1,44 +1,78 @@
User-agent: *
Disallow: /index.php
Disallow: /index.php?
Disallow: /skins/
Disallow: /wiki/Special:
Disallow: /zh/Special:
Disallow: /zh-hans/Special:
Disallow: /zh-hant/Special:
Disallow: /wiki/File:
Disallow: /zh/File:
Disallow: /zh-hans/File:
Disallow: /zh-hant/File:
Disallow: /wiki/User_talk:
Disallow: /zh/User_talk:
Disallow: /zh-hans/User_talk:
Disallow: /zh-hant/User_talk:
Disallow: /wiki/%E9%99%84%E5%BD%95:
Disallow: /zh/%E9%99%84%E5%BD%95:
Disallow: /zh-hans/%E9%99%84%E5%BD%95:
Disallow: /zh-hant/%E9%99%84%E5%BD%95:
Disallow: /wiki/附录:
Disallow: /zh/附录:
Disallow: /zh-hans/附录:
Disallow: /zh-hant/附录:
Disallow: /wiki/Pre:
Disallow: /zh/Pre:
Disallow: /zh-hans/Pre:
Disallow: /zh-hant/Pre:
Crawl-delay: 500

User-agent: Googlebot-Image
Disallow: /
User-agent: Baiduspider-image
Disallow: /
User-agent: msnbot-media
Disallow: /
User-agent: Mediapartners-Google*
Disallow: /
User-agent: Sogou Pic Spider
Disallow: /
User-agent: JikeSpider
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: *
Disallow: /index.php
Disallow: /index.php?
Disallow: /skins/
Disallow: /wiki/Special:
Disallow: /zh/Special:
Disallow: /zh-hans/Special:
Disallow: /zh-hant/Special:
Disallow: /wiki/File:
Disallow: /zh/File:
Disallow: /zh-hans/File:
Disallow: /zh-hant/File:
Disallow: /wiki/Talk:
Disallow: /zh/Talk:
Disallow: /zh-hans/Talk:
Disallow: /zh-hant/Talk:
Disallow: /wiki/User:
Disallow: /zh/User:
Disallow: /zh-hans/User:
Disallow: /zh-hant/User:
Disallow: /wiki/User_talk:
Disallow: /zh/User_talk:
Disallow: /zh-hans/User_talk:
Disallow: /zh-hant/User_talk:
Disallow: /wiki/%E9%99%84%E5%BD%95:
Disallow: /zh/%E9%99%84%E5%BD%95:
Disallow: /zh-hans/%E9%99%84%E5%BD%95:
Disallow: /zh-hant/%E9%99%84%E5%BD%95:
Disallow: /wiki/附录:
Disallow: /zh/附录:
Disallow: /zh-hans/附录:
Disallow: /zh-hant/附录:
Disallow: /wiki/Pre:
Disallow: /zh/Pre:
Disallow: /zh-hans/Pre:
Disallow: /zh-hant/Pre:
Disallow: /wiki/%E7%A5%9E%E5%A5%87%E5%AE%9D%E8%B4%9D%E7%99%BE%E7%A7%91_talk:
Disallow: /zh/%E7%A5%9E%E5%A5%87%E5%AE%9D%E8%B4%9D%E7%99%BE%E7%A7%91_talk:
Disallow: /zh-hans/%E7%A5%9E%E5%A5%87%E5%AE%9D%E8%B4%9D%E7%99%BE%E7%A7%91_talk:
Disallow: /zh-hant/%E7%A5%9E%E5%A5%87%E5%AE%9D%E8%B4%9D%E7%99%BE%E7%A7%91_talk:
Disallow: /wiki/神奇宝贝百科_talk:
Disallow: /zh/神奇宝贝百科_talk:
Disallow: /zh-hans/神奇宝贝百科_talk:
Disallow: /zh-hant/神奇宝贝百科_talk:
Crawl-delay: 500

User-agent: Googlebot-Image
Disallow: /
User-agent: Baiduspider-image
Disallow: /
User-agent: msnbot-media
Disallow: /
User-agent: Mediapartners-Google*
Disallow: /
User-agent: Sogou Pic Spider
Disallow: /
User-agent: JikeSpider
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: GPTBot
Disallow: /
User-agent: Google-Extended
Disallow: /
User-agent: Applebot-Extended
Disallow: /
User-agent: anthropic-ai
Disallow: /
User-agent: ClaudeBot
Disallow: /
User-agent: Diffbot
Disallow: /
User-agent: ImagesiftBot
Disallow: /
User-agent: PerplexityBot
Disallow: /
User-agent: cohere-ai
Disallow: /

0 comments on commit cdcf732

Please sign in to comment.