Turbinando o robots.txt



2008-05-05

Abaixo, nova versão do robots.txt utilizado pelo xoops brasil. Confira aqui a versão anterior, aqui para conferir todas as versões deste arquivo, e aqui para a versão atual. Trata-se de apenas um brainstorm para a criação de um robots.txt mais avançado, complexo, ideal. Mas confesso que não estou seguro que estas configurações estejam livres de falha... Ou seja, se desejar, use por sua conta e risco. :-)

O Código:

# XOOPS CUBE BRASIL - ROBOTS.TXT
# ------------------------------------------------------------------------------
# Robots like Google and Yahoo! understand the 2.0 specs of robots.txt,
# so we can allow them a bit more freedom about what to do.
# ------------------------------------------------------------------------------


User-agent: Googlebot

Disallow: /cache/
Disallow: /cgi-bin/
Disallow: /cgi/
Disallow: /core/
Disallow: /install/
Disallow: /kernel/
Disallow: /language/
Disallow: /preload/
Disallow: /settings/
Disallow: /temp/
Disallow: /templates_c/
Disallow: /tmp/

Disallow: /class/
Allow: /class/*.js
Allow: /class/*.gif
Allow: /class/*.jpg
Allow: /class/*.png
Allow: /class/*.swf

Disallow: /Frameworks/
Allow: /Frameworks/*.js
Allow: /Frameworks/*.gif
Allow: /Frameworks/*.jpg
Allow: /Frameworks/*.png
Allow: /Frameworks/*.swf

Disallow: /include/
Allow: /include/*.css
Allow: /include/*.js

Disallow: /*?PHPSESSID=
Disallow: /*?xoops_session=

Disallow: /index.php
Disallow: /admin.php
Disallow: /avatar.php
Disallow: /banners.php
Disallow: /edituser.php
Disallow: /footer.php
Disallow: /header.php
Disallow: /image.php
Disallow: /imagemanager.php
Disallow: /lostpass.php
Disallow: /mainfile.dist.php
Disallow: /mainfile.php
Disallow: /misc.php
Disallow: /modules/*/admin/
Disallow: /notifications.php
Disallow: /pmlite.php
Disallow: /readpmsg.php
Disallow: /register.php
Disallow: /user.php
Disallow: /viewpmsg.php

Disallow: /modules/legacy/
Disallow: /modules/legacyRender/
Disallow: /modules/pm/
Disallow: /modules/stdCache/
Disallow: /modules/system/
Disallow: /modules/tellafriend/


# XIGG
Disallow: /modules/*/*/voteform
Disallow: /modules/*/index.php/submit


# Disallow: /modules/*/*order=DESC

Disallow: /modules/*/*page=newtopic
Disallow: /modules/*/*page=print
Disallow: /modules/*/*page=reply
Disallow: /modules/*/action.transfer.php
Disallow: /modules/*/actions/print.php
Disallow: /modules/*/admin/
Disallow: /modules/*/broken.php
Disallow: /modules/*/brokenfile.php
Disallow: /modules/*/brokenlink.php
Disallow: /modules/*/brokenmovie.php
Disallow: /modules/*/class/
Disallow: /modules/*/deletefile.php
Disallow: /modules/*/email.php
Disallow: /modules/*/include/
Disallow: /modules/*/includes/
Disallow: /modules/*/index.php?page=brokenfile
Disallow: /modules/*/index.php?page=modfile
Disallow: /modules/*/index.php?page=ratefile
Disallow: /modules/*/index.php?page=visit
Disallow: /modules/*/language/
Disallow: /modules/*/modfile.php
Disallow: /modules/*/myheader.php
Disallow: /modules/*/print.php
Disallow: /modules/*/ratefile.php
Disallow: /modules/*/ratelink.php
Disallow: /modules/*/ratenews.php
Disallow: /modules/*/report.php
Disallow: /modules/*/review.php
Disallow: /modules/*/sql/
Disallow: /modules/*/tell_a_friend.php
Disallow: /modules/*/visit.php


# REVER:
# Disallow: /common/

# CONTEÚDO DUPLICADO PELO APACHE
# DUPLICATE CONTENT FROM APACHE

Disallow: /*?D=A
Disallow: /*?D=D
Disallow: /*?M=A
Disallow: /*?M=D
Disallow: /*?N=A
Disallow: /*?N=D
Disallow: /*?S=A
Disallow: /*?S=D

# REVER:
# Disallow: /*?sort=date
# Disallow: /*?sort=name
# Disallow: /*?sort=size

# CONTORNAR BLACK SEO / AVOID BLACK SEO
# Disallow: /*.php?*%20
# Disallow: /*.php?*%22
Disallow: /*.php?*http://
Disallow: /*.php?*www.*.*

User-agent: wget
Disallow: /*.php
Disallow: /*.htm

User-agent: sitecheck.internetseer.com
Disallow: /

User-agent: Zealbot
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: Fetch
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: TeleportPro
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: linko
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: Xenu
Disallow: /

User-agent: larbin
Disallow: /

User-agent: ZyBORG
Disallow: /

User-agent: Download Ninja
Disallow: /

User-agent: grub-client
Disallow: /

User-agent: k2spider
Disallow: /

User-agent: NPBot
Disallow: /

User-agent: WebReaper
Disallow: /


User-agent: *

Disallow: /cache/
Disallow: /cgi-bin/
Disallow: /cgi/
Disallow: /core/
Disallow: /install/
Disallow: /kernel/
Disallow: /language/
Disallow: /preload/
Disallow: /settings/
Disallow: /temp/
Disallow: /templates_c/
Disallow: /tmp/

Disallow: /class/
Allow: /class/*.js
Allow: /class/*.png
Allow: /class/*.gif
Allow: /class/*.jpg

Disallow: /include/
Allow: /include/*.css
Allow: /include/*.js

Disallow: /Frameworks/
Allow: /Frameworks/*.js
Allow: /Frameworks/*.gif
Allow: /Frameworks/*.jpg
Allow: /Frameworks/*.png
Allow: /Frameworks/*.swf


User-agent: ia_archiver
Disallow:
Allow: /

# MEDIA BOTS

# Google Image
User-agent: Googlebot-Image
Disallow:
Allow: /

# MSN Live Bot-media
User-agent: MSNBot-media
Disallow:
Allow: /

# EOF
Postar um comentário