飛び先のチェック (ver 0.04) by ぴょん基地の友達 解析

飛び先のURL
http://www.gigazine.net/robots.txt

code = 200
base = http://www.gigazine.net/robots.txt
body =
# /robots.txt file for Disallow: /
# 2008/04/07 11:51

User-agent: Mediapartners-Google
Crawl-delay: 100
Disallow: /index.php?/news/trackbacks/
Disallow: /Backnumber/
Disallow: /cgi-bin/
Disallow: /Images/
Disallow: /Images/
Disallow: /index.php?/news/2005/11/
Disallow: /index.php?/news/2005/12/
Disallow: /index.php?/news/2006/01/
Disallow: /index.php?/news/2006/02/
Disallow: /index.php?/news/2006/03/
Disallow: /index.php?/news/2006/04/
Disallow: /index.php?/news/2006/05/
Disallow: /index.php?/news/2006/06/
Disallow: /index.php?/news/2006/07/
Disallow: /index.php?/news/2006/08/
Disallow: /index.php?/news/2006/09/
Disallow: /index.php?/news/2006/10/
Disallow: /index.php?/news/2006/11/
Disallow: /index.php?/news/archives/
Disallow: /index.php?/news/C10/
Disallow: /index.php?/news/C11/
Disallow: /index.php?/news/C12/
Disallow: /index.php?/news/C13/
Disallow: /index.php?/news/C14/
Disallow: /index.php?/news/C15/
Disallow: /index.php?/news/C16/
Disallow: /index.php?/news/C17/
Disallow: /index.php?/news/C4/
Disallow: /index.php?/news/C5/
Disallow: /index.php?/news/C6/
Disallow: /index.php?/news/C7/
Disallow: /index.php?/news/C8/
Disallow: /index.php?/news/C9/
Disallow: /index.php?/news/categories/
Disallow: /index.php?/news/contact/
Disallow: /index.php?/search
Disallow: /Library/
Disallow: /Link/
Disallow: /News/

User-agent: Googlebot
Disallow: /index.php?/news/trackbacks/
Disallow: /Backnumber/
Disallow: /cgi-bin/
Disallow: /Images/
Disallow: /Images/
Disallow: /index.php?/news/2005/11/
Disallow: /index.php?/news/2005/12/
Disallow: /index.php?/news/2006/01/
Disallow: /index.php?/news/2006/02/
Disallow: /index.php?/news/2006/03/
Disallow: /index.php?/news/2006/04/
Disallow: /index.php?/news/2006/05/
Disallow: /index.php?/news/2006/06/
Disallow: /index.php?/news/2006/07/
Disallow: /index.php?/news/2006/08/
Disallow: /index.php?/news/2006/09/
Disallow: /index.php?/news/2006/10/
Disallow: /index.php?/news/2006/11/
Disallow: /index.php?/news/archives/
Disallow: /index.php?/news/C10/
Disallow: /index.php?/news/C11/
Disallow: /index.php?/news/C12/
Disallow: /index.php?/news/C13/
Disallow: /index.php?/news/C14/
Disallow: /index.php?/news/C15/
Disallow: /index.php?/news/C16/
Disallow: /index.php?/news/C17/
Disallow: /index.php?/news/C4/
Disallow: /index.php?/news/C5/
Disallow: /index.php?/news/C6/
Disallow: /index.php?/news/C7/
Disallow: /index.php?/news/C8/
Disallow: /index.php?/news/C9/
Disallow: /index.php?/news/categories/
Disallow: /index.php?/news/contact/
Disallow: /index.php?/search
Disallow: /Library/
Disallow: /Link/
Disallow: /News/

User-agent: msnbot
Disallow: /index.php?/news/trackbacks/
Disallow: /Backnumber/
Disallow: /cgi-bin/
Disallow: /Images/
Disallow: /Images/
Disallow: /index.php?/news/2005/11/
Disallow: /index.php?/news/2005/12/
Disallow: /index.php?/news/2006/01/
Disallow: /index.php?/news/2006/02/
Disallow: /index.php?/news/2006/03/
Disallow: /index.php?/news/2006/04/
Disallow: /index.php?/news/2006/05/
Disallow: /index.php?/news/2006/06/
Disallow: /index.php?/news/2006/07/
Disallow: /index.php?/news/2006/08/
Disallow: /index.php?/news/2006/09/
Disallow: /index.php?/news/2006/10/
Disallow: /index.php?/news/2006/11/
Disallow: /index.php?/news/archives/
Disallow: /index.php?/news/C10/
Disallow: /index.php?/news/C11/
Disallow: /index.php?/news/C12/
Disallow: /index.php?/news/C13/
Disallow: /index.php?/news/C14/
Disallow: /index.php?/news/C15/
Disallow: /index.php?/news/C16/
Disallow: /index.php?/news/C17/
Disallow: /index.php?/news/C4/
Disallow: /index.php?/news/C5/
Disallow: /index.php?/news/C6/
Disallow: /index.php?/news/C7/
Disallow: /index.php?/news/C8/
Disallow: /index.php?/news/C9/
Disallow: /index.php?/news/categories/
Disallow: /index.php?/news/contact/
Disallow: /index.php?/search
Disallow: /Library/
Disallow: /Link/
Disallow: /News/

User-Agent: Yahoo! Slurp
Disallow: /index.php?/news/trackbacks/
Disallow: /Backnumber/
Disallow: /cgi-bin/
Disallow: /Images/
Disallow: /Images/
Disallow: /index.php?/news/2005/11/
Disallow: /index.php?/news/2005/12/
Disallow: /index.php?/news/2006/01/
Disallow: /index.php?/news/2006/02/
Disallow: /index.php?/news/2006/03/
Disallow: /index.php?/news/2006/04/
Disallow: /index.php?/news/2006/05/
Disallow: /index.php?/news/2006/06/
Disallow: /index.php?/news/2006/07/
Disallow: /index.php?/news/2006/08/
Disallow: /index.php?/news/2006/09/
Disallow: /index.php?/news/2006/10/
Disallow: /index.php?/news/2006/11/
Disallow: /index.php?/news/archives/
Disallow: /index.php?/news/C10/
Disallow: /index.php?/news/C11/
Disallow: /index.php?/news/C12/
Disallow: /index.php?/news/C13/
Disallow: /index.php?/news/C14/
Disallow: /index.php?/news/C15/
Disallow: /index.php?/news/C16/
Disallow: /index.php?/news/C17/
Disallow: /index.php?/news/C4/
Disallow: /index.php?/news/C5/
Disallow: /index.php?/news/C6/
Disallow: /index.php?/news/C7/
Disallow: /index.php?/news/C8/
Disallow: /index.php?/news/C9/
Disallow: /index.php?/news/categories/
Disallow: /index.php?/news/contact/
Disallow: /index.php?/search
Disallow: /Library/
Disallow: /Link/
Disallow: /News/

User-agent: Yahoo-NewsCrawler
Disallow: /index.php?/news/trackbacks/
Disallow: /Backnumber/
Disallow: /cgi-bin/
Disallow: /Images/
Disallow: /Images/
Disallow: /index.php?/news/2005/11/
Disallow: /index.php?/news/2005/12/
Disallow: /index.php?/news/2006/01/
Disallow: /index.php?/news/2006/02/
Disallow: /index.php?/news/2006/03/
Disallow: /index.php?/news/2006/04/
Disallow: /index.php?/news/2006/05/
Disallow: /index.php?/news/2006/06/
Disallow: /index.php?/news/2006/07/
Disallow: /index.php?/news/2006/08/
Disallow: /index.php?/news/2006/09/
Disallow: /index.php?/news/2006/10/
Disallow: /index.php?/news/2006/11/
Disallow: /index.php?/news/archives/
Disallow: /index.php?/news/C10/
Disallow: /index.php?/news/C11/
Disallow: /index.php?/news/C12/
Disallow: /index.php?/news/C13/
Disallow: /index.php?/news/C14/
Disallow: /index.php?/news/C15/
Disallow: /index.php?/news/C16/
Disallow: /index.php?/news/C17/
Disallow: /index.php?/news/C4/
Disallow: /index.php?/news/C5/
Disallow: /index.php?/news/C6/
Disallow: /index.php?/news/C7/
Disallow: /index.php?/news/C8/
Disallow: /index.php?/news/C9/
Disallow: /index.php?/news/categories/
Disallow: /index.php?/news/contact/
Disallow: /index.php?/search
Disallow: /Library/
Disallow: /Link/
Disallow: /News/

User-agent: BecomeBot
Disallow: /index.php?/news/trackbacks/
Crawl-Delay: 100
Disallow: /Backnumber/
Disallow: /cgi-bin/
Disallow: /Images/
Disallow: /Images/
Disallow: /index.php?/news/2005/11/
Disallow: /index.php?/news/2005/12/
Disallow: /index.php?/news/2006/01/
Disallow: /index.php?/news/2006/02/
Disallow: /index.php?/news/2006/03/
Disallow: /index.php?/news/2006/04/
Disallow: /index.php?/news/2006/05/
Disallow: /index.php?/news/2006/06/
Disallow: /index.php?/news/2006/07/
Disallow: /index.php?/news/2006/08/
Disallow: /index.php?/news/2006/09/
Disallow: /index.php?/news/2006/10/
Disallow: /index.php?/news/2006/11/
Disallow: /index.php?/news/archives/
Disallow: /index.php?/news/C10/
Disallow: /index.php?/news/C11/
Disallow: /index.php?/news/C12/
Disallow: /index.php?/news/C13/
Disallow: /index.php?/news/C14/
Disallow: /index.php?/news/C15/
Disallow: /index.php?/news/C16/
Disallow: /index.php?/news/C17/
Disallow: /index.php?/news/C4/
Disallow: /index.php?/news/C5/
Disallow: /index.php?/news/C6/
Disallow: /index.php?/news/C7/
Disallow: /index.php?/news/C8/
Disallow: /index.php?/news/C9/
Disallow: /index.php?/news/categories/
Disallow: /index.php?/news/contact/
Disallow: /index.php?/search
Disallow: /Library/
Disallow: /Link/
Disallow: /News/

User-agent: Fasterfox
Disallow: /

User-agent: *
Crawl-delay: 100
Disallow: /index.php?/news/trackbacks/
Disallow: /Backnumber/
Disallow: /cgi-bin/
Disallow: /Images/
Disallow: /Images/
Disallow: /index.php?/news/2005/11/
Disallow: /index.php?/news/2005/12/
Disallow: /index.php?/news/2006/01/
Disallow: /index.php?/news/2006/02/
Disallow: /index.php?/news/2006/03/
Disallow: /index.php?/news/2006/04/
Disallow: /index.php?/news/2006/05/
Disallow: /index.php?/news/2006/06/
Disallow: /index.php?/news/2006/07/
Disallow: /index.php?/news/2006/08/
Disallow: /index.php?/news/2006/09/
Disallow: /index.php?/news/2006/10/
Disallow: /index.php?/news/2006/11/
Disallow: /index.php?/news/archives/
Disallow: /index.php?/news/C10/
Disallow: /index.php?/news/C11/
Disallow: /index.php?/news/C12/
Disallow: /index.php?/news/C13/
Disallow: /index.php?/news/C14/
Disallow: /index.php?/news/C15/
Disallow: /index.php?/news/C16/
Disallow: /index.php?/news/C17/
Disallow: /index.php?/news/C4/
Disallow: /index.php?/news/C5/
Disallow: /index.php?/news/C6/
Disallow: /index.php?/news/C7/
Disallow: /index.php?/news/C8/
Disallow: /index.php?/news/C9/
Disallow: /index.php?/news/categories/
Disallow: /index.php?/news/contact/
Disallow: /index.php?/search
Disallow: /Library/
Disallow: /Link/
Disallow: /News/

User-agent: Megalodon
Disallow: /