import re
from urlparse import urlparse, urljoin


from wfuzz.plugin_api.mixins import DiscoveryPluginMixin
from wfuzz.plugin_api.base import BasePlugin
from wfuzz.plugin_api.urlutils import check_content_type
from wfuzz.externals.moduleman.plugin import moduleman_plugin

@moduleman_plugin
class robots(BasePlugin, DiscoveryPluginMixin):
    name = "robots"
    author = ("Xavi Mendez (@xmendez)",)
    version = "0.1"
    summary = "Parses robots.txt looking for new content."
    description = ("Parses robots.txt looking for new content.",)
    category = ["default", "active", "discovery"]
    priority = 99

    parameters = (
    )

    def __init__(self):
        BasePlugin.__init__(self)

    def validate(self, fuzzresult):
	return fuzzresult.history.urlparse.ffname == "robots.txt" and fuzzresult.code == 200 and check_content_type(fuzzresult, 'text')

    def process(self, fuzzresult):
	# Shamelessly (partially) copied from w3af's plugins/discovery/robotsReader.py
	for line in fuzzresult.history.content.split('\n'):
	    line = line.strip()

	    if len(line) > 0 and line[0] != '#' and (line.upper().find('ALLOW') == 0 or\
	    line.upper().find('DISALLOW') == 0 or line.upper().find('SITEMAP') == 0):

		url = line[ line.find(':') + 1 : ]
		url = url.strip(" *")

		if url:
		    self.queue_url(urljoin(fuzzresult.url, url))

