class HreflocalizeSpider(scrapy.Spider):
name = "hrefLocalize"
allowed_domains = [URL]
start_urls = (
'URL_DE_BUSQUEDA',
)
rules = ( Rule(LinkExtractor(allow=()),callback='parse', follow=True))
settings.overrides['DEPTH_LIMIT']= 4 #Puse esto para forzar el cambio
settings.overrides['DEPTH_PRIORITY']= 4
def parse(self, response):
hxs = scrapy.Selector(response)
lines = hxs.xpath("//@href").extract()
linkPattern = re.compile("^(?:ftp|http|https):\/\/(?:[\w\.\-\+]+:{0,1}[\w\.\-\+]*@)?(?:[a-z0-9\-\.]+)(?::[0-9]+)?(?:\/|\/(?:[\w#!:\.\?\+=&%@!\-\/\(\)]+)|\?(?:[\w#!:\.\?\+=&%@!\-\/\(\)]+))?$")
for line in lines:
print line
if linkPattern.match(line):
yield Request(line,self.parse)
class HreflocalizeSpider(scrapy.Spider):
name = 'hrefLocalize'
allowed_domains = [URL]
start_urls = ('URL_DE_BUSQUEDA',)
rules = Rule(LinkExtractor(allow=()), callback='parse', follow=True)
settings.overrides['DEPTH_LIMIT'] = 4 # Puse esto para forzar el cambio
settings.overrides['DEPTH_PRIORITY'] = 4
def parse(self, response):
hxs = scrapy.Selector(response)
lines = hxs.xpath('//@href').extract()
linkPattern = re.compile(r'^(?:ftp|http|https):\/\/(?:[\w\.\-\+]+:{0,1}[\w\.\-\+]*@)?(?:[a-z0-9\-\.]+)(?::[0-9]+)?(?:\/|\/(?:[\w#!:\.\?\+=&%@!\-\/\(\)]+)|\?(?:[\w#!:\.\?\+=&%@!\-\/\(\)]+))?$')
for line in lines:
print(line)
if linkPattern.match(line):
yield Request(line, self.parse)
¿Alguien me podría echar una mano y decirme que estoy haciendo mal??