comunidad. estoy intentando extraer la informacion tecnica de cada uno de los vehiculos de esa pagina web, en un formato Json, pero al momento de ejecutar el codigo en consola, el resultado es el formato Json pero sin ningun dato, es decir no se guarda nada. Cuando ejecuto en la consola con el comando scrapy crawl project, todo parece salir bien, pero no se extrae ninguna informacion. Se crea el documento que solicite, se guarda pero sin ninguna informacion en el. Podrian ayudarme, soy bastante nuevo en este mundo, y quiero mejorar.
aqui el codigo:
#Links = ('//div[@class="figcaption"]/a[@href]/@href')
#names_vehicles ('//div[@class="figcaption"]/span[@class="code-name"]/text()').getall()')
#Vehicle_Descripstion_header //div[@class="inner"]//ul[@class="technical_info_nav"]/li[@class]/a[@href]/text()
#vehicle_description_body = div[@class="inner"]//div[@class="technical_info_content"]/div[contains(@class, "description")]//div[@class="spec-name col-lg-4 ng-binding"]//text()
import scrapy
class SpiderProject(scrapy.Spider):
name = 'project'
start_urls = [
'https://hitachicm.com.au/machinery/machinery/'
]
custom_settings = {
'FEED_URI': 'project.json',
'FEED_FORMAT': 'json',
'CONCURRENT_REQUESTS': 24,
'MEMUSAGE_LIMIT_MB': 2048,
'ROBOTSTXT_OBEY': True,
'USER_AGENT': 'LionSegura',
'FEED_EXPORT_ENCODING': 'utf-8'
}
def parse(self, response, **kwargs):
if kwargs:
vehicles = kwargs['vehicles']
description = kwargs['description']
description_header = kwargs['description header']
vehicles.extend(response.xpath('//div[@class="inner"]//ul[@class="technical_info_nav"]/li[@class]/a[@href]/text()').get())
description.extend(response.xpath('div[@class="inner"]//div[@class="technical_info_content"]/div[contains(@class, "description")]//div[@class="spec-name col-lg-4 ng-binding"]//text()').getall())
description_header.extend(response.xpath('//div[@class="inner"]//ul[@class="technical_info_nav"]/li[@class]/a[@href]/text()').getall())
links_vehicles = response.xpath('//div[@class="figcaption"]/a[@href]/@href').getall()
for link in links_vehicles:
yield response.follow(link, callback=self.parse, cb_kwargs={'vehicles' : vehicles, 'description header': description_header, 'description': description})
def parse_link(self, response, **kwards):
link=kwards['url']
names_vehicles = response.xpath('//div[@class="figcaption"]/span[@class="code-name"]/text()').getall()
vehicle_description_header = response.xpath('//div[@class="inner"]//ul[@class="technical_info_nav"]/li[@class]/a[@href]/text()').get()
vehicle_description_body = response.xpath('//div[@class="inner"]//div[@class="technical_info_content"]/div[contains(@class, "description")]//div[@class="spec-name col-lg-4 ng-binding"]//text()').getall()
yield{
'url': link,
'Kind of Vehicle' : names_vehicles,
'Description_header' : vehicle_description_header,
'Description body' : vehicle_description_body
}