Quiero obtener el texto de esta página web y los siguientes textos al hacer clic en el botón "Oui" o "No", que estará en el mismo lugar, y almacenarlos como un archivo json :
Intento con Python
from selenium import webdriver
from bs4 import BeautifulSoup
import pandas as pd
# driver = webdriver.Chrome(ChromeDriverManager().install())
driver = webdriver.Chrome(executable_path=r"C:\Programs\chromedriver.exe")
driver.get('http://google.com/')
# driver = webdriver.Chrome(r"C:\Programs\chromedriver")
questions=[] #List to store questions the survey
driver.get("http://www.leparisien.fr/elections/municipales/municipales-a-paris-notre-simulateur-pour-savoir-quel-candidat-vous-correspond-le-mieux-05-03-2020-8273238.php")
content = driver.page_source
soup = BeautifulSoup(content, features="lxml")
for a in soup.findAll('div',href=True, attrs={'id':'root'}):
question = a.find('div', attrs={'style': 'text-align: left; padding-right: 20px; padding-left: 20px; height: 50px; position: relative; margin-left: 18px; font-size: 20px;'})
print(question.text)
questions.append(question.text)
print("question: ",question)
Pero me devuelve:
(scr_env) C:\Users\antoi\Documents\Programming\Scraping\Python>python3 text_scraper.py
DevTools listening on ws://127.0.0.1:63980/devtools/browser/106c67f8-af8a-4230-8ec9-d7eb13fb8110
[30396:6884:0520/120628.687:ERROR:browser_switcher_service.cc(238)] XXX Init()
Intento con javascript
'use strict';
const puppeteer = require('puppeteer');
(async function main() {
try {
const browser = await puppeteer.launch();
const [page] = await browser.pages();
await page.goto('http://www.leparisien.fr/elections/municipales/municipales-a-paris-notre-simulateur-pour-savoir-quel-candidat-vous-correspond-le-mieux-05-03-2020-8273238.php');
const pollFrame = page.frames().find(
frame => frame.url() === 'https://livemixr-assets.s3-eu-west-1.amazonaws.com/quel-candidat/index.html'
);
const data = await pollFrame.evaluate(
() => document.querySelector('html > body > div > div > div > div > div:nth-child(5) > h4').innerText
);
console.log(data);
await browser.close();
} catch (err) {
console.error(err);
}
})();
Pero solo me devuelve el primero
So how to extract text from a caroussel?