Skip to main content
4 de 17
título editado

Iterar en las tarjetas de una página web para extraer los titulos-interrogaciones y las respuestas con Selenium

Intento scrapar las preguntas y respuestas de google forms que se encontran en un archivo csv. Desafortunadamente parece que repaso los cuestionarios sin obtener las preguntas y respuestas. En efecto, con mi codigo :

from collections import defaultdict
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import ElementClickInterceptedException
import time
from selenium.webdriver.chrome.options import Options
from selenium.common.exceptions import NoSuchElementException
import pandas as pd
from selenium.common.exceptions import ElementNotInteractableException, NoSuchElementException
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
import pickle

def setup_chromedriver():
    chrome_options = Options()
    chrome_options.add_argument("--headless")
    # driver = webdriver.Chrome(ChromeDriverManager().install())
    driver = webdriver.Chrome("C:\Programs\chromedriver.exe")
    return driver


def load_data(survey):
    df = pd.read_csv("cuestionarios.csv")
    filter_col = ["Link"]
    return df, filter_col


def extract(driver, df, column):
    locations = df[column].iteritems()
    count = 0
    result = []
    for location in locations:
        questionaire = {}
        if "docs.google.com" in str(location[1]):
            location = location[1]
            driver.get(location)
            containers = driver.find_elements_by_class_name(
                "freebirdFormeditorViewItemcardRoot"
                )
            print("containers: ", containers)
            # btns = driver.find_elements_by_css_selector(".appsMaterialWizButtonEl")
            # if not btns:
            #     print("btns: ", btns)
            #     break
            for container in containers:
                try:
                    question = container.find_element_by_class_name(
                        "appsMaterialWizTextinputTextareaInput.exportTextarea"
                        )
                except NoSuchElementException:
                    continue
                responses = container.find_elements_by_class_name(
                    "docssharedWizToggleLabeledLabelText.exportLabel.freebirdFormviewerViewItemsRadioLabel"
                    )
                extracted_text = [response.text for response in responses]
                print("extracted_text: ", extracted_text)
                questionaire[question.text] = extracted_text
        result.append({count: questionaire})
        count += 1
    return result

if __name__ == '__main__':
    """ Need to log on to the google account to access certain questionaires. Also Setup chromedriver to run in 
    headless state """
    driver = setup_chromedriver()
    """ Load CSV download of Google Sheet """
    sources = ["Airline XYZ", "Hotel Chain ABC", "The Ministry of Tourism of France"]
    for s in sources:
        df, columns = load_data(s)
        for column in columns:
            result = extract(driver, df, column)
            s = s.replace(" ", "_")
            pickle_out = open(s + "_c.p", "wb")
            pickle.dump(result, pickle_out)
            pickle_out.close()

Obtengo:

(scr_env) C:\Users\antoi\Documents\Programming\Scraping\Python>python3 new_scraper.py

DevTools listening on ws://127.0.0.1:50709/devtools/browser/292151c8-3d44-42ba-9bbe-25b91590c41f
containers:  [<selenium.webdriver.remote.webelement.WebElement (session="6edc7531b31e8306a2e31e4ba4d09496", element="971e8229-ae70-4ec3-ab9a-5a0dcae0d103")>, <selenium.webdriver.remote.webelement.WebElement (sessi
on="6edc7531b31e8306a2e31e4ba4d09496", element="77a13f38-5f04-422c-8a76-34975f66c0d9")>, <selenium.webdriver.remote.webelement.WebElement (session="6edc7531b31e8306a2e31e4ba4d09496", element="e4dd154a-b1d8-4818-89
5e-8d70ed6cd8ca")>, ..., <selenium.webdriver.remote.webelement.WebElement (session="6edc7
531b31e8306a2e31e4ba4d09496", element="67c440f9-dc07-4bb8-80d6-e01c4cd10c58")>]
extracted_text:  []
extracted_text:  []
...