Estoy tratando de adaptar para poder hacer scraping a multiples urls desde python con selenium web driver y requests.
Las urls a scrapear comienza así:
http://mi.dominio.net/Report?server=xxx.xxx.xxx.xx&machine=1
El ultimo valor es variable entre 1 a 100, entonces trato de adaptar el siguiente codigo:
from selenium.webdriver.common.desired_capabilities import
DesiredCapabilities
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import os
from pandas import ExcelWriter
import pandas as pd
import requests
from bs4 import BeautifulSoup
mipath = "C:/test"
desired_capabilities = DesiredCapabilities.CHROME
desired_capabilities["pageLoadStrategy"] = "none"
driver = webdriver.Chrome('/Users/JDani/Documents/Proyectos/chromedriver/chromedriver.exe')
wait = WebDriverWait(driver, 20)
''' ADAPTACION '''
baseurl = requests.get('http://mi.dominio.net/Report?server=xxx.xxx.xxx.xx&PC=')
valid_url = '1,2,3,4,5'
for n in range(len(valid_url)):
url = f'{baseurl}{valid_url[n]}'
driver.get(url)
print(url)
print(pages.title)
''' Recorrer todas las urls y guardar los datos '''
try:
wait.until(EC.presence_of_all_elements_located((By.CLASS_NAME, "ValorCorte")))
'''os.stat(mipath)'''
except TimeoutException:
print('Nope')
else:
'''os.mkdir(mipath)'''
driver.execute_script("window.stop();")
content = driver.find_elements_by_class_name("ValorCorte")
codigos = [element.text for element in content]
'''Save data in Text file'''
file = open(mipath + "/mytext.txt", "a")
file.write('\n' + str(codigos))
file.close()
'''print(codigos)'''
driver.close()
Pero la adaptación que estoy intentando me arroja lo siguiente: C:\Users\JDani\AppData\Local\Programs\Python\Python37-32\python.exe C:/Users/JDani/.PyCharmCE2019.1/config/scratches/scratch_7.py 1 None , None 2 None , None 3 None , None 4 None , None 5 None
valid_url = '1,2,3,4,5'
lo tenes con comillas ?, si es así, es unstring
. Intenta quitandoselas