Cliquez sur « Afficher plus » jusqu'à ce que toutes les données du tableau sont affichées et récupérer toutes les données de la table

Question

je dois récupérer toutes les données de la table sur cette page https://www.nets.eu/dk-da/l%C3%B8sninger/Registreringsnumre

Mais je dois cliquer sur le lien « Afficher plus » bouton jusqu'à ce que toutes les données sont affichées ..

Mais peu importe le nombre de fois que je clique sur le bouton "afficher plus", la table continue d'avoir 30 lignes ..!?

import sys 
import time 
from pyvirtualdisplay import Display 
from selenium import webdriver 
from selenium.webdriver.common.by import By 
from selenium.webdriver.support.ui import WebDriverWait 
from selenium.webdriver.support import expected_conditions as EC 
from selenium.common.exceptions import TimeoutException 
import json 

class Scrape: 
    display = None 
    driver = None 

    def __init__(self): 
     # Start display 
     self.display = Display(visible=0, size=(800, 600)) 
     self.display.start() 

     # Init driver 
     self.driver = webdriver.Firefox() 
     self.driver.wait = WebDriverWait(self.driver, 5) 

     self.load_page() 

     time.sleep(5) 

     self.close() 

    def load_page(self): 
     data = [] 
     url = 'https://www.nets.eu/dk-da/l%C3%B8sninger/Registreringsnumre' 
     xpath = '//table[@class="itera-DataTable"]/tbody/tr' 
     self.driver.get(url) 

     try: 
      table = self.driver.wait.until(EC.presence_of_element_located(
       (By.CLASS_NAME, 'itera-DataTable'))) 

      print 'Table found!' 

      i = 1 
      while True: 
       button = self.driver.wait.until(EC.presence_of_element_located(
        (By.CLASS_NAME, 'itera-nextbatchbox'))) 

       print 'Button %d found!' % (i) 

       row_count = len(self.driver.find_elements_by_xpath(xpath)) 
       print row_count 

       button.click() 

       i += 1 

       if i > 5: 
        break 

      i = 1 
      for tr in self.driver.find_elements_by_xpath(xpath): 
       print 'TR %d' % (i) 
       tr_data = [] 
       tds = tr.find_elements_by_tag_name('td') 
       if tds: 
        tr_data.append([td.text for td in tds]) 
        data.append(tr_data) 

       i += 1 

      #print json.dumps(data) 

     except TimeoutException: 
      self.error('Table not found') 

    def error(self, str): 
     self.close() 

     print>>sys.stderr, str 
     sys.exit(1) 

    def close(self): 
     if self.driver is not None: 
      self.driver.quit() 
     self.display.stop() 

if __name__ == '__main__': 
    Scrape() 

Answer 1

Il semble que la cause principale soit que le bouton ne fonctionne pas. Utilisez JavaScriptExecutor pour cliquer sur le bouton pour résoudre ce problème. Voir mon code ci-dessous.

from selenium import webdriver 
from selenium.webdriver.common.by import By 
from selenium.webdriver.support.ui import WebDriverWait 
from selenium.webdriver.support import expected_conditions as EC 
from selenium.common.exceptions import TimeoutException 
import time 

driver = webdriver.Chrome() 
data = [] 
url = 'https://www.nets.eu/dk-da/l%C3%B8sninger/Registreringsnumre' 
xpath = '//table[@class="itera-DataTable"]/tbody/tr' 
driver.get(url) 

try: 
    table = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CLASS_NAME, 'itera-DataTable'))) 
    print('Table found!') 
except TimeoutException: 
    print('Table not found') 


row_count = len(driver.find_elements_by_xpath(xpath)) 
print(row_count) 
while True: 
    try: 
     button = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, '.itera-nextbatchbox'))) 
     print('Button found!') 
    except TimeoutException: 
     break 

    driver.execute_script("arguments[0].click();", button) 
    time.sleep(1) 

    try: 
     WebDriverWait(driver, 10).until(lambda driver: len(driver.find_elements_by_xpath(xpath)) > row_count) 
     time.sleep(1) 
     row_count = len(driver.find_elements_by_xpath(xpath)) 
     print(row_count) 
    except TimeoutException: 
     print('No more rows. Rows count: ' + str(len(driver.find_elements_by_xpath(xpath))))