Skip to main content
 首页 » 编程设计

python之在 Python 中使用 Selenium 抓取无限滚动网站

2025年05月04日13jackei

我想删除这个网站有滚动条的内容: http://stocktwits.com/symbol/AAPL?q=AAPL

我在 Stactoverflow 中找到了一个类似问题的答案: scrape websites with infinite scrolling

这是从那里复制的代码:

from selenium import webdriver 
from selenium.webdriver.common.by import By 
from selenium.webdriver.common.keys import Keys 
from selenium.webdriver.support.ui import Select 
from selenium.webdriver.support.ui import WebDriverWait 
from selenium.common.exceptions import TimeoutException 
from selenium.webdriver.support import expected_conditions as EC 
from selenium.common.exceptions import NoSuchElementException 
from selenium.common.exceptions import NoAlertPresentException 
import sys 
 
import unittest, time, re 
 
class Sel(unittest.TestCase): 
    def setUp(self): 
        self.driver = webdriver.Firefox() 
        self.driver.implicitly_wait(30) 
        self.base_url = "https://twitter.com" 
        self.verificationErrors = [] 
        self.accept_next_alert = True 
    def test_sel(self): 
        driver = self.driver 
        delay = 3 
        driver.get(self.base_url + "/search?q=stckoverflow&src=typd") 
        driver.find_element_by_link_text("All").click() 
        for i in range(1,100): 
            self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight);") 
            time.sleep(4) 
        html_source = driver.page_source 
        data = html_source.encode('utf-8') 
 
 
if __name__ == "__main__": 
    unittest.main() 

现在我想废弃 Stocktwits 网站而不是 twitter(链接在上面)。

我把上面的代码修改成这样:

from selenium import webdriver 
from selenium.webdriver.common.by import By 
from selenium.webdriver.common.keys import Keys 
from selenium.webdriver.support.ui import Select 
from selenium.webdriver.support.ui import WebDriverWait 
from selenium.common.exceptions import TimeoutException 
from selenium.webdriver.support import expected_conditions as EC 
from selenium.common.exceptions import NoSuchElementException 
from selenium.common.exceptions import NoAlertPresentException 
import sys 
 
import unittest, time, re 
 
class Sel(unittest.TestCase): 
    def setUp(self): 
        self.driver = webdriver.Firefox() 
        self.driver.implicitly_wait(30) 
        self.base_url = "http://stocktwits.com/symbol/AAPL?q=AAPL" 
        self.verificationErrors = [] 
        self.accept_next_alert = True 
    def test_sel(self): 
        driver = self.driver 
        delay = 3 
        driver.get(self.base_url) 
        driver.find_element_by_link_text("All").click() 
        for i in range(1,100): 
            self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight);") 
            time.sleep(4) 
        html_source = driver.page_source 
        data = html_source.encode('utf-8') 
 
 
if __name__ == "__main__": 
    unittest.main() 

但是当我运行代码时出现这个错误:

NoSuchElementException: Message: Unable to locate element: {"method":"link text","selector":"All"} 

我感谢任何帮助找出问题的帮助。

请您参考如下方法:

看起来问题出在这一行:

driver.find_element_by_link_text("All").click() 

您期待一个带有链接文本 "All" 的元素,但不存在。