Skip to main content
 首页 » 编程设计

python之试用PyQt4,无法抓取多个网页

2025年05月04日10unruledboy

我正在尝试更改 this code一次抓取多个 URL 并捕获抛出的特定 javascript 标记。这是我的代码:

#!/usr/bin/env python 
 
import sys 
import signal 
import urllib 
import re 
import csv 
import time 
 
from optparse import OptionParser 
from PyQt4 import QtCore 
from PyQt4.QtCore import * 
from PyQt4.QtGui import * 
from PyQt4.QtWebKit import QWebPage 
 
class Crawler( QWebPage ): 
    def __init__(self, url, file): 
        QWebPage.__init__( self ) 
        self._url = url 
        #print(url) 
        self._file = file 
        #print(file) 
 
    def crawl( self ): 
        signal.signal( signal.SIGINT, signal.SIG_DFL ) 
        self.connect( self, SIGNAL( 'loadFinished(bool)' ), self._finished_loading ) 
        #print(self._url) 
        self.mainFrame().load( QUrl( self._url ) ) 
 
    def _finished_loading( self, result ): 
        print(self._url) 
        file = open( self._file, 'a' ) 
        s = QtCore.QString('aJavascriptKeyword') 
        val = (self.mainFrame().toHtml().indexOf(s) >= 0) 
        #print val 
        file.write( self._url + '^' + str(val) ) 
        #print( self._url + '^' + str(val) ) 
        file.close() 
        sys.exit( 0 ) 
 
def main(): 
    app = QApplication( sys.argv ) 
 
    urls = open(r'urls.txt','r') 
    #output=open(r'C:\Users\me\output.txt','w') 
    for url in urls: 
        #print(url) 
        crawler = Crawler( url, "output.txt" ) 
        crawler.crawl() 
    sys.exit( app.exec_() ) 
 
 
if __name__ == '__main__': 
    main() 

在这种情况下,urls.txt 只是一个文本文件,每行一个 url。当我使用包含 1 个 url 的文件运行它时,它运行得很好,但是当我使用 > 1 url 运行时,只有最终 url 被传递给 finished_loading()。我做错了什么?

请您参考如下方法:

问题是

for url in urls: 
    ... 
    crawler = Crawler( url, "output.txt" ) 
    crawler.crawl() 

在没有运行主 PyQt 应用程序的情况下完成。这意味着每次循环迭代时都会重置爬虫,最后它只是具有最后一个 url 的爬虫。

一个解决方案可能是保存实例化爬虫的实例

def main(): 
    app = QApplication( sys.argv ) 
 
    # Get all the urls 
    with open(r'urls.txt','r') as urls: 
        my_urls = [u.rstrip("\n") for u in urls] 
 
    # Instantiate all the Crawlers 
    crawlers = [Crawler(u, "output.txt") for u in my_urls] 
 
    # Call all of the crawlers' crawl methdos 
    for crawler in crawlers: 
        crawler.crawl() 
 
    sys.exit( app.exec_() )