python之Python 网络爬虫的文件存储问题

我正在使用网络爬虫进行屏幕抓取数据并将结果(来自 Twitter 页面的推文)存储为我正在抓取的每个用户的单独 html 文件。我打算稍后解析html文件并将数据存储到数据库中以供分析。但是，我遇到了一个奇怪的问题。

当我运行以下程序时——整个爬虫的一小段——我能够为每个关注者获得一个单独的 html 文件:

import re 
import urllib2 
import twitter 
 
start_follower = "NYTimesKrugman" 
depth = 3 
 
searched = set() 
 
api = twitter.Api() 
 
def crawl(follower, in_depth): 
    if in_depth > 0: 
        searched.add(follower) 
        directory = "C:\\Python28\\Followertest1\\" + follower + ".html" 
        output = open(directory, 'a') 
        output.write(follower) 
        output.write('\n\n') 
        users = api.GetFriends(follower) 
        names = set([str(u.screen_name) for u in users]) 
        names -= searched 
        for name in list(names)[0:5]: 
            crawl(name, in_depth-1)  
 
crawl(start_follower, depth) 
 
for x in searched: 
    print x 
print "Program is completed." 

但是，当我运行完整的爬虫时，我没有为每个关注者获得一个单独的文件:

import twitter 
import urllib 
from BeautifulSoup import BeautifulSoup 
import re 
import time 
 
start_follower = "NYTimeskrugman"  
depth = 2 
searched = set() 
 
api = twitter.Api() 
 
 
def add_to_U(user): 
    U.append(user) 
 
def site(follower): #creates a twitter site url in string format based on the follower username 
    followersite = "http://mobile.twitter.com/" + follower 
    return followersite 
 
def getPage(follower): #obtains access to a webapge 
    url = site(follower) 
    response = urllib.urlopen(url) 
    return response 
 
def getSoup(response): #creates the parsing module 
    html = response.read() 
    soup = BeautifulSoup(html) 
    return soup 
 
def gettweets(soup, output): 
    tags = soup.findAll('div', {'class' : "list-tweet"})#to obtain tweet of a follower 
    for tag in tags:  
        a = tag.renderContents() 
        b = str (a) 
        output.write(b) 
        output.write('\n\n') 
 
def are_more_tweets(soup):#to check whether there is more than one page on mobile twitter  
    links = soup.findAll('a', {'href': True}, {id: 'more_link'}) 
    for link in links: 
        b = link.renderContents() 
        test_b = str(b) 
        if test_b.find('more') != -1: 
            return True 
    return False 
 
def getnewlink(soup): #to get the link to go to the next page of tweets on twitter  
    links = soup.findAll('a', {'href': True}, {id : 'more_link'}) 
    for link in links: 
        b = link.renderContents() 
        if str(b) == 'more': 
            c = link['href'] 
            d = 'http://mobile.twitter.com' +c 
            return d 
 
def crawl(follower, in_depth): #main method of sorts 
    if in_depth > 0: 
        searched.add(follower) 
        directory = "C:\\Python28\\Followertest2\\" + follower + ".html" 
        output = open(directory, 'a') 
        output.write(follower) 
        output.write('\n\n') 
        a = getPage(follower) 
        soup = getSoup(a) 
        gettweets(soup, output) 
        tweets = are_more_tweets(soup) 
        while(tweets):  
            b = getnewlink(soup) 
            red = urllib.urlopen(b) 
            html = red.read() 
            soup = BeautifulSoup(html) 
            gettweets(soup, output) 
            tweets = are_more_tweets(soup) 
        users = api.GetFriends(follower) 
        names = set([str(u.screen_name) for u in users]) 
        names -= searched 
        for name in list(names)[0:5]: 
            print name 
            crawl(name, in_depth - 1) 
 
crawl(start_follower, depth) 
print("Program done. Look at output file.") 

更具体地说，我似乎为大约前五个关注者获得了一个单独的 html 文件，然后似乎没有创建新文件。任何帮助，将不胜感激!

请您参考如下方法:

depth 值在片段和完整代码之间是不同的(您只会在完整代码中获得一级递归)。此外，您只从关注者列表中获取前五个名字: for name in list(names)[0:5]: 所以您总共得到六个人:起始关注者和他们的前五个 friend 。