Skip to main content
 首页 » 编程设计

python之在 Python 脚本中使用 Scrapy Spider 输出的问题

2024年11月24日7shihaiming

我想在 python 脚本中使用蜘蛛的输出。为此,我根据另一个 thread 编写了以下代码.

我面临的问题是函数 spider_results() 只会一遍又一遍地返回最后一项的列表,而不是包含所有找到的项目的列表。当我使用 scrapy crawl 命令手动运行同一个蜘蛛时,我得到了所需的输出。脚本的输出、手动 json 输出和蜘蛛本身如下。

我的代码有什么问题?

from scrapy import signals 
from scrapy.crawler import CrawlerProcess 
from scrapy.utils.project import get_project_settings 
from circus.spiders.circus import MySpider 
 
from scrapy.signalmanager import dispatcher 
 
 
def spider_results(): 
    results = [] 
 
    def crawler_results(signal, sender, item, response, spider): 
        results.append(item) 
 
 
    dispatcher.connect(crawler_results, signal=signals.item_passed) 
 
    process = CrawlerProcess(get_project_settings()) 
    process.crawl(MySpider) 
    process.start()  # the script will block here until the crawling is finished 
    return results 
 
 
if __name__ == '__main__': 
    print(spider_results()) 

脚本输出:

{'away_odds': 1.44, 
 'away_team': 'Los Angeles Dodgers', 
 'event_time': datetime.datetime(2019, 6, 8, 2, 15), 
 'home_odds': 2.85, 
 'home_team': 'San Francisco Giants', 
 'last_update': datetime.datetime(2019, 6, 6, 20, 58, 41, 655497), 
 'league': 'MLB'}, {'away_odds': 1.44, 
 'away_team': 'Los Angeles Dodgers', 
 'event_time': datetime.datetime(2019, 6, 8, 2, 15), 
 'home_odds': 2.85, 
 'home_team': 'San Francisco Giants', 
 'last_update': datetime.datetime(2019, 6, 6, 20, 58, 41, 655497), 
 'league': 'MLB'}, {'away_odds': 1.44, 
 'away_team': 'Los Angeles Dodgers', 
 'event_time': datetime.datetime(2019, 6, 8, 2, 15), 
 'home_odds': 2.85, 
 'home_team': 'San Francisco Giants', 
 'last_update': datetime.datetime(2019, 6, 6, 20, 58, 41, 655497), 
 'league': 'MLB'}] 

使用 scrapy 抓取的 Json 输出:

[ 
{"home_team": "Los Angeles Angels", "away_team": "Seattle Mariners", "event_time": "2019-06-08 02:07:00", "home_odds": 1.58, "away_odds": 2.4, "last_update": "2019-06-06 20:48:16", "league": "MLB"}, 
{"home_team": "San Diego Padres", "away_team": "Washington Nationals", "event_time": "2019-06-08 02:10:00", "home_odds": 1.87, "away_odds": 1.97, "last_update": "2019-06-06 20:48:16", "league": "MLB"}, 
{"home_team": "San Francisco Giants", "away_team": "Los Angeles Dodgers", "event_time": "2019-06-08 02:15:00", "home_odds": 2.85, "away_odds": 1.44, "last_update": "2019-06-06 20:48:16", "league": "MLB"} 
] 

我的蜘蛛:

from scrapy.spiders import Spider 
from ..items import MatchItem 
import json 
import datetime 
import dateutil.parser 
 
class MySpider(Spider): 
    name = 'first_spider' 
 
    start_urls = ["https://websiteXYZ.com"] 
 
    def parse(self, response): 
        item = MatchItem() 
 
        timestamp = datetime.datetime.utcnow() 
 
        response_json = json.loads(response.body) 
 
        for event in response_json["el"]: 
            for team in event["epl"]: 
                if team["so"] == 1: item["home_team"] = team["pn"] 
                if team["so"] == 2: item["away_team"] = team["pn"] 
 
            for market in event["ml"]: 
                if market["mn"] == "Match result": 
                    item["event_time"] = dateutil.parser.parse(market["dd"]).replace(tzinfo=None) 
                    for outcome in market["msl"]: 
                        if outcome["mst"] == "1": item["home_odds"] = outcome["msp"] 
                        if outcome["mst"] == "X": item["draw_odds"] = outcome["msp"] 
                        if outcome["mst"] == "2": item["away_odds"] = outcome["msp"] 
 
                if market["mn"] == 'Moneyline': 
                    item["event_time"] = dateutil.parser.parse(market["dd"]).replace(tzinfo=None) 
                    for outcome in market["msl"]: 
                        if outcome["mst"] == "1": item["home_odds"] = outcome["msp"] 
                        #if outcome["mst"] == "X": item["draw_odds"] = outcome["msp"] 
                        if outcome["mst"] == "2": item["away_odds"] = outcome["msp"] 
 
 
            item["last_update"] = timestamp 
            item["league"] = event["scn"] 
 
            yield item 

编辑:

根据下面的答案,我尝试了以下两个脚本:

Controller .py

import json 
from scrapy import signals 
from scrapy.crawler import CrawlerRunner 
from twisted.internet import reactor, defer 
from betsson_controlled.spiders.betsson import Betsson_Spider 
from scrapy.utils.project import get_project_settings 
 
 
class MyCrawlerRunner(CrawlerRunner): 
    def crawl(self, crawler_or_spidercls, *args, **kwargs): 
        # keep all items scraped 
        self.items = [] 
 
        # create crawler (Same as in base CrawlerProcess) 
        crawler = self.create_crawler(crawler_or_spidercls) 
 
        # handle each item scraped 
        crawler.signals.connect(self.item_scraped, signals.item_scraped) 
 
        # create Twisted.Deferred launching crawl 
        dfd = self._crawl(crawler, *args, **kwargs) 
 
        # add callback - when crawl is done cal return_items 
        dfd.addCallback(self.return_items) 
        return dfd 
 
    def item_scraped(self, item, response, spider): 
        self.items.append(item) 
 
    def return_items(self, result): 
        return self.items 
 
def return_spider_output(output): 
    return json.dumps([dict(item) for item in output]) 
 
settings = get_project_settings() 
runner = MyCrawlerRunner(settings) 
spider = Betsson_Spider() 
deferred = runner.crawl(spider) 
deferred.addCallback(return_spider_output) 
 
 
reactor.run() 
print(deferred) 

当我执行 controller.py 时,我得到:

<Deferred at 0x7fb046e652b0 current result: '[{"home_team": "St. Louis Cardinals", "away_team": "Pittsburgh Pirates", "home_odds": 1.71, "away_odds": 2.19, "league": "MLB"}, {"home_team": "St. Louis Cardinals", "away_team": "Pittsburgh Pirates", "home_odds": 1.71, "away_odds": 2.19, "league": "MLB"}, {"home_team": "St. Louis Cardinals", "away_team": "Pittsburgh Pirates", "home_odds": 1.71, "away_odds": 2.19, "league": "MLB"}, {"home_team": "St. Louis Cardinals", "away_team": "Pittsburgh Pirates", "home_odds": 1.71, "away_odds": 2.19, "league": "MLB"}, {"home_team": "St. Louis Cardinals", "away_team": "Pittsburgh Pirates", "home_odds": 1.71, "away_odds": 2.19, "league": "MLB"}, {"home_team": "St. Louis Cardinals", "away_team": "Pittsburgh Pirates", "home_odds": 1.71, "away_odds": 2.19, "league": "MLB"}, {"home_team": "St. Louis Cardinals", "away_team": "Pittsburgh Pirates", "home_odds": 1.71, "away_odds": 2.19, "league": "MLB"}, {"home_team": "St. Louis Cardinals", "away_team": "Pittsburgh Pirates", "home_odds": 1.71, "away_odds": 2.19, "league": "MLB"}]'> 

请您参考如下方法:

最近的编辑:阅读后CrawlerProcess vs CrawlerRunner我意识到您可能需要 CrawlerProcess。我必须使用 runner,因为我需要 klein 才能使用延迟对象。 Process 只期望 scrapy,而 runner 期望其他脚本/程序与之交互。希望这对您有所帮助。

您需要修改 CrawlerRunner/Process 并使用信号和/或回调将项目传递到 CrawlerRunner 中的脚本中。

How to integrate Flask & Scrapy?如果您查看顶部答案中的选项,带有扭曲的 klein 和 scrapy 的选项就是您正在寻找的示例,因为它在做同样的事情,只是在抓取后将其发送到 Klein http 服务器。您可以使用 CrawlerRunner 设置类似的方法,以便在脚本抓取时将每个项目发送到您的脚本。注意:此特定问题在收集项目后将结果发送到 Klein Web 服务器。答案是制作一个 API,它收集结果并等待直到抓取完成并将其作为转储发送到 JSON,但您可以将同样的方法应用于您的情况。主要要看的是 CrawlerRunner 如何被子类化和扩展以添加额外的功能。

您想要做的是拥有一个单独的脚本,您可以执行该脚本导入您的 Spider 并扩展 CrawlerRunner。然后执行此脚本,它将启动您的 Twisted react 器并使用您的自定义运行器启动爬网过程。

就是说——这个问题或许可以在元素管道中解决。创建自定义项目管道并在返回项目之前将项目传递到脚本中。

# main.py 
 
import json 
from scrapy import signals 
from scrapy.crawler import CrawlerProcess 
from twisted.internet import reactor, defer # import we missed 
from myproject.spiders.mymodule import MySpiderName 
from scrapy.utils.project import get_project_settings 
 
 
class MyCrawlerProcess(CrawlerProcess): 
    def crawl(self, crawler_or_spidercls, *args, **kwargs): 
        # keep all items scraped 
        self.items = [] 
 
        crawler = self.create_crawler(crawler_or_spidercls) 
 
        crawler.signals.connect(self.item_scraped, signals.item_scraped) 
 
        dfd = self._crawl(crawler, *args, **kwargs) 
 
        dfd.addCallback(self.return_items) 
        return dfd 
 
    def item_scraped(self, item, response, spider): 
        self.items.append(item) 
 
    def return_items(self, result): 
        return self.items 
 
 
def return_spider_output(output): 
    return json.dumps([dict(item) for item in output]) 
 
 
process = MyCrawlerProcess() 
deferred = process.crawl(MySpider) 
deferred.addCallback(return_spider_output) 
 
 
process.start() - Script should block here again but I'm not sure if it will work right without using reactor.run() 
print(deferred) 

同样,这段代码是我猜测的,我还没有测试过。我希望它能让你朝着更好的方向前进。

引用资料: