Skip to main content
 首页 » 编程设计

python之如何获取 Selenium 中当前标签的网址

2024年10月24日14freeliver54

一切正常,但我无法在每次点击时提取新标签页的网址?我缺少的是我已经尝试了所有可能的资源 - current_url,response.url,getCurrentUrl 似乎都不起作用。在这种情况下最好的解决方案是什么?

import scrapy 
from scrapy.contrib.spiders import CrawlSpider, Rule 
from scrapy.selector import Selector 
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor 
from selenium import webdriver 
from urlparse import urljoin 
import time 
from selenium.webdriver.common.keys import Keys 
 
class CompItem(scrapy.Item): 
    model_name = scrapy.Field() 
    model_link = scrapy.Field() 
    url  =scrapy.Field() 
 
class criticspider(CrawlSpider): 
    name = "paytm_l" 
    allowed_domains = ["paytm.com"] 
    start_urls = ["https://paytm.com/shop/g/electronics/mobile-accessories/mobiles"] 
 
 
    def __init__(self, *args, **kwargs): 
        super(criticspider, self).__init__(*args, **kwargs) 
        self.download_delay = 0.25 
        self.browser = webdriver.Firefox() 
 
        self.browser.implicitly_wait(2) 
 
    def parse_start_url(self, response): 
        self.browser.get(response.url) 
        #sites = response.xpath('//div[@class="single-review"]/div[@class="review-header"]') 
        self.browser.implicitly_wait(30) 
 
        items = [] 
        time.sleep(20) 
    #   for i in range(0,200): 
            #self.browser.execute_script("window.scrollTo(0, document.body.scrollHeight);") 
 
 
        sel = Selector(text=self.browser.page_source) 
        sites = sel.xpath('//div[contains(@class,"overflow-hidden")]') 
 
        item = CompItem() 
 
        for r in range(1,5): 
 
 
                    #item['model_name'] = site.xpath('.//p[contains(@ng-if,"applyLimit")]/text()') 
                    button = self.browser.find_element_by_xpath("/html/body/div[5]/div[5]/div/div[5]/div[3]/ul/li[%d]/a"%r) 
                    main_window = self.browser.current_window_handle 
                    button.send_keys(Keys.CONTROL + Keys.RETURN) 
                    self.browser.find_element_by_tag_name('body').send_keys(Keys.CONTROL + Keys.TAB) 
                    time.sleep(5) 
                    self.browser.find_element_by_tag_name('body').send_keys(Keys.CONTROL + 'l') 
                    self.browser.find_element_by_tag_name('body').send_keys(Keys.CONTROL + 'c') 
 
                    item["url"]=self.browser.current_url() 
                    time.sleep(10) 
                    self.browser.switch_to_window(main_window) 
                    time.sleep(10) 
 
                    self.browser.find_element_by_tag_name('body').send_keys(Keys.CONTROL + 'w') 
                    self.browser.switch_to_window(main_window) 
 
                    #item['model_link'] = site.xpath('//a[contains(@class,"{"na": !productClasses(product)}"]/@href').extract()[0] 
 
                    yield item 

请您参考如下方法:

通过在选项卡上更改驱动程序的焦点,我能够获取 url。

import scrapy 
from scrapy.contrib.spiders import CrawlSpider, Rule 
from scrapy.selector import Selector 
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor 
from selenium import webdriver 
from urlparse import urljoin 
import time 
from selenium.webdriver.common.keys import Keys 
 
class CompItem(scrapy.Item): 
    model_name = scrapy.Field() 
    model_link = scrapy.Field() 
    url  =scrapy.Field() 
 
class criticspider(CrawlSpider): 
    name = "paytm_l" 
    allowed_domains = ["paytm.com"] 
    start_urls = ["https://paytm.com/shop/g/electronics/mobile-accessories/mobiles"] 
 
 
    def __init__(self, *args, **kwargs): 
        super(criticspider, self).__init__(*args, **kwargs) 
        self.download_delay = 0.25 
        self.browser = webdriver.Firefox() 
 
        self.browser.implicitly_wait(2) 
 
    def parse_start_url(self, response): 
        self.browser.get(response.url) 
        #sites = response.xpath('//div[@class="single-review"]/div[@class="review-header"]') 
        self.browser.implicitly_wait(30) 
 
        items = [] 
        time.sleep(20) 
    #   for i in range(0,200): 
            #self.browser.execute_script("window.scrollTo(0, document.body.scrollHeight);") 
 
 
        sel = Selector(text=self.browser.page_source) 
        sites = sel.xpath('//div[contains(@class,"overflow-hidden")]') 
 
        item = CompItem() 
 
        for r in range(1,5): 
 
 
                    #item['model_name'] = site.xpath('.//p[contains(@ng-if,"applyLimit")]/text()') 
                    button = self.browser.find_element_by_xpath("/html/body/div[5]/div[5]/div/div[5]/div[3]/ul/li[%d]/a"%r) 
                    main_window = self.browser.current_window_handle 
                    button.send_keys(Keys.CONTROL + Keys.RETURN) 
                    self.browser.find_element_by_tag_name('body').send_keys(Keys.CONTROL + Keys.TAB) 
                    time.sleep(5) 
                    self.browser.switch_to_window(main_window) 
 
 
                    item["url"]=self.browser.current_url 
                    time.sleep(10) 
 
                    time.sleep(10) 
 
                    self.browser.find_element_by_tag_name('body').send_keys(Keys.CONTROL + 'w') 
                    self.browser.switch_to_window(main_window) 
 
                    #item['model_link'] = site.xpath('//a[contains(@class,"{"na": !productClasses(product)}"]/@href').extract()[0] 
 
                    yield item