Skip to main content
 首页 » 编程设计

python之如何在不覆盖结果的情况下抓取多个网页

2024年11月24日13thcjp

刚开始尝试从 Transfermarkt 抓取多个网页而不覆盖之前的网页。

知道之前已经有人问过这个问题,但我无法让它适用于这种情况。

from bs4 import BeautifulSoup as bs 
import requests 
import re 
import pandas as pd 
import itertools 
 
headers = {'User-Agent' : 'Mozilla/5.0'} 
df_headers = ['position_number' , 'position_description' , 'name' , 'dob' , 'nationality' , 'height' , 'foot' , 'joined' , 'signed_from' , 'contract_until'] 
urls = ['https://www.transfermarkt.com/fc-bayern-munich-u17/kader/verein/21058/saison_id/2018/plus/1', 'https://www.transfermarkt.com/fc-hennef-05-u17/kader/verein/48776/saison_id/2018/plus/1'] 
 
for url in urls: 
    r = requests.get(url,  headers = headers) 
    soup = bs(r.content, 'html.parser') 
 
 
    position_number = [item.text for item in soup.select('.items .rn_nummer')] 
    position_description = [item.text for item in soup.select('.items td:not([class])')] 
    name = [item.text for item in soup.select('.hide-for-small .spielprofil_tooltip')] 
    dob = [item.text for item in soup.select('.zentriert:nth-of-type(3):not([id])')] 
    nationality = ['/'.join([i['title'] for i in item.select('[title]')]) for item in soup.select('.zentriert:nth-of-type(4):not([id])')] 
    height = [item.text for item in soup.select('.zentriert:nth-of-type(5):not([id])')] 
    foot = [item.text for item in soup.select('.zentriert:nth-of-type(6):not([id])')] 
    joined = [item.text for item in soup.select('.zentriert:nth-of-type(7):not([id])')] 
    signed_from = ['/'.join([item.find('img')['title'].lstrip(': '), item.find('img')['alt']]) if item.find('a') else '' 
                   for item in soup.select('.zentriert:nth-of-type(8):not([id])')] 
    contract_until = [item.text for item in soup.select('.zentriert:nth-of-type(9):not([id])')] 
 
df = pd.DataFrame(list(zip(position_number, position_description, name, dob, nationality, height, foot, joined, signed_from, contract_until)), columns = df_headers) 
print(df) 
 
df.to_csv(r'Uljanas-MacBook-Air-2:~ uljanadufour$\bayern-munich123.csv') 

在抓取后能够区分网页也会很有帮助。

如有任何帮助,我们将不胜感激。

请您参考如下方法:

您上面的代码为每个 URL 抓取数据,在不将其放入数据帧的情况下对其进行解析,然后移动到下一个 URL。由于您对 pd.DataFrame() 的调用发生在循环之外,因此您正在从 urls 中的最后一个 URL 构建页面数据的数据框。

您需要在 for 循环之外创建一个数据框,然后将每个 URL 的传入数据附加到此数据框。

from bs4 import BeautifulSoup as bs 
import requests 
import re 
import pandas as pd 
import itertools 
 
headers = {'User-Agent' : 'Mozilla/5.0'} 
df_headers = ['position_number' , 'position_description' , 'name' , 'dob' , 'nationality' , 'height' , 'foot' , 'joined' , 'signed_from' , 'contract_until'] 
urls = ['https://www.transfermarkt.com/fc-bayern-munich-u17/kader/verein/21058/saison_id/2018/plus/1', 'https://www.transfermarkt.com/fc-hennef-05-u17/kader/verein/48776/saison_id/2018/plus/1'] 
 
#### Add this before for-loop. #### 
# Create empty dataframe with expected column names. 
df_full = pd.DataFrame(columns = df_headers) 
 
for url in urls: 
    r = requests.get(url,  headers = headers) 
    soup = bs(r.content, 'html.parser') 
 
 
    position_number = [item.text for item in soup.select('.items .rn_nummer')] 
    position_description = [item.text for item in soup.select('.items td:not([class])')] 
    name = [item.text for item in soup.select('.hide-for-small .spielprofil_tooltip')] 
    dob = [item.text for item in soup.select('.zentriert:nth-of-type(3):not([id])')] 
    nationality = ['/'.join([i['title'] for i in item.select('[title]')]) for item in soup.select('.zentriert:nth-of-type(4):not([id])')] 
    height = [item.text for item in soup.select('.zentriert:nth-of-type(5):not([id])')] 
    foot = [item.text for item in soup.select('.zentriert:nth-of-type(6):not([id])')] 
    joined = [item.text for item in soup.select('.zentriert:nth-of-type(7):not([id])')] 
    signed_from = ['/'.join([item.find('img')['title'].lstrip(': '), item.find('img')['alt']]) if item.find('a') else '' 
                   for item in soup.select('.zentriert:nth-of-type(8):not([id])')] 
    contract_until = [item.text for item in soup.select('.zentriert:nth-of-type(9):not([id])')] 
 
 
    #### Add this to for-loop. #### 
 
    # Create a dataframe for page data. 
    df = pd.DataFrame(list(zip(position_number, position_description, name, dob, nationality, height, foot, joined, signed_from, contract_until)), columns = df_headers) 
 
    # Add page URL to index of page data. 
    df.index = [url] * len(df) 
 
    # Append page data to full data. 
    df_full = df_full.append(df) 
 
print(df_full)