Skip to main content
 首页 » 编程设计

python之如何使用管道和 FeatureUnion 添加功能

2024年11月24日7开发

在下面的代码中,我使用高音扬声器数据集来执行情感分析。我使用执行以下过程的管道:

1) 执行一些基本的文本预处理

2) 向量化推文文本

3) 添加一个额外的特征(文本长度)

4)分类

我想再添加一项功能,即缩放的关注者数量。我写了一个函数,它将整个数据框 (df) 作为输入,并返回一个新的数据框,其中包含按比例缩放的关注者数量。但是,我发现在管道中添加此过程具有挑战性,例如使用 sklearn 管道将此功能添加到其他功能。

任何有关此问题的帮助或建议将不胜感激。

下面的问题和代码的灵感来自于 Ryan 的帖子:pipelines

 
import nltk 
import re 
from sklearn.feature_extraction.text import CountVectorizer 
from sklearn.linear_model import LogisticRegression 
from sklearn.model_selection import train_test_split 
from sklearn.preprocessing import MinMaxScaler 
 
def import_data(filename,sep,eng,header = None,skiprows=1): 
    #read csv 
    dataset = pd.read_csv(filename,sep=sep,engine=eng,header = header,skiprows=skiprows) 
    #rename columns 
    dataset.columns = ['text','followers','sentiment'] 
    return dataset 
 
df = import_data('apple_v3.txt','\t','python') 
X, y = df.text, df.sentiment 
X_train, X_test, y_train, y_test = train_test_split(X, y) 
 
tokenizer = nltk.casual.TweetTokenizer(preserve_case=False, reduce_len=True) 
count_vect = CountVectorizer(tokenizer=tokenizer.tokenize)  
classifier = LogisticRegression() 
 
def get_scalled_followers(df): 
    scaler = MinMaxScaler() 
    df[['followers']] = df[['followers']].astype(float) 
    df[['followers']] = scaler.fit_transform(df[['followers']]) 
    followers = df['followers'].values 
    followers_reshaped = followers.reshape((len(followers),1)) 
    return df 
 
def get_tweet_length(text): 
    return len(text) 
import numpy as np 
 
def genericize_mentions(text): 
    return re.sub(r'@[\w_-]+', 'thisisanatmention', text) 
 
def reshape_a_feature_column(series): 
    return np.reshape(np.asarray(series), (len(series), 1)) 
 
def pipelinize_feature(function, active=True): 
    def list_comprehend_a_function(list_or_series, active=True): 
        if active: 
            processed = [function(i) for i in list_or_series] 
            processed = reshape_a_feature_column(processed) 
            return processed 
 
        else: 
            return reshape_a_feature_column(np.zeros(len(list_or_series))) 
 
from sklearn.pipeline import FeatureUnion, Pipeline 
from sklearn_helpers import pipelinize, genericize_mentions, train_test_and_evaluate 
 
 
sentiment_pipeline = Pipeline([ 
        ('genericize_mentions', pipelinize(genericize_mentions, active=True)), 
        ('features', FeatureUnion([ 
                    ('vectorizer', count_vect), 
                    ('post_length', pipelinize_feature(get_tweet_length, active=True)) 
                ])), 
        ('classifier', classifier) 
    ]) 
 
sentiment_pipeline, confusion_matrix = train_test_and_evaluate(sentiment_pipeline, X_train, y_train, X_test, y_test) 
 

请您参考如下方法:

到目前为止,我找到的最佳解释是在以下帖子中:pipelines

我的数据包含异构特征,以下分步方法效果很好且易于理解:

from sklearn.base import BaseEstimator, TransformerMixin 
from sklearn.pipeline import Pipeline, FeatureUnion 
 
#step1 - select data from dataframe and split the dataset in train and test sets 
 
features= [c for c in df.columns.values if c  not in ['sentiment']] 
numeric_features= [c for c in df.columns.values if c  not in ['text','sentiment']] 
target = 'sentiment' 
 
X_train, X_test, y_train, y_test = train_test_split(df[features], df[target], test_size=0.33, random_state=42) 
 
#step2 - create a number selector class and text selector class. These classes allow to select specific columns from the dataframe 
 
class NumberSelector(BaseEstimator, TransformerMixin): 
 
    def __init__(self, key): 
        self.key = key 
 
    def fit(self, X, y=None): 
        return self 
 
    def transform(self, X): 
        return X[[self.key]] 
 
class TextSelector(BaseEstimator, TransformerMixin): 
 
    def __init__(self, key): 
        self.key = key 
 
    def fit(self, X, y=None): 
        return self 
 
    def transform(self, X): 
        return X[self.key] 
 
#step 3 create one pipeline for the text data and one for the numerical data 
 
 
text = Pipeline([ 
                ('selector', TextSelector(key='content')), 
                ('tfidf', TfidfVectorizer( stop_words='english')) 
            ]) 
 
text.fit_transform(X_train) 
 
followers =  Pipeline([ 
                ('selector', NumberSelector(key='followers')), 
                ('standard', MinMaxScaler()) 
            ]) 
 
followers.fit_transform(X_train) 
 
#step 4 - features union 
 
feats = FeatureUnion([('text', text),  
                      ('length', followers)]) 
 
feature_processing = Pipeline([('feats', feats)]) 
feature_processing.fit_transform(X_train) 
 
# step 5 - add the classifier and predict  
 
pipeline = Pipeline([ 
    ('features',feats), 
    ('classifier', SVC(kernel = 'linear', probability=True, C=1, class_weight = 'balanced')) 
]) 
 
pipeline.fit(X_train, y_train) 
 
preds = pipeline.predict(X_test) 
np.mean(preds == y_test) 
 
# step 6 use the model to predict new data not included in the test set 
# in my example the pipeline expects a dataframe as an input which should have a column called 'text' and a column called 'followers' 
 
array = [["@apple is amazing",25000]] 
dfObj = pd.DataFrame(array,columns = ['text' , 'followers'])  
 
#prints the expected class e.g. positive or negative sentiment 
print(pipeline.predict(dfObj)) 
 
#print the probability for each class 
print(pipeline.predict_proba(dfObj))