Skip to main content
 首页 » 编程设计

python之在 Python 中快速计算频率

2024年10月25日8cloudgamer

我需要计算语料库中单词的出现频率。通常我使用 collections 包中的 Counter 类。

from collections import Counter 
list_of_words = ['one', 'two', 'three', 'three'] 
freqs = Counter(list_of_words) 

但是,我正在分析的语料库由几百万个单词组成,所以如果有更快的方法来计算这些分数会很好吗?

下面是读入单词的代码:

from read_cg3 import read_cg3 
 
test = read_cg3('/Users/arashsaidi/Work/Corpus/DUO_Corpus/Bokmaal-tagged-random/DUO_BM_0.txt') 
count = 0 
word_list = [] 
for sentence in test: 
    for word in sentence: 
        count += 1 
        word_list.append(word) 
print count 

read_cg3 是一个读取已解析文件并返回句子列表的模块。这是模块:

import re 
 
 
def is_number(s): 
    try: 
        float(s) 
        return True 
    except ValueError: 
        return False 
 
 
def read_cg3(cg3_file): 
    """ 
    Reads a cg3 file and returns a list of each sentence with Token, parsed, and one tag 
    :param cg3_file: path to file 
    :return: list of words + attributes 
    """ 
    rx_token = re.compile("^\"<(.+?)>\"$") 
    rx_attributes = re.compile("^\s+\".+?\"\s+.+$") 
    rx_eos = re.compile("^\s*$") 
 
    curr_token = None 
    curr_word = [] 
    curr_sentence = [] 
    result = [] 
 
    with open(cg3_file) as cg3_file: 
        for line in cg3_file: 
 
            if rx_token.match(line): 
                curr_token = "\"%s\"" % rx_token.match(line).group(1) 
                # print curr_token 
 
            if rx_attributes.match(line): 
                curr_word = line.split() 
                # print curr_word[0], curr_word[1] 
                # print curr_word 
                if curr_token and curr_word: 
                    # to get more tags uncomment this and comment below 
                    # curr_sentence += [[curr_token] + curr_word] 
                    if '$' not in curr_word[0] and not is_number(curr_word[0].strip('"').replace('.', '')) \ 
                            and len(curr_word[0]) < 30: 
                        # curr_sentence += [[curr_token.strip('"')] + 
                        # [curr_word[0].lower().strip('"')] + [curr_word[1]]] 
                        curr_sentence += [curr_word[0].lower().strip('"')] 
                    curr_token = None 
                    curr_word = [] 
 
            if rx_eos.match(line): 
                # print curr_sentence 
                if curr_sentence: 
                    result += [curr_sentence] 
                curr_sentence = [] 
                curr_token = None 
                curr_word = [] 
 
    # cleanup if last sentence not EOL 
    if curr_token and curr_word: 
        print 'cg3 reached end of file and did some cleanup on file {}'.format(cg3_file) 
        curr_sentence += [[curr_token] + curr_word] 
 
    if curr_sentence: 
        print 'cg3 reached end of file and did some cleanup on file {}'.format(cg3_file) 
        result += curr_sentence 
 
    return result 

这是 read_cg3 读取文件的方式:

"<TEKNOLOGI>" 
    "teknologi" subst appell mask ub ent  
"<OG>" 
    "og" konj <*>  
"<UNDERVISNING>" 
    "undervisning" subst appell fem ub ent <*>  
"<|>" 
    "$|" clb <overskrift> <<<  
 
"<En>" 
    "en" det mask ent kvant  
"<intervjuunders¯kelse>" 
    "intervjuunders¯kelse" subst appell mask ub ent  
"<av>" 
    "av" prep  
"<musikklÊreres>" 
    "musikklÊrer" subst appell mask ub fl gen  
"<didaktiske>" 
    "didaktisk" adj fl pos  
"<bruk>" 
    "bruk" subst appell mask ub ent  
"<av>" 
    "av" prep  
"<digitale>" 
    "digital" adj fl pos  
"<verkt¯y>" 
    "verkt¯y" subst appell n¯yt ub fl <*¯y>  
"<i>" 
    "i" prep  
"<undervisningsfaget>" 
    "undervisningsfag" subst appell n¯yt be ent  
"<komposisjon>" 
    "komposisjon" subst appell mask ub ent  
"<i>" 
    "i" prep  
"<videregÂende>" 
    "videregÂende" adj ub m/f ent pos  
"<skole>" 
    "skole" subst appell mask ub ent  
"<|>" 
    "$|" clb <overskrift> <<<  
 
"<Markus>" 
    "Markus" subst prop mask  
"<A.>" 
    "A." subst prop fork <*>  
"<SkjÊrstad>" 
    "SkjÊrstad" subst prop <*stad> <*>  
"<|>" 
    "$|" clb <overskrift> <<<  

我的方法只读入一个文件,这是为了测试,语料库由大约30000个文件组成。

请您参考如下方法:

看起来您不需要使用标记,并且您的正则表达式可以取消。这将计算每个单词在每个文件中出现的次数:

import multiprocessing as mp 
import os 
import itertools 
 
def wordCounter(qIn, qOut): 
    answer = {} 
    for fname, words in iter(qIn.get, None): 
        for word in words: 
            if fname not in answer: 
                answer[fname] = {} 
            if word not in answer[fname]: 
                answer[fname][word] = 0  
            answer[fname][word] += 1 
    qOut.put(answer) 
 
 
def getLines(corpusPath, qIn, numProcs): 
    for fname in os.listdir(corpusPath): 
        with open(os.path.join(corpusPath, fname)) as infile: 
            for i, (k,lines) in enumerate(itertools.groupby((l.strip() for l in infile), lambda line : bool(line) and not line.startswith('"<') and "$" not in line.split(None,1)[0])): 
                if not k: 
                    continue 
                qIn.put((fname, [line.split(None,1)[0].strip('"').strip().lower() for line in lines])) 
 
    for _ in range(numProcs): 
        qIn.put(None) 
 
 
def main(corpusPath): 
    qIn, qOut = [mp.Queue() for _ in range(2)] 
    procs = [mp.Process(target=wordCounter, args=(qIn, qOut)) for _ in range(mp.cpu_count() -1)] 
 
    lineGetter = mp.Process(target=getLines, args=(corpusPath, qIn, len(procs))) 
    lineGetter.start() 
 
    for p in procs: 
        p.start() 
 
    answer = {} 
    for _ in range(len(procs)): 
        for fname, wdict in qOut.get().items(): 
            if fname not in answer: 
                answer[fname] = {} 
            for word,count in wdict.items(): 
                if word not in answer[fname]: 
                    answer[fname][word] = 0  
                answer[fname][word] += count 
 
    for fname in sorted(answer): 
        for word in sorted(answer[fname]): 
            print("{} appeared in {} {} times".format(word, fname, answer[fname][word])) 
 
    for p in procs: 
        p.terminate() 
    lineGetter.terminate() 

整个过程只用了不到一秒钟的时间来处理您的测试文件。请注意,其中一些是由于设置子进程的开销造成的,因此这应该可以更好地扩展到大型语料库。

希望对你有帮助