Skip to main content
 首页 » 编程设计

python之将 QuadgramCollationFinder 转换为 PentagramCollationFinder

2025年05月04日14bluestorm

我尝试将 NLTK 工具包中的 QuadgramCollat​​ionFinde 类转换为 PentagramCollocationFinder one,即使用 5 个项目,不仅是 4 个项目,而且确实有效。

这是nltk类

import nltk 
from nltk.compat import iteritems 
from nltk.probability import FreqDist 
from nltk.util import ngrams 
from nltk.metrics import ContingencyMeasures, BigramAssocMeasures,TrigramAssocMeasures 
from nltk.metrics.spearman import ranks_from_scores, spearman_correlation 
from __future__ import print_function 
 
class QuadgramCollocationFinder(AbstractCollocationFinder): 
    """A tool for the finding and ranking of quadgram collocations or other association measures. 
    It is often useful to use from_words() rather than constructing an instance directly. 
    """ 
    default_ws = 4 
 
    def __init__(self, word_fd, quadgram_fd, ii, iii, ixi, ixxi, iixi, ixii): 
        """Construct a QuadgramCollocationFinder, given FreqDists for appearances of words, 
        bigrams, trigrams, two words with one word and two words between them, three words 
        with a word between them in both variations. 
        """ 
        AbstractCollocationFinder.__init__(self, word_fd, quadgram_fd) 
        self.iii = iii 
        self.ii = ii 
        self.ixi = ixi 
        self.ixxi = ixxi 
        self.iixi = iixi 
        self.ixii = ixii 
 
 
    def from_words(cls, words, window_size=4): 
        if window_size < 4: 
            raise ValueError("Specify window_size at least 4") 
        ixxx = FreqDist() 
        iiii = FreqDist() 
        ii = FreqDist() 
        iii = FreqDist() 
        ixi = FreqDist() 
        ixxi = FreqDist() 
        iixi = FreqDist() 
        ixii = FreqDist() 
 
        for window in ngrams(words, window_size, pad_right=True): 
            w1 = window[0] 
            if w1 is None:  
                continue 
            for w2, w3, w4 in _itertools.combinations(window[1:], 3): 
                ixxx[w1] += 1 
                if w2 is None: 
                    continue 
                ii[(w1, w2)] += 1 
                if w3 is None: 
                    continue 
                iii[(w1, w2, w3)] += 1 
                ixi[(w1, w3)] += 1 
                if w4 is None: 
                    continue 
                iiii[(w1, w2, w3, w4)] += 1 
                ixxi[(w1, w4)] += 1 
                ixii[(w1, w3, w4)] += 1 
                iixi[(w1, w2, w4)] += 1 
 
        return cls(ixxx, iiii, ii, iii, ixi, ixxi, iixi, ixii) 
 
 
    def score_ngram(self, score_fn, w1, w2, w3, w4): 
        n_all = self.word_fd.N() 
        n_iiii = self.ngram_fd[(w1, w2, w3, w4)] 
        if not n_iiii: 
            return 
        n_iiix = self.iii[(w1, w2, w3)] 
        n_xiii = self.iii[(w2, w3, w4)] 
        n_iixi = self.iixi[(w1, w2, w4)] 
        n_ixii = self.ixii[(w1, w3, w4)] 
 
        n_iixx = self.ii[(w1, w2)] 
        n_xxii = self.ii[(w3, w4)] 
        n_xiix = self.ii[(w2, w3)] 
        n_ixix = self.ixi[(w1, w3)] 
        n_ixxi = self.ixxi[(w1, w4)] 
        n_xixi = self.ixi[(w2, w4)] 
 
        n_ixxx = self.word_fd[w1] 
        n_xixx = self.word_fd[w2] 
        n_xxix = self.word_fd[w3] 
        n_xxxi = self.word_fd[w4] 
        return score_fn(n_iiii, 
                        (n_iiix, n_iixi, n_ixii, n_xiii), 
                        (n_iixx, n_ixix, n_ixxi, n_xixi, n_xxii, n_xiix), 
                        (n_ixxx, n_xixx, n_xxix, n_xxxi), 
                        n_all) 

这就是我的想法

class fivegramCollocationFinder(AbstractCollocationFinder): 
"""A tool for the finding and ranking of quadgram collocations or other association measures. 
It is often useful to use from_words() rather than constructing an instance directly. 
""" 
default_ws = 4 
 
def __init__(self, word_fd, quingram_fd, ii, iii, ixi,iiii,ixxi, ixii, iixi, ixxxi, iixxi, ixixi, ixxii, iiixi, ixiii, iixii): 
    """Construct a QuadgramCollocationFinder, given FreqDists for appearances of words, 
    bigrams, trigrams, two words with one word and two words between them, three words 
    with a word between them in both variations. 
    """ 
    AbstractCollocationFinder.__init__(self, word_fd, quingram_fd) 
    self.iiii = iiii 
    self.iii = iii 
    self.ii = ii 
    self.ixi = ixi 
    self.ixxi = ixxi 
    self.iixi = iixi 
    self.ixii = ixii 
    self.ixxxi = ixxxi 
    self.iixxi = iixxi 
    self.ixixi = ixixi 
    self.ixxii = ixxii 
    self.iiixi = iiixi 
    self.ixiii = ixiii 
    self.iixii = iixii 
 
@classmethod 
 
def from_words(cls, words, window_size=5): 
    if window_size < 5: 
        raise ValueError("Specify window_size at least 4") 
    ixxxx = FreqDist() 
    ii =  FreqDist() 
    iii = FreqDist() 
    iiii = FreqDist() 
    iiiii = FreqDist() 
    ixi = FreqDist() 
    ixxi = FreqDist() 
    iixi = FreqDist() 
    ixii = FreqDist() 
    ixxxi = FreqDist() 
    iixxi  = FreqDist() 
    ixixi  = FreqDist() 
    ixxii = FreqDist() 
    iiixi = FreqDist() 
    ixiii = FreqDist() 
    iixii   = FreqDist() 
 
 
    for window in ngrams(words, window_size, pad_right=True): 
        w1 = window[0] 
        if w1 is None:  
            continue 
        for w2, w3, w4, w5 in _itertools.combinations(window[1:], 4): 
            ixxxx[w1] += 1 
            if w2 is None: 
                continue 
            ii[(w1, w2)] += 1 
            if w3 is None: 
                continue 
            iii[(w1, w2, w3)] += 1 
            ixi[(w1, w3)] += 1 
            if w4 is None: 
                continue 
            iiii[(w1, w2, w3, w4)] += 1 
            ixxi[(w1, w4)] += 1 
            ixii[(w1, w3, w4)] += 1 
            iixi[(w1, w2, w4)] += 1 
            if w5 is None: 
                continue 
            iiiii[(w1, w2, w3, w4, w5)] += 1 
            ixxxi[(w1, w5)] += 1 
            iixxi[(w1, w2, w5)] += 1 
            ixixi[(w1, w3, w5)] += 1 
            ixxii[(w1, w4, w5)] += 1 
            iiixi[(w1, w2, w3, w5)] +=1 
            ixiii[(w1, w3, w4, w5)] +=1 
            iixii[(w1, w2, w4, w5)] +=1 
 
 
    return cls(ixxxx, ii,iii, ixi, ixxi, iixi, ixii,iiii, iiiii, ixxxi, iixxi, ixixi, ixxii, iiixi, ixiii, iixii) 
 
 
def score_ngram(self, score_fn, w1, w2, w3, w4, w5): 
    n_all = self.word_fd.N() 
    n_iiiii = self.ngram_fd[(w1, w2, w3, w4, w5)] 
    if not n_iiiii: 
        return 
 
 
    n_iiixi = self.iiixi[(w1, w2, w3, w5)] 
    n_ixiii = self.ixiii[(w1, w3, w4, w5)] 
    n_iixii = self.iixii[(w1, w2, w4, w5)] 
    n_iiiix = self.iiii[(w1,w2,w3,w4)] 
    n_xiiii = self.iiii[(w2,w3,w4,w5)] 
 
    n_iixix = self.iixix[(w1, w2, w4)] 
    n_iixxi = self.iixxi[(w1, w2, w5)] 
    n_ixixi = self.ixixi[(w1, w3, w5)] 
    n_ixxii = self.ixxii[(w1, w4, w5)] 
    n_xiixi = self.xiixi[(w2, w3, w5)] 
    n_xixii = self.xixii[(w2, w4, w5)] 
    n_ixiix = self.ixiix[(w1, w3, w4)] 
    n_iiixx = self.iii[(w1, w2, w3)] 
    n_xiiix = self.iii[(w2, w3, w4)] 
    n_xxiii = self.iii[(w3, w4, w5)] 
 
 
 
 
    n_ixixx = self.ixixx[(w1, w3)] 
    n_ixxix = self.ixxix[(w1, w4)] 
    n_ixxxi = self.ixxxi[(w1, w5)] 
    n_xixix = self.xixix[(w2, w4)] 
    n_xixxi = self.xixxi[(w2, w5)] 
    n_xxixi = self.xxixi[(w2, w5)] 
    n_iixxx = self.ii[(w1, w2)] 
    n_xiixx = self.ii[(w2, w3)] 
    n_xxiix= self.ii[(w3, w4)] 
    n_xxxii= self.ii[(w4, w5)] 
 
 
    n_ixxxx = self.word_fd[w1] 
    n_xixxx = self.word_fd[w2] 
    n_xxixx = self.word_fd[w3] 
    n_xxxix = self.word_fd[w4] 
    n_xxxxi = self.word_fd[w5] 
 
    return score_fn(n_iiiii, 
                    (n_iiixi, n_ixiii, n_iixii, n_iiiix, n_xiiii),  
                    (n_iixix,n_iixxi, n_ixixi, n_ixxii, n_xiixi, n_xixii,n_ixiix, n_iiixx, n_xiiix, n_xxiii), 
                    (n_ixixx, n_ixxix,n_ixxxi, n_xixix, n_xixxi, n_xxixi, n_iixxx, n_xiixx,n_xxiix,n_xxxii), 
                    (n_ixxxx,n_xixxx, n_xxixx, n_xxxix, n_xxxxi), 
                    n_all) 

因此需要更新类以检测 PentagramCollocationFinder

有人可以帮忙吗?

请您参考如下方法:

构建模式似乎有些问题,所以这里有一些代码可以构建所有合法的 i-patterns,以及要使用的 n-patterns。

import collections 
 
def make_ngram_ipatterns(n): 
    """Make all needed patterns used by *gramCollocationFinder up to n words""" 
 
    i_patterns = [] 
 
    for i in xrange(1, n+1): 
        if i <= 2: 
            i_patterns.append('i' * i) 
 
        else: 
            for j in xrange(2**(i-2)): 
                 bin_str = '{0:0{1}b}'.format(j, i-2) 
                 ix_pattern = bin_str.replace('0', 'x').replace('1', 'i') 
                 i_patterns.append('i{}i'.format(ix_pattern)) 
 
    return i_patterns 
 
def make_ngram_npatterns(n): 
    """Make all needed n-patterings used by *gramCollocationFinder up to n words""" 
    all_ipatterns = make_ngram_ipatterns(n) 
 
    npatterns = [] 
 
    for ipattern in all_ipatterns: 
         i_order = sum(c == 'i' for c in ipattern) 
         i_length = len(ipattern) 
         for j in xrange(n - i_length+1): 
             npattern = 'n_{}{}{}'.format('x'* j, 
                                           ipattern , 
                                           'x'* (n - i_length - j)) 
 
             npatterns.append((i_order, ipattern, npattern)) 
 
    return sorted(npatterns) 
 
 
def main(): 
 
    n = 5 
 
    all_ipatterns = make_ngram_ipatterns(n) 
 
    print '\n'.join(make_ngram_ipatterns(n)) 
 
    for order, ipattern, npattern in make_ngram_npatterns(n): 
         wparams = ', '.join('w{}'.format(i+1) 
                                for i, c in enumerate(npattern[2:]) 
                                if c == 'i' 
                            ) 
         print('order: {1:2}   ipattern: {2:{0}s}   npattern: {3}' 
               ' ->  {3} = self.{2}({4})'.format( 
                   n, order, ipattern, npattern, wparams)) 
 
 
if __name__ == '__main__': 
    main() 

n=5 的输出是:

i 
ii 
ixi 
iii 
ixxi 
ixii 
iixi 
iiii 
ixxxi 
ixxii 
ixixi 
ixiii 
iixxi 
iixii 
iiixi 
iiiii 
order:  1   ipattern: i       npattern: n_ixxxx ->  n_ixxxx = self.i(w1) 
order:  1   ipattern: i       npattern: n_xixxx ->  n_xixxx = self.i(w2) 
order:  1   ipattern: i       npattern: n_xxixx ->  n_xxixx = self.i(w3) 
order:  1   ipattern: i       npattern: n_xxxix ->  n_xxxix = self.i(w4) 
order:  1   ipattern: i       npattern: n_xxxxi ->  n_xxxxi = self.i(w5) 
order:  2   ipattern: ii      npattern: n_iixxx ->  n_iixxx = self.ii(w1, w2) 
order:  2   ipattern: ii      npattern: n_xiixx ->  n_xiixx = self.ii(w2, w3) 
order:  2   ipattern: ii      npattern: n_xxiix ->  n_xxiix = self.ii(w3, w4) 
order:  2   ipattern: ii      npattern: n_xxxii ->  n_xxxii = self.ii(w4, w5) 
order:  2   ipattern: ixi     npattern: n_ixixx ->  n_ixixx = self.ixi(w1, w3) 
order:  2   ipattern: ixi     npattern: n_xixix ->  n_xixix = self.ixi(w2, w4) 
order:  2   ipattern: ixi     npattern: n_xxixi ->  n_xxixi = self.ixi(w3, w5) 
order:  2   ipattern: ixxi    npattern: n_ixxix ->  n_ixxix = self.ixxi(w1, w4) 
order:  2   ipattern: ixxi    npattern: n_xixxi ->  n_xixxi = self.ixxi(w2, w5) 
order:  2   ipattern: ixxxi   npattern: n_ixxxi ->  n_ixxxi = self.ixxxi(w1, w5) 
order:  3   ipattern: iii     npattern: n_iiixx ->  n_iiixx = self.iii(w1, w2, w3) 
order:  3   ipattern: iii     npattern: n_xiiix ->  n_xiiix = self.iii(w2, w3, w4) 
order:  3   ipattern: iii     npattern: n_xxiii ->  n_xxiii = self.iii(w3, w4, w5) 
order:  3   ipattern: iixi    npattern: n_iixix ->  n_iixix = self.iixi(w1, w2, w4) 
order:  3   ipattern: iixi    npattern: n_xiixi ->  n_xiixi = self.iixi(w2, w3, w5) 
order:  3   ipattern: iixxi   npattern: n_iixxi ->  n_iixxi = self.iixxi(w1, w2, w5) 
order:  3   ipattern: ixii    npattern: n_ixiix ->  n_ixiix = self.ixii(w1, w3, w4) 
order:  3   ipattern: ixii    npattern: n_xixii ->  n_xixii = self.ixii(w2, w4, w5) 
order:  3   ipattern: ixixi   npattern: n_ixixi ->  n_ixixi = self.ixixi(w1, w3, w5) 
order:  3   ipattern: ixxii   npattern: n_ixxii ->  n_ixxii = self.ixxii(w1, w4, w5) 
order:  4   ipattern: iiii    npattern: n_iiiix ->  n_iiiix = self.iiii(w1, w2, w3, w4) 
order:  4   ipattern: iiii    npattern: n_xiiii ->  n_xiiii = self.iiii(w2, w3, w4, w5) 
order:  4   ipattern: iiixi   npattern: n_iiixi ->  n_iiixi = self.iiixi(w1, w2, w3, w5) 
order:  4   ipattern: iixii   npattern: n_iixii ->  n_iixii = self.iixii(w1, w2, w4, w5) 
order:  4   ipattern: ixiii   npattern: n_ixiii ->  n_ixiii = self.ixiii(w1, w3, w4, w5) 
order:  5   ipattern: iiiii   npattern: n_iiiii ->  n_iiiii = self.iiiii(w1, w2, w3, w4, w5) 

改变到一个新的维度现在是使用和设置所有 i-patterns 为低阶类,替换 n-patterns,并将所有相同顺序的 n-patterns 整理到 score_fn() 集。

编辑:使用适当的 w#

完成了 n 模式的设置