Combinaison de w2vec et de la sélection de fonctionnalités dans le pipeline

Question

Basé sur cet article: http://nadbordrozd.github.io/blog/2016/05/20/text-classification-with-word2vec/ J'essaye d'implémenter un modèle de word2vec de gensim avec les vecteurs pré-chargés de GloVe dans une tâche de classification de texte. Cependant, je voudrais faire FeatureSelection également dans mes données de texte. J'ai essayé plusieurs séquences dans le pipeline mais je reçois rapidement une erreur de mémoire qui pointe vers la partie de transformation de TfidfEmbeddingVectorizer.

return np.array([ 
       np.mean([self.word2vec[w] * self.word2weight[w] 
         for w in words if w in self.word2vec] or 
         [np.zeros(self.dim)], axis=0) 
       for words in X 

Si je remplace la classe TfidfEmbeddingVectorizer avec une TfIdfVectorizer régulière, il fonctionne correctement. Existe-t-il un moyen de combiner SelectFromModel et W2vec dans le pipeline?

from sklearn.base import BaseEstimator, TransformerMixin 
import pandas as pd 
import numpy as np 
import itertools 
from sklearn.feature_extraction.text import TfidfVectorizer 
from sklearn.pipeline import Pipeline 
from sklearn.feature_extraction.text import TfidfTransformer 
from sklearn.feature_extraction.text import CountVectorizer 
from sklearn.metrics import precision_recall_fscore_support as score, f1_score 
import pickle 
from sklearn.externals import joblib 
from sklearn.ensemble import RandomForestClassifier 
from sklearn.feature_selection import SelectFromModel 
from sklearn.pipeline import FeatureUnion 
from sklearn.feature_extraction import DictVectorizer 
from sklearn.ensemble import ExtraTreesClassifier 
from sklearn.svm import LinearSVC 
import gensim 
import collections 

class ItemSelector(BaseEstimator, TransformerMixin): 

    def __init__(self, column): 
     self.column = column 

    def fit(self, X, y=None, **fit_params): 
     return self 

    def transform(self, X): 
     return (X[self.column]) 




class TextStats(BaseEstimator, TransformerMixin): 
    """Extract features from each document for DictVectorizer""" 

    def fit(self, x, y=None): 
     return self 

    def transform(self, posts): 
     return [{'REPORT_M': text} 
       for text in posts] 


class TfidfEmbeddingVectorizer(object): 
    def __init__(self, word2vec): 
    self.word2vec = word2vec 
    self.word2weight = None 
    self.dim = len(word2vec.values()) 

    def fit(self, X, y): 
    tfidf = TfidfVectorizer(analyzer=lambda x: x) 
    tfidf.fit(X) 
    # if a word was never seen - it must be at least as infrequent 
    # as any of the known words - so the default idf is the max of 
    # known idf's 
    max_idf = max(tfidf.idf_) 
    self.word2weight = collections.defaultdict(
     lambda: max_idf, 
     [(w, tfidf.idf_[i]) for w, i in tfidf.vocabulary_.items()]) 

    return self 

    def transform(self, X): 
    return np.array([ 
      np.mean([self.word2vec[w] * self.word2weight[w] 
        for w in words if w in self.word2vec] or 
        [np.zeros(self.dim)], axis=0) 
      for words in X 
     ]) 


# training model 
def train(data_train, data_val): 

    with open("glove.6B/glove.6B.50d.txt", "rb") as lines: 
     w2v = {line.split()[0]: np.array(map(float, line.split()[1:])) 
       for line in lines} 
    classifier = Pipeline([ 
        ('union', FeatureUnion([ 

          ('text', Pipeline([ 
           ('selector', ItemSelector(column='TEXT')), 
           ("word2vec vectorizer", TfidfEmbeddingVectorizer(w2v)), 
           ('feature_selection', SelectFromModel(LinearSVC(penalty="l1", dual=False),threshold=0.01)) 
          ])), 

          ('category', Pipeline([ 
           ('selector', ItemSelector(column='category')), 
           ('stats', TextStats()), 
           ('vect', DictVectorizer()) 
          ])) 
    ])), 
        ('clf',ExtraTreesClassifier(n_estimators=200, max_depth=500, min_samples_split=6, class_weight= 'balanced'))]) 

    classifier.fit(data_train,data_train.CLASSES) 
    predicted = classifier.predict(data_val) 

Answer 1

Je pense ici self.dim = len(word2vec.values()) vous devez spécifier la dimension du modèle. Si vous utilisez glove.6B.50d.txt, la cote doit être 50.

len(word2vec.values()) est le nombre total de mots, créant ainsi une matrice énorme, c'est-à-dire une erreur de mémoire.