Natural language processing : bag of words

[SamVanhoutte]

Provide support for bag of words translation of documents
Bag of words is a method used to convert to convert unstructured text to vector values. This is commonly used in NLP. An explanation of the concept can be found here.

import scipy as sp


def GetBagOfWords(train_set, test_set, text_property_name, additional_feature_names=None):
    # Omzetten naar bag-of-words
    train_text_values = train_set[text_property_name]
    test_text_values = test_set[text_property_name]

    count_vectorizer = CountVectorizer(binary=False)
    count_vectorizer.fit(train_text_values)
    train_set_bow = count_vectorizer.transform(train_text_values)
    test_set_bow = count_vectorizer.transform(test_text_values)

    transformer = TfidfTransformer()
    transformer = TfidfTransformer(use_idf=True).fit(train_set_bow)
    train_set_bow = transformer.transform(train_set_bow)
    test_set_bow = transformer.transform(test_set_bow)

    if(additional_feature_names):
        train_set_bow = sp.sparse.hstack((train_set_bow,
                                          train_set[additional_feature_names].values),
                                         format='csr').toarray()
        test_set_bow = sp.sparse.hstack((test_set_bow,
                                         test_set[additional_feature_names].values),
                                        format='csr')

    return (train_set_bow, test_set_bow)