Bag of words

Author: André Almada

bag-of-words.py

@@ -8,9 +8,11 @@
 from nltk.tokenize import word_tokenize
 from nltk.corpus import stopwords
 from nltk.stem.porter import PorterStemmer
+from sklearn.feature_extraction.text import CountVectorizer
 
 path = './txt/'
 stemmer = PorterStemmer()
+cv = CountVectorizer(max_features=1000, encoding='latin1')
 
 def get_class (filename):
     return filename.split('-')[0].split('\\')[1]
@@ -29,14 +31,11 @@ def get_data (filename):
         for i in range(len(data)):
             data[i] = stemmer.stem(data[i])
 
-        return data
+        plain_text = " ".join(data)
+        return plain_text
 
 names = [f for f in glob.glob(os.path.join(path, '*.txt'))]
 
-dataset = pd.DataFrame({'journal' : [get_class(f) for f in names], 'data' : [get_data(f) for f in names]})
-
-# Tira caracteres não alfabéticos e deixa o texto inteiro na minúscula  
-#dataset.data = dataset.data.map(lambda x: re.sub('[^A-Za-z]', ' ', x).lower())
-
-#dataset.data = pre_processing (dataset.data)
+dataset = pd.DataFrame({'themes' : [get_class(f) for f in names], 'data' : [get_data(f) for f in names]})
 
+bag_of_words = cv.fit_transform(dataset.data).toarray()