fix some typo and docstring (alfredfrancis#88)

* utf8 language issue fixes
* pep8 fixes
* made word2vec support configurable via config.py

Author: hadifar

app/agents/controllers.py

@@ -26,7 +26,6 @@ def set_config(bot_name):
 def get_config(bot_name):
     """
     Update bot config
-    :param json:
     :return:
     """
     bot = Bot.objects.get(name=bot_name)

app/endpoint/controllers.py

@@ -8,7 +8,6 @@
 from app import app
 from app.agents.models import Bot
 from app.commons import build_response
-from app.commons.logger import logger
 from app.endpoint.utils import SilentUndefined
 from app.endpoint.utils import call_api
 from app.endpoint.utils import get_synonyms
@@ -57,8 +56,7 @@ def api():
 
     if request_json:
 
-        context = {}
-        context["context"] = request_json["context"]
+        context = {"context": request_json["context"]}
 
         if app.config["DEFAULT_WELCOME_INTENT_NAME"] in request_json.get(
                 "input"):
@@ -73,10 +71,10 @@ def api():
                 undefined=SilentUndefined)
             result_json["speechResponse"] = split_sentence(template.render(**context))
 
-            logger.info(request_json.get("input"), extra=result_json)
+            app.logger.info(request_json.get("input"), extra=result_json)
             return build_response.build_json(result_json)
 
-        intent_id, confidence, suggetions = predict(request_json.get("input"))
+        intent_id, confidence, suggestions = predict(request_json.get("input"))
         app.logger.info("intent_id => %s" % intent_id)
         intent = Intent.objects.get(intentId=intent_id)
 
@@ -145,9 +143,8 @@ def api():
 
                 if len(result_json["missingParameters"]) == 0:
                     result_json["complete"] = True
-                    context = {}
-                    context["parameters"] = result_json["extractedParameters"]
-                    context["context"] = request_json["context"]
+                    context = {"parameters": result_json["extractedParameters"],
+                               "context": request_json["context"]}
                 else:
                     missing_parameter = result_json["missingParameters"][0]
                     result_json["complete"] = False
@@ -194,7 +191,7 @@ def api():
                 template = Template(intent.speechResponse,
                                     undefined=SilentUndefined)
                 result_json["speechResponse"] = split_sentence(template.render(**context))
-        logger.info(request_json.get("input"), extra=result_json)
+        app.logger.info(request_json.get("input"), extra=result_json)
         return build_response.build_json(result_json)
     else:
         return abort(400)
@@ -211,15 +208,20 @@ def update_model(app, message, **extra):
     """
     global sentence_classifier
 
-    sentence_classifier = EmbeddingIntentClassifier.load(app.config["MODELS_DIR"])
+    sentence_classifier = EmbeddingIntentClassifier.load(
+        app.config["MODELS_DIR"], app.config["USE_WORD_VECTORS"])
+
     synonyms = get_synonyms()
+
     global entity_extraction
+
     entity_extraction = EntityExtractor(synonyms)
+
     app.logger.info("Intent Model updated")
 
 
 with app.app_context():
-    update_model(app, "Modles updated")
+    update_model(app, "Models updated")
 
 model_updated_signal.connect(update_model, app)
 

app/endpoint/utils.py

@@ -35,36 +35,25 @@ def call_api(url, type, headers={}, parameters={}, is_json=False):
     :param is_json:
     :return:
     """
-    app.logger.info("Initiating API Call with following info:"
-                    " url => {} payload => {}".format(url, parameters))
-
+    app.logger.info("Initiating API Call with following info: url => {} payload => {}".format(url, parameters))
     if "GET" in type:
-        response = requests.get(url, headers=headers,
-                                params=parameters, timeout=5)
+        response = requests.get(url, headers=headers, params=parameters, timeout=5)
     elif "POST" in type:
         if is_json:
-            response = requests.post(url, headers=headers,
-                                     json=parameters, timeout=5)
+            response = requests.post(url, headers=headers, json=parameters, timeout=5)
         else:
-            response = requests.post(url, headers=headers,
-                                     params=parameters, timeout=5)
+            response = requests.post(url, headers=headers, params=parameters, timeout=5)
     elif "PUT" in type:
         if is_json:
-            response = requests.put(url, headers=headers,
-                                    json=parameters, timeout=5)
+            response = requests.put(url, headers=headers, json=parameters, timeout=5)
         else:
-            response = requests.put(url, headers=headers,
-                                    params=parameters, timeout=5)
+            response = requests.put(url, headers=headers, params=parameters, timeout=5)
     elif "DELETE" in type:
-        response = requests.delete(url, headers=headers,
-                                   params=parameters, timeout=5)
+        response = requests.delete(url, headers=headers, params=parameters, timeout=5)
     else:
         raise Exception("unsupported request method.")
-
     result = json.loads(response.text)
-
     app.logger.info("API response => %s", result)
-
     return result
 
 

app/entities/controllers.py

@@ -14,7 +14,6 @@
 def create_entity():
     """
     Create a story from the provided json
-    :param json:
     :return:
     """
     content = request.get_json(silent=True)
@@ -60,8 +59,7 @@ def read_entity(id):
 def update_entity(id):
     """
     Update a story from the provided json
-    :param intent_id:
-    :param json:
+    :param id:
     :return:
     """
     json_data = loads(request.get_data())

app/intents/controllers.py

@@ -70,8 +70,8 @@ def read_intents():
     find list of intents for the agent
     :return:
     """
-    intents = Intent.objects
-    return build_response.sent_json(intents.to_json())
+    all_intents = Intent.objects
+    return build_response.sent_json(all_intents.to_json())
 
 
 @intents.route('/<id>')
@@ -83,8 +83,7 @@ def read_intent(id):
     """
     return Response(response=dumps(
         Intent.objects.get(
-            id=ObjectId(
-                id)).to_mongo().to_dict()),
+            id=ObjectId(id)).to_mongo().to_dict()),
         status=200,
         mimetype="application/json")
 
@@ -93,8 +92,6 @@ def read_intent(id):
 def update_intent(id):
     """
     Update a story from the provided json
-    :param intent_id:
-    :param json:
     :return:
     """
     json_data = loads(request.get_data())
@@ -118,7 +115,7 @@ def delete_intent(id):
     except BaseException:
         pass
 
-    # remove NER model for the deleted stoy
+    # remove NER model for the deleted story
     try:
         os.remove("{}/{}.model".format(app.config["MODELS_DIR"], id))
     except OSError:
@@ -154,18 +151,18 @@ def import_intents():
     if 'file' not in request.files:
         abort(400, 'No file part')
     json_file = request.files['file']
-    intents = import_json(json_file)
+    all_intents = import_json(json_file)
 
-    return build_response.build_json({"num_intents_created": len(intents)})
+    return build_response.build_json({"num_intents_created": len(all_intents)})
 
 
 def import_json(json_file):
     json_data = json_file.read()
     # intents = Intent.objects.from_json(json_data)
-    intents = loads(json_data)
+    all_intents = loads(json_data)
 
     creates_intents = []
-    for intent in intents:
+    for intent in all_intents:
         new_intent = Intent()
         new_intent = update_document(new_intent, intent)
         new_intent.save()

app/nlu/classifiers/sklearn_intent_classifer.py

@@ -6,6 +6,7 @@
 from nltk.corpus import stopwords
 from sklearn.feature_extraction.stop_words import ENGLISH_STOP_WORDS
 from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.model_selection import GridSearchCV
 from sklearn.pipeline import Pipeline
 from sklearn.svm import SVC
 
@@ -76,8 +77,6 @@ def build(X, y=None):
                             probability=True, class_weight='balanced')
                  )])
 
-            from sklearn.model_selection import GridSearchCV
-
             items, counts = np.unique(y, return_counts=True)
 
             cv_splits = max(2, min(5, np.min(counts) // 5))
@@ -108,7 +107,7 @@ def build(X, y=None):
 
     def load(self, PATH):
         """
-        load trained model froom given path
+        load trained model from given path
         :param PATH:
         :return:
         """
@@ -121,9 +120,6 @@ def load(self, PATH):
     def predict(self, text, return_all=False, INTENT_RANKING_LENGTH=5):
         """
         Predict class label for given model
-        :param text:
-        :param PATH:
-        :return:
         """
         return self.process(text, return_all, INTENT_RANKING_LENGTH)
 
@@ -135,8 +131,6 @@ def predict_proba(self, X):
         :return: tuple of first, the most probable label
         and second, its probability"""
 
-        import numpy as np
-
         pred_result = self.model.predict_proba(X)
         print(pred_result)
         # sort the probabilities retrieving the indices of the elements

app/nlu/classifiers/starspace_intent_classifier.py

@@ -7,12 +7,14 @@
 
 import io
 import os
+import re
 
 import cloudpickle as pickle
 import numpy as np
 import spacy
 import tensorflow as tf
 from flask import current_app as app
+from sklearn.feature_extraction.text import CountVectorizer
 
 
 class EmbeddingIntentClassifier:
@@ -197,12 +199,16 @@ def _create_intent_dict(training_data):
     def _create_intent_token_dict(intents, intent_split_symbol):
         """Create intent token dictionary"""
 
-        distinct_tokens = set([token
-                               for intent in intents
-                               for token in intent.split(intent_split_symbol)
-                               ])
+        distinct_tokens = set()
+
+        for intent in intents:
+            for token in intent.split(intent_split_symbol):
+                distinct_tokens.add(token)
+
+        distinct_tokens = sorted(distinct_tokens)
+
         return {token: idx for idx, token in
-                enumerate(sorted(distinct_tokens))}
+                enumerate(distinct_tokens)}
 
     def _create_encoded_intents(self, intent_dict):
         """Create matrix with intents encoded in rows as bag of words,
@@ -408,9 +414,6 @@ def _lemmatize(self, message):
 
     def prepare_training_data(self, X, y):
 
-        from sklearn.feature_extraction.text import CountVectorizer
-        import re
-
         training_data = {
             "intent_examples": []
         }
@@ -420,8 +423,7 @@ def prepare_training_data(self, X, y):
             token_pattern=r'(?u)\b\w\w+\b',
             strip_accents=None,
             stop_words=None,
-            ngram_range=(1,
-                         1),
+            ngram_range=(1, 1),
             max_df=1.0,
             min_df=1,
             max_features=None,
@@ -564,7 +566,9 @@ def process(self, query, INTENT_RANKING_LENGTH=5):
                           "confidence": message_sim[0]}
 
                 ranking = list(zip(list(intent_ids), message_sim))
+
                 ranking = ranking[:INTENT_RANKING_LENGTH]
+
                 intent_ranking = [{"intent": self.inv_intent_dict[intent_idx],
                                    "confidence": score}
                                   for intent_idx, score in ranking]
@@ -627,6 +631,7 @@ def load(cls, model_dir=None, use_word_vectors=False):
             app.logger.warning("Failed to load nlu model. Maybe path {} "
                                "doesn't exist"
                                "".format(os.path.abspath(model_dir)))
+
             return EmbeddingIntentClassifier()
 
     def persist(self, model_dir):

app/nlu/classifiers/tf_intent_classifer.py

@@ -26,6 +26,7 @@ def train(self, X, y, models_dir=None, verbose=True):
         Train intent classifier for given training data
         :param X:
         :param y:
+        :param models_dir:
         :param verbose:
         :return:
         """
@@ -115,7 +116,6 @@ def predict(self, text):
         """
         Predict class label for given model
         :param text:
-        :param PATH:
         :return:
         """
         return self.process(text)

app/nlu/entity_extractor.py

@@ -126,7 +126,7 @@ def train(self, train_sentences, model_name):
         trainer.train('model_files/%s.model' % model_name)
         return True
 
-    # Extract Labeles from BIO tagged sentence
+    # Extract Labels from BIO tagged sentence
     def crf2json(self, tagged_sentence):
         """
         Extract label-value pair from NER prediction output

app/nlu/tasks.py

@@ -49,7 +49,7 @@ def train_intent_classifier(intents):
             X.append(example.get("text"))
             y.append(str(intent.intentId.encode('utf8')))
 
-    intent_classifier = EmbeddingIntentClassifier()
+    intent_classifier = EmbeddingIntentClassifier(use_word_vectors=app.config['USE_WORD_VECTORS'])
     intent_classifier.train(X, y)
     intent_classifier.persist(model_dir=app.config["MODELS_DIR"])
 

app/train/controllers.py

@@ -24,7 +24,7 @@ def save_training_data(story_id):
 @train.route('/<story_id>/data', methods=['GET'])
 def get_training_data(story_id):
     """
-    retrive training data for a given story
+    retrieve training data for a given story
     :param story_id:
     :return:
     """

config.py

@@ -1,6 +1,3 @@
-import os
-
-
 class Config(object):
     DEBUG = False
     MONGODB_DB = "iky-ai"
@@ -9,11 +6,12 @@ class Config(object):
     MONGODB_USERNAME = ""
     MONGODB_USERNAME = ""
 
-    # Intent Classifier model detials
+    # Intent Classifier model details
     MODELS_DIR = "model_files/"
     INTENT_MODEL_NAME = "intent.model"
     DEFAULT_FALLBACK_INTENT_NAME = "fallback"
     DEFAULT_WELCOME_INTENT_NAME = "init_conversation"
+    USE_WORD_VECTORS = True
 
 
 class Development(Config):