updated intent classifier module, added confidence tuning

Author: alfredfrancis

Diff for: README.md

@@ -17,6 +17,7 @@ After any of next methods, you will need to [import default intents](#restore),
 ```sh
 docker-compose build
 docker-compose up -d
+docker-compose exec iky_backend python manage.py init
 ```
 
 ### Using Docker
@@ -29,6 +30,9 @@ docker build -t iky_gateway:3.0.0 frontend/.
 # start iky backend
 docker run --name=iky_backend -e="APPLICATION_ENV=Production" iky_backend:3.0.0
 
+# setup default intents
+docker exec -it python manage.py init
+
 # start iky gateway with frontend
 docker run --name=iky_gateway --link iky_backend:iky_backend -p 8080:80 iky_gateway:3.0.0
 
@@ -43,6 +47,8 @@ docker run --name=iky_gateway --link iky_backend:iky_backend -p 8080:80 iky_gate
 make setup
 
 make run_dev
+
+source venv/bin/activate && python manage.py init
 ```
 * Production
 ```sh

Diff for: app/__init__.py

@@ -30,13 +30,15 @@
 def not_found(error):
     return "Not found", 404
 
-
+from app.agents.controllers import bots
 from app.nlu.controllers import nlu
 from app.intents.controllers import intents
 from app.train.controllers import train
 from app.endpoint.controllers import endpoint
 
+
 app.register_blueprint(nlu)
 app.register_blueprint(intents)
 app.register_blueprint(train)
 app.register_blueprint(endpoint)
+app.register_blueprint(bots)

Diff for: app/agents/__init__.py

@@ -0,0 +1,34 @@
+from flask import Blueprint, request
+
+from app.agents.models import Bot
+from app.commons import build_response
+
+bots = Blueprint('bots_blueprint', __name__,
+                    url_prefix='/agents/<bot_name>')
+
+
+
+@bots.route('/config', methods=['PUT'])
+def set_config(bot_name):
+    """
+    Read bot config
+    :param bot_name:
+    :return:
+    """
+
+    content = request.get_json(silent=True)
+    bot = Bot.objects.get(name=bot_name)
+    bot.config = content
+    bot.save()
+    return build_response.sent_ok()
+
+@bots.route('/config', methods=['GET'])
+def get_config(bot_name):
+    """
+    Update bot config
+    :param json:
+    :return:
+    """
+    bot = Bot.objects.get(name=bot_name)
+
+    return build_response.build_json(bot.config)

Diff for: app/agents/controllers.py

@@ -0,0 +1,7 @@
+from mongoengine.fields import StringField , Document,DictField
+
+class Bot(Document):
+    name = StringField(max_length=100, required=True, unique=True)
+    config = DictField(required=True,default={
+        "confidence_threshold": .70
+    })

Diff for: app/agents/models.py

@@ -73,34 +73,36 @@ def call_api(url, type, parameters, is_json=False):
 
 from app.nlu.intent_classifer import IntentClassifier
 
-PATH = "{}/{}".format(app.config["MODELS_DIR"],
-                      app.config["INTENT_MODEL_NAME"])
+with app.app_context():
+    PATH = "{}/{}".format(app.config["MODELS_DIR"],
+                          app.config["INTENT_MODEL_NAME"])
 
-sentence_classifier = IntentClassifier()
-sentence_classifier.load(PATH)
+    sentence_classifier = IntentClassifier()
+    sentence_classifier.load(PATH)
+    print("Intent Model loaded.")
 
 
 def update_model(app, message, **extra):
-    print(message)
     sentence_classifier.load(PATH)
-    print("model updated")
+    print("Intent Model updated")
 
 from app.nlu.tasks import model_updated_signal
 model_updated_signal.connect(update_model, app)
 
+from app.agents.models import Bot
 def predict(sentence):
     """
     Predict Intent using Intent classifier
     :param sentence:
     :return:
     """
+    bot = Bot.objects.get(name="default")
     predicted = sentence_classifier.predict(sentence)
-
-    if not predicted:
-        return Intent.objects(
-            intentName=app.config["DEFAULT_FALLBACK_INTENT_NAME"]).first().id
+    print(predicted)
+    if predicted["confidence"] < bot.config.get("confidence_threshold",.90):
+        return Intent.objects(intentId=app.config["DEFAULT_FALLBACK_INTENT_NAME"]).first().id,1.0
     else:
-        return predicted["class"]
+        return predicted["intent"],predicted["confidence"]
 
 
 # Request Handler
@@ -135,7 +137,7 @@ def api():
             logger.info(request_json.get("input"), extra=result_json)
             return build_response.build_json(result_json)
 
-        intent_id = predict(request_json.get("input"))
+        intent_id,confidence = predict(request_json.get("input"))
         intent = Intent.objects.get(id=ObjectId(intent_id))
 
         if intent.parameters:
@@ -147,6 +149,7 @@ def api():
                 request_json.get("complete") is True)):
             result_json["intent"] = {
                 "name": intent.name,
+                "confidence":confidence,
                 "id": str(intent.id)
             }
 

Diff for: app/endpoint/controllers.py

@@ -71,6 +71,7 @@ class ApiDetails(EmbeddedDocument):
 
 class Intent(Document):
     name = StringField(max_length=100, required=True, unique=True)
+    userDefined = BooleanField(default=True)
     intentId = StringField(required=True)
     apiTrigger = BooleanField(required=True)
     apiDetails = EmbeddedDocumentField(ApiDetails)

Diff for: app/intents/models.py

@@ -1,10 +1,8 @@
-
 import cloudpickle
 from sklearn import preprocessing
 from sklearn.feature_extraction.text import TfidfVectorizer
-from sklearn.multiclass import OneVsRestClassifier
+from sklearn.svm import SVC
 from sklearn.pipeline import Pipeline
-from sklearn.svm import LinearSVC
 from app.nlu.nltk_preprocessor import NLTKPreprocessor
 
 
@@ -28,6 +26,7 @@ def train(self, X, y, outpath=None, verbose=True):
         :param verbose:
         :return:
         """
+
         def build(X, y=None):
             """
             Inner build function that builds a single model.
@@ -39,17 +38,27 @@ def build(X, y=None):
                 ('preprocessor', NLTKPreprocessor()),
                 ('vectorizer', TfidfVectorizer(
                     tokenizer=self.identity, preprocessor=None, lowercase=False)),
-                ('clf', OneVsRestClassifier(LinearSVC()))])
+                ('clf', SVC(C=1,
+                            probability=True,
+                            class_weight='balanced'))])
+
+            from sklearn.model_selection import GridSearchCV
+
+            Cs = [1, 2, 5, 10, 20, 100]
+            param_grid = {'clf__C': Cs, 'clf__kernel': ["linear"]}
+            grid_search = GridSearchCV(model,
+                                       param_grid=param_grid,
+                                       scoring='f1_weighted',
+                                       verbose=1)
+            grid_search.fit(X, y)
 
-            model.fit(X, y)
+            model = grid_search
             return model
 
-        # Label encode the targets
-        labels = preprocessing.MultiLabelBinarizer()
-        y = labels.fit_transform(y)
+        print(X)
+        print(len(y))
 
         model = build(X, y)
-        model.labels_ = labels
 
         if outpath:
             with open(outpath, 'wb') as f:
@@ -74,14 +83,42 @@ def predict(self, text):
         :param PATH:
         :return:
         """
+        return self.process(text)
+
+    def predict_proba(self, X):
+        """Given a bow vector of an input text, predict most probable label. Returns only the most likely label.
+
+        :param X: bow of input text
+        :return: tuple of first, the most probable label and second, its probability"""
+
+        import numpy as np
 
-        yhat = self.model.predict([
-            text
-        ])
-        if yhat.any():
-            return {
-                "class": self.model.labels_.inverse_transform(yhat)[0][0],
-                "accuracy": 1
-            }
+        pred_result = self.model.predict_proba(X)
+        # sort the probabilities retrieving the indices of the elements in sorted order
+        sorted_indices = np.fliplr(np.argsort(pred_result, axis=1))
+        return sorted_indices, pred_result[:, sorted_indices]
+
+    def process(self, x, return_type="intent", INTENT_RANKING_LENGTH=5):
+        """Returns the most likely intent and its probability for the input text."""
+
+        if not self.model:
+            print("no class")
+            intent = None
+            intent_ranking = []
         else:
-            return False
+            intents, probabilities = self.predict_proba([x])
+            intents, probabilities = [self.model.classes_[intent] for intent in
+                                      intents.flatten()], probabilities.flatten()
+
+            if len(intents) > 0 and len(probabilities) > 0:
+                ranking = list(zip(list(intents), list(probabilities)))[:INTENT_RANKING_LENGTH]
+
+                intent = {"intent": intents[0], "confidence": probabilities[0]}
+                intent_ranking = [{"intent": intent_name, "confidence": score} for intent_name, score in ranking]
+            else:
+                intent = {"name": None, "confidence": 0.0}
+                intent_ranking = []
+        if return_type == "intent":
+            return intent
+        else:
+            return intent_ranking

Diff for: app/nlu/intent_classifer.py

@@ -16,7 +16,7 @@ def __init__(self, stopwords=None, punct=None,
                  lower=True, strip=True):
         self.lower = lower
         self.strip = strip
-        #self.stopwords  = stopwords or set(sw.words('english'))
+        self.stopwords  = stopwords or set(sw.words('english'))
         self.punct = punct or set(string.punctuation)
         self.lemmatizer = WordNetLemmatizer()
 
@@ -30,7 +30,6 @@ def transform(self, X):
         return [
             list(self.tokenize(doc)) for doc in X
         ]
-
     def tokenize(self, document):
         # Break the document into sentences
         for sent in sent_tokenize(document):
@@ -43,8 +42,8 @@ def tokenize(self, document):
                 token = token.strip('*') if self.strip else token
 
                 # If stopword, ignore token and continue
-                # if token in self.stopwords:
-                #     continue
+                if token in self.stopwords:
+                    continue
 
                 # If punctuation, ignore token and continue
                 if all(char in self.punct for char in token):

Diff for: app/nlu/nltk_preprocessor.py

@@ -39,8 +39,10 @@ def train_intent_classifier(intents):
     for intent in intents:
         training_data = intent.trainingData
         for example in training_data:
+            if example.get("text").strip() == "":
+                continue
             X.append(example.get("text"))
-            y.append([str(intent.id)])
+            y.append(str(intent.id))
 
     PATH = "{}/{}".format(app.config["MODELS_DIR"],
                           app.config["INTENT_MODEL_NAME"])

Diff for: app/nlu/tasks.py

@@ -1,20 +1,23 @@
 version: '2'
 services:
   iky_backend:
+    container_name: iky_backend
     build: .
     environment:
       APPLICATION_ENV: Production
     depends_on:
       - mongodb
 
   iky_gateway:
+    container_name: iky_gateway
     build: frontend/
     ports:
         - "8080:80"
     depends_on:
       - iky_backend
 
   mongodb:
+    container_name: mongodb
     image: mongo
     hostname: mongodb
     volumes:

Diff for: docker-compose.yml

@@ -1 +1 @@
-[{"_id": {"$oid": "59aae7bd26f6f60007b06fb3"}, "name": "Default Fallback intent", "intentId": "fallback", "apiTrigger": false, "speechResponse": "Sorry. I'm having trouble understanding you."}, {"name": "cancel", "trainingData": [{"text": "cancel that", "entities": []}, {"text": "cancel", "entities": []}], "intentId": "cancel", "apiTrigger": false, "_id": {"$oid": "59aae7bd26f6f60007b06fb5"}, "speechResponse": "Ok. Canceled."}, {"_id": {"$oid": "59aae7bd26f6f60007b06fb7"}, "name": "Welcome message", "intentId": "init_conversation", "apiTrigger": false, "speechResponse": "Hi, What can i do for you ?"}]
+[{"name": "Default Fallback intent", "userDefined": false, "apiTrigger": false, "intentId": "fallback", "_id": {"$oid": "59aae7bd26f6f60007b06fb3"}, "speechResponse": "Sorry. I'm having trouble understanding you."}, {"name": "cancel", "trainingData": [{"text": "i want to cancel", "entities": []}, {"text": "cancel that", "entities": []}, {"text": "cancel", "entities": []}], "userDefined": false, "apiTrigger": false, "intentId": "cancel", "_id": {"$oid": "59aae7bd26f6f60007b06fb5"}, "speechResponse": "Ok. Canceled."}, {"name": "Welcome message", "userDefined": false, "apiTrigger": false, "intentId": "init_conversation", "_id": {"$oid": "59aae7bd26f6f60007b06fb7"}, "speechResponse": "Hi, What can i do for you ?"}]

Diff for: examples/default_intents.json

@@ -1 +1 @@
-[{"name": "restaurant search", "parameters": [{"prompt": "Tell me your location?", "required": true, "type": "free_text", "id": {"$oid": "5adb289f07440e00128fcfd6"}, "name": "location"}, {"prompt": "Ok. What type cuisine are you looking for?", "required": true, "type": "free_text", "id": {"$oid": "5adb289f07440e00128fcfd7"}, "name": "cuisine"}], "trainingData": [{"text": "i'm looking for a place in new york", "entities": [{"begin": 27, "end": 35, "name": "location", "value": "new york"}]}, {"text": "im looking for a place in banglore", "entities": [{"begin": 26, "end": 34, "name": "location", "value": "banglore"}]}, {"text": "looking for indian cuisine in new york", "entities": [{"begin": 12, "end": 18, "name": "cuisine", "value": "indian"}, {"begin": 30, "end": 38, "name": "location", "value": "new york"}]}, {"text": "central indian restaurant", "entities": [{"begin": 0, "end": 7, "name": "location", "value": "central"}, {"begin": 8, "end": 14, "name": "cuisine", "value": "indian"}]}, {"text": "I am looking for mexican indian fusion", "entities": [{"begin": 17, "end": 38, "name": "cuisine", "value": "mexican indian fusion"}]}, {"text": "I am looking a restaurant in 29432", "entities": [{"begin": 29, "end": 34, "name": "location", "value": "29432"}]}, {"text": "I am looking for asian fusion food", "entities": [{"begin": 17, "end": 29, "name": "cuisine", "value": "asian fusion"}]}, {"text": "anywhere near 18328", "entities": [{"begin": 14, "end": 19, "name": "location", "value": "18328"}]}, {"text": "anywhere in the west", "entities": [{"begin": 16, "end": 20, "name": "location", "value": "west"}]}, {"text": "search for restaurants", "entities": []}, {"text": "i am looking for an indian spot called olaolaolaolaolaola", "entities": [{"begin": 39, "end": 57, "name": "location", "value": "olaolaolaolaolaola"}, {"begin": 20, "end": 26, "name": "cuisine", "value": "indian"}]}, {"text": "show me a mexican place in the centre", "entities": [{"begin": 10, "end": 17, "name": "cuisine", "value": "mexican"}, {"begin": 31, "end": 37, "name": "location", "value": "centre"}]}, {"text": "show me chines restaurants in the north", "entities": [{"begin": 8, "end": 14, "name": "cuisine", "value": "chines"}, {"begin": 34, "end": 39, "name": "location", "value": "north"}]}, {"text": "show me chinese restaurants", "entities": [{"begin": 8, "end": 15, "name": "cuisine", "value": "chinese"}]}, {"text": "i'm looking for a place in the north of town", "entities": [{"begin": 31, "end": 44, "name": "location", "value": "north of town"}]}, {"text": "I am searching for a dinner spot", "entities": []}, {"text": "I want to grab lunch", "entities": []}, {"text": "i'm looking for a place to eat", "entities": []}], "apiTrigger": false, "intentId": "restaurant_search", "_id": {"$oid": "5adb265507440e00128fcfa1"}, "speechResponse": "Ok, I found following restaurants matching your query. Location => {{parameters[\"location\"] }} and Cuisine => {{parameters[\"cuisine\"] }}"}]
+[{"name": "Restaurant search", "parameters": [{"prompt": "Tell me your location?", "required": true, "type": "free_text", "id": {"$oid": "5adca3734ae5354f655fed86"}, "name": "location"}, {"prompt": "Ok. What type cuisine are you looking for?", "required": true, "type": "free_text", "id": {"$oid": "5adca3734ae5354f655fed87"}, "name": "cuisine"}], "userDefined": true, "trainingData": [{"text": "im looking for a place near banglore", "entities": [{"begin": 28, "end": 36, "name": "location", "value": "banglore"}]}, {"text": "i'm looking for a place to eat near down town la", "entities": [{"begin": 36, "end": 48, "name": "location", "value": "down town la"}]}, {"text": "i'm looking for a place in new york", "entities": [{"begin": 27, "end": 35, "name": "location", "value": "new york"}]}, {"text": "im looking for a place in banglore", "entities": [{"begin": 26, "end": 34, "name": "location", "value": "banglore"}]}, {"text": "looking for indian cuisine in new york", "entities": [{"begin": 12, "end": 18, "name": "cuisine", "value": "indian"}, {"begin": 30, "end": 38, "name": "location", "value": "new york"}]}, {"text": "central indian restaurant", "entities": [{"begin": 0, "end": 7, "name": "location", "value": "central"}, {"begin": 8, "end": 14, "name": "cuisine", "value": "indian"}]}, {"text": "I am looking for mexican indian fusion", "entities": [{"begin": 17, "end": 38, "name": "cuisine", "value": "mexican indian fusion"}]}, {"text": "I am looking a restaurant in 29432", "entities": [{"begin": 29, "end": 34, "name": "location", "value": "29432"}]}, {"text": "I am looking for asian fusion food", "entities": [{"begin": 17, "end": 29, "name": "cuisine", "value": "asian fusion"}]}, {"text": "anywhere near 18328", "entities": [{"begin": 14, "end": 19, "name": "location", "value": "18328"}]}, {"text": "anywhere in the west", "entities": [{"begin": 16, "end": 20, "name": "location", "value": "west"}]}, {"text": "search for restaurants", "entities": []}, {"text": "i am looking for an indian spot called olaolaolaolaolaola", "entities": [{"begin": 39, "end": 57, "name": "location", "value": "olaolaolaolaolaola"}, {"begin": 20, "end": 26, "name": "cuisine", "value": "indian"}]}, {"text": "show me a mexican place in the centre", "entities": [{"begin": 10, "end": 17, "name": "cuisine", "value": "mexican"}, {"begin": 31, "end": 37, "name": "location", "value": "centre"}]}, {"text": "show me chines restaurants in the north", "entities": [{"begin": 8, "end": 14, "name": "cuisine", "value": "chines"}, {"begin": 34, "end": 39, "name": "location", "value": "north"}]}, {"text": "show me chinese restaurants", "entities": [{"begin": 8, "end": 15, "name": "cuisine", "value": "chinese"}]}, {"text": "i'm looking for a place in the north of town", "entities": [{"begin": 31, "end": 44, "name": "location", "value": "north of town"}]}, {"text": "I am searching for a dinner spot", "entities": []}, {"text": "I want to grab lunch", "entities": []}, {"text": "i'm looking for a place to eat", "entities": []}], "apiTrigger": false, "intentId": "restaurant_search", "_id": {"$oid": "5adb265507440e00128fcfa1"}, "speechResponse": "Ok, I found following restaurants matching your query. Location => {{parameters[\"location\"] }} and Cuisine => {{parameters[\"cuisine\"] }}"}, {"name": "Greet", "trainingData": [{"text": "dear sir", "entities": []}, {"text": "good evening\n", "entities": []}, {"text": "good morning\n", "entities": []}, {"text": "hi\n", "entities": []}, {"text": "hello\n", "entities": []}, {"text": "hey there\n", "entities": []}, {"text": "howdy\n", "entities": []}, {"text": "hey", "entities": []}], "userDefined": true, "apiTrigger": false, "intentId": "greet", "_id": {"$oid": "5adc89ad4ae5353f40be85bd"}, "speechResponse": "Hello Sir"}, {"name": "Affirm", "trainingData": [{"text": "sounds really good", "entities": []}, {"text": "great choice\n", "entities": []}, {"text": "correct\n", "entities": []}, {"text": "right, thank you\n", "entities": []}, {"text": "great\n", "entities": []}, {"text": "ok\n", "entities": []}, {"text": "that's right\n", "entities": []}, {"text": "indeed\n", "entities": []}, {"text": "yeah\n", "entities": []}, {"text": "yep\n", "entities": []}, {"text": "yes\n", "entities": []}], "userDefined": true, "apiTrigger": false, "intentId": "affirm", "_id": {"$oid": "5adc8a3d4ae5353f40be85ca"}, "speechResponse": "I'm glad that you agree"}, {"name": "Goodbye", "trainingData": [{"text": "have a good one", "entities": []}, {"text": "Bye bye\n", "entities": []}, {"text": "farewell\n", "entities": []}, {"text": "end\n", "entities": []}, {"text": "stop\n", "entities": []}, {"text": "good bye\n", "entities": []}, {"text": "goodbye\n", "entities": []}, {"text": "bye", "entities": []}], "userDefined": true, "apiTrigger": false, "intentId": "goodbye", "_id": {"$oid": "5adc8a9c4ae5353f40be85cf"}, "speechResponse": "It was nice talking to you sir"}]