Merge pull request #165 from janhq/164-bug-caching-still-not-properly-implemented

hotfix: caching

Author: tikikun

controllers/llamaCPP.cc

@@ -157,14 +157,17 @@ void llamaCPP::chatCompletion(
   // To set default value
 
   if (jsonBody) {
+    // Default values to enable auto caching
+    data["cache_prompt"] = true;
+    data["n_keep"] = -1;
+
     data["stream"] = (*jsonBody).get("stream", false).asBool();
     data["n_predict"] = (*jsonBody).get("max_tokens", 500).asInt();
     data["top_p"] = (*jsonBody).get("top_p", 0.95).asFloat();
     data["temperature"] = (*jsonBody).get("temperature", 0.8).asFloat();
     data["frequency_penalty"] =
         (*jsonBody).get("frequency_penalty", 0).asFloat();
     data["presence_penalty"] = (*jsonBody).get("presence_penalty", 0).asFloat();
-    data["cache_prompt"] = true;
     const Json::Value &messages = (*jsonBody)["messages"];
     for (const auto &message : messages) {
       std::string input_role = message["role"].asString();