Update key generation for KV cache.

Author: zhongkaifu

Seq2SeqSharp/Applications/GPT.cs

@@ -20,6 +20,7 @@
 using Seq2SeqSharp.Tools;
 using Seq2SeqSharp.Utils;
 using TensorSharp;
+using ManagedCuda.BasicTypes;
 
 namespace Seq2SeqSharp.Applications
 {
@@ -144,13 +145,28 @@ private bool CreateTrainableParameters(IModel model)
                     m_posEmbedding?.GetNetworkOnDevice(deviceIdIdx));
         }
 
+        /// <summary>
+        /// Generate key for kv cache. </s> will be removed from the key if it exist
+        /// </summary>
+        /// <param name="strs"></param>
+        /// <returns></returns>
         private string GenerateCacheKey(List<List<string>> strs)
         {
             List<string> r = new List<string>();
 
             foreach (var str in strs)
             {
-                r.Add(string.Join(" ", str));
+                List<string> normStr = new List<string>();
+                foreach (string word in str)
+                {
+                    if (word == "</s>")
+                    {
+                        continue;
+                    }
+                    normStr.Add(word);
+                }
+
+                r.Add(string.Join(" ", normStr));
             }
 
             return string.Join("\t", r);