src/dict: make query generators handle whitespace and newlines

Makes exactquerygenerator and deconjugationquerygenerator handle
newlines and whitespace as if they weren't there. This is useful since
these methods fail to capture results if it is split across multiple
lines as can sometimes happen with expressions. MeCab handles this
internally, so Memento should as well.

Author: ripose-jp

src/dict/deconjugator.cpp

@@ -672,20 +672,36 @@ static QString formatDerivation(QList<WordForm> derivations)
 
 QList<ConjugationInfo> deconjugate(const QString query, bool sentenceMode)
 {
+    static const QRegularExpression WHITESPACE_REGEX("\\s");
+
     QList<ConjugationInfo> results;
     if (sentenceMode)
     {
-        QString word = query;
-        while (!word.isEmpty())
+        QString conjugated = query;
+        while (!conjugated.isEmpty())
         {
-            ConjugationInfo detail = { word, word, QList<WordForm>(), "" };
+            QString word = conjugated;
+            word.remove(WHITESPACE_REGEX);
+            ConjugationInfo detail = {
+                word, conjugated, QList<WordForm>(), ""
+            };
             deconjugateRecursive(detail, results);
-            word.chop(1);
+            do
+            {
+                conjugated.chop(1);
+            }
+            while (
+                !conjugated.isEmpty() &&
+                WHITESPACE_REGEX.match(conjugated.back()).hasMatch()
+            );
+
         }
     }
     else
     {
-        ConjugationInfo detail = { query, query, QList<WordForm>(), ""};
+        QString word = query;
+        word.remove(WHITESPACE_REGEX);
+        ConjugationInfo detail = { word, query, QList<WordForm>(), ""};
         deconjugateRecursive(detail, results);
     }
 

src/dict/exactquerygenerator.cpp

@@ -28,11 +28,18 @@ std::vector<SearchQuery> ExactQueryGenerator::generateQueries(
     QString query = text;
     while (!query.isEmpty())
     {
-        SearchQuery sq;
+        SearchQuery sq{};
         sq.deconj = query;
         sq.surface = query;
         sq.source = SearchQuery::Source::exact;
-        queries.emplace_back(std::move(sq));
+        queries.emplace_back(sq);
+
+        sq.deconj.remove(m_whitespaceRegex);
+        if (sq.deconj != query)
+        {
+            queries.emplace_back(std::move(sq));
+        }
+
         query.chop(1);
     }
 

src/dict/exactquerygenerator.h

@@ -22,6 +22,8 @@
 
 #include "dict/querygenerator.h"
 
+#include <QRegularExpression>
+
 /**
  * @brief A class that generates queries from substrings exactly as they appear
  * in the source text.
@@ -64,4 +66,7 @@ class ExactQueryGenerator final : public QueryGenerator
     [[nodiscard]]
     std::vector<SearchQuery> generateQueries(
         const QString &text) const override;
+
+private:
+    const QRegularExpression m_whitespaceRegex{"\\s"};
 };