OPENNLP-1886: Pin factory state and supplementary input

Author: krickert

opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/light/FinnishLightStemmer.java

@@ -78,6 +78,9 @@
 @ThreadSafe
 public final class FinnishLightStemmer extends AbstractCharArrayStemmer
     implements StemmerFactory {
+
+  private static final String LLA_SUFFIX = "lla";
+
   /** {@inheritDoc} */
   @Override
   public Stemmer newStemmer() {
@@ -108,6 +111,7 @@ int stem(char[] s, int len) {
     return len;
   }
 
+  /** Applies the first suffix-removal pass and returns the remaining length. */
   private int step1(char[] s, int len) {
     if (len > 8) {
       if (endsWith(s, len, "kin")) return step1(s, len - 3);
@@ -121,9 +125,10 @@ private int step1(char[] s, int len) {
     return len;
   }
 
+  /** Applies the second suffix-removal pass and returns the remaining length. */
   private int step2(char[] s, int len) {
     if (len > 5) {
-      if (endsWith(s, len, "lla") || endsWith(s, len, "tse") || endsWith(s, len, "sti"))
+      if (endsWith(s, len, LLA_SUFFIX) || endsWith(s, len, "tse") || endsWith(s, len, "sti"))
         return len - 3;
 
       if (endsWith(s, len, "ni")) return len - 2;
@@ -134,6 +139,7 @@ private int step2(char[] s, int len) {
     return len;
   }
 
+  /** Applies the final suffix-removal pass and returns the remaining length. */
   private int step3(char[] s, int len) {
     if (len > 8) {
       if (endsWith(s, len, "nnen")) {
@@ -172,7 +178,7 @@ private int step3(char[] s, int len) {
 
       if (endsWith(s, len, "ssa")
           || endsWith(s, len, "sta")
-          || endsWith(s, len, "lla")
+          || endsWith(s, len, LLA_SUFFIX)
           || endsWith(s, len, "lta")
           || endsWith(s, len, "tta")
           || endsWith(s, len, "ksi")
@@ -198,6 +204,7 @@ private int step3(char[] s, int len) {
     return len;
   }
 
+  /** Applies the first terminal-character normalization and returns the remaining length. */
   private int norm1(char[] s, int len) {
     if (len > 5 && endsWith(s, len, "hde")) {
       s[len - 3] = 'k';
@@ -223,6 +230,7 @@ private int norm1(char[] s, int len) {
     return len;
   }
 
+  /** Applies the final terminal-character normalization and returns the remaining length. */
   private int norm2(char[] s, int len) {
     if (len > 8) {
       if (s[len - 1] == 'e' || s[len - 1] == 'o' || s[len - 1] == 'u') len--;

opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/light/FrenchLightStemmer.java

@@ -77,6 +77,9 @@
 @ThreadSafe
 public final class FrenchLightStemmer extends AbstractCharArrayStemmer
     implements StemmerFactory {
+
+  private static final String IVE_SUFFIX = "ive";
+
   /** {@inheritDoc} */
   @Override
   public Stemmer newStemmer() {
@@ -109,7 +112,7 @@ int stem(char[] s, int len) {
 
     if (len > 6 && endsWith(s, len, "ement")) {
       len -= 4;
-      if (len > 3 && endsWith(s, len, "ive")) {
+      if (len > 3 && endsWith(s, len, IVE_SUFFIX)) {
         len--;
         s[len - 1] = 'f';
       }
@@ -191,7 +194,7 @@ int stem(char[] s, int len) {
       return norm(s, len);
     }
 
-    if (len > 7 && endsWith(s, len, "ive")) {
+    if (len > 7 && endsWith(s, len, IVE_SUFFIX)) {
       len--;
       s[len - 1] = 'f';
       return norm(s, len);
@@ -233,6 +236,7 @@ int stem(char[] s, int len) {
     return norm(s, len);
   }
 
+  /** Normalizes terminal letters after suffix removal and returns the remaining length. */
   private int norm(char[] s, int len) {
     if (len > 4) {
       for (int i = 0; i < len; i++)

opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/light/GermanLightStemmer.java

@@ -114,6 +114,7 @@ int stem(char[] s, int len) {
     return step2(s, len);
   }
 
+  /** Returns whether {@code ch} can precede the suffix {@code st}. */
   private boolean stEnding(char ch) {
     switch (ch) {
       case 'b':
@@ -132,6 +133,7 @@ private boolean stEnding(char ch) {
     }
   }
 
+  /** Applies the first suffix-removal pass and returns the remaining length. */
   private int step1(char[] s, int len) {
     if (len > 5 && s[len - 3] == 'e' && s[len - 2] == 'r' && s[len - 1] == 'n') return len - 3;
 
@@ -151,6 +153,7 @@ private int step1(char[] s, int len) {
     return len;
   }
 
+  /** Applies the second suffix-removal pass and returns the remaining length. */
   private int step2(char[] s, int len) {
     if (len > 5 && s[len - 3] == 'e' && s[len - 2] == 's' && s[len - 1] == 't') return len - 3;
 

opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/light/HungarianLightStemmer.java

@@ -119,6 +119,7 @@ int stem(char[] s, int len) {
     return normalize(s, len);
   }
 
+  /** Removes case endings and returns the remaining length. */
   private int removeCase(char[] s, int len) {
     if (len > 6 && endsWith(s, len, "kent")) return len - 4;
 
@@ -172,6 +173,7 @@ private int removeCase(char[] s, int len) {
     return len;
   }
 
+  /** Removes possessive endings and returns the remaining length. */
   private int removePossessive(char[] s, int len) {
     if (len > 6) {
       if (!isVowel(s[len - 5])
@@ -227,6 +229,7 @@ private int removePossessive(char[] s, int len) {
   }
 
   @SuppressWarnings("fallthrough")
+  /** Removes plural endings and returns the remaining length. */
   private int removePlural(char[] s, int len) {
     if (len > 3 && s[len - 1] == 'k')
       switch (s[len - 2]) {
@@ -240,6 +243,7 @@ private int removePlural(char[] s, int len) {
     return len;
   }
 
+  /** Normalizes terminal vowels and returns the remaining length. */
   private int normalize(char[] s, int len) {
     if (len > 3)
       switch (s[len - 1]) {

opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/light/NorwegianLightStemmer.java

@@ -94,6 +94,7 @@ public NorwegianLightStemmer(NorwegianVariety first, NorwegianVariety... more) {
     useNynorsk = varieties.contains(NorwegianVariety.NYNORSK);
   }
 
+  /** Initializes a stemmer from validated variety flags. */
   private NorwegianLightStemmer(boolean useBokmaal, boolean useNynorsk) {
     this.useBokmaal = useBokmaal;
     this.useNynorsk = useNynorsk;

opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/light/NorwegianMinimalStemmer.java

@@ -93,6 +93,7 @@ public NorwegianMinimalStemmer(NorwegianVariety first, NorwegianVariety... more)
     useNynorsk = varieties.contains(NorwegianVariety.NYNORSK);
   }
 
+  /** Initializes a stemmer from validated variety flags. */
   private NorwegianMinimalStemmer(boolean useBokmaal, boolean useNynorsk) {
     this.useBokmaal = useBokmaal;
     this.useNynorsk = useNynorsk;

opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/light/PortugueseLightStemmer.java

@@ -142,6 +142,7 @@ int stem(char[] s, int len) {
     return len;
   }
 
+  /** Removes a plural suffix and returns the remaining length. */
   private int removeSuffix(char[] s, int len) {
     if (len > 4 && endsWith(s, len, "es"))
       switch (s[len - 3]) {
@@ -192,6 +193,7 @@ private int removeSuffix(char[] s, int len) {
     return len;
   }
 
+  /** Normalizes feminine endings and returns the remaining length. */
   private int normFeminine(char[] s, int len) {
     if (len > 7
         && (endsWith(s, len, "inha") || endsWith(s, len, "iaca") || endsWith(s, len, "eira"))) {

opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/light/RussianLightStemmer.java

@@ -89,6 +89,7 @@ int stem(char[] s, int len) {
     return normalize(s, len);
   }
 
+  /** Normalizes terminal characters and returns the remaining length. */
   private int normalize(char[] s, int len) {
     if (len > 3)
       switch (s[len - 1]) {
@@ -101,6 +102,7 @@ private int normalize(char[] s, int len) {
     return len;
   }
 
+  /** Removes case endings and returns the remaining length. */
   private int removeCase(char[] s, int len) {
     if (len > 6
         && (endsWith(s, len, "\u0438\u044F\u043C\u0438")

opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/light/SpanishMinimalStemmer.java

@@ -233,10 +233,12 @@ int stem(char[] s, int len) {
     return len;
   }
 
+  /** Returns whether {@code c} is an unaccented Spanish vowel. */
   private boolean isVowel(char c) {
     return c == 'a' || c == 'e' || c == 'i' || c == 'o' || c == 'u';
   }
 
+  /** Replaces accented vowels in the stem with their unaccented forms. */
   private void removeAccents(char[] s, int len) {
     for (int i = 0; i < len; i++) {
       switch (s[i]) {

opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/light/LightStemmerContractTest.java

@@ -32,6 +32,7 @@
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 /**
  * The shared API contract of the light and minimal stemmers: null rejection, identity on empty
@@ -90,6 +91,15 @@ void testDoesNotNormalizeDecomposedInput(Stemmer stemmer) {
     assertEquals(decomposed, stemmer.stem(decomposed).toString());
   }
 
+  @ParameterizedTest
+  @MethodSource("stemmers")
+  void testSupplementaryPrefixRemainsIntact(Stemmer stemmer) {
+    final String result = stemmer.stem("\uD83D\uDE00tests").toString();
+    assertTrue(result.length() >= 2);
+    assertTrue(Character.isSurrogatePair(result.charAt(0), result.charAt(1)));
+    assertEquals(0x1F600, result.codePointAt(0));
+  }
+
   @ParameterizedTest
   @MethodSource("stemmers")
   void testConcurrentCallsMatchSerialResults(Stemmer stemmer) throws Exception {