OPENNLP-1934: Speed up the tokenizer alphanumeric optimization for plain character-class patterns

opennlp-api/src/main/java/opennlp/tools/util/CompatibilityMode.java

@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.util;
+
+import java.util.Locale;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Selects, for the classes that corrected their output in 3.0.0, between the corrected output
+ * and the output of the 1.x/2.x releases. A model trained under the old output may depend on
+ * it until it is retrained. Each class that consults the mode documents what differs.
+ * <p>
+ * Resolved from the {@value #MODE_PROPERTY} system property when this class is initialized
+ * and shared process-wide, so a model is trained and decoded under one mode. Tests and
+ * embedders may override the mode via {@link #setActive(CompatibilityMode)} and
+ * {@link #reset()}. The mode is independent of {@link WhitespaceMode}.
+ *
+ * @since 3.0.0
+ */
+public enum CompatibilityMode {
+
+  /**
+   * The output of OpenNLP 1.x/2.x. Restores byte-identical output for models trained with
+   * those releases in the classes that consult the mode.
+   */
+  LEGACY,
+
+  /**
+   * The corrected output. The default from 3.0 onward.
+   */
+  CURRENT;
+
+  /**
+   * System property that selects the active {@link CompatibilityMode} at startup. Accepts
+   * {@code LEGACY} or {@code CURRENT}, case-insensitive; unset or blank resolves to
+   * {@link #CURRENT}, any other value raises an {@link IllegalArgumentException} when the
+   * mode is resolved.
+   */
+  public static final String MODE_PROPERTY = "opennlp.compat.mode";
+
+  private static final Logger logger = LoggerFactory.getLogger(CompatibilityMode.class);
+  private static final AtomicBoolean LEGACY_WARNED = new AtomicBoolean();
+
+  private static volatile CompatibilityMode active = fromProperty();
+
+  /**
+   * Returns the active {@link CompatibilityMode}: the value resolved from the
+   * {@value #MODE_PROPERTY} system property when this class was initialized, or the value
+   * most recently passed to {@link #setActive(CompatibilityMode)}.
+   *
+   * @return The active {@link CompatibilityMode}.
+   */
+  public static CompatibilityMode current() {
+    return active;
+  }
+
+  /**
+   * Overrides the active {@link CompatibilityMode} for the whole process, taking precedence
+   * over the {@value #MODE_PROPERTY} system property. Intended for tests and embedders;
+   * callers pinning a mode temporarily should call {@link #reset()} afterward.
+   *
+   * @param mode The {@link CompatibilityMode} to activate. Must not be {@code null}.
+   *
+   * @throws IllegalArgumentException Thrown if {@code mode} is {@code null}.
+   */
+  public static void setActive(CompatibilityMode mode) {
+    if (mode == null) {
+      throw new IllegalArgumentException("mode must not be null");
+    }
+    active = mode;
+    warnIfLegacy(mode);
+  }
+
+  /**
+   * Discards any override set via {@link #setActive(CompatibilityMode)} and re-resolves the
+   * active mode from the {@value #MODE_PROPERTY} system property.
+   *
+   * @throws IllegalArgumentException Thrown if the property holds a value other than
+   *     {@code LEGACY} or {@code CURRENT} (case-insensitive); the previous mode is retained.
+   */
+  public static void reset() {
+    active = fromProperty();
+  }
+
+  /**
+   * Resolves the mode from the {@value #MODE_PROPERTY} system property; unset or blank
+   * resolves to {@link #CURRENT}. Warns once per process when {@link #LEGACY} is selected.
+   */
+  private static CompatibilityMode fromProperty() {
+    String value = System.getProperty(MODE_PROPERTY);
+    CompatibilityMode mode;
+    if (value == null || value.isBlank()) {
+      mode = CURRENT;
+    } else {
+      try {
+        mode = CompatibilityMode.valueOf(value.trim().toUpperCase(Locale.ROOT));
+      } catch (IllegalArgumentException e) {
+        throw new IllegalArgumentException("Invalid value '" + value + "' for system property '"
+            + MODE_PROPERTY + "': expected LEGACY or CURRENT", e);
+      }
+    }
+    warnIfLegacy(mode);
+    return mode;
+  }
+
+  /**
+   * Logs the legacy-mode removal warning, once per process.
+   */
+  private static void warnIfLegacy(CompatibilityMode mode) {
+    if (mode == LEGACY && LEGACY_WARNED.compareAndSet(false, true)) {
+      logger.warn("Using the legacy (pre-3.0) output of the classes that consult " + MODE_PROPERTY
+          + ". This compatibility mode is scheduled for removal in 4.0.");
+    }
+  }
+}

opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java

@@ -31,6 +31,8 @@ public class StringUtil {
 
   private static final Logger logger = LoggerFactory.getLogger(StringUtil.class);
 
+  private static final String MUST_NOT_BE_NULL = " must not be null";
+
   /**
    * The ten ASCII digit strings {@code "0"} to {@code "9"}, indexed by digit value. Precomputed so
    * code folding digits to ASCII does not allocate a new single-character string per digit; the
@@ -127,7 +129,7 @@ public static boolean isUnicodeWhitespace(int charCode) {
    */
   public static String[] splitOnUnicodeWhitespace(CharSequence input) {
     if (input == null) {
-      throw new IllegalArgumentException("input must not be null");
+      throw new IllegalArgumentException("input" + MUST_NOT_BE_NULL);
     }
     final List<String> terms = new ArrayList<>();
     final int n = input.length();
@@ -225,6 +227,157 @@ public static String[] split(CharSequence input, char separator, int limit) {
     return fields.toArray(new String[0]);
   }
 
+  /**
+   * Tests whether {@code input} contains an ASCII capital letter, {@code A} to {@code Z}.
+   * Capital letters outside ASCII do not count.
+   *
+   * @param input The text to check. Must not be {@code null}.
+   * @return {@code true} if at least one character is an ASCII capital letter.
+   * @throws IllegalArgumentException If {@code input} is {@code null}.
+   */
+  public static boolean containsAsciiUpperCase(CharSequence input) {
+    if (input == null) {
+      throw new IllegalArgumentException("input" + MUST_NOT_BE_NULL);
+    }
+    for (int i = 0; i < input.length(); i++) {
+      if (isAsciiUpperCase(input.charAt(i))) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  /**
+   * Tests whether {@code input} contains an ASCII digit, {@code 0} to {@code 9}. Digits outside
+   * ASCII do not count.
+   *
+   * @param input The text to check. Must not be {@code null}.
+   * @return {@code true} if at least one character is an ASCII digit.
+   * @throws IllegalArgumentException If {@code input} is {@code null}.
+   */
+  public static boolean containsAsciiDigit(CharSequence input) {
+    if (input == null) {
+      throw new IllegalArgumentException("input" + MUST_NOT_BE_NULL);
+    }
+    for (int i = 0; i < input.length(); i++) {
+      if (isAsciiDigit(input.charAt(i))) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  /**
+   * Tests for an ASCII letter, {@code a} to {@code z} or {@code A} to {@code Z}.
+   *
+   * @param codePoint The code point to test.
+   * @return {@code true} if {@code codePoint} is an ASCII letter.
+   */
+  public static boolean isAsciiLetter(int codePoint) {
+    return isAsciiLowerCase(codePoint) || isAsciiUpperCase(codePoint);
+  }
+
+  /**
+   * Tests for an ASCII lowercase letter, {@code a} to {@code z}.
+   *
+   * @param codePoint The code point to test.
+   * @return {@code true} if {@code codePoint} is an ASCII lowercase letter.
+   */
+  public static boolean isAsciiLowerCase(int codePoint) {
+    return codePoint >= 'a' && codePoint <= 'z';
+  }
+
+  /**
+   * Tests for an ASCII capital letter, {@code A} to {@code Z}.
+   *
+   * @param codePoint The code point to test.
+   * @return {@code true} if {@code codePoint} is an ASCII capital letter.
+   */
+  private static boolean isAsciiUpperCase(int codePoint) {
+    return codePoint >= 'A' && codePoint <= 'Z';
+  }
+
+  /**
+   * Tests for an ASCII digit, {@code 0} to {@code 9}.
+   *
+   * @param codePoint The code point to test.
+   * @return {@code true} if {@code codePoint} is an ASCII digit.
+   */
+  public static boolean isAsciiDigit(int codePoint) {
+    return codePoint >= '0' && codePoint <= '9';
+  }
+
+  /**
+   * Finds the end of the run of ASCII digits that starts at {@code from}.
+   *
+   * @param text The text to scan. Must not be {@code null}.
+   * @param from The offset the run starts at, between {@code 0} and {@code text.length()}.
+   * @return The offset after the last digit of the run, or {@code from} if no digit is there.
+   * @throws IllegalArgumentException If {@code text} is {@code null} or {@code from} is out of
+   *         range.
+   */
+  public static int endOfAsciiDigits(CharSequence text, int from) {
+    requireOffset(text, from);
+    int i = from;
+    while (i < text.length() && isAsciiDigit(text.charAt(i))) {
+      i++;
+    }
+    return i;
+  }
+
+  /**
+   * Tests for a line terminator: line feed {@code U+000A}, carriage return {@code U+000D},
+   * next line {@code U+0085}, line separator {@code U+2028}, or paragraph separator
+   * {@code U+2029}. This is the set {@link java.util.regex.Pattern} treats as line terminators
+   * outside {@code UNIX_LINES} mode. It is narrower than
+   * {@link opennlp.tools.util.normalizer.UnicodeWhitespace#lineBreakCodePointSet()}, which
+   * also holds vertical tab {@code U+000B} and form feed {@code U+000C}.
+   *
+   * @param codePoint The code point to test.
+   * @return {@code true} if {@code codePoint} ends a line.
+   */
+  public static boolean isLineTerminator(int codePoint) {
+    return codePoint == '\n' || codePoint == '\r' || codePoint == '\u0085'
+        || codePoint == '\u2028' || codePoint == '\u2029';
+  }
+
+  /**
+   * Finds the first line terminator at or after {@code from}, as defined by
+   * {@link #isLineTerminator(int)}.
+   *
+   * @param text The text to scan. Must not be {@code null}.
+   * @param from The offset to start at, between {@code 0} and {@code text.length()}.
+   * @return The offset of the first line terminator, or {@code -1} if there is none.
+   * @throws IllegalArgumentException If {@code text} is {@code null} or {@code from} is out of
+   *         range.
+   */
+  public static int indexOfLineTerminator(CharSequence text, int from) {
+    requireOffset(text, from);
+    for (int i = from; i < text.length(); i++) {
+      if (isLineTerminator(text.charAt(i))) {
+        return i;
+      }
+    }
+    return -1;
+  }
+
+  /**
+   * Validates a text and an offset into it.
+   *
+   * @param text The text. Must not be {@code null}.
+   * @param from The offset, between {@code 0} and {@code text.length()}.
+   * @throws IllegalArgumentException If {@code text} is {@code null} or {@code from} is out of
+   *         range.
+   */
+  private static void requireOffset(CharSequence text, int from) {
+    if (text == null) {
+      throw new IllegalArgumentException("text" + MUST_NOT_BE_NULL);
+    }
+    if (from < 0 || from > text.length()) {
+      throw new IllegalArgumentException("from must be between 0 and " + text.length());
+    }
+  }
+
   /**
    * Trims leading and trailing runs of Unicode {@code White_Space}, the same set
    * {@link #splitOnUnicodeWhitespace(CharSequence)} breaks terms on.
@@ -235,7 +388,7 @@ public static String[] split(CharSequence input, char separator, int limit) {
    */
   public static String trimUnicodeWhitespace(CharSequence input) {
     if (input == null) {
-      throw new IllegalArgumentException("input must not be null");
+      throw new IllegalArgumentException("input" + MUST_NOT_BE_NULL);
     }
     int start = 0;
     int end = input.length();