Merge refreshed PR apache#1165 into OPENNLP-1833 gRPC helper

# Conflicts:
#	opennlp-api/src/main/java/opennlp/tools/tokenize/BertTokenizer.java
#	opennlp-api/src/main/java/opennlp/tools/tokenize/SubwordPiece.java
#	opennlp-api/src/main/java/opennlp/tools/tokenize/SubwordTokenizer.java
#	opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceEncoder.java
#	opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceTokenizer.java
#	opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/AbstractDL.java
#	opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/vectors/SentenceVectorsDL.java
#	opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/CreateTokenizerTest.java
#	opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/BertTokenizerTest.java
#	opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/WordpieceEncoderReferenceSequencesTest.java
#	opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/WordpieceEncoderTest.java

Author: krickert

opennlp-api/src/main/java/opennlp/tools/tokenize/BertNormalization.java

@@ -24,6 +24,9 @@
  */
 final class BertNormalization {
 
+  /** Default maximum word length used by BERT wordpiece tokenizers. */
+  static final int DEFAULT_MAX_WORD_CODE_POINTS = 100;
+
   private BertNormalization() {
   }
 
@@ -70,6 +73,12 @@ static boolean isWhitespace(int codePoint) {
     return Character.getType(codePoint) == Character.SPACE_SEPARATOR;
   }
 
+  /** Tests whether a code point is a Unicode line or paragraph separator. */
+  static boolean isLineOrParagraphSeparator(int codePoint) {
+    final int type = Character.getType(codePoint);
+    return type == Character.LINE_SEPARATOR || type == Character.PARAGRAPH_SEPARATOR;
+  }
+
   /**
    * A punctuation character in the BERT sense: any non-alphanumeric ASCII
    * character that is not whitespace, or any Unicode punctuation category.

opennlp-api/src/main/java/opennlp/tools/tokenize/BertTokenizer.java

@@ -16,34 +16,36 @@
  */
 package opennlp.tools.tokenize;
 
-import opennlp.tools.util.Span;
-
 /**
- * One subword unit produced by a {@link SubwordTokenizer}, with its vocabulary representation
+ * One subword unit produced by a {@link SubwordTokenizer}, including the model representation
  * and source range.
  *
- * <p>The piece string is in the tokenizer's normalized form, so it is generally not a substring of
- * the input. {@code start} and {@code end} are UTF-16 offsets into the original text, so the
- * surface that produced this piece is {@code text.subSequence(start, end)}. Pieces without source
- * text, such as control symbols or the fill bytes of a byte-fallback expansion,
- * report an empty span with {@code start == end}.</p>
+ * <p>The piece string is in the tokenizer's normalized form and need not equal the input.
+ * {@code start} and {@code end} are UTF-16 offsets into the original text, so the
+ * surface associated with this piece is {@code text.subSequence(start, end)}. A span can include
+ * adjacent source characters when normalization reorders characters. Pieces without source text,
+ * such as control symbols, report an empty span with {@code start == end}.</p>
  *
  * @param piece The piece in the vocabulary's normalized form; must not be {@code null} or empty.
  * @param id    The non-negative vocabulary id of the piece.
  * @param start The inclusive start offset in the original text.
- * @param end   The exclusive end offset in the original text; not less than {@code start}.
+ * @param end   The exclusive end offset in the original text; at least {@code start}.
+ * @since 3.0.0
  */
 public record SubwordPiece(String piece, int id, int start, int end) {
 
   /**
    * Instantiates a {@link SubwordPiece}.
    *
-   * @throws IllegalArgumentException Thrown if {@code piece} is {@code null} or empty, {@code id} is
-   *     negative, or the span is negative or inverted.
+   * @throws IllegalArgumentException Thrown if {@code piece} is {@code null} or empty,
+   *     {@code id} is negative, or the span is negative or inverted.
    */
   public SubwordPiece {
-    if (piece == null || piece.isEmpty()) {
-      throw new IllegalArgumentException("piece must not be null or empty");
+    if (piece == null) {
+      throw new IllegalArgumentException("piece must not be null");
+    }
+    if (piece.isEmpty()) {
+      throw new IllegalArgumentException("piece must not be empty");
     }
     if (id < 0) {
       throw new IllegalArgumentException("id must not be negative");
@@ -52,12 +54,8 @@ public record SubwordPiece(String piece, int id, int start, int end) {
       throw new IllegalArgumentException("start must not be negative");
     }
     if (end < start) {
-      throw new IllegalArgumentException("end must not be less than start");
+      throw new IllegalArgumentException("end must be at least start");
     }
   }
 
-  /** {@return the original-text span of this piece as a {@link Span}} */
-  public Span span() {
-    return new Span(start, end);
-  }
 }

opennlp-api/src/main/java/opennlp/tools/tokenize/SubwordPiece.java

@@ -19,22 +19,27 @@
 import java.util.List;
 
 /**
- * Splits text into subword units against a fixed vocabulary, reporting for every unit its
- * vocabulary id and the exact span of the original text it covers.
+ * Splits text into subword units from a fixed model vocabulary, reporting the model id and
+ * original-text span for each unit.
  *
- * <p>The segmentation is vocabulary-driven rather than linguistic, and each piece is in the
- * model's normalized form, so a piece is generally not a substring of the input. The offsets in
- * each {@link SubwordPiece} always refer to the caller's original text.</p>
+ * <p>Segmentation follows model entries, not linguistic token boundaries. Each piece is in the
+ * model's normalized form and need not equal the input. Offsets in each {@link SubwordPiece}
+ * refer to the original input text.</p>
+ *
+ * <p>An implementation may include model control pieces with empty source spans. Their presence
+ * and placement are part of that tokenizer's contract, not this interface.</p>
  *
  * <p>Thread safety is implementation specific.</p>
+ *
+ * @since 3.0.0
  */
 public interface SubwordTokenizer {
 
   /**
    * Encodes text into subword pieces.
    *
    * @param text The text to encode; must not be {@code null}.
-   * @return The pieces in text order; empty when no units can be encoded.
+   * @return The pieces in model order; may be empty.
    * @throws IllegalArgumentException Thrown if {@code text} is {@code null}.
    */
   List<SubwordPiece> encode(CharSequence text);
@@ -43,7 +48,7 @@ public interface SubwordTokenizer {
    * Encodes text into vocabulary ids.
    *
    * @param text The text to encode; must not be {@code null}.
-   * @return The ids in text order; empty when no units can be encoded.
+   * @return The ids from {@link #encode(CharSequence)}, in the same order.
    * @throws IllegalArgumentException Thrown if {@code text} is {@code null}.
    */
   default int[] encodeToIds(CharSequence text) {
@@ -59,7 +64,7 @@ default int[] encodeToIds(CharSequence text) {
    * Encodes text into piece strings in the vocabulary's normalized form.
    *
    * @param text The text to encode; must not be {@code null}.
-   * @return The pieces in text order; empty when no units can be encoded.
+   * @return The piece strings from {@link #encode(CharSequence)}, in the same order.
    * @throws IllegalArgumentException Thrown if {@code text} is {@code null}.
    */
   default String[] encodeToPieces(CharSequence text) {