OPENNLP-1930: Read AD whitespace by the Unicode definition and tidy the scans

The tree and markup scans, the guillemet fix, the POS tag join, the
contraction split, and the NER tag split use the Unicode White_Space
property through StringUtil.isUnicodeWhitespace and
splitOnUnicodeWhitespace, so the samples no longer depend on the
whitespace mode or on the shared WhitespaceTokenizer instance.

The scans return records instead of arrays, the level prefix and the
next closing parenthesis are computed once per line, the memo of the
leaf scan is allocated only when a quoted lemma follows a tag, and the
character literals of the grammar are constants. A fallback line with
no text after its last equals sign is skipped, an id that does not fit
into an int is invalid metadata, a contraction lexeme of underscores
only has no left part, and the helpers are instance methods.

The manual states the whitespace definition, the id rules, and the
self-closing tags.

Author: krickert

opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADMetadata.java

@@ -27,36 +27,45 @@ final class ADMetadata {
 
   private static final String PARAGRAPH_PREFIX = "p=";
   private static final String SOURCE_PREFIX = "source=\"";
+  private static final char HYPHEN = '-';
+  private static final char QUOTE = '"';
 
-  private ADMetadata() {
+  /**
+   * The ids of a sentence.
+   *
+   * @param text The text id.
+   * @param paragraph The paragraph id.
+   */
+  record TextAndParagraph(int text, int paragraph) {
   }
 
   /**
-   * Parses the text id and the paragraph id: the text id is the ASCII digit run after any
-   * leading ASCII letters and hyphens, the paragraph id the digit run after the first
-   * {@code p=} that at least one digit follows.
-   *
-   * @param meta The metadata.
-   * @return The text id and the paragraph id, or {@code null} if either is missing.
+   * Where the digit runs of the two ids lie in the metadata, each as an inclusive start and
+   * an exclusive end.
    */
-  static int[] parseTextAndParagraph(String meta) {
-    int[] spans = scanTextAndParagraph(meta);
-    if (spans == null) {
-      return null;
-    }
-    return new int[] {Integer.parseInt(meta.substring(spans[0], spans[1])),
-        Integer.parseInt(meta.substring(spans[2], spans[3]))};
+  private record IdSpans(int textStart, int textEnd, int paragraphStart, int paragraphEnd) {
+  }
+
+  private ADMetadata() {
   }
 
   /**
-   * Reads the digits of the text id, see {@link #parseTextAndParagraph(String)}.
+   * Parses the text id and the paragraph id: the text id is the ASCII digit run that directly
+   * follows the leading ASCII letters and hyphens, the paragraph id the digit run after the
+   * first {@code p=} that at least one digit follows.
    *
    * @param meta The metadata.
-   * @return The digits, or {@code null} if the text id or the paragraph id is missing.
+   * @return The two ids, or {@code null} if either is missing or does not fit into an
+   *         {@code int}.
    */
-  static String textId(String meta) {
-    int[] spans = scanTextAndParagraph(meta);
-    return spans == null ? null : meta.substring(spans[0], spans[1]);
+  static TextAndParagraph parseTextAndParagraph(String meta) {
+    IdSpans spans = scanTextAndParagraph(meta);
+    if (spans == null) {
+      return null;
+    }
+    int text = parseDigits(meta, spans.textStart(), spans.textEnd());
+    int paragraph = parseDigits(meta, spans.paragraphStart(), spans.paragraphEnd());
+    return text == -1 || paragraph == -1 ? null : new TextAndParagraph(text, paragraph);
   }
 
   /**
@@ -68,8 +77,8 @@ static String textId(String meta) {
    *         missing.
    */
   static String textPrefix(String meta) {
-    int[] spans = scanTextAndParagraph(meta);
-    return spans == null || spans[0] == 0 ? null : meta.substring(0, spans[0]);
+    IdSpans spans = scanTextAndParagraph(meta);
+    return spans == null || spans.textStart() == 0 ? null : meta.substring(0, spans.textStart());
   }
 
   /**
@@ -84,28 +93,26 @@ static String source(String meta) {
       return null;
     }
     start += SOURCE_PREFIX.length();
-    int end = meta.indexOf('"', start);
+    int end = meta.indexOf(QUOTE, start);
     return end == -1 ? null : meta.substring(start, end);
   }
 
   /**
    * Scans the text id and the paragraph id, see {@link #parseTextAndParagraph(String)}.
    *
    * @param meta The metadata.
-   * @return The start and end of the text id and the start and end of the paragraph id, or
-   *         {@code null} if either is missing.
+   * @return The spans of the two ids, or {@code null} if either is missing.
    */
-  private static int[] scanTextAndParagraph(String meta) {
+  private static IdSpans scanTextAndParagraph(String meta) {
     int i = 0;
-    while (i < meta.length() && (StringUtil.isAsciiLetter(meta.charAt(i)) || meta.charAt(i) == '-')) {
+    while (i < meta.length() && (StringUtil.isAsciiLetter(meta.charAt(i)) || meta.charAt(i) == HYPHEN)) {
       i++;
     }
     int textStart = i;
-    i = StringUtil.endOfAsciiDigits(meta, i);
-    if (i == textStart) {
+    int textEnd = StringUtil.endOfAsciiDigits(meta, i);
+    if (textEnd == textStart) {
       return null;
     }
-    int textEnd = i;
     int from = textEnd;
     while (true) {
       int prefix = meta.indexOf(PARAGRAPH_PREFIX, from);
@@ -115,10 +122,30 @@ private static int[] scanTextAndParagraph(String meta) {
       int paragraphStart = prefix + PARAGRAPH_PREFIX.length();
       int paragraphEnd = StringUtil.endOfAsciiDigits(meta, paragraphStart);
       if (paragraphEnd > paragraphStart) {
-        return new int[] {textStart, textEnd, paragraphStart, paragraphEnd};
+        return new IdSpans(textStart, textEnd, paragraphStart, paragraphEnd);
       }
       from = prefix + 1;
     }
   }
 
+  /**
+   * Reads a run of ASCII digits as a number.
+   *
+   * @param meta The metadata.
+   * @param start The inclusive start of the run.
+   * @param end The exclusive end of the run.
+   * @return The number, or -1 if it does not fit into an {@code int}.
+   */
+  private static int parseDigits(String meta, int start, int end) {
+    int value = 0;
+    for (int i = start; i < end; i++) {
+      int digit = meta.charAt(i) - '0';
+      if (value > (Integer.MAX_VALUE - digit) / 10) {
+        return -1;
+      }
+      value = value * 10 + digit;
+    }
+    return value;
+  }
+
 }

opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java

@@ -32,11 +32,11 @@
 import opennlp.tools.formats.ad.ADSentenceStream.SentenceParser.Node;
 import opennlp.tools.formats.ad.ADSentenceStream.SentenceParser.TreeElement;
 import opennlp.tools.namefind.NameSample;
-import opennlp.tools.tokenize.WhitespaceTokenizer;
 import opennlp.tools.util.InputStreamFactory;
 import opennlp.tools.util.ObjectStream;
 import opennlp.tools.util.PlainTextByLineStream;
 import opennlp.tools.util.Span;
+import opennlp.tools.util.StringUtil;
 
 /**
  * Parser for Floresta Sita(c)tica Arvores Deitadas corpus, output to for the
@@ -59,6 +59,9 @@
  * Detailed info about the
  * <a href="http://beta.visl.sdu.dk/visl/pt/info/portsymbol.html#semtags_names">NER tagset</a>.
  * <p>
+ * Whitespace inside the tags of a leaf and in a contraction is the Unicode White_Space property,
+ * see {@link StringUtil#isUnicodeWhitespace(char)}, independent of the whitespace mode.
+ * <p>
  * <b>Note:</b>
  * Do not use this class, internal use only!
  */
@@ -70,6 +73,16 @@ public class ADNameSampleStream implements ObjectStream<NameSample> {
    */
   private static final Map<String, String> HAREM;
 
+  private static final String NER_PREFIX = "NER:";
+  private static final String HYPHEN = "-";
+  private static final char HYPHEN_CHAR = '-';
+  private static final char UNDERSCORE = '_';
+  private static final char TAG_OPEN = '<';
+  private static final char TAG_CLOSE = '>';
+  private static final String LITERARY_PREFIX = "LIT";
+  private static final String SCIENTIFIC_PREFIX = "CIE";
+  private static final String INVALID_METADATA = "Invalid metadata: ";
+
   static {
     Map<String, String> harem = new HashMap<>();
 
@@ -243,7 +256,7 @@ private void processLeaf(Leaf leaf, List<String> sentence, List<Span> names) {
       String c = PortugueseContractionUtility.toContraction(
           leftContractionPart, right);
       if (c != null) {
-        String[] parts = WhitespaceTokenizer.INSTANCE.tokenize(c);
+        String[] parts = StringUtil.splitOnUnicodeWhitespace(c);
         sentence.addAll(Arrays.asList(parts));
         alreadyAdded = true;
       } else {
@@ -266,7 +279,7 @@ private void processLeaf(Leaf leaf, List<String> sentence, List<Span> names) {
         if (lexemes.length > 1) {
           sentence.addAll(Arrays.asList(lexemes).subList(0, lexemes.length - 1));
         }
-        leftContractionPart = lexemes[lexemes.length - 1];
+        leftContractionPart = lexemes.length == 0 ? null : lexemes[lexemes.length - 1];
         return;
       }
       if (leafTag.contains("<NER2>")) {
@@ -335,12 +348,12 @@ private List<String> processTok(String tok) {
     }
 
     // lets split all hyphens
-    if (this.splitHyphenatedTokens && tok.contains("-") && tok.length() > 1) {
+    if (this.splitHyphenatedTokens && tok.contains(HYPHEN) && tok.length() > 1) {
       String[] parts = matchHyphenatedToken(tok);
 
       if (parts != null) {
         addIfNotEmpty(parts[0], out);
-        addIfNotEmpty("-", out);
+        addIfNotEmpty(HYPHEN, out);
         addIfNotEmpty(parts[1], out);
         addIfNotEmpty(parts[2], out);
         tokAdded = true;
@@ -372,11 +385,17 @@ private void addIfNotEmpty(String firstTok, List<String> out) {
    * @param s The lexeme.
    * @return The parts in order; empty when the lexeme has no character other than underscores.
    */
-  static String[] splitOnUnderscores(String s) {
+  String[] splitOnUnderscores(String s) {
+    if (s.isEmpty()) {
+      return new String[0];
+    }
+    if (s.indexOf(UNDERSCORE) == -1) {
+      return new String[] {s};
+    }
     List<String> tokens = new ArrayList<>();
     int start = -1;
     for (int i = 0; i < s.length(); i++) {
-      if (s.charAt(i) == '_') {
+      if (s.charAt(i) == UNDERSCORE) {
         if (start >= 0) {
           tokens.add(s.substring(start, i));
           start = -1;
@@ -398,7 +417,7 @@ static String[] splitOnUnderscores(String s) {
    * @return {@code true} if the token is non-empty and every code point is a letter or a
    *         decimal digit.
    */
-  static boolean isAlphaNumeric(String tok) {
+  boolean isAlphaNumeric(String tok) {
     if (tok.isEmpty()) {
       return false;
     }
@@ -422,23 +441,20 @@ static boolean isAlphaNumeric(String tok) {
    * @return The first token, second token, and rest, each {@code null} when absent, or
    *         {@code null} if the token has none of the three shapes.
    */
-  static String[] matchHyphenatedToken(String tok) {
+  String[] matchHyphenatedToken(String tok) {
     int len = tok.length();
-    // (\p{L}+)-$
-    if (len > 1 && tok.charAt(len - 1) == '-' && isAllLetters(tok, 0, len - 1)) {
+    if (len > 1 && tok.charAt(len - 1) == HYPHEN_CHAR && lettersEnd(tok, 0) == len - 1) {
       return new String[] {tok.substring(0, len - 1), null, null};
     }
-    // ^-(\p{L}+)(.*)
-    if (tok.charAt(0) == '-') {
+    if (tok.charAt(0) == HYPHEN_CHAR) {
       int lettersEnd = lettersEnd(tok, 1);
       if (lettersEnd > 1) {
         return new String[] {null, tok.substring(1, lettersEnd), tok.substring(lettersEnd)};
       }
       return null;
     }
-    // (\p{L}+)-(\p{L}+)(.*)
     int firstEnd = lettersEnd(tok, 0);
-    if (firstEnd > 0 && firstEnd + 1 < len && tok.charAt(firstEnd) == '-') {
+    if (firstEnd > 0 && firstEnd + 1 < len && tok.charAt(firstEnd) == HYPHEN_CHAR) {
       int secondEnd = lettersEnd(tok, firstEnd + 1);
       if (secondEnd > firstEnd + 1) {
         return new String[] {tok.substring(0, firstEnd),
@@ -455,7 +471,7 @@ static String[] matchHyphenatedToken(String tok) {
    * @param from The start offset.
    * @return The offset after the run, or {@code from} if no letter starts there.
    */
-  private static int lettersEnd(String s, int from) {
+  private int lettersEnd(String s, int from) {
     int i = from;
     while (i < s.length()) {
       int cp = s.codePointAt(i);
@@ -467,38 +483,18 @@ private static int lettersEnd(String s, int from) {
     return i;
   }
 
-  /**
-   * Tests whether a range holds letters only.
-   *
-   * @param s The text.
-   * @param from The inclusive start.
-   * @param to The exclusive end.
-   * @return {@code true} if every code point in the range is a letter.
-   */
-  private static boolean isAllLetters(String s, int from, int to) {
-    int i = from;
-    while (i < to) {
-      int cp = s.codePointAt(i);
-      if (!Character.isLetter(cp)) {
-        return false;
-      }
-      i += Character.charCount(cp);
-    }
-    return true;
-  }
-
   /**
    * Extracts the content of a NER tag in Arvores Deitadas format, between the optional
    * {@code NER:} prefix and the closing angle bracket.
    *
    * @param t The tag.
    * @return The content, or {@code null} if {@code t} is not enclosed in angle brackets.
    */
-  static String tagContent(String t) {
-    if (t.length() < 2 || t.charAt(0) != '<' || t.charAt(t.length() - 1) != '>') {
+  String tagContent(String t) {
+    if (t.length() < 2 || t.charAt(0) != TAG_OPEN || t.charAt(t.length() - 1) != TAG_CLOSE) {
       return null;
     }
-    int start = t.startsWith("NER:", 1) ? 5 : 1;
+    int start = t.startsWith(NER_PREFIX, 1) ? 1 + NER_PREFIX.length() : 1;
     return t.substring(start, t.length() - 1);
   }
 
@@ -508,11 +504,11 @@ static String tagContent(String t) {
    * @param tags The NER tag in Arvores Deitadas format.
    * @return The NER tag, or {@code null} if not a NER tag in Arvores Deitadas format.
    */
-  private static String getNER(String tags) {
+  private String getNER(String tags) {
     if (tags.contains("<NER2>")) {
       return null;
     }
-    String[] tag = WhitespaceTokenizer.INSTANCE.tokenize(tags);
+    String[] tag = StringUtil.splitOnUnicodeWhitespace(tags);
     for (String t : tag) {
       String ner = tagContent(t);
       if (ner != null && HAREM.containsKey(ner)) {
@@ -532,29 +528,31 @@ public void close() throws IOException {
     adSentenceStream.close();
   }
 
+  /**
+   * Reads the id of the text a sentence belongs to; adaptive data is cleared when it changes. In
+   * the Amazonia corpus it is the text id of the metadata. In the literary and scientific corpora
+   * the text name stands in for it, and the id is the same for all sentences (OPENNLP-1951).
+   *
+   * @param paragraph The sentence.
+   * @return The id.
+   * @throws RuntimeException If the metadata has no id or one that does not fit into an
+   *                          {@code int}.
+   */
   private int getTextID(Sentence paragraph) {
-
     final String meta = paragraph.metadata();
-    int textIdMeta2 = -1;
-    String textMeta2 = "";
-
-    if (meta.startsWith("LIT") || meta.startsWith("CIE")) {
-      String textId = meta.startsWith("LIT") ? ADMetadata.textPrefix(meta) : ADMetadata.source(meta);
-      if (textId == null) {
-        throw new RuntimeException("Invalid metadata: " + meta);
-      }
-      if (!textId.equals(textMeta2)) {
-        textIdMeta2++;
-        textMeta2 = textId;
+    boolean literary = meta.startsWith(LITERARY_PREFIX);
+    if (literary || meta.startsWith(SCIENTIFIC_PREFIX)) {
+      String textName = literary ? ADMetadata.textPrefix(meta) : ADMetadata.source(meta);
+      if (textName == null) {
+        throw new RuntimeException(INVALID_METADATA + meta);
       }
-      return textIdMeta2;
+      return textName.isEmpty() ? -1 : 0;
     }
-    // Amazonia
-    String textId = ADMetadata.textId(meta);
-    if (textId == null) {
-      throw new RuntimeException("Invalid metadata: " + meta);
+    ADMetadata.TextAndParagraph ids = ADMetadata.parseTextAndParagraph(meta);
+    if (ids == null) {
+      throw new RuntimeException(INVALID_METADATA + meta);
     }
-    return Integer.parseInt(textId);
+    return ids.text();
   }
 
 }