Skip to content

Commit 425e799

Browse files
krickertkrickert
committed
OPENNLP-1928: Add opennlp.compat.mode and keep the old emoji output under LEGACY
CompatibilityMode in opennlp-api selects, for the classes that corrected their output in 3.0.0, between the corrected output and the output of the 1.x/2.x releases, built like WhitespaceMode and independent of it. Under LEGACY the EmojiCharSequenceNormalizer replaces hyphens, code points from U+D83C to U+10FC00 and unpaired surrogates in that range, as the old pattern did, so language detector models trained with an earlier release keep their n-grams until they are retrained. A test compares the legacy output with the old pattern on all code points. The manual describes the property in a new section.
1 parent 8fbbd3f commit 425e799

7 files changed

Lines changed: 398 additions & 16 deletions

File tree

‎opennlp-api/src/main/java/opennlp/tools/util/CompatibilityMode.java‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package opennlp.tools.util;
19+
20+
import java.util.Locale;
21+
import java.util.concurrent.atomic.AtomicBoolean;
22+
23+
import org.slf4j.Logger;
24+
import org.slf4j.LoggerFactory;
25+
26+
/**
27+
* Selects, for the classes that corrected their output in 3.0.0, between the corrected output
28+
* and the output of the 1.x/2.x releases. A model trained under the old output may depend on
29+
* it until it is retrained. Each class that consults the mode documents what differs.
30+
* <p>
31+
* Resolved from the {@value #MODE_PROPERTY} system property when this class is initialized
32+
* and shared process-wide, so a model is trained and decoded under one mode. Tests and
33+
* embedders may override the mode via {@link #setActive(CompatibilityMode)} and
34+
* {@link #reset()}. The mode is independent of {@link WhitespaceMode}.
35+
*
36+
* @since 3.0.0
37+
*/
38+
public enum CompatibilityMode {
39+
40+
/**
41+
* The output of OpenNLP 1.x/2.x. Restores byte-identical output for models trained with
42+
* those releases in the classes that consult the mode.
43+
*/
44+
LEGACY,
45+
46+
/**
47+
* The corrected output. The default from 3.0 onward.
48+
*/
49+
CURRENT;
50+
51+
/**
52+
* System property that selects the active {@link CompatibilityMode} at startup. Accepts
53+
* {@code LEGACY} or {@code CURRENT}, case-insensitive; unset or blank resolves to
54+
* {@link #CURRENT}, any other value raises an {@link IllegalArgumentException} when the
55+
* mode is resolved.
56+
*/
57+
public static final String MODE_PROPERTY = "opennlp.compat.mode";
58+
59+
private static final Logger logger = LoggerFactory.getLogger(CompatibilityMode.class);
60+
private static final AtomicBoolean LEGACY_WARNED = new AtomicBoolean();
61+
62+
private static volatile CompatibilityMode active = fromProperty();
63+
64+
/**
65+
* Returns the active {@link CompatibilityMode}: the value resolved from the
66+
* {@value #MODE_PROPERTY} system property when this class was initialized, or the value
67+
* most recently passed to {@link #setActive(CompatibilityMode)}.
68+
*
69+
* @return The active {@link CompatibilityMode}.
70+
*/
71+
public static CompatibilityMode current() {
72+
return active;
73+
}
74+
75+
/**
76+
* Overrides the active {@link CompatibilityMode} for the whole process, taking precedence
77+
* over the {@value #MODE_PROPERTY} system property. Intended for tests and embedders;
78+
* callers pinning a mode temporarily should call {@link #reset()} afterward.
79+
*
80+
* @param mode The {@link CompatibilityMode} to activate. Must not be {@code null}.
81+
*
82+
* @throws IllegalArgumentException Thrown if {@code mode} is {@code null}.
83+
*/
84+
public static void setActive(CompatibilityMode mode) {
85+
if (mode == null) {
86+
throw new IllegalArgumentException("mode must not be null");
87+
}
88+
active = mode;
89+
warnIfLegacy(mode);
90+
}
91+
92+
/**
93+
* Discards any override set via {@link #setActive(CompatibilityMode)} and re-resolves the
94+
* active mode from the {@value #MODE_PROPERTY} system property.
95+
*
96+
* @throws IllegalArgumentException Thrown if the property holds a value other than
97+
* {@code LEGACY} or {@code CURRENT} (case-insensitive); the previous mode is retained.
98+
*/
99+
public static void reset() {
100+
active = fromProperty();
101+
}
102+
103+
/**
104+
* Resolves the mode from the {@value #MODE_PROPERTY} system property; unset or blank
105+
* resolves to {@link #CURRENT}. Warns once per process when {@link #LEGACY} is selected.
106+
*/
107+
private static CompatibilityMode fromProperty() {
108+
String value = System.getProperty(MODE_PROPERTY);
109+
CompatibilityMode mode;
110+
if (value == null || value.isBlank()) {
111+
mode = CURRENT;
112+
} else {
113+
try {
114+
mode = CompatibilityMode.valueOf(value.trim().toUpperCase(Locale.ROOT));
115+
} catch (IllegalArgumentException e) {
116+
throw new IllegalArgumentException("Invalid value '" + value + "' for system property '"
117+
+ MODE_PROPERTY + "': expected LEGACY or CURRENT", e);
118+
}
119+
}
120+
warnIfLegacy(mode);
121+
return mode;
122+
}
123+
124+
/**
125+
* Logs the legacy-mode removal warning, once per process.
126+
*/
127+
private static void warnIfLegacy(CompatibilityMode mode) {
128+
if (mode == LEGACY && LEGACY_WARNED.compareAndSet(false, true)) {
129+
logger.warn("Using the legacy (pre-3.0) output of the classes that consult " + MODE_PROPERTY
130+
+ ". This compatibility mode is scheduled for removal in 4.0.");
131+
}
132+
}
133+
}

‎opennlp-api/src/test/java/opennlp/tools/util/CompatibilityModeTest.java‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package opennlp.tools.util;
19+
20+
import org.junit.jupiter.api.AfterEach;
21+
import org.junit.jupiter.api.BeforeAll;
22+
import org.junit.jupiter.api.Test;
23+
import org.junit.jupiter.params.ParameterizedTest;
24+
import org.junit.jupiter.params.provider.CsvSource;
25+
26+
import static org.junit.jupiter.api.Assertions.assertEquals;
27+
import static org.junit.jupiter.api.Assertions.assertThrows;
28+
import static org.junit.jupiter.api.Assertions.assertTrue;
29+
30+
/**
31+
* Tests for the {@link CompatibilityMode} class.
32+
*/
33+
public class CompatibilityModeTest {
34+
35+
/**
36+
* Initializes {@link CompatibilityMode} while the property is unset, so tests that set an
37+
* invalid value exercise {@link CompatibilityMode#reset()} rather than class initialization.
38+
*/
39+
@BeforeAll
40+
static void initializeWithCleanProperty() {
41+
System.clearProperty(CompatibilityMode.MODE_PROPERTY);
42+
CompatibilityMode.reset();
43+
}
44+
45+
/**
46+
* Restores property resolution after each test, so no mode or property state leaks.
47+
*/
48+
@AfterEach
49+
void resetCompatibilityMode() {
50+
System.clearProperty(CompatibilityMode.MODE_PROPERTY);
51+
CompatibilityMode.reset();
52+
}
53+
54+
@Test
55+
void testDefaultsToCurrent() {
56+
CompatibilityMode.reset();
57+
assertEquals(CompatibilityMode.CURRENT, CompatibilityMode.current());
58+
}
59+
60+
@Test
61+
void testBlankPropertyDefaultsToCurrent() {
62+
System.setProperty(CompatibilityMode.MODE_PROPERTY, " ");
63+
CompatibilityMode.reset();
64+
assertEquals(CompatibilityMode.CURRENT, CompatibilityMode.current());
65+
}
66+
67+
@ParameterizedTest
68+
@CsvSource({"LEGACY, LEGACY", "legacy, LEGACY", " Legacy , LEGACY", "CURRENT, CURRENT",
69+
"current, CURRENT"})
70+
void testPropertyResolvesCaseInsensitively(String value, CompatibilityMode expected) {
71+
System.setProperty(CompatibilityMode.MODE_PROPERTY, value);
72+
CompatibilityMode.reset();
73+
assertEquals(expected, CompatibilityMode.current());
74+
}
75+
76+
@ParameterizedTest
77+
@CsvSource({"sloppy", "UNICODE", "LEGACY CURRENT"})
78+
void testInvalidPropertyValueThrows(String value) {
79+
CompatibilityMode before = CompatibilityMode.current();
80+
System.setProperty(CompatibilityMode.MODE_PROPERTY, value);
81+
IllegalArgumentException e =
82+
assertThrows(IllegalArgumentException.class, CompatibilityMode::reset);
83+
assertTrue(e.getMessage().contains(CompatibilityMode.MODE_PROPERTY));
84+
assertTrue(e.getMessage().contains(value));
85+
assertEquals(before, CompatibilityMode.current());
86+
}
87+
88+
@Test
89+
void testCurrentCachesUntilReset() {
90+
CompatibilityMode.reset();
91+
assertEquals(CompatibilityMode.CURRENT, CompatibilityMode.current());
92+
93+
System.setProperty(CompatibilityMode.MODE_PROPERTY, "LEGACY");
94+
assertEquals(CompatibilityMode.CURRENT, CompatibilityMode.current());
95+
96+
CompatibilityMode.reset();
97+
assertEquals(CompatibilityMode.LEGACY, CompatibilityMode.current());
98+
}
99+
100+
@Test
101+
void testSetActiveOverridesProperty() {
102+
System.setProperty(CompatibilityMode.MODE_PROPERTY, "CURRENT");
103+
CompatibilityMode.reset();
104+
CompatibilityMode.setActive(CompatibilityMode.LEGACY);
105+
assertEquals(CompatibilityMode.LEGACY, CompatibilityMode.current());
106+
}
107+
108+
@Test
109+
void testSetActiveRejectsNull() {
110+
assertThrows(IllegalArgumentException.class, () -> CompatibilityMode.setActive(null));
111+
}
112+
113+
@Test
114+
void testIndependentOfWhitespaceMode() {
115+
WhitespaceMode before = WhitespaceMode.current();
116+
try {
117+
CompatibilityMode.setActive(CompatibilityMode.LEGACY);
118+
assertEquals(before, WhitespaceMode.current());
119+
WhitespaceMode.setActive(WhitespaceMode.LEGACY);
120+
CompatibilityMode.setActive(CompatibilityMode.CURRENT);
121+
assertEquals(WhitespaceMode.LEGACY, WhitespaceMode.current());
122+
} finally {
123+
WhitespaceMode.reset();
124+
}
125+
}
126+
}

‎opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/langdetect/LanguageDetectorFactory.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,8 @@ public class LanguageDetectorFactory extends BaseToolFactory {
4343
* @return Retrieves a {@link LanguageDetectorContextGenerator}. The deprecated emoji
4444
* normalizer stays in this chain because existing models were trained with it. Since
4545
* 3.0.0 it keeps hyphens and Basic Multilingual Plane characters (OPENNLP-1928), so
46-
* models trained with an earlier release see those characters in their n-grams.
46+
* models trained with an earlier release see those characters in their n-grams
47+
* unless {@link opennlp.tools.util.CompatibilityMode#LEGACY} is active.
4748
*/
4849
@SuppressWarnings("deprecation")
4950
public LanguageDetectorContextGenerator getContextGenerator() {

‎opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizer.java‎

Lines changed: 40 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,17 @@
1616
*/
1717
package opennlp.tools.util.normalizer;
1818

19+
import opennlp.tools.util.CompatibilityMode;
20+
1921
/**
2022
* A {@link CharSequenceNormalizer} that replaces every run of supplementary-plane code points,
2123
* {@code U+10000} and above, with a single space. Emoji outside the Basic Multilingual Plane are
2224
* replaced along with every other supplementary character, CJK Extension B ideographs included;
2325
* BMP characters and unpaired surrogates are kept. Since 3.0.0 hyphens and BMP characters are no
24-
* longer replaced (OPENNLP-1928).
26+
* longer replaced (OPENNLP-1928). Under {@link CompatibilityMode#LEGACY} the output of the
27+
* 1.x/2.x releases is produced instead: a run of hyphens, of code points from {@code U+D83C}
28+
* to {@code U+10FC00}, unpaired surrogates in that range included, becomes one space, and
29+
* code points above {@code U+10FC00} are kept.
2530
*
2631
* @deprecated Replaces every supplementary-plane code point with a space, not only emoji. Use
2732
* {@link EmojiToEmoticonCharSequenceNormalizer} instead.
@@ -31,6 +36,10 @@ public class EmojiCharSequenceNormalizer implements CharSequenceNormalizer {
3136

3237
private static final long serialVersionUID = 4553401197981667914L;
3338

39+
private static final int HYPHEN = '-';
40+
private static final int LEGACY_RANGE_FIRST = 0xD83C;
41+
private static final int LEGACY_RANGE_LAST = 0x10FC00;
42+
3443
private static final EmojiCharSequenceNormalizer INSTANCE = new EmojiCharSequenceNormalizer();
3544

3645
public static EmojiCharSequenceNormalizer getInstance() {
@@ -39,50 +48,67 @@ public static EmojiCharSequenceNormalizer getInstance() {
3948

4049
/**
4150
* {@inheritDoc}
42-
* Every run of supplementary-plane code points becomes one space; text without one is
43-
* returned as it is.
51+
* Every run of replaced code points becomes one space; text without one is returned as
52+
* it is. Which code points are replaced depends on {@link CompatibilityMode#current()}.
4453
*/
4554
@Override
4655
public CharSequence normalize(CharSequence text) {
4756
if (text == null) {
4857
throw new IllegalArgumentException("The text must not be null.");
4958
}
50-
int i = indexOfSupplementary(text);
59+
final boolean legacy = CompatibilityMode.current() == CompatibilityMode.LEGACY;
60+
int i = indexOfReplaced(text, legacy);
5161
if (i == -1) {
5262
return text;
5363
}
5464
StringBuilder normalized = new StringBuilder(text.length()).append(text, 0, i);
5565
boolean inRun = false;
5666
while (i < text.length()) {
5767
int cp = Character.codePointAt(text, i);
58-
if (Character.isSupplementaryCodePoint(cp)) {
68+
if (isReplaced(cp, legacy)) {
5969
if (!inRun) {
6070
normalized.append(' ');
6171
inRun = true;
6272
}
63-
i += 2;
6473
}
6574
else {
66-
normalized.append((char) cp);
75+
normalized.appendCodePoint(cp);
6776
inRun = false;
68-
i++;
6977
}
78+
i += Character.charCount(cp);
7079
}
7180
return normalized.toString();
7281
}
7382

7483
/**
75-
* Finds the first supplementary-plane code point.
84+
* Tells whether a code point is replaced by a space.
85+
*
86+
* @param codePoint The code point, an unpaired surrogate read as its own code point included.
87+
* @param legacy {@code true} for the output of the 1.x/2.x releases.
88+
* @return {@code true} if the code point is replaced.
89+
*/
90+
private boolean isReplaced(int codePoint, boolean legacy) {
91+
if (legacy) {
92+
return codePoint == HYPHEN
93+
|| (codePoint >= LEGACY_RANGE_FIRST && codePoint <= LEGACY_RANGE_LAST);
94+
}
95+
return Character.isSupplementaryCodePoint(codePoint);
96+
}
97+
98+
/**
99+
* Finds the first replaced code point.
76100
*
77101
* @param text The text.
78-
* @return The offset of its high surrogate, or {@code -1} if there is none.
102+
* @param legacy {@code true} for the output of the 1.x/2.x releases.
103+
* @return The offset of its first char, or {@code -1} if there is none.
79104
*/
80-
private int indexOfSupplementary(CharSequence text) {
81-
for (int i = 0; i < text.length(); i++) {
82-
if (Character.isHighSurrogate(text.charAt(i)) && i + 1 < text.length()
83-
&& Character.isLowSurrogate(text.charAt(i + 1))) {
105+
private int indexOfReplaced(CharSequence text, boolean legacy) {
106+
for (int i = 0; i < text.length();) {
107+
int cp = Character.codePointAt(text, i);
108+
if (isReplaced(cp, legacy)) {
84109
return i;
85110
}
111+
i += Character.charCount(cp);
86112
}
87113
return -1;
88114
}

0 commit comments

Comments
 (0)