Skip to content

Commit 917e4be

Browse files
krickertkrickert
committed
OPENNLP-1887: Synset similarity measures and hypernym-anchored word typing
SynsetSimilarity scores noun synset pairs with the path, Wu-Palmer, and Leacock-Chodorow measures over the knowledge base seam. HypernymTyper labels a word by walking its senses' hypernym and instance-hypernym chains to the nearest caller-registered anchor concept, so a knowledge base can type names as person, organization, or location without a model. Blank checks follow the toolkit whitespace definition.
1 parent cbd1645 commit 917e4be

5 files changed

Lines changed: 649 additions & 0 deletions

File tree

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
.claude
12
*.iml
23
.idea
34
target

‎opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/HypernymTyper.java‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package opennlp.wordnet;
19+
20+
import java.util.ArrayDeque;
21+
import java.util.Deque;
22+
import java.util.HashMap;
23+
import java.util.HashSet;
24+
import java.util.LinkedHashMap;
25+
import java.util.List;
26+
import java.util.Map;
27+
import java.util.Optional;
28+
import java.util.Set;
29+
30+
import opennlp.tools.util.StringUtil;
31+
import opennlp.tools.wordnet.LexicalKnowledgeBase;
32+
import opennlp.tools.wordnet.Synset;
33+
import opennlp.tools.wordnet.WordNetPOS;
34+
import opennlp.tools.wordnet.WordNetRelation;
35+
36+
/**
37+
* Types a noun by walking its hypernym chain to the nearest registered anchor: the
38+
* caller names anchor concepts by lemma, {@code person}, {@code organization},
39+
* {@code location}, and any word whose senses lead up to an anchor's synsets receives
40+
* that anchor's label. The nearest anchor wins, so a more specific registered concept
41+
* beats a general one.
42+
*
43+
* <p>Anchors are resolved against the knowledge base at construction and follow its
44+
* sense inventory; nothing beyond the caller's anchor choice is built in. Words with no
45+
* sense reaching an anchor get no type.</p>
46+
*
47+
* <p>The typer reads only immutable state and is safe to share between threads.</p>
48+
*
49+
* @since 3.0.0
50+
*/
51+
public class HypernymTyper {
52+
53+
/** The relations that lead from a synset to its generalizations. */
54+
private static final List<WordNetRelation> UPWARD_RELATIONS =
55+
List.of(WordNetRelation.HYPERNYM, WordNetRelation.INSTANCE_HYPERNYM);
56+
57+
private final LexicalKnowledgeBase knowledgeBase;
58+
private final Map<String, String> labelBySynsetId;
59+
60+
/**
61+
* Initializes the typer.
62+
*
63+
* @param knowledgeBase The knowledge base to walk. Must not be {@code null}.
64+
* @param anchors The anchor lemmas mapped to the labels they confer, for example
65+
* {@code person} to {@code person}. Every lemma is resolved as a noun;
66+
* all its senses anchor. Must not be {@code null} or empty, and no
67+
* lemma or label may be blank.
68+
* @throws IllegalArgumentException Thrown if a parameter is {@code null},
69+
* {@code anchors} is empty or holds a blank entry, or an anchor lemma is
70+
* unknown to the knowledge base.
71+
*/
72+
public HypernymTyper(LexicalKnowledgeBase knowledgeBase, Map<String, String> anchors) {
73+
if (knowledgeBase == null) {
74+
throw new IllegalArgumentException("knowledgeBase must not be null");
75+
}
76+
if (anchors == null || anchors.isEmpty()) {
77+
throw new IllegalArgumentException("anchors must not be null or empty");
78+
}
79+
this.knowledgeBase = knowledgeBase;
80+
final Map<String, String> labels = new LinkedHashMap<>();
81+
for (final Map.Entry<String, String> anchor : anchors.entrySet()) {
82+
if (anchor.getKey() == null || StringUtil.isBlank(anchor.getKey())
83+
|| anchor.getValue() == null || StringUtil.isBlank(anchor.getValue())) {
84+
throw new IllegalArgumentException("anchors must not contain blank entries");
85+
}
86+
final List<Synset> senses = knowledgeBase.lookup(anchor.getKey(), WordNetPOS.NOUN);
87+
if (senses.isEmpty()) {
88+
throw new IllegalArgumentException(
89+
"anchor lemma is unknown to the knowledge base: " + anchor.getKey());
90+
}
91+
for (final Synset sense : senses) {
92+
labels.putIfAbsent(sense.id(), anchor.getValue());
93+
}
94+
}
95+
this.labelBySynsetId = Map.copyOf(labels);
96+
}
97+
98+
/**
99+
* Types a noun by its nearest anchored hypernym.
100+
*
101+
* @param lemma The noun lemma to type. Must not be {@code null} or blank.
102+
* @return The label of the nearest anchor over all senses, or empty when no sense
103+
* reaches an anchor.
104+
* @throws IllegalArgumentException Thrown if {@code lemma} is {@code null} or blank.
105+
*/
106+
public Optional<String> type(String lemma) {
107+
if (lemma == null || StringUtil.isBlank(lemma)) {
108+
throw new IllegalArgumentException("lemma must not be null or blank");
109+
}
110+
String bestLabel = null;
111+
int bestDistance = Integer.MAX_VALUE;
112+
for (final Synset sense : knowledgeBase.lookup(lemma, WordNetPOS.NOUN)) {
113+
final int[] distance = new int[1];
114+
final String label = nearestAnchor(sense.id(), distance);
115+
if (label != null && distance[0] < bestDistance) {
116+
bestDistance = distance[0];
117+
bestLabel = label;
118+
}
119+
}
120+
return Optional.ofNullable(bestLabel);
121+
}
122+
123+
/**
124+
* Types a specific synset by its nearest anchored hypernym.
125+
*
126+
* @param synsetId The synset identifier. Must not be {@code null}.
127+
* @return The nearest anchor's label, or empty when no ancestor is anchored.
128+
* @throws IllegalArgumentException Thrown if {@code synsetId} is {@code null}.
129+
*/
130+
public Optional<String> typeSynset(String synsetId) {
131+
if (synsetId == null) {
132+
throw new IllegalArgumentException("synsetId must not be null");
133+
}
134+
return Optional.ofNullable(nearestAnchor(synsetId, new int[1]));
135+
}
136+
137+
/** Breadth-first walk up the hypernym graph to the closest anchored synset. */
138+
private String nearestAnchor(String synsetId, int[] distanceOut) {
139+
final Set<String> visited = new HashSet<>();
140+
final Deque<String> queue = new ArrayDeque<>();
141+
final Map<String, Integer> depths = new HashMap<>();
142+
queue.add(synsetId);
143+
visited.add(synsetId);
144+
depths.put(synsetId, 0);
145+
while (!queue.isEmpty()) {
146+
final String current = queue.remove();
147+
final String label = labelBySynsetId.get(current);
148+
if (label != null) {
149+
distanceOut[0] = depths.get(current);
150+
return label;
151+
}
152+
for (final WordNetRelation relation : UPWARD_RELATIONS) {
153+
for (final String parent : knowledgeBase.related(current, relation)) {
154+
if (visited.add(parent)) {
155+
depths.put(parent, depths.get(current) + 1);
156+
queue.add(parent);
157+
}
158+
}
159+
}
160+
}
161+
return null;
162+
}
163+
}

‎opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/SynsetSimilarity.java‎

Lines changed: 202 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,202 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package opennlp.wordnet;
19+
20+
import java.util.ArrayDeque;
21+
import java.util.ArrayList;
22+
import java.util.Deque;
23+
import java.util.HashMap;
24+
import java.util.List;
25+
import java.util.Map;
26+
27+
import opennlp.tools.wordnet.LexicalKnowledgeBase;
28+
import opennlp.tools.wordnet.WordNetRelation;
29+
30+
/**
31+
* Taxonomy-based similarity between synsets: measures over the hypernym graph of a
32+
* {@link LexicalKnowledgeBase}, computed on demand with no precomputed tables.
33+
*
34+
* <p>Path similarity is {@code 1 / (1 + d)} for the shortest hypernym-graph distance
35+
* {@code d} through a common ancestor. Wu-Palmer similarity relates the depth of the
36+
* deepest common ancestor to the depths of both synsets. Leacock-Chodorow scales the
37+
* shortest path against a caller-supplied taxonomy depth, since the knowledge base
38+
* interface does not enumerate the taxonomy. Unrelated synsets, those sharing no
39+
* ancestor, score zero everywhere. Information-content measures need corpus counts and
40+
* are not provided here.</p>
41+
*
42+
* <p>Both plain and instance hypernyms count as taxonomy edges. The measures read only
43+
* the knowledge base and hold no mutable state, so instances are as thread-safe as
44+
* their knowledge base.</p>
45+
*
46+
* @since 3.0.0
47+
*/
48+
public class SynsetSimilarity {
49+
50+
private final LexicalKnowledgeBase knowledgeBase;
51+
52+
/**
53+
* Initializes the measures.
54+
*
55+
* @param knowledgeBase The knowledge base to walk. Must not be {@code null}.
56+
* @throws IllegalArgumentException Thrown if {@code knowledgeBase} is {@code null}.
57+
*/
58+
public SynsetSimilarity(LexicalKnowledgeBase knowledgeBase) {
59+
if (knowledgeBase == null) {
60+
throw new IllegalArgumentException("knowledgeBase must not be null");
61+
}
62+
this.knowledgeBase = knowledgeBase;
63+
}
64+
65+
/**
66+
* Computes path similarity: {@code 1 / (1 + d)} over the shortest hypernym-graph
67+
* distance.
68+
*
69+
* @param synsetId The first synset identifier. Must not be {@code null}.
70+
* @param otherSynsetId The second synset identifier. Must not be {@code null}.
71+
* @return The similarity in {@code (0, 1]}, or {@code 0} when the synsets share no
72+
* ancestor.
73+
* @throws IllegalArgumentException Thrown if an identifier is {@code null}.
74+
*/
75+
public double path(String synsetId, String otherSynsetId) {
76+
final int distance = shortestDistance(synsetId, otherSynsetId);
77+
return distance < 0 ? 0.0 : 1.0 / (1.0 + distance);
78+
}
79+
80+
/**
81+
* Computes Wu-Palmer similarity: {@code 2 * depth(lcs) / (depth(a) + depth(b))},
82+
* with depths counted from the taxonomy root and the deepest common ancestor as the
83+
* lcs.
84+
*
85+
* @param synsetId The first synset identifier. Must not be {@code null}.
86+
* @param otherSynsetId The second synset identifier. Must not be {@code null}.
87+
* @return The similarity in {@code (0, 1]}, or {@code 0} when the synsets share no
88+
* ancestor.
89+
* @throws IllegalArgumentException Thrown if an identifier is {@code null}.
90+
*/
91+
public double wuPalmer(String synsetId, String otherSynsetId) {
92+
final Map<String, Integer> up = depthsAbove(synsetId);
93+
final Map<String, Integer> otherUp = depthsAbove(otherSynsetId);
94+
double best = 0.0;
95+
for (final Map.Entry<String, Integer> common : up.entrySet()) {
96+
final Integer otherDistance = otherUp.get(common.getKey());
97+
if (otherDistance == null) {
98+
continue;
99+
}
100+
final int rootDepth = depthFromRoot(common.getKey());
101+
final int depthA = rootDepth + common.getValue();
102+
final int depthB = rootDepth + otherDistance;
103+
if (depthA + depthB == 0) {
104+
continue;
105+
}
106+
final double score = 2.0 * rootDepth / (depthA + depthB);
107+
best = Math.max(best, score);
108+
}
109+
return best;
110+
}
111+
112+
/**
113+
* Computes Leacock-Chodorow similarity:
114+
* {@code -log((d + 1) / (2 * taxonomyDepth))} over the shortest hypernym-graph
115+
* distance {@code d}.
116+
*
117+
* @param synsetId The first synset identifier. Must not be {@code null}.
118+
* @param otherSynsetId The second synset identifier. Must not be {@code null}.
119+
* @param taxonomyDepth The maximum depth of the taxonomy the synsets live in. Must
120+
* be positive.
121+
* @return The similarity, higher for closer synsets, or {@code 0} when the synsets
122+
* share no ancestor.
123+
* @throws IllegalArgumentException Thrown if an identifier is {@code null} or
124+
* {@code taxonomyDepth} is not positive.
125+
*/
126+
public double leacockChodorow(String synsetId, String otherSynsetId,
127+
int taxonomyDepth) {
128+
if (taxonomyDepth <= 0) {
129+
throw new IllegalArgumentException(
130+
"taxonomyDepth must be positive: " + taxonomyDepth);
131+
}
132+
final int distance = shortestDistance(synsetId, otherSynsetId);
133+
if (distance < 0) {
134+
return 0.0;
135+
}
136+
return -Math.log((distance + 1.0) / (2.0 * taxonomyDepth));
137+
}
138+
139+
/**
140+
* Finds the shortest distance between two synsets through a common ancestor.
141+
*
142+
* @param synsetId The first synset identifier. Must not be {@code null}.
143+
* @param otherSynsetId The second synset identifier. Must not be {@code null}.
144+
* @return The edge count of the shortest connecting path, or {@code -1} when no
145+
* common ancestor exists.
146+
* @throws IllegalArgumentException Thrown if an identifier is {@code null}.
147+
*/
148+
public int shortestDistance(String synsetId, String otherSynsetId) {
149+
final Map<String, Integer> up = depthsAbove(synsetId);
150+
final Map<String, Integer> otherUp = depthsAbove(otherSynsetId);
151+
int best = -1;
152+
for (final Map.Entry<String, Integer> common : up.entrySet()) {
153+
final Integer otherDistance = otherUp.get(common.getKey());
154+
if (otherDistance != null) {
155+
final int total = common.getValue() + otherDistance;
156+
if (best < 0 || total < best) {
157+
best = total;
158+
}
159+
}
160+
}
161+
return best;
162+
}
163+
164+
/** Collects every ancestor with its minimal upward distance, the synset included. */
165+
private Map<String, Integer> depthsAbove(String synsetId) {
166+
if (synsetId == null) {
167+
throw new IllegalArgumentException("synset identifiers must not be null");
168+
}
169+
final Map<String, Integer> depths = new HashMap<>();
170+
final Deque<String> queue = new ArrayDeque<>();
171+
depths.put(synsetId, 0);
172+
queue.add(synsetId);
173+
while (!queue.isEmpty()) {
174+
final String current = queue.remove();
175+
final int depth = depths.get(current);
176+
for (final String parent : hypernyms(current)) {
177+
if (!depths.containsKey(parent) || depths.get(parent) > depth + 1) {
178+
depths.put(parent, depth + 1);
179+
queue.add(parent);
180+
}
181+
}
182+
}
183+
return depths;
184+
}
185+
186+
/** Measures a synset's depth from its taxonomy root, the shortest way up. */
187+
private int depthFromRoot(String synsetId) {
188+
final Map<String, Integer> above = depthsAbove(synsetId);
189+
int deepest = 0;
190+
for (final int distance : above.values()) {
191+
deepest = Math.max(deepest, distance);
192+
}
193+
return deepest;
194+
}
195+
196+
private Iterable<String> hypernyms(String synsetId) {
197+
final List<String> parents = new ArrayList<>(
198+
knowledgeBase.related(synsetId, WordNetRelation.HYPERNYM));
199+
parents.addAll(knowledgeBase.related(synsetId, WordNetRelation.INSTANCE_HYPERNYM));
200+
return parents;
201+
}
202+
}

0 commit comments

Comments
 (0)