From 7da7ec655c061d58f9456c924dacc78e34248d3d Mon Sep 17 00:00:00 2001 From: Dima Chechetkin Date: Wed, 18 Sep 2019 18:16:01 -0400 Subject: [PATCH 1/3] refactoring --- build.gradle | 5 +- .../com/onkiup/linker/parser/EnumRule.java | 6 + .../onkiup/linker/parser/ParserLocation.java | 27 ++ .../onkiup/linker/parser/PatternMatcher.java | 6 +- .../onkiup/linker/parser/TerminalMatcher.java | 2 +- .../onkiup/linker/parser/TokenMatcher.java | 2 +- .../onkiup/linker/parser/TokenTestResult.java | 8 +- .../linker/parser/annotation/AcceptAll.java | 12 - .../linker/parser/token/CompoundToken.java | 94 +++++ .../linker/parser/token/ConsumingToken.java | 141 +++++++- .../onkiup/linker/parser/token/EnumToken.java | 137 +++++++ .../linker/parser/token/PartialToken.java | 199 +++++------ .../onkiup/linker/parser/token/RuleToken.java | 334 ++++++------------ .../linker/parser/token/TerminalToken.java | 145 +++----- .../linker/parser/util/ParserError.java | 10 +- 15 files changed, 667 insertions(+), 461 deletions(-) create mode 100644 src/main/java/com/onkiup/linker/parser/EnumRule.java delete mode 100644 src/main/java/com/onkiup/linker/parser/annotation/AcceptAll.java create mode 100644 src/main/java/com/onkiup/linker/parser/token/CompoundToken.java create mode 100644 src/main/java/com/onkiup/linker/parser/token/EnumToken.java diff --git a/build.gradle b/build.gradle index 3606543..2c00a9e 100644 --- a/build.gradle +++ b/build.gradle @@ -1,4 +1,4 @@ - /* +/* * This build file was auto generated by running the Gradle 'init' task * by 'chedim' at '7/5/19 1:04 PM' with Gradle 3.2.1 * @@ -9,9 +9,9 @@ // Apply the java plugin to add support for Java plugins { - id "java" id "maven-publish" id "eclipse" + id "java" } project.group = 'com.onkiup' @@ -27,6 +27,7 @@ repositories { // Use 'jcenter' for resolving your dependencies. // You can declare any Maven/Ivy/file repository here. jcenter() + mavenLocal() } // In this section you declare the dependencies for your production and test code diff --git a/src/main/java/com/onkiup/linker/parser/EnumRule.java b/src/main/java/com/onkiup/linker/parser/EnumRule.java new file mode 100644 index 0000000..8be3639 --- /dev/null +++ b/src/main/java/com/onkiup/linker/parser/EnumRule.java @@ -0,0 +1,6 @@ +package com.onkiup.linker.parser; + +public interface EnumRule { + String getTerminal(); +} + diff --git a/src/main/java/com/onkiup/linker/parser/ParserLocation.java b/src/main/java/com/onkiup/linker/parser/ParserLocation.java index c5c814c..ac65fa6 100644 --- a/src/main/java/com/onkiup/linker/parser/ParserLocation.java +++ b/src/main/java/com/onkiup/linker/parser/ParserLocation.java @@ -1,5 +1,7 @@ package com.onkiup.linker.parser; +import java.util.Objects; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -11,6 +13,21 @@ public class ParserLocation { private final int line, column, position; private final String name; + public static ParserLocation endOf(CharSequence text) { + int lines = 0; + int column = 0; + for (int i = 0; i < text.length(); i++) { + if (text.charAt(i) == '\n') { + line++; + column = 0; + } else { + column++; + } + } + + return new ParserLocation(null, text.length(), lines, column); + } + public ParserLocation(String name, int position, int line, int column) { if (position < 0) { throw new IllegalArgumentException("Position cannot be negative"); @@ -75,5 +92,15 @@ public ParserLocation advance(CharSequence source) { logger.debug("Advanced from {} to {} using chars: '{}'", this, result, source); return result; } + + public ParserLocation add(ParserLocation another) { + if (another.name() != null && !Objects.equals(name(), another.name())) { + throw new IllegalArgumentException("Unable to add parser location with a different name"); + } + int anotherLines = another.line(); + int resultLine = line + anotherLines; + int resultColumn = anotherLines == 0 ? column + another.column() : another.column(); + return new ParserLocation(name, resultLine, resultColumn, position + another.position()); + } } diff --git a/src/main/java/com/onkiup/linker/parser/PatternMatcher.java b/src/main/java/com/onkiup/linker/parser/PatternMatcher.java index 471c4cc..4d532e5 100644 --- a/src/main/java/com/onkiup/linker/parser/PatternMatcher.java +++ b/src/main/java/com/onkiup/linker/parser/PatternMatcher.java @@ -39,7 +39,7 @@ public PatternMatcher(CapturePattern pattern) { } @Override - public TokenTestResult apply(StringBuilder buffer) { + public TokenTestResult apply(CharSequence buffer) { matcher.reset(buffer); boolean matches = matcher.matches(), lookingAt = matcher.lookingAt(), @@ -54,7 +54,7 @@ public TokenTestResult apply(StringBuilder buffer) { matcher.appendReplacement(result, replacement); return TestResult.match(matcher.end(), result.toString()); } else { - String token = buffer.substring(0, matcher.end()); + String token = buffer.subSequence(0, matcher.end()).toString(); return TestResult.match(matcher.end(), token); } } else { @@ -68,7 +68,7 @@ public TokenTestResult apply(StringBuilder buffer) { } else if (lookingAt) { return TestResult.fail(); } else { - String token = buffer.substring(0, matcher.start()); + String token = buffer.subSequence(0, matcher.start()).toString(); return TestResult.match(matcher.start(), token); } } else { diff --git a/src/main/java/com/onkiup/linker/parser/TerminalMatcher.java b/src/main/java/com/onkiup/linker/parser/TerminalMatcher.java index 36f44f5..c94bcbe 100644 --- a/src/main/java/com/onkiup/linker/parser/TerminalMatcher.java +++ b/src/main/java/com/onkiup/linker/parser/TerminalMatcher.java @@ -11,7 +11,7 @@ public TerminalMatcher(String pattern) { } @Override - public TokenTestResult apply(StringBuilder buffer) { + public TokenTestResult apply(CharSequence buffer) { int bufferLen = buffer.length(); int charsToCompare = Math.min(patternLen, bufferLen); for (int i = 0; i < charsToCompare; i++) { diff --git a/src/main/java/com/onkiup/linker/parser/TokenMatcher.java b/src/main/java/com/onkiup/linker/parser/TokenMatcher.java index 69c0d08..179c788 100644 --- a/src/main/java/com/onkiup/linker/parser/TokenMatcher.java +++ b/src/main/java/com/onkiup/linker/parser/TokenMatcher.java @@ -7,7 +7,7 @@ import com.onkiup.linker.parser.annotation.CapturePattern; @FunctionalInterface -public interface TokenMatcher extends Function { +public interface TokenMatcher extends Function { public static TokenMatcher forField(Field field) { Class type = field.getType(); diff --git a/src/main/java/com/onkiup/linker/parser/TokenTestResult.java b/src/main/java/com/onkiup/linker/parser/TokenTestResult.java index 4bec2df..8c9ed67 100644 --- a/src/main/java/com/onkiup/linker/parser/TokenTestResult.java +++ b/src/main/java/com/onkiup/linker/parser/TokenTestResult.java @@ -1,11 +1,11 @@ package com.onkiup.linker.parser; -public class TokenTestResult { +public class TokenTestResult { private TestResult result; - private Object token; + private X token; private int length; - protected TokenTestResult(TestResult result, int length, Object token) { + protected TokenTestResult(TestResult result, int length, X token) { this.result = result; this.length = length; this.token = token; @@ -23,7 +23,7 @@ public int getTokenLength() { return length; } - public Object getToken() { + public X getToken() { return token; } diff --git a/src/main/java/com/onkiup/linker/parser/annotation/AcceptAll.java b/src/main/java/com/onkiup/linker/parser/annotation/AcceptAll.java deleted file mode 100644 index f398365..0000000 --- a/src/main/java/com/onkiup/linker/parser/annotation/AcceptAll.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.onkiup.linker.parser.annotation; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -@Target(ElementType.TYPE) -@Retention(RetentionPolicy.RUNTIME) -public @interface AcceptAll { -} - diff --git a/src/main/java/com/onkiup/linker/parser/token/CompoundToken.java b/src/main/java/com/onkiup/linker/parser/token/CompoundToken.java new file mode 100644 index 0000000..8214730 --- /dev/null +++ b/src/main/java/com/onkiup/linker/parser/token/CompoundToken.java @@ -0,0 +1,94 @@ +package com.onkiup.linker.parser.token; + +import java.util.Arrays; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Consumer; + +import com.onkiup.linker.parser.Rule; + +public interface CompoundToken extends PartialToken { + + void onChildPopulated(); + + void onChildFailed(); + + PartialToken[] children(); + void children(PartialToken[] children); + + PartialToken nextChild(); + + CharSequence traceback(); + + /** + * advances to the next child token or parent + * @return next token to populate or null if this is a root token and it has no further tokens to populate + */ + default Optional> advance() { + if (isPopulated() || isFailed()) { + return parent().map(p -> p); + } + + return Optional.of(nextChild()); + } + + /** + * @return String containing all characters to ignore for this token + */ + default String ignoredCharacters() { + return ""; + } + + /** + * @return number of alternatives for this token, including its children + */ + @Override + default int alternativesLeft() { + return Arrays.stream(children()) + .filter(Objects::nonNull) + .mapToInt(PartialToken::alternativesLeft) + .sum(); + } + + @Override + default int basePriority() { + int result = PartialToken.super.basePriority(); + + for (PartialToken child : children()) { + if (child != null && child.propagatePriority()) { + result += child.basePriority(); + } + } + + return result; + } + + default void rotate() { + } + + default boolean rotatable() { + return false; + } + + default void unrotate() { + } + + @Override + default CharSequence source() { + StringBuilder result = new StringBuilder(); + Arrays.stream(children()) + .filter(Objects::nonNull) + .map(PartialToken::source) + .forEach(result::append); + return result; + } + + @Override + default void visit(Consumer> visitor) { + Arrays.stream(children()) + .filter(Objects::nonNull) + .forEach(child -> child.visit(visitor)); + PartialToken.super.visit(visitor); + } +} + diff --git a/src/main/java/com/onkiup/linker/parser/token/ConsumingToken.java b/src/main/java/com/onkiup/linker/parser/token/ConsumingToken.java index f77068c..c3d4e25 100644 --- a/src/main/java/com/onkiup/linker/parser/token/ConsumingToken.java +++ b/src/main/java/com/onkiup/linker/parser/token/ConsumingToken.java @@ -1,12 +1,149 @@ package com.onkiup.linker.parser.token; import java.util.Optional; +import java.util.WeakHashMap; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +import com.onkiup.linker.parser.ParserLocation; +import com.onkiup.linker.parser.TestResult; +import com.onkiup.linker.parser.TokenMatcher; +import com.onkiup.linker.parser.TokenTestResult; +import com.onkiup.linker.parser.util.ParserError; public interface ConsumingToken extends PartialToken { + + default void setTokenMatcher(TokenMatcher matcher) { + CharSequence ignoredCharacters = parent().map(CompoundToken::ignoredCharacters).orElse(null); + ConsumptionState.create(this, ignoredCharacters, matcher); + } + + void onConsumeSuccess(Object token) { + parent().ifPresent(CompoundToken::onChildPopulated); + } + + /** * Attempts to consume next character - * @returns null if character was consumed, otherwise returns a StringBuilder with failed characters + * @return null if character was consumed, otherwise returns a CharSequence with failed characters */ - Optional consume(char character, boolean last); + default CharSequence consume(char character, boolean last) { + ConsumptionState consumption = ConsumptionState.of(this); + consumption.consume(character); + + if (isPopulated() || consumption.failed()) { + if (!lookahead(consumption.buffer())) { + onFail(); + // not accepting new characters at this time + return parent() + .map(CompoundToken::traceback) + .map(StringBuilder::new) + .map(sb -> sb.append(consumption.consumed())) + .orElseGet(consumption::consumed); + } + // performing lookahead so, reporting successfull consumption (despite that the token has already failed) + return null; + } + + TokenTestResult result = consumption.test(); + + if (result.isFailed() || (result.isContinue() && last)) { + // switching to lookahead mode + consumption.setFailed(); + return null; + } else if (result.isMatch() || (result.isMatchContinue() && last)) { + int tokenLength = result.getTokenLength(); + StringBuilder buffer = consumption.buffer(); + CharSequence excess = buffer.substring(tokenLength); + onConsumeSuccess(result.getToken()); + onPopulated(consumption.end()); + return excess; + } + + return null; + } + + @Override + default void invalidate() { + PartialToken.super.invalidate(); + ConsumptionState.discard(this); + } + + @Override + default CharSequence source() { + if (isFailed()) { + return ""; + } + return ConsumptionState.of(this).consumed(); + } + + class ConsumptionState { + private static final ConcurrentHashMap states = new ConcurrentHashMap<>(); + + private static synchronized ConsumptionState of(ConsumingToken token) { + if (!states.containsKey(token)) { + throw new ParserError("No consumption state available for token " + token + " (create one by calling ConsumingToken::setMatcher first?)", token); + } + return states.get(token); + } + + private static void create(ConsumingToken token, CharSequence ignoredCharacters, Function tester) { + states.put(token, new ConsumptionState(ignoredCharacters, tester)); + } + + private static void discard(ConsumingToken token) { + states.remove(token); + } + + private final StringBuilder buffer = new StringBuilder(); + private final StringBuilder consumed = new StringBuilder(); + private final CharSequence ignoredCharacters; + private final Function tester; + private ParserLocation end; + private boolean failed; + + private ConsumptionState(CharSequence ignoredCharacters, Function tester) { + this.ignoredCharacters = ignoredCharacters; + this.tester = tester; + } + + protected StringBuilder buffer() { + return buffer; + } + + protected StringBuilder consumed() { + return consumed; + } + + protected ParserLocation end() { + return ParserLocation.endOf(consumed()); + } + + private boolean ignored(int character) { + return ignoredCharacters != null && ignoredCharacters.chars().anyMatch(ignored -> ignored == character); + } + + private void consume(char character) { + consumed.append(character); + if (buffer.length() > 0 || !ignored(character)) { + buffer.append(character); + } + } + + private TokenTestResult test() { + if (buffer.length() == 0) { + return TestResult.continueNoMatch(); + } + return tester.apply(buffer); + } + + private void setFailed() { + failed = true; + } + + private boolean failed() { + return failed; + } + } } diff --git a/src/main/java/com/onkiup/linker/parser/token/EnumToken.java b/src/main/java/com/onkiup/linker/parser/token/EnumToken.java new file mode 100644 index 0000000..3a67f14 --- /dev/null +++ b/src/main/java/com/onkiup/linker/parser/token/EnumToken.java @@ -0,0 +1,137 @@ +package com.onkiup.linker.parser.token; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import com.onkiup.linker.parser.ParserLocation; +import com.onkiup.linker.parser.PatternMatcher; +import com.onkiup.linker.parser.TestResult; +import com.onkiup.linker.parser.TokenTestResult; +import com.onkiup.linker.parser.annotation.CapturePattern; +import com.onkiup.linker.parser.util.ParserError; + +public class EnumToken implements ConsumingToken { + + private Class enumType; + private int nextVariant = 0; + private CompoundToken parent; + private Field field; + private ParserLocation location, end; + private Map variants = new HashMap<>(); + private X token; + private boolean failed, optional, populated; + private String ignoreCharacters; + + public EnumToken(CompoundToken parent, Field field, Class enumType, ParserLocation location) { + this.enumType = enumType; + this.location = location; + this.parent = parent; + this.field = field; + + for (X variant : enumType.getEnumConstants()) { + try { + Field variantField = enumType.getDeclaredField(variant.name()); + CapturePattern annotation = field.getAnnotation(CapturePattern.class); + if (annotation == null) { + throw new ParserError("Unable to use enum value " + variant + ": no @CapturePattern present", this); + } + PatternMatcher matcher = new PatternMatcher(annotation); + variants.put(variant, matcher); + } catch (ParserError pe) { + throw pe; + } catch (Exception e) { + throw new ParserError("Failed to read field for enum value " + variant, this, e); + } + } + + setTokenMatcher(buffer -> { + if (variants.size() == 0) { + return TestResult.fail(); + } + + List failed = new ArrayList<>(); + for (Map.Entry entry : variants.entrySet()) { + TokenTestResult result = entry.getValue().apply(buffer); + if (result.isMatch()) { + return new TestResult.match(result.getTokenLength(), entry.getKey()); + } else if (result.isFailed()) { + failed.add(entry.getKey()); + } + } + + failed.forEach(variants::remove); + + if (variants.size() == 0) { + return TestResult.fail(); + } + + return TestResult.continueNoMatch(); + }); + } + + @Override + public void onPopulated(ParserLocation end) { + this.end = end; + this.populated = true; + } + + @Override + public void markOptional() { + this.optional = true; + } + + @Override + public ParserLocation location() { + return location; + } + + @Override + public ParserLocation end() { + return end != null ? end : location; + } + + @Override + public Optional targetField() { + return Optional.ofNullable(field); + } + + @Override + public X token() { + return token; + } + + @Override + public Class tokenType() { + return enumType; + } + + @Override + public void onConsumeSuccess(Object value) { + token = (X) value; + } + + @Override + public boolean isPopulated() { + return token != null; + } + + @Override + public boolean isFailed() { + return failed; + } + + @Override + public boolean isOptional() { + return optional; + } + + @Override + public Optional> parent() { + return Optional.ofNullable(parent); + } +} + diff --git a/src/main/java/com/onkiup/linker/parser/token/PartialToken.java b/src/main/java/com/onkiup/linker/parser/token/PartialToken.java index 31c4f9d..ff8c887 100644 --- a/src/main/java/com/onkiup/linker/parser/token/PartialToken.java +++ b/src/main/java/com/onkiup/linker/parser/token/PartialToken.java @@ -15,11 +15,13 @@ import com.onkiup.linker.parser.SyntaxError; import com.onkiup.linker.parser.TokenGrammar; import com.onkiup.linker.parser.annotation.AdjustPriority; +import com.onkiup.linker.parser.annotation.OptionalToken; +import com.onkiup.linker.parser.annotation.SkipIfFollowedBy; import com.onkiup.linker.parser.util.ParserError; public interface PartialToken { - static PartialToken forField(PartialToken parent, Field field, ParserLocation position) { + static PartialToken forField(CompoundToken parent, Field field, ParserLocation position) { if (position == null) { throw new ParserError("Child token position cannot be null", parent); @@ -27,12 +29,12 @@ static PartialToken forField(PartialToken parent, Field field, ParserLocation po Class fieldType = field.getType(); if (fieldType.isArray()) { - return new CollectionToken(parent, field, position); + return new CollectionToken(parent, field, fieldType, position); } else if (Rule.class.isAssignableFrom(fieldType)) { if (!TokenGrammar.isConcrete(fieldType)) { - return new VariantToken(parent, fieldType, position); + return new VariantToken(parent, field, fieldType, position); } else { - return new RuleToken(parent, fieldType, position); + return new RuleToken(parent, field, fieldType, position); } } else if (fieldType == String.class) { return new TerminalToken(parent, field, position); @@ -40,77 +42,111 @@ static PartialToken forField(PartialToken parent, Field field, ParserLocation po throw new IllegalArgumentException("Unsupported field type: " + fieldType); } - static PartialToken forClass(PartialToken parent, Class type, ParserLocation position) { + static PartialToken forClass(Class type, ParserLocation position) { if (position == null) { - throw new ParserError("Child token location cannot be null", parent); + position = new ParserLocation(null, 0, 0, 0); } if (TokenGrammar.isConcrete(type)) { - return new RuleToken(parent, type, position); + return new RuleToken(null, null, type, position); } else { - return new VariantToken(parent, type, position); + return new VariantToken(null, null, type, position); } } + static CharSequence getOptionalCondition(Field field) { + if (field == null) { + return null; + } + if (field.isAnnotationPresent(OptionalToken.class)) { + return field.getAnnotation(OptionalToken.class).whenFollowedBy(); + } else if (field.isAnnotationPresent(SkipIfFollowedBy.class)) { + return field.getAnnotation(SkipIfFollowedBy.class).value(); + } + return null; + } + + static boolean isOptional(Field field) { + return field != null && field.isAnnotationPresent(OptionalToken.class); + } + /** - * advances this token using given subToken and returns next token to populate - * null sub-token indicates that current token did not match - * @param subToken populated sub-token - * @returns next token to populate or null if this is a root token and it has no further tokens to populate + * @return Java representation of populated token */ - Optional advance(boolean forcePopulate) throws SyntaxError; - - /** - * Rollbacks the token until last junction - * Called by failed child token on a parent - * @returns StringBuilder with characters to be returned to the buffer or null + X token(); + + Class tokenType(); + + boolean isPopulated(); + boolean isFailed(); + boolean isOptional(); + + Optional> parent(); + Optional targetField(); + + ParserLocation location(); + ParserLocation end(); + + void markOptional(); + void onPopulated(ParserLocation end); + + /** + * @return all characters consumed by the token and its children */ - default Optional pushback(boolean force) { - return Optional.empty(); - } + CharSequence source(); /** - * Pullbacks token characters - * Called by parent token on a child that was failed by latter child - * @returns StringBuilder with characters to be returned to the buffer of empty + * Called upon token failures */ - Optional pullback(); + default void onFail() { + invalidate(); + parent().ifPresent(CompoundToken::onChildFailed); + } /** - * @returns Java representation of populated token + * Called on failed tokens + * @return true if the token should continue consumption, false otherwise */ - X getToken(); - - Class getTokenType(); - - boolean isPopulated(); + default boolean lookahead(CharSequence buffer) { + CharSequence optionalCondition = PartialToken.getOptionalCondition(targetField().orElse(null)); + final CharSequence[] parentBuffer = new CharSequence[] { buffer }; + boolean conditionPresent = optionalCondition != null && optionalCondition.length() > 0; + boolean myResult = true; + + if (!isOptional() && conditionPresent) { + if (buffer.length() >= optionalCondition.length()) { + CharSequence test = buffer.subSequence(0, optionalCondition.length()); + if (Objects.equals(test, optionalCondition)) { + parentBuffer[0] = buffer.subSequence(optionalCondition.length(), buffer.length()); + markOptional(); + } + myResult = false; + } + } else if (conditionPresent) { + parentBuffer[0] = buffer.subSequence(optionalCondition.length(), buffer.length()); + } - Optional getParent(); + return myResult || parent() + // delegating lookahead call to parent + .flatMap(p -> p.lookahead(parentBuffer[0]) ? Optional.of(true) : Optional.empty()) + .isPresent(); + } - default Optional findInTree(Predicate comparator) { + default Optional> findInTree(Predicate comparator) { if (comparator.test(this)) { return Optional.of(this); } - return getParent() + return parent() .flatMap(parent -> parent.findInTree(comparator)); } - default List getPath() { - LinkedList result = new LinkedList<>(); - getParent().ifPresent(parent -> result.addAll(parent.getPath())); + default List> path() { + LinkedList> result = new LinkedList<>(); + parent().ifPresent(parent -> result.addAll(parent.path())); result.add(this); return result; } - /** - * @returns String containing all characters to ignore for this token - */ - default String getIgnoredCharacters() { - return ""; - } - - ParserLocation location(); - default int position() { ParserLocation location = location(); if (location == null) { @@ -119,44 +155,18 @@ default int position() { return location.position(); } - int consumed(); - - default int alternativesLeft() { - return Arrays.stream(getChildren()) - .filter(Objects::nonNull) - .mapToInt(PartialToken::alternativesLeft) - .sum(); - } - - default void rotate() { - } - - default boolean rotatable() { - return false; - } - - default void unrotate() { - } - default int basePriority() { int result = 0; - Class tokenType = getTokenType(); + Class tokenType = tokenType(); if (tokenType.isAnnotationPresent(AdjustPriority.class)) { AdjustPriority adjustment = tokenType.getAnnotation(AdjustPriority.class); result += adjustment.value(); } - - for (PartialToken child : getChildren()) { - if (child != null && child.propagatePriority()) { - result += child.basePriority(); - } - } - return result; } default boolean propagatePriority() { - Class tokenType = getTokenType(); + Class tokenType = tokenType(); if (tokenType.isAnnotationPresent(AdjustPriority.class)) { return tokenType.getAnnotation(AdjustPriority.class).propagate(); } @@ -168,45 +178,29 @@ default void sortPriorities() { } - default PartialToken[] getChildren() { - return new PartialToken[0]; - } - - default void setChildren(PartialToken[] children) { - throw new RuntimeException("setChildren is unsupported for " + this); - } - default PartialToken replaceCurrentToken() { throw new RuntimeException("Unsupported"); } - default void setToken(X token) { - throw new RuntimeException("Unsupported"); + default void token(X token) { + throw new RuntimeException("Unsupported"); } default void invalidate() { - } - ParserLocation end(); - - StringBuilder source(); - - default String tag() { - return "UNKNOWN"; + default void visit(Consumer> visitor) { + visitor.accept(this); } - default void visit(Consumer visitor) { - Arrays.stream(getChildren()) - .filter(Objects::nonNull) - .forEach(child -> child.visit(visitor)); - visitor.accept(this); + default int alternativesLeft() { + return 0; } - default PartialToken root() { - PartialToken current = this; + default PartialToken root() { + PartialToken current = this; while(true) { - PartialToken parent = (PartialToken) current.getParent().orElse(null); + PartialToken parent = current.parent().orElse(null); if (parent == null) { return current; } @@ -214,14 +208,13 @@ default PartialToken root() { } } - default String tail(int length) { - String source = source().toString(); + default CharSequence tail(int length) { + CharSequence source = source(); if (source.length() > length) { - source = source.substring(source.length() - length); + source = source.subSequence(source.length() - length, length); } return String.format("%" + length + "s", source); } - PartialToken expected(); } diff --git a/src/main/java/com/onkiup/linker/parser/token/RuleToken.java b/src/main/java/com/onkiup/linker/parser/token/RuleToken.java index e5fd016..5499826 100644 --- a/src/main/java/com/onkiup/linker/parser/token/RuleToken.java +++ b/src/main/java/com/onkiup/linker/parser/token/RuleToken.java @@ -13,27 +13,31 @@ import com.onkiup.linker.parser.ParserLocation; import com.onkiup.linker.parser.Rule; -import com.onkiup.linker.parser.SyntaxError; import com.onkiup.linker.parser.annotation.IgnoreCharacters; -import com.onkiup.linker.parser.annotation.OptionalToken; -public class RuleToken implements PartialToken { +public class RuleToken implements CompoundToken { private X token; private Class tokenType; private Field[] fields; private PartialToken[] values; - private PartialToken parent; + private int nextChild = 0; + private CompoundToken parent; + private Field field; private String ignoreCharacters = ""; private int nextField; private boolean rotated = false; + private boolean populated = false; private boolean failed = false; + private boolean optional; + private CharSequence optionalCondition; private final ParserLocation location; private ParserLocation lastTokenEnd; private final Logger logger; - public RuleToken(PartialToken parent, Class type, ParserLocation location) { + public RuleToken(CompoundToken parent, Field field, Class type, ParserLocation location) { this.tokenType = type; this.parent = parent; + this.field = field; this.location = location; this.logger = LoggerFactory.getLogger(type); @@ -44,13 +48,13 @@ public RuleToken(PartialToken parent, Class type, ParserLocat } fields = Arrays.stream(type.getDeclaredFields()) - .filter(field -> !Modifier.isTransient(field.getModifiers())) + .filter(childField -> !Modifier.isTransient(childField.getModifiers())) .toArray(Field[]::new); values = new PartialToken[fields.length]; if (parent != null) { - ignoreCharacters += parent.getIgnoredCharacters(); + ignoreCharacters += parent.ignoredCharacters(); } if (type.isAnnotationPresent(IgnoreCharacters.class)) { @@ -60,214 +64,117 @@ public RuleToken(PartialToken parent, Class type, ParserLocat } ignoreCharacters += type.getAnnotation(IgnoreCharacters.class).value(); } + + optionalCondition = PartialToken.getOptionalCondition(field); + optional = optionalCondition == null && PartialToken.isOptional(field); } @Override - public Optional pushback(boolean force) { - logger.info("Received pushback request"); - StringBuilder result = new StringBuilder(); - Field lastField = fields[nextField - 1]; - - if (force || !isOptional(nextField - 1)) { - int backField; - for (backField = nextField - 1; backField > -1; backField--) { - PartialToken token = values[backField]; - Field field = fields[backField]; - - logger.debug("Trying to rotate"); - if (token.rotatable()) { - token.rotate(); - logger.debug("Rotated successfully"); - rotated = true; - break; - } - - if (token.alternativesLeft() > 0) { - logger.debug("Found alternatives at field {}", field.getName()); - token.pushback(false).ifPresent(result::append); - break; + public void sortPriorities() { + if (rotatable()) { + if (values[0] instanceof CompoundToken) { + CompoundToken child = (CompoundToken) values[0]; + if (child.rotatable()) { + int myPriority = basePriority(); + int childPriority = child.basePriority(); + logger.debug("Verifying priority order for tokens; parent: {} child: {}", myPriority, childPriority); + if (childPriority < myPriority) { + logger.debug("Fixing priority order"); + unrotate(); + } } } - - if (backField < 0) { - logger.debug("FAILED, pushing parent"); - failed = true; - getParent() - .flatMap(p -> p.pushback(false)) - .ifPresent(b -> result.append(b)); - lastTokenEnd = location; - return Optional.of(result); - } - - - for (int i = backField + 1; i < nextField - 1; i++) { - Field field = fields[i]; - logger.debug("pulling back field {}", this, tokenType.getSimpleName(), field.getName()); - values[i].pullback().ifPresent(b -> { - logger.debug("pulled back '{}' from field {}", b, field.getName()); - result.append(b); - }); - } - - nextField = backField + 1; - lastTokenEnd = values[backField].end(); - - return Optional.of(result); - } else { - logger.debug("Not propagating pushback call from optional field"); - return values[nextField - 1].pullback(); } } @Override - public Optional pullback() { - logger.debug("Pullback request received"); - StringBuilder result = new StringBuilder(); - for (int i = 0; i < nextField; i++) { - PartialToken token = values[i]; - if (token == null) { - continue; - } - Field field = fields[i]; - logger.debug("Pulling back field {}", field.getName()); - token.pullback().ifPresent(b -> { - logger.debug("Field {} return characters: '{}'", field.getName(), b); - result.append(b); - }); - logger.debug("Pulled back field {}", field.getName()); - } - - failed = true; - return Optional.of(result); + public void markOptional() { + optional = true; } @Override - public Optional advance(boolean force) throws SyntaxError { - - if (failed) { - logger.debug("Short-curcuiting advance request on failed token to parent"); - return parent == null ? Optional.empty() : parent.advance(force); - } + public boolean isOptional() { + return optional; + } - if (nextField > 0) { - // checking if the last token was successfully populated - int currentField = nextField - 1; - Field field = fields[currentField]; - PartialToken token = values[currentField]; - logger.debug("Verifying results for field {}", field.getName()); - - if (rotated) { - logger.info("Last token was rotated, not advancing"); - rotated = false; - return Optional.of(token); - } if (token != null && token.isPopulated()) { - set(field, token.getToken()); - lastTokenEnd = token.end(); - } else if (token != null && token.alternativesLeft() > 0) { - logger.debug("last token still has some alternatives, not advancing"); - return Optional.of(token); - } else if (token != null && !isOptional(currentField)){ - logger.debug("FAILED!"); - failed = true; - if (parent == null) { - throw new SyntaxError("Expected " + field, this, null); - } else { - return parent == null ? Optional.empty() : parent.advance(force); - } - } - } + @Override + public Optional targetField() { + return Optional.ofNullable(field); + } - if (force && nextField < fields.length) { - logger.debug("Force-advancing!"); - // checking if all unpopulated fields are optional - // and fast-forwarding to populate this token if possible - do { - Field field = fields[nextField]; - if (!isOptional(nextField)) { - break; - } - set(field, null); - } while (++nextField < fields.length); - - if (isPopulated()) { - logger.debug("Force-populated!"); - sortPriorities(); - Rule.Metadata.metadata(token, this); - return parent == null ? Optional.empty() : parent.advance(force); - } - - logger.debug("Force-populate failed"); - return pushback(false).map(b -> new FailedToken(parent, b, location())); - } + @Override + public X token() { + return token; + } - if (nextField < fields.length && nextField > -1) { - Field field = fields[nextField]; - logger.info("Populating field {}", field.getName()); - return Optional.of(values[nextField++] = createChild(field, end())); - } + @Override + public Class tokenType() { + return tokenType; + } - logger.debug("Populated"); - sortPriorities(); - Rule.Metadata.metadata(token, this); - logger.debug("Advancing parent token"); - return parent == null ? Optional.empty() : parent.advance(force); + @Override + public Optional> parent() { + return Optional.ofNullable(parent); } - protected PartialToken createChild(Field field, ParserLocation location) { - return PartialToken.forField(this, field, location); + @Override + public boolean isPopulated() { + return populated; } @Override - public void sortPriorities() { - if (rotatable()) { - PartialToken child = values[0]; - if (child != null && child.rotatable()) { - int myPriority = basePriority(); - int childPriority = child.basePriority(); - logger.debug("Verifying priority order for tokens; parent: {} child: {}", myPriority, childPriority); - if (childPriority < myPriority) { - logger.debug("Fixing priority order"); - unrotate(); - } - } - } + public boolean isFailed() { + return failed; } @Override - public X getToken() { - return token; + public String ignoredCharacters() { + return ignoreCharacters; } - @Override - public Class getTokenType() { - return tokenType; + @Override + public PartialToken nextChild() { + Field childField = fields[nextChild]; + PartialToken result = PartialToken.forField(this, childField, lastTokenEnd); + values[nextChild++] = result; + return result; } @Override - public Optional getParent() { - return Optional.ofNullable(parent); + public void onChildPopulated() { + PartialToken child = values[nextChild - 1]; + Field field = fields[nextChild - 1]; + set(field, child.token()); + lastTokenEnd = child.end(); } @Override - public boolean isPopulated() { - for (int i = values.length - 1; i > -1; i--) { - PartialToken lastToken = values[i]; - if (lastToken != null && lastToken.isPopulated()) { - return true; - } - Field field = fields[i]; - if (!isOptional(i)) { - logger.debug("Not populated -- field {} is not optional", field.getName()); - return false; - } + public void onChildFailed() { + PartialToken child = values[nextChild - 1]; + if (alternativesLeft() == 0 && !child.isOptional()) { + failed = true; } - // empty class? - return true; } @Override - public String getIgnoredCharacters() { - return ignoreCharacters; + public void onPopulated(ParserLocation end) { + this.populated = true; + lastTokenEnd = end; + } + + @Override + public CharSequence traceback() { + StringBuilder result = new StringBuilder(); + for (int i = nextChild - 1; i > -1; i--) { + PartialToken child = values[i]; + result.append(child.source()); + if (child.alternativesLeft() > 0) { + nextChild = i; + lastTokenEnd = child.location(); + break; + } + } + return result; } private void set(Field field, Object value) { @@ -287,12 +194,6 @@ private void set(Field field, Object value) { } } - private boolean isOptional(int position) throws SyntaxError { - Field field = fields[position]; - PartialToken token = values[position]; - return field.isAnnotationPresent(OptionalToken.class) || (token instanceof TerminalToken && ((TerminalToken)token).skipped()); - } - protected T convert(Class into, Object what) { if (into.isArray()) { Object[] collection = (Object[]) what; @@ -351,11 +252,6 @@ public ParserLocation location() { return location; } - @Override - public int consumed() { - return lastTokenEnd == null ? 0 : lastTokenEnd.position() - position(); - } - @Override public boolean rotatable() { if (fields.length < 3) { @@ -393,61 +289,63 @@ public Field getCurrentField() { public void rotate() { logger.info("Rotating"); token.invalidate(); - RuleToken wrap = new RuleToken(this, tokenType, location); - wrap.nextField = nextField; - nextField = 1; - PartialToken[] wrapValues = wrap.values; + RuleToken wrap = new RuleToken(this, fields[0], fields[0].getType(), location); + wrap.nextChild = nextChild; + nextChild = 1; + PartialToken[] wrapValues = wrap.values; wrap.values = values; values = wrapValues; values[0] = wrap; - X wrapToken = (X) wrap.token; + X wrapToken = (X) wrap.token(); wrap.token = token; token = wrapToken; - setChildren(values); } @Override public void unrotate() { logger.debug("Un-rotating"); PartialToken firstToken = values[0]; - PartialToken kiddo = firstToken; - if (kiddo instanceof VariantToken) { - kiddo = ((VariantToken)kiddo).resolvedAs(); + + CompoundToken kiddo; + if (firstToken instanceof VariantToken) { + kiddo = (CompoundToken)((VariantToken)kiddo).resolvedAs(); + } else { + kiddo = (CompoundToken) firstToken; } - Rule childToken = (Rule) kiddo.getToken(); - Class childTokenType = kiddo.getTokenType(); + Rule childToken = (Rule) kiddo.token(); + Class childTokenType = kiddo.tokenType(); invalidate(); kiddo.invalidate(); - PartialToken[] grandChildren = kiddo.getChildren(); + PartialToken[] grandChildren = kiddo.children(); values[0] = grandChildren[grandChildren.length - 1]; - set(fields[0], values[0].getToken()); - kiddo.setToken(token); - kiddo.setChildren(values); + set(fields[0], values[0].token()); + kiddo.token(token); + kiddo.children(values); values = grandChildren; values[values.length - 1] = kiddo; tokenType = null; token = (X) childToken; tokenType = (Class) childTokenType; - setChildren(values); - set(fields[fields.length - 1], values[values.length - 1].getToken()); + children(values); + set(fields[fields.length - 1], values[values.length - 1].token()); } @Override - public PartialToken[] getChildren() { + public PartialToken[] children() { return values; } @Override - public void setToken(X token) { + public void token(X token) { this.token = token; } @Override - public void setChildren(PartialToken[] children) { + public void children(PartialToken[] children) { this.values = children; } @@ -472,21 +370,5 @@ public StringBuilder source() { return result; } - @Override - public PartialToken expected() { - - for (int field = nextField; field > 0; field--) { - if (field < fields.length && !isOptional(field) && values[field] != null) { - return values[field].expected(); - } - } - - return this; - } - - @Override - public String tag() { - return tokenType.getSimpleName(); - } } diff --git a/src/main/java/com/onkiup/linker/parser/token/TerminalToken.java b/src/main/java/com/onkiup/linker/parser/token/TerminalToken.java index f669087..816509f 100644 --- a/src/main/java/com/onkiup/linker/parser/token/TerminalToken.java +++ b/src/main/java/com/onkiup/linker/parser/token/TerminalToken.java @@ -1,6 +1,7 @@ package com.onkiup.linker.parser.token; import java.lang.reflect.Field; +import java.util.Objects; import java.util.Optional; import org.slf4j.Logger; @@ -21,23 +22,20 @@ public class TerminalToken implements PartialToken, ConsumingToken parent; private TokenMatcher matcher; private TokenTestResult lastTestResult; - private String token; - private StringBuilder buffer = new StringBuilder(); - private StringBuilder cleanBuffer = new StringBuilder(); - private StringBuilder ignoredCharacters = new StringBuilder(); + private CharSequence token; private boolean failed = false; private boolean isOptional; - private boolean isSkippable; private boolean skipped; - private String optionalCondition = ""; - private ParserLocation location; + private CharSequence optionalCondition = ""; + private final ParserLocation start; + private ParserLocation end; private Logger logger; public TerminalToken(PartialToken parent, Field field, ParserLocation location) { this.parent = parent; this.field = field; this.logger = LoggerFactory.getLogger(field.getDeclaringClass().getName() + "$" + field.getName()); - this.location = location; + this.start = this.end = location; this.matcher = TokenMatcher.forField(field); if (field.isAnnotationPresent(OptionalToken.class)) { OptionalToken optional = field.getAnnotation(OptionalToken.class); @@ -50,15 +48,22 @@ public TerminalToken(PartialToken parent, Field field, ParserLoc throw new IllegalStateException("Tokens cannot be both optional and skippable (" + field.getDeclaringClass().getSimpleName() + "." + field.getName() + ")"); } SkipIfFollowedBy condition = field.getAnnotation(SkipIfFollowedBy.class); - this.isSkippable = true; + this.isOptional = true; this.optionalCondition = condition.value(); } + + this.setTokenMatcher(matcher); } public TerminalToken(PartialToken parent, TokenMatcher matcher) { this.parent = parent; this.matcher = matcher; - this.location = new ParserLocation("unknown", 0, 0, 0); + this.start = this.end = new ParserLocation("unknown", 0, 0, 0); + } + + @Override + public boolean isFailed() { + return failed; } @Override @@ -68,109 +73,41 @@ public Optional advance(boolean force) throws SyntaxError { } @Override - public Optional consume(char character, boolean last) { - - if (token != null) { - return Optional.of(new StringBuilder().append(character)); - } + public void onPopulate(CharSequence value) { + logger.debug("-- MATCHED"); + this.token = value; + this.end = + } - String ignoreCharacters = parent == null ? null : parent.getIgnoredCharacters(); - - buffer.append(character); - - if (ignoreCharacters != null) { - if (cleanBuffer.length() == 0) { - if (ignoreCharacters.indexOf(character) == -1) { - logger.debug("Did not find character {} in ignored character list", (int) character); - cleanBuffer.append(character); - } else if (last) { - cleanBuffer.append(character); - StringBuilder ret = cleanBuffer; - cleanBuffer = new StringBuilder(); - return Optional.of(ret); - } else { - logger.debug("Ignoring character with code " + (int) character); - ignoredCharacters.append(character); - location = location.advance("" + character); - } - } else { - logger.debug("Clean buffer is not empty, not testing if the character should be ignored"); - cleanBuffer.append(character); - } - } else { - logger.info("Accepting all characters as ignored characters list was empty"); - cleanBuffer.append(character); - } + @Override + public boolean onFail(CharSequence on) { + logger.debug("-- FAILED"); + failed = true; + return !lookahead(on); + } - if (cleanBuffer.length() == 0) { - return Optional.empty(); + @Override + public boolean lookahead(CharSequence buffer) { + if (optionalCondition == null || optionalCondition.length() == 0) { + logger.debug("the token is unconditionally optional"); + skipped = true; + return false; } - - if (!failed) { - lastTestResult = matcher.apply(cleanBuffer); + + if (optionalCondition.length() > buffer.length()) { + logger.debug("unable to resolve optionality conditions -- not enough characters -- continuting lookahead"); + return true; } - if (failed || lastTestResult.isFailed()) { - failed = true; - logger.debug("Test failed on buffer '{}' using matcher {}", cleanBuffer, matcher); - StringBuilder returnBuffer = new StringBuilder(); - - boolean callParent = !(isOptional || isSkippable); - if (optionalCondition.length() > 0) { - logger.debug("Testing optional condition '{}'", optionalCondition); - String followed = cleanBuffer.toString(); - if (!optionalCondition.equals(followed)) { - if (optionalCondition.startsWith(followed)) { - logger.debug("Need to consume more characters to decide if token is optional"); - return Optional.empty(); - } - logger.debug("Token is not optional as it is followed by '{}' and not '{}'", cleanBuffer, optionalCondition); - callParent = true; - } - } - - if (callParent) { - logger.debug("Token is not optional; pushing back to parent"); - getParent() - .flatMap(p -> p.pushback(true)) - .ifPresent(b -> { - logger.debug("Received characters from pushed back parent: '{}'", b); - returnBuffer.insert(0, b); - }); - } else { - logger.debug("Token is optional; returning consumed characters without notifying parent"); - skipped = true; - } - - returnBuffer.append(buffer); - buffer = new StringBuilder(); - logger.debug("Returning back to parser buffer: '{}'", returnBuffer); - return Optional.of(returnBuffer); - } else if (lastTestResult.isMatch() || (lastTestResult.isMatchContinue() && last)) { - logger.debug("Test suceeded (forced: {}) on buffer '{}' using matcher {}", last, cleanBuffer, matcher); - - if (isSkippable && !last) { - String after = cleanBuffer.substring(lastTestResult.getTokenLength()); - if (after.length() < optionalCondition.length()) { - logger.debug("Token is skippable -- need more input to detect if it should be skipped"); - return Optional.empty(); - } else if (after.startsWith(optionalCondition)) { - logger.debug("Skipping this token as it is followed by '{}'", optionalCondition); - StringBuilder returnBuffer = buffer; - buffer = new StringBuilder(); - return Optional.of(returnBuffer); - } - } - - StringBuilder result = new StringBuilder(populate(lastTestResult).toString()); - buffer = new StringBuilder(); - return Optional.of(result); + if (Objects.equals(optionalCondition, buffer.subSequence(0, optionalCondition.length()))) { + logger.debug("parser input satisfies token optionality condition"); + skipped = true; } - return Optional.empty(); + return false; } private boolean isOptional() { - return field.isAnnotationPresent(OptionalToken.class); + return skipped; } @Override diff --git a/src/main/java/com/onkiup/linker/parser/util/ParserError.java b/src/main/java/com/onkiup/linker/parser/util/ParserError.java index 1b91039..750b2d6 100644 --- a/src/main/java/com/onkiup/linker/parser/util/ParserError.java +++ b/src/main/java/com/onkiup/linker/parser/util/ParserError.java @@ -5,10 +5,14 @@ public class ParserError extends RuntimeException { private PartialToken source; - private String message; public ParserError(String msg, PartialToken source) { - this.message = msg; + super(message); + this.source = source; + } + + public ParserError(String msg, PartialToken source, Throwable cause) { + super(message, cause); this.source = source; } @@ -17,7 +21,7 @@ public String toString() { StringBuilder result = new StringBuilder("Parser error at position "); result.append(source.position()) .append(": ") - .append(message) + .append(getMessage()) .append("\n"); PartialToken parent = source; From f7ffb85b7c4a3ca9aada72719242a6da479dced5 Mon Sep 17 00:00:00 2001 From: Dima Chechetkin Date: Fri, 27 Sep 2019 23:52:11 -0400 Subject: [PATCH 2/3] 0.8 - Major refactoring - Adds OptionalToken.whenFieldIsNull option - Logging improvements - Staticstics-based token variant selection should improve parser's performance by picking more frequent tokens first - VariantTokens now will short-curcuit all thier children if same variant base type is already tested at current position - First implementation of EnumToken support --- build.gradle | 2 +- .../com/onkiup/linker/parser/EnumRule.java | 3 +- .../onkiup/linker/parser/EvaluationError.java | 2 +- .../onkiup/linker/parser/NumberMatcher.java | 4 +- .../onkiup/linker/parser/ParserLocation.java | 4 +- .../java/com/onkiup/linker/parser/Rule.java | 21 +- .../com/onkiup/linker/parser/SyntaxError.java | 9 +- .../onkiup/linker/parser/TerminalMatcher.java | 4 +- .../onkiup/linker/parser/TokenGrammar.java | 336 +++++++++++++----- .../onkiup/linker/parser/TokenMatcher.java | 6 +- .../parser/annotation/OptionalToken.java | 1 + .../linker/parser/token/AbstractToken.java | 117 ++++++ .../linker/parser/token/CollectionToken.java | 197 +++++----- .../linker/parser/token/CompoundToken.java | 80 ++++- .../linker/parser/token/ConsumingToken.java | 98 +++-- .../onkiup/linker/parser/token/EnumToken.java | 79 +--- .../linker/parser/token/FailedToken.java | 113 ------ .../linker/parser/token/PartialToken.java | 167 +++++---- .../onkiup/linker/parser/token/Rotatable.java | 10 + .../onkiup/linker/parser/token/RuleToken.java | 203 +++++------ .../linker/parser/token/TerminalToken.java | 184 +--------- .../linker/parser/token/VariantToken.java | 234 +++++------- .../linker/parser/util/LoggerLayout.java | 52 +++ .../linker/parser/util/ParserError.java | 18 +- src/main/java/resources/log4j.properties | 48 +++ .../linker/parser/NumberMatcherTest.java | 0 .../linker/parser/PatternMatcherTest.java | 95 ----- .../linker/parser/TerminalMatcherTest.java | 20 -- .../linker/parser/TokenGrammarTest.java | 179 ---------- .../parser/token/AbstractTokenTest.java | 94 +++++ .../parser/token/CollectionTokenTest.java | 154 ++++---- .../parser/token/CompoundTokenTest.java | 120 +++++++ .../linker/parser/token/EnumTokenTest.java | 51 +++ .../linker/parser/token/PartialTokenTest.java | 214 +++++++++++ .../linker/parser/token/RuleTokenTest.java | 62 ---- .../parser/token/TerminalTokenTest.java | 50 --- .../linker/parser/token/UnrotationTest.java | 100 ------ .../linker/parser/token/VariantTokenTest.java | 63 ---- 38 files changed, 1588 insertions(+), 1606 deletions(-) create mode 100644 src/main/java/com/onkiup/linker/parser/token/AbstractToken.java delete mode 100644 src/main/java/com/onkiup/linker/parser/token/FailedToken.java create mode 100644 src/main/java/com/onkiup/linker/parser/token/Rotatable.java create mode 100644 src/main/java/com/onkiup/linker/parser/util/LoggerLayout.java create mode 100644 src/main/java/resources/log4j.properties delete mode 100644 src/test/java/com/onkiup/linker/parser/NumberMatcherTest.java delete mode 100644 src/test/java/com/onkiup/linker/parser/PatternMatcherTest.java delete mode 100644 src/test/java/com/onkiup/linker/parser/TerminalMatcherTest.java delete mode 100644 src/test/java/com/onkiup/linker/parser/TokenGrammarTest.java create mode 100644 src/test/java/com/onkiup/linker/parser/token/AbstractTokenTest.java create mode 100644 src/test/java/com/onkiup/linker/parser/token/CompoundTokenTest.java create mode 100644 src/test/java/com/onkiup/linker/parser/token/EnumTokenTest.java create mode 100644 src/test/java/com/onkiup/linker/parser/token/PartialTokenTest.java delete mode 100644 src/test/java/com/onkiup/linker/parser/token/TerminalTokenTest.java delete mode 100644 src/test/java/com/onkiup/linker/parser/token/UnrotationTest.java delete mode 100644 src/test/java/com/onkiup/linker/parser/token/VariantTokenTest.java diff --git a/build.gradle b/build.gradle index 2c00a9e..fd39f48 100644 --- a/build.gradle +++ b/build.gradle @@ -15,7 +15,7 @@ plugins { } project.group = 'com.onkiup' -project.version = '0.7.1' +project.version = '0.8' compileJava { sourceCompatibility = '1.8' diff --git a/src/main/java/com/onkiup/linker/parser/EnumRule.java b/src/main/java/com/onkiup/linker/parser/EnumRule.java index 8be3639..20dcaf9 100644 --- a/src/main/java/com/onkiup/linker/parser/EnumRule.java +++ b/src/main/java/com/onkiup/linker/parser/EnumRule.java @@ -1,6 +1,5 @@ package com.onkiup.linker.parser; public interface EnumRule { - String getTerminal(); -} +} diff --git a/src/main/java/com/onkiup/linker/parser/EvaluationError.java b/src/main/java/com/onkiup/linker/parser/EvaluationError.java index 1424094..c3d3b86 100644 --- a/src/main/java/com/onkiup/linker/parser/EvaluationError.java +++ b/src/main/java/com/onkiup/linker/parser/EvaluationError.java @@ -5,7 +5,7 @@ public class EvaluationError extends RuntimeException { public EvaluationError(PartialToken token, Object context, Exception cause) { - super("Failed to evaluate token " + token.getTokenType(), cause); + super("Failed to evaluate token " + token.tokenType(), cause); } } diff --git a/src/main/java/com/onkiup/linker/parser/NumberMatcher.java b/src/main/java/com/onkiup/linker/parser/NumberMatcher.java index 945c928..ff63acf 100644 --- a/src/main/java/com/onkiup/linker/parser/NumberMatcher.java +++ b/src/main/java/com/onkiup/linker/parser/NumberMatcher.java @@ -18,7 +18,7 @@ public NumberMatcher(Class type) { } @Override - public TokenTestResult apply(StringBuilder buffer) { + public TokenTestResult apply(CharSequence buffer) { try { pattern.newInstance(buffer.toString()); return TestResult.matchContinue(buffer.length(), buffer.toString()); @@ -35,7 +35,7 @@ public TokenTestResult apply(StringBuilder buffer) { try { char drop = buffer.charAt(buffer.length() - 1); if (drop != '.') { - Number token = pattern.newInstance(buffer.substring(0, buffer.length() - 1)); + Number token = pattern.newInstance(buffer.subSequence(0, buffer.length())); return TestResult.match(buffer.length() - 1, token); } } catch (InvocationTargetException nfe2) { diff --git a/src/main/java/com/onkiup/linker/parser/ParserLocation.java b/src/main/java/com/onkiup/linker/parser/ParserLocation.java index ac65fa6..a8a4110 100644 --- a/src/main/java/com/onkiup/linker/parser/ParserLocation.java +++ b/src/main/java/com/onkiup/linker/parser/ParserLocation.java @@ -18,7 +18,7 @@ public static ParserLocation endOf(CharSequence text) { int column = 0; for (int i = 0; i < text.length(); i++) { if (text.charAt(i) == '\n') { - line++; + lines++; column = 0; } else { column++; @@ -100,7 +100,7 @@ public ParserLocation add(ParserLocation another) { int anotherLines = another.line(); int resultLine = line + anotherLines; int resultColumn = anotherLines == 0 ? column + another.column() : another.column(); - return new ParserLocation(name, resultLine, resultColumn, position + another.position()); + return new ParserLocation(name, position + another.position(), resultLine, resultColumn); } } diff --git a/src/main/java/com/onkiup/linker/parser/Rule.java b/src/main/java/com/onkiup/linker/parser/Rule.java index 237f4dd..e6dc93b 100644 --- a/src/main/java/com/onkiup/linker/parser/Rule.java +++ b/src/main/java/com/onkiup/linker/parser/Rule.java @@ -35,22 +35,21 @@ static void remove(Rule rule) { } /** - * @returns parent token or null if this token is root token + * @return parent token or null if this token is root token */ default Optional parent() { - PartialToken meta = Metadata.metadata(this).get(); - do { - meta = (PartialToken) meta.getParent().orElse(null); - } while (meta != null && !(meta instanceof RuleToken)); - - if (meta != null) { - return Optional.of((R) meta.getToken()); - } - return Optional.empty(); + return Metadata.metadata(this) + .map(meta -> { + do { + meta = (PartialToken) meta.parent().orElse(null); + } while (meta instanceof VariantToken); + return meta; + }) + .flatMap(PartialToken::token); } /** - * @returns true if this token was successfully populated; false if parser is still working on some of the token's fields + * @return true if this token was successfully populated; false if parser is still working on some of the token's fields */ default boolean populated() { return Metadata.metadata(this) diff --git a/src/main/java/com/onkiup/linker/parser/SyntaxError.java b/src/main/java/com/onkiup/linker/parser/SyntaxError.java index 6ead289..3786571 100644 --- a/src/main/java/com/onkiup/linker/parser/SyntaxError.java +++ b/src/main/java/com/onkiup/linker/parser/SyntaxError.java @@ -11,19 +11,18 @@ public class SyntaxError extends RuntimeException { - private PartialToken lastToken; + private PartialToken expected; private StringBuilder source; private String message; - public SyntaxError(String message, PartialToken lastToken, StringBuilder source) { + public SyntaxError(String message, PartialToken expected, StringBuilder source) { this.message = message; - this.lastToken = lastToken; + this.expected = expected; this.source = source; } @Override public String toString() { - PartialToken expected = lastToken.expected(); StringBuilder result = new StringBuilder("Parser error:") .append(message) .append("\n") @@ -36,7 +35,7 @@ public String toString() { .append("\n\n\tTraceback:\n\t\t"); PartialToken parent = expected; - while (null != (parent = (PartialToken) parent.getParent().orElse(null))) { + while (null != (parent = (PartialToken) parent.parent().orElse(null))) { result.append(parent.toString().replace("\n", "\n\t\t")); } diff --git a/src/main/java/com/onkiup/linker/parser/TerminalMatcher.java b/src/main/java/com/onkiup/linker/parser/TerminalMatcher.java index c94bcbe..fec2555 100644 --- a/src/main/java/com/onkiup/linker/parser/TerminalMatcher.java +++ b/src/main/java/com/onkiup/linker/parser/TerminalMatcher.java @@ -21,9 +21,9 @@ public TokenTestResult apply(CharSequence buffer) { } if (patternLen <= bufferLen) { - return TestResult.MATCH.token(patternLen, pattern); + return TestResult.match(patternLen, pattern); } - return TestResult.matchContinue(bufferLen, buffer.toString()); + return TestResult.continueNoMatch(); } @Override diff --git a/src/main/java/com/onkiup/linker/parser/TokenGrammar.java b/src/main/java/com/onkiup/linker/parser/TokenGrammar.java index 8945661..19027b1 100644 --- a/src/main/java/com/onkiup/linker/parser/TokenGrammar.java +++ b/src/main/java/com/onkiup/linker/parser/TokenGrammar.java @@ -1,23 +1,31 @@ package com.onkiup.linker.parser; +import java.io.IOException; import java.io.Reader; import java.io.StringReader; import java.lang.reflect.Modifier; import java.util.Collection; +import java.util.Enumeration; +import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import org.apache.log4j.Appender; +import org.apache.log4j.Layout; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.onkiup.linker.parser.token.CompoundToken; import com.onkiup.linker.parser.token.ConsumingToken; -import com.onkiup.linker.parser.token.FailedToken; import com.onkiup.linker.parser.token.PartialToken; +import com.onkiup.linker.parser.util.LoggerLayout; +import com.onkiup.linker.parser.util.ParserError; // at 0.2.2: // - replaced all "evaluate" flags with context public class TokenGrammar { - private static final Logger logger = LoggerFactory.getLogger(TokenGrammar.class); + private static final Logger logger = LoggerFactory.getLogger("PARSER LOOP"); + private static final ThreadLocal BUFFER = new ThreadLocal<>(); private Class type; private String ignoreTrail; @@ -75,130 +83,266 @@ public X tokenize(Reader source) throws SyntaxError { } public X tokenize(String sourceName, Reader source) throws SyntaxError { - PartialToken rootToken = PartialToken.forClass(null, type, new ParserLocation(sourceName, 0, 0, 0)); - PartialToken token = rootToken; - PartialToken lastToken = token; + CompoundToken rootToken = CompoundToken.forClass(type, new ParserLocation(sourceName, 0, 0, 0)); + CompoundToken parent = rootToken; + ConsumingToken consumer = nextConsumingToken(parent).orElseThrow(() -> new ParserError("No possible consuming tokens found", parent)); StringBuilder buffer = new StringBuilder(); - boolean hitEnd = false; int line = 0, col = 0; - AtomicInteger position = new AtomicInteger(0); + AtomicInteger position = new AtomicInteger(0); try { + setupLoggingLayouts(buffer); do { - logger.debug("----------------------------------------------------------------------------------------"); - logger.debug("BUFFER = '{}'", buffer); - logger.debug("POSITION = {}", position.get()); - logger.debug("----------------------------------------------------------------------------------------"); if (logger.isDebugEnabled()) { - Collection path = token.getPath(); + System.out.println("|----------------------------------------------------------------------------------------"); + System.out.println("|----------------------------------------------------------------------------------------"); + System.out.println("|----------------------------------------------------------------------------------------"); + Collection path = consumer.path(); - String trace = (String) path.stream() + String trace = path.stream() .map(Object::toString) - .collect(Collectors.joining("\n")); - logger.debug("Trace:\n{}", trace); + .collect(Collectors.joining("\n|-")); + System.out.println("|-" + trace); + System.out.println("|----------------------------------------------------------------------------------------"); + System.out.println("|----------------------------------------------------------------------------------------"); } - while (buffer.length() < 2 && !hitEnd) { - int nextChar = source.read(); - hitEnd = nextChar < 0; - if (!hitEnd) { - buffer.append((char) nextChar); - - position.getAndIncrement(); - col++; + ConsumingToken lastConsumer = consumer; - if (nextChar == '\n') { - line++; - col = 0; + try { + boolean hitEnd = processConsumingToken(source, buffer, consumer); + if (hitEnd) { + logger.debug("Hit end while processing {}", consumer.tag()); + if (!consumer.isPopulated() && !consumer.isFailed()) { + consumer.onFail(); } } - } - if (buffer.length() > 0 && token instanceof ConsumingToken) { - logger.debug("Feeding character '{}' to {}", buffer.charAt(0), token); - final ConsumingToken consumingToken = (ConsumingToken) token; - boolean consumed = (Boolean)consumingToken.consume(buffer.charAt(0), buffer.length() == 1) - .map(Object::toString) - .map(returned -> { - buffer.replace(0, 1, (String) returned); - position.addAndGet(1 - ((String)returned).length()); - logger.debug("Token {} returned characters: '{}'; buffer = '{}'", consumingToken, returned, buffer); - return false; - }) - .orElseGet(() -> { - logger.debug("Discarding consumed by token {} character '{}'", consumingToken, buffer.charAt(0)); - buffer.delete(0, 1); - position.incrementAndGet(); - return true; - }); - - if (consumed) { - // prevents parser from advancing to next token - continue; + if (consumer.isFailed()) { + logger.debug("!!! CONSUMER FAILED !!! {}", consumer.tag()); + //if (!consumer.isOptional()) { + consumer = processTraceback(consumer, buffer).orElse(null); + //} + } else if (consumer.isPopulated()) { + logger.debug("consumer populated: {}", consumer.tag()); + consumer = onPopulated(consumer) + .flatMap(TokenGrammar::nextConsumingToken) + .orElse(null); } - } - - lastToken = token; - do { - logger.debug("Advancing. Buffer size: {}", buffer.length()); - token = (PartialToken) token.advance(buffer.length() == 0).orElse(null); - logger.debug("Advanced to token {}", token); - if (token instanceof FailedToken) { - String returned = (String) token.getToken(); - logger.info("Received from failed token: '{}'", returned); - buffer.insert(0, returned); + if (consumer == lastConsumer) { + consumer = nextConsumingToken(consumer).orElse(null); } - } while (token instanceof FailedToken); + if (consumer == null || buffer.length() == 0) { + if (rootToken.isPopulated()) { + if (!hitEnd) { + consumer = processEarlyPopulation(rootToken, source, buffer).orElse(null); + } else { + logger.debug("Perfectly parsed into: {}", rootToken.tag()); + return rootToken.token().get(); + } + } else if (consumer != null) { + logger.debug("Hit end and root token is not populated -- trying to traceback..."); + do { + consumer.onFail(); + consumer = processTraceback(consumer, buffer).orElse(null); + } while (buffer.length() == 0 && consumer != null); - if (token == null) { - if (buffer.length() > 0 || !hitEnd) { - logger.debug("Trying to rotate root token to avoid unmatched characters..."); - if (rootToken.rotatable()) { - rootToken.rotate(); - logger.debug("Rotated root token"); - token = rootToken; - continue; - } - - int alternativesLeft = rootToken.alternativesLeft(); - logger.debug("Alternatives left for {}: {}", rootToken, alternativesLeft); - if (alternativesLeft > 0) { - logger.debug("Hit end but root token had alternatives left; backtracking the token and continuing with next variant"); - rootToken.pullback().ifPresent(b -> buffer.insert(0, b)); - token = rootToken; - continue; - } - - int nextChar; - while (-1 != (nextChar = source.read())) { - buffer.append((char)nextChar); - } - - if (ignoreTrail != null) { - for (int i = 0; i < buffer.length(); i++) { - if (ignoreTrail.indexOf(buffer.charAt(i)) < 0) { - throw new SyntaxError("Unmatched trailing characters", rootToken, new StringBuilder(buffer.substring(i))); - } + if (buffer.length() != 0 && rootToken.isPopulated()) { + consumer = processEarlyPopulation(rootToken, source, buffer).orElse(null); + } else if (rootToken.isPopulated()) { + return rootToken.token().get(); } - rootToken.sortPriorities(); - return rootToken.getToken(); } - throw new SyntaxError("Unmatched trailing characters", rootToken, buffer); - } else { - rootToken.sortPriorities(); - return rootToken.getToken(); } + } catch (IOException ioe) { + throw new RuntimeException("Failed to read source data", ioe); } - } while(buffer.length() > 0); + + } while(consumer != null && buffer.length() > 0); + + if (rootToken.isPopulated()) { + return rootToken.token().orElse(null); + } throw new SyntaxError("Unexpected end of input", rootToken, buffer); } catch (SyntaxError se) { throw new RuntimeException("Syntax error at line " + line + ", column " + col, se); } catch (Exception e) { throw new RuntimeException(e); + } finally { + restoreLoggingLayouts(); + } + } + + private Optional> processEarlyPopulation(CompoundToken rootToken, Reader source, StringBuilder buffer) throws IOException { + if (validateTrailingCharacters(source, buffer)) { + logger.debug("Successfully parsed (with valid trailing characters '{}') into: {}", buffer, rootToken.tag()); + buffer.delete(0, buffer.length()); + return Optional.empty(); + } else if (rootToken.rotatable()) { + rootToken.rotate(); + return nextConsumingToken(rootToken); + } else if (rootToken.alternativesLeft() > 0) { + logger.info("Root token populated too early, failing it..."); + rootToken.traceback().ifPresent(returned -> { + logger.debug("Returned by root token: '{}'", LoggerLayout.sanitize(returned)); + buffer.insert(0, returned); + }); + rootToken.onFail(); + return nextConsumingToken(rootToken); + } else { + return Optional.empty(); + } + } + + private static Optional> onPopulated(PartialToken child) { + return child.parent().flatMap(parent -> { + parent.onChildPopulated(); + if (parent.isPopulated()) { + return onPopulated(parent); + } + return Optional.of(parent); + }); + } + + private static Optional> processTraceback(PartialToken child, StringBuilder buffer) { + return child.parent().flatMap(parent -> { + if (child.isFailed()) { + logger.debug("^^^--- TRACEBACK: {} <- {}", parent.tag(), child.tag()); + parent.onChildFailed(); + if (parent.isFailed() || parent.isPopulated()) { + return processTraceback(parent, buffer); + } + + parent.traceback().ifPresent(returned -> buffer.insert(0, returned.toString())); + return nextConsumingToken(parent); + } else { + logger.debug("|||--- TRACEBACK: (self) <- {}", child.tag()); + return firstUnfilledParent(child).flatMap(TokenGrammar::nextConsumingToken); + } + }); + } + + private static Optional> firstUnfilledParent(PartialToken child) { + logger.debug("traversing back to first unfilled parent from {}", child.tag()); + if (child instanceof CompoundToken && ((CompoundToken)child).unfilledChildren() > 0) { + logger.debug("<<<--- NEXT UNFILLED: (self) <--- {}", child.tag()); + return Optional.of((CompoundToken)child); + } + + return Optional.ofNullable( + child.parent().flatMap(parent -> { + logger.debug("parent: {}", parent.tag()); + parent.onChildPopulated(); + if (parent.isPopulated()) { + logger.debug("^^^--- NEXT UNFILLED: {} <-?- {}", parent.tag(), child.tag()); + return firstUnfilledParent(parent); + } else { + logger.debug("<<<--- NEXT UNFILLED: {} <--- {}", parent.tag(), child.tag()); + return Optional.of(parent); + } + }).orElseGet(() -> { + if (child instanceof CompoundToken) { + logger.debug("XXX NO NEXT UNFILLED: XXX <--- {} (compound: true, unfilled children: {}", child, ((CompoundToken)child).unfilledChildren()); + } else { + logger.debug("XXX NO NEXT UNFILLED: XXX <--- {}", child); + } + return null; + }) + ); + } + + public static Optional> nextConsumingToken(CompoundToken from) { + while (from != null) { + PartialToken child = from.nextChild().orElse(null); + if (child instanceof ConsumingToken) { + logger.debug("--->>> NEXT CONSUMER: {} ---> {}", from.tag(), child.tag()); + return Optional.of((ConsumingToken)child); + } else if (child instanceof CompoundToken) { + from = (CompoundToken)child; + } else if (child == null) { + from = from.parent().orElse(null); + } else { + throw new RuntimeException("Unknown child type: " + child.getClass()); + } + } + logger.debug("---XXX NEXT UNFILLED: {} ---> XXX (not found)", from); + return Optional.empty(); + } + + private static Optional> nextConsumingToken(ConsumingToken from) { + return from.parent().flatMap(TokenGrammar::nextConsumingToken); + } + + private boolean processConsumingToken(Reader source, StringBuilder buffer, ConsumingToken token) throws IOException { + boolean accepted = true; + while (accepted) { + if (populateBuffer(source, buffer)) { + char character = buffer.charAt(0); + accepted = token.consume(buffer.charAt(0)).map(CharSequence::toString).map(returned -> { + logger.debug("------ RETURN: '{}' +++ {} = '{}'", LoggerLayout.sanitize(String.valueOf(character)), token.tag(), LoggerLayout.sanitize(returned)); + buffer.replace(0, 1, returned); + return false; + }).orElseGet(() -> { + logger.debug("---+++ CONSUME: '{}' +++ {}", LoggerLayout.sanitize(String.valueOf(character)), token.tag()); + buffer.delete(0, 1); + return true; + }); + } else { + return true; + } } + return !populateBuffer(source, buffer); } + private boolean populateBuffer(Reader source, StringBuilder buffer) throws IOException { + if (buffer.length() == 0) { + int character = source.read(); + if (character < 0) { + return false; + } else { + buffer.append((char)character); + } + } + return true; + } + + private boolean validateTrailingCharacters(Reader source, StringBuilder buffer) throws IOException { + boolean hitEnd = false; + do { + populateBuffer(source, buffer); + if (buffer.length() > 0) { + char test = buffer.charAt(0); + if (ignoreTrail != null && ignoreTrail.indexOf(test) < 0) { + return false; + } + buffer.delete(0, 1); + populateBuffer(source, buffer); + } + } while (buffer.length() > 0); + return true; + } + + private void setupLoggingLayouts(StringBuilder buffer) { + Enumeration appenders = org.apache.log4j.Logger.getRootLogger().getAllAppenders(); + while(appenders.hasMoreElements()) { + Appender appender = appenders.nextElement(); + LoggerLayout loggerLayout = new LoggerLayout(appender.getLayout(), buffer); + appender.setLayout(loggerLayout); + } + } + + private void restoreLoggingLayouts() { + Enumeration appenders = org.apache.log4j.Logger.getRootLogger().getAllAppenders(); + while(appenders.hasMoreElements()) { + Appender appender = appenders.nextElement(); + Layout layout = appender.getLayout(); + if (layout instanceof LoggerLayout) { + LoggerLayout loggerLayout = (LoggerLayout) layout; + appender.setLayout(loggerLayout.parent()); + } + } + } } diff --git a/src/main/java/com/onkiup/linker/parser/TokenMatcher.java b/src/main/java/com/onkiup/linker/parser/TokenMatcher.java index 179c788..547269d 100644 --- a/src/main/java/com/onkiup/linker/parser/TokenMatcher.java +++ b/src/main/java/com/onkiup/linker/parser/TokenMatcher.java @@ -8,9 +8,13 @@ @FunctionalInterface public interface TokenMatcher extends Function { - + public static TokenMatcher forField(Field field) { Class type = field.getType(); + return forField(field, type); + } + + public static TokenMatcher forField(Field field, Class type) { if (type.isArray()) { throw new IllegalArgumentException("Array fields should be handled as ArrayTokens"); } else if (Rule.class.isAssignableFrom(type)) { diff --git a/src/main/java/com/onkiup/linker/parser/annotation/OptionalToken.java b/src/main/java/com/onkiup/linker/parser/annotation/OptionalToken.java index 797d4ef..6b9f6c8 100644 --- a/src/main/java/com/onkiup/linker/parser/annotation/OptionalToken.java +++ b/src/main/java/com/onkiup/linker/parser/annotation/OptionalToken.java @@ -9,5 +9,6 @@ @Target(ElementType.FIELD) public @interface OptionalToken { String whenFollowedBy() default ""; + String whenFieldIsNull() default ""; } diff --git a/src/main/java/com/onkiup/linker/parser/token/AbstractToken.java b/src/main/java/com/onkiup/linker/parser/token/AbstractToken.java new file mode 100644 index 0000000..c50ecc8 --- /dev/null +++ b/src/main/java/com/onkiup/linker/parser/token/AbstractToken.java @@ -0,0 +1,117 @@ +package com.onkiup.linker.parser.token; + +import java.lang.reflect.Field; +import java.util.Optional; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.onkiup.linker.parser.ParserLocation; + +public abstract class AbstractToken implements PartialToken { + + private CompoundToken parent; + private Field field; + private ParserLocation location, end; + private boolean optional, populated, failed; + private CharSequence optionalCondition; + private Logger logger; + + public AbstractToken(CompoundToken parent, Field targetField, ParserLocation location) { + this.parent = parent; + this.field = targetField; + this.location = location; + + readFlags(field); + } + + @Override + public void markOptional() { + log("marked optional"); + this.optional = true; + } + + @Override + public boolean isOptional() { + return optional; + } + + @Override + public boolean isPopulated() { + return populated; + } + + protected void dropPopulated() { + populated = false; + } + + @Override + public boolean isFailed() { + return failed; + } + + @Override + public ParserLocation location() { + return location; + } + + @Override + public ParserLocation end() { + return this.end == null ? this.location : this.end; + } + + @Override + public Optional> parent() { + return Optional.ofNullable(parent); + } + + @Override + public Optional targetField() { + return Optional.ofNullable(field); + } + + @Override + public void onPopulated(ParserLocation end) { + log("populated"); + populated = true; + this.end = end; + } + + @Override + public Logger logger() { + if (logger == null) { + logger = LoggerFactory.getLogger(tag()); + } + return logger; + } + + @Override + public String tag() { + return targetField() + .map(field -> field.getDeclaringClass().getName() + "$" + field.getName()) + .orElseGet(super::toString); + } + + @Override + public String toString() { + return targetField() + .map(field -> String.format("%-50.50s || %s (position: %d)", tail(50), field.getDeclaringClass().getName() + "$" + field.getName(), position())) + .orElseGet(super::toString); + } + + protected void readFlags(Field field) { + optionalCondition = PartialToken.getOptionalCondition(field).orElse(null); + optional = optionalCondition == null && PartialToken.hasOptionalAnnotation(field); + } + + @Override + public void onFail() { + failed = true; + PartialToken.super.onFail(); + } + + public Optional optionalCondition() { + return Optional.ofNullable(optionalCondition); + } +} + diff --git a/src/main/java/com/onkiup/linker/parser/token/CollectionToken.java b/src/main/java/com/onkiup/linker/parser/token/CollectionToken.java index 10f1ecc..33111c4 100644 --- a/src/main/java/com/onkiup/linker/parser/token/CollectionToken.java +++ b/src/main/java/com/onkiup/linker/parser/token/CollectionToken.java @@ -1,38 +1,26 @@ package com.onkiup.linker.parser.token; +import java.lang.reflect.Array; import java.lang.reflect.Field; +import java.util.Arrays; import java.util.LinkedList; import java.util.Optional; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.onkiup.linker.parser.ParserLocation; -import com.onkiup.linker.parser.Rule; -import com.onkiup.linker.parser.SyntaxError; import com.onkiup.linker.parser.annotation.CaptureLimit; -import com.onkiup.linker.parser.util.ParserError; -public class CollectionToken implements PartialToken { - private PartialToken parent; - private Field field; +public class CollectionToken extends AbstractToken implements CompoundToken { private Class fieldType; private Class memberType; private LinkedList children = new LinkedList<>(); private PartialToken current; private CaptureLimit captureLimit; - private boolean populated = false; - private final ParserLocation location; private ParserLocation lastTokenEnd; - private final Logger logger; - public CollectionToken(PartialToken parent, Field field, ParserLocation location) { - this.parent = parent; - this.field = field; - logger = LoggerFactory.getLogger(field.getDeclaringClass().getName() + "$" + field.getName() + "[]"); - this.location = location; + public CollectionToken(CompoundToken parent, Field field, Class tokenType, ParserLocation location) { + super(parent, field, location); lastTokenEnd = location; - this.fieldType = (Class) field.getType(); + this.fieldType = tokenType; this.memberType = fieldType.getComponentType(); if (field.isAnnotationPresent(CaptureLimit.class)) { captureLimit = field.getAnnotation(CaptureLimit.class); @@ -40,154 +28,129 @@ public CollectionToken(PartialToken parent, Field field, ParserL } @Override - public Optional pushback(boolean force) { - int size = children.size(); - - if (captureLimit != null && size <= captureLimit.min()) { - logger.debug("failed to satisfy all requirements --> passing rollback call to the parent"); - return parent.pushback(false); + public void onChildPopulated() { + if (current == null) { + throw new RuntimeException("OnChildPopulated called when there is no child!"); + } + children.add(current); + current = null; + if (captureLimit != null && children.size() >= captureLimit.max()) { + onPopulated(current.end()); } - - logger.info("Pulling back failed (last) member {} and marking collection token as populated", current); - populated = true; - PartialToken lastMember = current; - return lastMember.pullback(); - } - - @Override - public Optional pullback() { - logger.debug("received pullback request, propagating to children"); - StringBuilder result = new StringBuilder(); - children.stream() - .map(PartialToken::pullback) - .forEach(o -> o.ifPresent(result::append)); - - logger.debug("pullback result: '{}'", result); - return Optional.of(result); } @Override - public Optional advance(boolean force) throws SyntaxError { - logger.debug("received advance request"); - int size = children.size(); - if (current != null && !current.isPopulated()) { - logger.debug("Last collection token failed"); - if (captureLimit == null || (size >= captureLimit.min() || size <= captureLimit.max())) { - logger.debug("Marking collection as populated"); - populated = true; - } - } else if (current != null) { - children.add(current); - lastTokenEnd = current.end(); - populated = (captureLimit == null) ? false : ++size == captureLimit.max(); - logger.info("Consumed token {}; populated: {}", current.getTokenType().getSimpleName(), populated); + public void onChildFailed() { + if (current == null) { + throw new IllegalStateException("No child is currently populated yet onChildFailed was called"); } - if (force) { - logger.info("Force-populating collection token"); - populated = true; - } - - if (populated) { - return parent == null ? Optional.empty() : parent.advance(force); + int size = children.size(); + if (captureLimit != null && size < captureLimit.min()) { + log("Child failed and collection is underpopulated -- failing the whole collection"); + onFail(); } else { - current = PartialToken.forClass(this, memberType, lastTokenEnd); - logger.debug("Advancing to next collection member"); - return Optional.of(current); + log("Child failed and collection has enough elements (or no lower limit) -- marking collection as populated"); + onPopulated(lastTokenEnd); } } @Override - public String getIgnoredCharacters() { - if (parent != null) { - return parent.getIgnoredCharacters(); - } - return ""; - } - - @Override - public Optional getParent() { - return Optional.ofNullable(parent); + public Class tokenType () { + return fieldType; } @Override - public boolean isPopulated() { - if (captureLimit != null) { - int size = children.size(); - logger.debug("Testing if collection is populated (size: {}; min: {})", size, captureLimit.min()); - return size >= captureLimit.min(); + public Optional token() { + if (!isPopulated()) { + return Optional.empty(); } - return populated; - } - @Override - public Class getTokenType () { - return fieldType; + return Optional.of((X) children.stream() + .map(PartialToken::token) + .map(o -> o.orElse(null)) + .toArray(size -> newArray(memberType, size))); } - @Override - public X getToken() { - if (!populated) { - throw new IllegalStateException("Not populated"); - } - - return (X) children.stream() - .map(PartialToken::getToken) - .toArray(); + private static final M[] newArray(Class memberType, int size) { + return (M[]) Array.newInstance(memberType, size); } @Override - public int consumed() { - return lastTokenEnd == null ? 0 : lastTokenEnd.position() - position(); + public String tag() { + return fieldType.getName() + "[]"; } @Override public String toString() { - return memberType.getName() + "[" + children.size() + "]"; + return String.format("%-50.50s || %s[%d] (position: %d)", tail(50), fieldType.getName(), children.size(), position()); } @Override - public ParserLocation location() { - if (children.size() > 0) { - return children.get(0).location(); + public CharSequence source() { + StringBuilder result = new StringBuilder(); + for (int i = 0; i < children.size(); i++) { + result.append(children.get(i).source()); + } + if (current != null) { + result.append(current.source()); } - return location; + return result; } @Override - public ParserLocation end() { - return lastTokenEnd == null ? location : lastTokenEnd; + public Optional> nextChild() { + if (captureLimit == null || captureLimit.max() > children.size()) { + return Optional.of(current = PartialToken.forField(this, targetField().orElse(null), memberType, lastTokenEnd)); + } + return Optional.empty(); } @Override - public StringBuilder source() { - StringBuilder result = new StringBuilder(); - - for (int i = 0; i < children.size(); i++) { - result.append(children.get(i).source()); + public PartialToken[] children() { + PartialToken[] result; + if (current != null) { + result = new PartialToken[children.size() + 1]; + result = children.toArray(result); + result[result.length - 1] = current; + } else { + result = new PartialToken[children.size()]; + result = children.toArray(result); } return result; } @Override - public PartialToken[] getChildren() { - PartialToken[] result = new PartialToken[children.size()]; - return children.toArray(result); + public int unfilledChildren() { + if (isPopulated()) { + return 0; + } + if (captureLimit == null) { + return 1; + } + + return captureLimit.max() - children.size(); } + @Override + public int currentChild() { + return children.size() - 1; + } @Override - public PartialToken expected() { - return current.expected(); + public void nextChild(int newIndex) { + children = new LinkedList<>(children.subList(0, newIndex)); + current = children.peekLast(); } - + @Override - public String tag() { - return "Collection<" + memberType.getSimpleName() + ">"; + public void children(PartialToken[] children) { + this.children = new LinkedList<>(Arrays.asList(children)); + current = null; } @Override public int alternativesLeft() { - return 0; + return current == null ? 0 : current.alternativesLeft(); } } diff --git a/src/main/java/com/onkiup/linker/parser/token/CompoundToken.java b/src/main/java/com/onkiup/linker/parser/token/CompoundToken.java index 8214730..63cb795 100644 --- a/src/main/java/com/onkiup/linker/parser/token/CompoundToken.java +++ b/src/main/java/com/onkiup/linker/parser/token/CompoundToken.java @@ -5,31 +5,91 @@ import java.util.Optional; import java.util.function.Consumer; +import com.onkiup.linker.parser.ParserLocation; import com.onkiup.linker.parser.Rule; +import com.onkiup.linker.parser.TokenGrammar; -public interface CompoundToken extends PartialToken { +public interface CompoundToken extends PartialToken { + + static CompoundToken forClass(Class type, ParserLocation position) { + if (position == null) { + position = new ParserLocation(null, 0, 0, 0); + } + if (TokenGrammar.isConcrete(type)) { + return new RuleToken(null, null, type, position); + } else { + return new VariantToken(null, null, type, position); + } + } void onChildPopulated(); void onChildFailed(); + int unfilledChildren(); + + int currentChild(); + void nextChild(int newIndex); + + /** + * @return all previously created children, optionally excluding any possible future children + */ PartialToken[] children(); + + /** + * @param children an array of PartialToken objects to replace current token's children with + */ void children(PartialToken[] children); - PartialToken nextChild(); + Optional> nextChild(); - CharSequence traceback(); - /** - * advances to the next child token or parent - * @return next token to populate or null if this is a root token and it has no further tokens to populate + * Walks through token's children in reverse order removing them until the first child with alternativesLeft() > 0 + * If no such child found, then returns full token source + * @return source for removed tokens */ - default Optional> advance() { - if (isPopulated() || isFailed()) { - return parent().map(p -> p); + default Optional traceback() { + PartialToken[] children = children(); + if (children.length == 0) { + invalidate(); + onFail(); + return Optional.empty(); + } + StringBuilder result = new StringBuilder(); + int newSize = 0; + for (int i = children.length - 1; i > -1; i--) { + PartialToken child = children[i]; + if (child == null) { + continue; + } + if (child instanceof CompoundToken) { + ((CompoundToken)child).traceback().ifPresent(returned -> result.insert(0, returned.toString())); + } else { + result.insert(0, child.source().toString()); + } + + child.invalidate(); + + if (child.alternativesLeft() > 0) { + newSize = i + 1; + break; + } + + child.onFail(); + } + + if (newSize > 0) { + PartialToken[] newChildren = new PartialToken[newSize]; + System.arraycopy(children, 0, newChildren, 0, newSize); + children(newChildren); + nextChild(newSize - 1); + log("Traced back to child #{}: {}", newSize, newChildren[newSize-1]); + } else { + invalidate(); + onFail(); } - return Optional.of(nextChild()); + return Optional.of(result); } /** diff --git a/src/main/java/com/onkiup/linker/parser/token/ConsumingToken.java b/src/main/java/com/onkiup/linker/parser/token/ConsumingToken.java index c3d4e25..d36deae 100644 --- a/src/main/java/com/onkiup/linker/parser/token/ConsumingToken.java +++ b/src/main/java/com/onkiup/linker/parser/token/ConsumingToken.java @@ -1,7 +1,6 @@ package com.onkiup.linker.parser.token; import java.util.Optional; -import java.util.WeakHashMap; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; @@ -18,49 +17,54 @@ default void setTokenMatcher(TokenMatcher matcher) { ConsumptionState.create(this, ignoredCharacters, matcher); } - void onConsumeSuccess(Object token) { - parent().ifPresent(CompoundToken::onChildPopulated); - } - + void onConsumeSuccess(Object token); /** * Attempts to consume next character * @return null if character was consumed, otherwise returns a CharSequence with failed characters */ - default CharSequence consume(char character, boolean last) { - ConsumptionState consumption = ConsumptionState.of(this); + default Optional consume(char character) { + ConsumptionState consumption = ConsumptionState.of(this).orElseThrow(() -> new ParserError("No consumption state found (call ConsumingToken::setTokenMatcher to create it first)", this)); + consumption.consume(character); - if (isPopulated() || consumption.failed()) { + if (consumption.failed()) { if (!lookahead(consumption.buffer())) { + log("Lokahead complete"); onFail(); - // not accepting new characters at this time - return parent() - .map(CompoundToken::traceback) - .map(StringBuilder::new) - .map(sb -> sb.append(consumption.consumed())) - .orElseGet(consumption::consumed); + CharSequence consumed = consumption.consumed(); + consumption.clear(); + return Optional.of(consumed); } - // performing lookahead so, reporting successfull consumption (despite that the token has already failed) - return null; + log("performing lookahead so, reporting successfull consumption (despite that the token has already failed)"); + return Optional.empty(); } TokenTestResult result = consumption.test(); - if (result.isFailed() || (result.isContinue() && last)) { - // switching to lookahead mode + if (result.isFailed()) { + log("failed; switching to lookahead mode"); consumption.setFailed(); - return null; - } else if (result.isMatch() || (result.isMatchContinue() && last)) { + return Optional.empty(); + } else if (result.isMatch()) { int tokenLength = result.getTokenLength(); - StringBuilder buffer = consumption.buffer(); - CharSequence excess = buffer.substring(tokenLength); + CharSequence excess = consumption.trim(tokenLength); + log("matched"); + onConsumeSuccess(result.getToken()); + onPopulated(location().add(consumption.end())); + parent() + .filter(p -> p.unfilledChildren() == 0) + .map(p -> p.lookahead(excess)); + return Optional.of(excess); + } + + if (result.isMatchContinue()) { + log("matched; continuing..."); onConsumeSuccess(result.getToken()); onPopulated(consumption.end()); - return excess; } - return null; + return Optional.empty(); } @Override @@ -69,28 +73,35 @@ default void invalidate() { ConsumptionState.discard(this); } + @Override + default Optional traceback() { + return ConsumptionState.of(this).map(ConsumptionState::consumed) + .filter(consumed -> consumed.length() > 0); + } + @Override default CharSequence source() { if (isFailed()) { return ""; } - return ConsumptionState.of(this).consumed(); + return ConsumptionState.of(this).map(ConsumptionState::consumed).orElse(""); } class ConsumptionState { private static final ConcurrentHashMap states = new ConcurrentHashMap<>(); - private static synchronized ConsumptionState of(ConsumingToken token) { - if (!states.containsKey(token)) { - throw new ParserError("No consumption state available for token " + token + " (create one by calling ConsumingToken::setMatcher first?)", token); - } - return states.get(token); + private static synchronized Optional of(ConsumingToken token) { + return Optional.ofNullable(states.get(token)); } private static void create(ConsumingToken token, CharSequence ignoredCharacters, Function tester) { states.put(token, new ConsumptionState(ignoredCharacters, tester)); } + static void inject(ConsumingToken token, ConsumptionState state) { + states.put(token, state); + } + private static void discard(ConsumingToken token) { states.remove(token); } @@ -107,12 +118,19 @@ private ConsumptionState(CharSequence ignoredCharacters, Function implements ConsumingToken { +public class EnumToken extends AbstractToken implements ConsumingToken { private Class enumType; private int nextVariant = 0; - private CompoundToken parent; - private Field field; - private ParserLocation location, end; - private Map variants = new HashMap<>(); + private Map variants = new HashMap<>(); private X token; - private boolean failed, optional, populated; + private boolean failed, populated; private String ignoreCharacters; - public EnumToken(CompoundToken parent, Field field, Class enumType, ParserLocation location) { + public EnumToken(CompoundToken parent, Field field, Class enumType, ParserLocation location) { + super(parent, field, location); this.enumType = enumType; - this.location = location; - this.parent = parent; - this.field = field; for (X variant : enumType.getEnumConstants()) { try { Field variantField = enumType.getDeclaredField(variant.name()); - CapturePattern annotation = field.getAnnotation(CapturePattern.class); - if (annotation == null) { - throw new ParserError("Unable to use enum value " + variant + ": no @CapturePattern present", this); - } - PatternMatcher matcher = new PatternMatcher(annotation); + CapturePattern annotation = variantField.getAnnotation(CapturePattern.class); + TokenMatcher matcher = annotation == null ? new TerminalMatcher(variant.toString()) : new PatternMatcher(annotation); variants.put(variant, matcher); } catch (ParserError pe) { throw pe; @@ -54,10 +49,10 @@ public EnumToken(CompoundToken parent, Field field, Class enumType, Parser } List failed = new ArrayList<>(); - for (Map.Entry entry : variants.entrySet()) { + for (Map.Entry entry : variants.entrySet()) { TokenTestResult result = entry.getValue().apply(buffer); if (result.isMatch()) { - return new TestResult.match(result.getTokenLength(), entry.getKey()); + return TestResult.match(result.getTokenLength(), entry.getKey()); } else if (result.isFailed()) { failed.add(entry.getKey()); } @@ -74,34 +69,8 @@ public EnumToken(CompoundToken parent, Field field, Class enumType, Parser } @Override - public void onPopulated(ParserLocation end) { - this.end = end; - this.populated = true; - } - - @Override - public void markOptional() { - this.optional = true; - } - - @Override - public ParserLocation location() { - return location; - } - - @Override - public ParserLocation end() { - return end != null ? end : location; - } - - @Override - public Optional targetField() { - return Optional.ofNullable(field); - } - - @Override - public X token() { - return token; + public Optional token() { + return Optional.ofNullable(token); } @Override @@ -112,26 +81,8 @@ public Class tokenType() { @Override public void onConsumeSuccess(Object value) { token = (X) value; + this.populated = true; } - @Override - public boolean isPopulated() { - return token != null; - } - - @Override - public boolean isFailed() { - return failed; - } - - @Override - public boolean isOptional() { - return optional; - } - - @Override - public Optional> parent() { - return Optional.ofNullable(parent); - } } diff --git a/src/main/java/com/onkiup/linker/parser/token/FailedToken.java b/src/main/java/com/onkiup/linker/parser/token/FailedToken.java deleted file mode 100644 index d9e2a30..0000000 --- a/src/main/java/com/onkiup/linker/parser/token/FailedToken.java +++ /dev/null @@ -1,113 +0,0 @@ -package com.onkiup.linker.parser.token; - -import java.util.Optional; - -import com.onkiup.linker.parser.ParserLocation; - -/** - * A special token used to return characters from force-failed tokens - */ -public class FailedToken implements PartialToken, ConsumingToken { - - private PartialToken parent; - private StringBuilder data; - private ParserLocation location; - private ParserLocation end; - - public FailedToken(PartialToken parent, StringBuilder data, ParserLocation location) { - this.parent = parent; - this.location = location; - this.data = data; - int line = location.line(), column = location.column(); - for(int i = 0; i < data.length(); i++) { - char chr = data.charAt(i); - if(chr == '\n') { - line++; - column = 0; - } else { - column++; - } - } - this.end = new ParserLocation( - location.name(), - location.position() + data.length(), - location.line() + line, - column - ); - } - - @Override - public int consumed() { - return data.length(); - } - - @Override - public ParserLocation location() { - return location; - } - - @Override - public ParserLocation end() { - return end; - } - - @Override - public Optional getParent() { - return Optional.ofNullable(parent); - } - - @Override - public boolean isPopulated() { - return false; - } - - @Override - public Class getTokenType() { - return String.class; - } - - @Override - public String getToken() { - return data.toString(); - } - - @Override - public Optional pullback() { - throw new RuntimeException("Not supported"); - } - - @Override - public Optional advance(boolean last) { - return getParent() - .flatMap(p -> p.advance(last)); - } - - @Override - public Optional consume(char character, boolean last) { - return Optional.of(new StringBuilder().append(data).append(character)); - } - - @Override - public StringBuilder source() { - return new StringBuilder(data.toString()); - } - - @Override - public PartialToken expected() { - return parent.expected(); - } - - @Override - public String toString() { - return new StringBuilder() - .append("'") - .append(data.length() > 10 ? data.substring(0, 5) + "<..>" + data.substring(data.length() - 6) : data) - .append("'") - .append(" <-- FailedToken@") - .append(location) - .append(" --> ") - .append(parent) - .toString(); - } -} - diff --git a/src/main/java/com/onkiup/linker/parser/token/PartialToken.java b/src/main/java/com/onkiup/linker/parser/token/PartialToken.java index ff8c887..6873724 100644 --- a/src/main/java/com/onkiup/linker/parser/token/PartialToken.java +++ b/src/main/java/com/onkiup/linker/parser/token/PartialToken.java @@ -2,32 +2,36 @@ import java.lang.reflect.Field; import java.util.Arrays; -import java.util.Comparator; import java.util.LinkedList; -import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.function.Consumer; import java.util.function.Predicate; +import org.slf4j.Logger; + import com.onkiup.linker.parser.ParserLocation; import com.onkiup.linker.parser.Rule; -import com.onkiup.linker.parser.SyntaxError; import com.onkiup.linker.parser.TokenGrammar; import com.onkiup.linker.parser.annotation.AdjustPriority; import com.onkiup.linker.parser.annotation.OptionalToken; import com.onkiup.linker.parser.annotation.SkipIfFollowedBy; +import com.onkiup.linker.parser.util.LoggerLayout; import com.onkiup.linker.parser.util.ParserError; public interface PartialToken { - static PartialToken forField(CompoundToken parent, Field field, ParserLocation position) { + static PartialToken forField(CompoundToken parent, Field field, ParserLocation position) { if (position == null) { throw new ParserError("Child token position cannot be null", parent); } Class fieldType = field.getType(); + return forField(parent, field, fieldType, position); + } + + static PartialToken forField(CompoundToken parent, Field field, Class fieldType, ParserLocation position) { if (fieldType.isArray()) { return new CollectionToken(parent, field, fieldType, position); } else if (Rule.class.isAssignableFrom(fieldType)) { @@ -37,45 +41,68 @@ static PartialToken forField(CompoundToken parent, Field field, ParserLocatio return new RuleToken(parent, field, fieldType, position); } } else if (fieldType == String.class) { - return new TerminalToken(parent, field, position); + return (PartialToken) new TerminalToken(parent, field, fieldType, position); + } else if (fieldType.isEnum()) { + return (PartialToken) new EnumToken(parent, field, fieldType, position); } throw new IllegalArgumentException("Unsupported field type: " + fieldType); } - static PartialToken forClass(Class type, ParserLocation position) { - if (position == null) { - position = new ParserLocation(null, 0, 0, 0); - } - if (TokenGrammar.isConcrete(type)) { - return new RuleToken(null, null, type, position); - } else { - return new VariantToken(null, null, type, position); - } - } - - static CharSequence getOptionalCondition(Field field) { + static Optional getOptionalCondition(Field field) { if (field == null) { - return null; + return Optional.empty(); } + CharSequence result = null; if (field.isAnnotationPresent(OptionalToken.class)) { - return field.getAnnotation(OptionalToken.class).whenFollowedBy(); + result = field.getAnnotation(OptionalToken.class).whenFollowedBy(); } else if (field.isAnnotationPresent(SkipIfFollowedBy.class)) { - return field.getAnnotation(SkipIfFollowedBy.class).value(); + result = field.getAnnotation(SkipIfFollowedBy.class).value(); } - return null; + + return Optional.ofNullable(result == null || result.length() == 0 ? null : result); } - static boolean isOptional(Field field) { - return field != null && field.isAnnotationPresent(OptionalToken.class); + static boolean hasOptionalAnnotation(Field field) { + return field != null && (field.isAnnotationPresent(OptionalToken.class) || field.isAnnotationPresent(SkipIfFollowedBy.class)); + } + + /** + * Context-aware field optionality checks + * @param owner Context to check + * @param field Field to check + * @return true if the field should be optional in this context + */ + static boolean isOptional(CompoundToken owner, Field field) { + try { + if (field.isAnnotationPresent(OptionalToken.class)) { + OptionalToken optionalToken = field.getAnnotation(OptionalToken.class); + if (optionalToken.whenFieldIsNull().length() != 0) { + final String fieldName = optionalToken.whenFieldIsNull(); + Field targetField = owner.tokenType().getField(fieldName); + targetField.setAccessible(true); + return targetField.get(owner.token()) == null; + } + + return optionalToken.whenFollowedBy().length() != 0; + } + + return false; + } catch (Exception e) { + throw new ParserError("Failed to determine if field " + field.getName() + " should be optional", owner); + } } /** * @return Java representation of populated token */ - X token(); + Optional token(); Class tokenType(); + /** + * Called by parser to detect if this token is populated + * The result of this method should always be calculated + */ boolean isPopulated(); boolean isFailed(); boolean isOptional(); @@ -87,19 +114,33 @@ static boolean isOptional(Field field) { ParserLocation end(); void markOptional(); - void onPopulated(ParserLocation end); + void onPopulated(ParserLocation end); + String tag(); + + Optional traceback(); /** * @return all characters consumed by the token and its children */ CharSequence source(); + Logger logger(); + + default void log(CharSequence message, Object... arguments) { + logger().debug(message.toString(), arguments); + } + + default void error(CharSequence message, Throwable error) { + logger().error(message.toString(), error); + } + + /** * Called upon token failures */ default void onFail() { + log("!!! FAILED !!!"); invalidate(); - parent().ifPresent(CompoundToken::onChildFailed); } /** @@ -107,28 +148,38 @@ default void onFail() { * @return true if the token should continue consumption, false otherwise */ default boolean lookahead(CharSequence buffer) { - CharSequence optionalCondition = PartialToken.getOptionalCondition(targetField().orElse(null)); - final CharSequence[] parentBuffer = new CharSequence[] { buffer }; - boolean conditionPresent = optionalCondition != null && optionalCondition.length() > 0; - boolean myResult = true; - - if (!isOptional() && conditionPresent) { - if (buffer.length() >= optionalCondition.length()) { - CharSequence test = buffer.subSequence(0, optionalCondition.length()); - if (Objects.equals(test, optionalCondition)) { - parentBuffer[0] = buffer.subSequence(optionalCondition.length(), buffer.length()); - markOptional(); + log("performing lookahead"); + return targetField() + .flatMap(PartialToken::getOptionalCondition) + .map(condition -> { + log("Loookahead '{}' on '{}'", condition, buffer); + final CharSequence[] parentBuffer = new CharSequence[] { buffer }; + boolean myResult = true; + if (!isOptional()) { + if (buffer.length() >= condition.length()) { + CharSequence test = buffer.subSequence(0, condition.length()); + if (Objects.equals(test, condition)) { + log("Optional condition match: '{}' == '{}'", condition, test); + parentBuffer[0] = buffer.subSequence(condition.length(), buffer.length()); + markOptional(); + } + myResult = false; + } else if (!condition.subSequence(0, buffer.length()).equals(buffer)) { + parentBuffer[0] = buffer; + myResult = false; + } + } else { + parentBuffer[0] = buffer.subSequence(condition.length(), buffer.length()); } - myResult = false; - } - } else if (conditionPresent) { - parentBuffer[0] = buffer.subSequence(optionalCondition.length(), buffer.length()); - } - return myResult || parent() - // delegating lookahead call to parent - .flatMap(p -> p.lookahead(parentBuffer[0]) ? Optional.of(true) : Optional.empty()) - .isPresent(); + return myResult || isOptional() && parent() + .filter(p -> p.unfilledChildren() == 1) + .filter(p -> p.lookahead(parentBuffer[0])) + .isPresent(); + }).orElseGet(() -> targetField() + .flatMap(field -> parent().map(parent -> isOptional(parent, field))) + .orElse(false) + ); } default Optional> findInTree(Predicate comparator) { @@ -140,13 +191,6 @@ default Optional> findInTree(Predicate comparator) .flatMap(parent -> parent.findInTree(comparator)); } - default List> path() { - LinkedList> result = new LinkedList<>(); - parent().ifPresent(parent -> result.addAll(parent.path())); - result.add(this); - return result; - } - default int position() { ParserLocation location = location(); if (location == null) { @@ -178,10 +222,6 @@ default void sortPriorities() { } - default PartialToken replaceCurrentToken() { - throw new RuntimeException("Unsupported"); - } - default void token(X token) { throw new RuntimeException("Unsupported"); } @@ -209,12 +249,15 @@ default PartialToken root() { } default CharSequence tail(int length) { - CharSequence source = source(); - if (source.length() > length) { - source = source.subSequence(source.length() - length, length); - } - return String.format("%" + length + "s", source); + return LoggerLayout.ralign(LoggerLayout.sanitize(source().toString()), length); } + default LinkedList path() { + LinkedList path = parent() + .map(PartialToken::path) + .orElseGet(LinkedList::new); + path.add(this); + return path; + } } diff --git a/src/main/java/com/onkiup/linker/parser/token/Rotatable.java b/src/main/java/com/onkiup/linker/parser/token/Rotatable.java new file mode 100644 index 0000000..61f2903 --- /dev/null +++ b/src/main/java/com/onkiup/linker/parser/token/Rotatable.java @@ -0,0 +1,10 @@ +package com.onkiup.linker.parser.token; + +public interface Rotatable { + + boolean rotatable(); + + void rotateForth(); + void rotateBack(); +} + diff --git a/src/main/java/com/onkiup/linker/parser/token/RuleToken.java b/src/main/java/com/onkiup/linker/parser/token/RuleToken.java index 5499826..cb8b266 100644 --- a/src/main/java/com/onkiup/linker/parser/token/RuleToken.java +++ b/src/main/java/com/onkiup/linker/parser/token/RuleToken.java @@ -8,38 +8,25 @@ import java.util.Arrays; import java.util.Optional; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.onkiup.linker.parser.ParserLocation; import com.onkiup.linker.parser.Rule; import com.onkiup.linker.parser.annotation.IgnoreCharacters; +import com.onkiup.linker.parser.util.LoggerLayout; -public class RuleToken implements CompoundToken { +public class RuleToken extends AbstractToken implements CompoundToken, Rotatable { private X token; private Class tokenType; private Field[] fields; private PartialToken[] values; private int nextChild = 0; - private CompoundToken parent; - private Field field; private String ignoreCharacters = ""; - private int nextField; private boolean rotated = false; - private boolean populated = false; - private boolean failed = false; - private boolean optional; - private CharSequence optionalCondition; - private final ParserLocation location; private ParserLocation lastTokenEnd; - private final Logger logger; - public RuleToken(CompoundToken parent, Field field, Class type, ParserLocation location) { + public RuleToken(CompoundToken parent, Field field, Class type, ParserLocation location) { + super(parent, field, location); this.tokenType = type; - this.parent = parent; - this.field = field; - this.location = location; - this.logger = LoggerFactory.getLogger(type); + this.lastTokenEnd = location; try { this.token = type.newInstance(); @@ -54,7 +41,7 @@ public RuleToken(CompoundToken parent, Field field, Class type, ParserLoca values = new PartialToken[fields.length]; if (parent != null) { - ignoreCharacters += parent.ignoredCharacters(); + ignoreCharacters = parent.ignoredCharacters(); } if (type.isAnnotationPresent(IgnoreCharacters.class)) { @@ -64,9 +51,6 @@ public RuleToken(CompoundToken parent, Field field, Class type, ParserLoca } ignoreCharacters += type.getAnnotation(IgnoreCharacters.class).value(); } - - optionalCondition = PartialToken.getOptionalCondition(field); - optional = optionalCondition == null && PartialToken.isOptional(field); } @Override @@ -77,10 +61,10 @@ public void sortPriorities() { if (child.rotatable()) { int myPriority = basePriority(); int childPriority = child.basePriority(); - logger.debug("Verifying priority order for tokens; parent: {} child: {}", myPriority, childPriority); + log("Verifying priority order for tokens; parent: {} child: {}", myPriority, childPriority); if (childPriority < myPriority) { - logger.debug("Fixing priority order"); - unrotate(); + log("Fixing priority order"); + rotateBack(); } } } @@ -88,23 +72,11 @@ public void sortPriorities() { } @Override - public void markOptional() { - optional = true; - } - - @Override - public boolean isOptional() { - return optional; - } - - @Override - public Optional targetField() { - return Optional.ofNullable(field); - } - - @Override - public X token() { - return token; + public Optional token() { + return Optional.ofNullable(token).map(token -> { + Rule.Metadata.metadata(token, this); + return token; + }); } @Override @@ -112,82 +84,66 @@ public Class tokenType() { return tokenType; } - @Override - public Optional> parent() { - return Optional.ofNullable(parent); - } - - @Override - public boolean isPopulated() { - return populated; - } - - @Override - public boolean isFailed() { - return failed; - } - @Override public String ignoredCharacters() { return ignoreCharacters; } @Override - public PartialToken nextChild() { - Field childField = fields[nextChild]; - PartialToken result = PartialToken.forField(this, childField, lastTokenEnd); - values[nextChild++] = result; - return result; + public Optional> nextChild() { + if (nextChild >= fields.length) { + return Optional.empty(); + } + if (values[nextChild] == null || values[nextChild].isFailed() || values[nextChild].isPopulated()) { + Field childField = fields[nextChild]; + PartialToken result = PartialToken.forField(this, childField, lastTokenEnd); + return Optional.of(values[nextChild++] = result); + } else { + return Optional.of(values[nextChild++]); + } } @Override public void onChildPopulated() { PartialToken child = values[nextChild - 1]; Field field = fields[nextChild - 1]; - set(field, child.token()); + set(field, child.token().orElse(null)); lastTokenEnd = child.end(); - } - - @Override - public void onChildFailed() { - PartialToken child = values[nextChild - 1]; - if (alternativesLeft() == 0 && !child.isOptional()) { - failed = true; + if (nextChild >= fields.length) { + onPopulated(lastTokenEnd); } } - @Override - public void onPopulated(ParserLocation end) { - this.populated = true; - lastTokenEnd = end; + public Field[] fields() { + return fields; } @Override - public CharSequence traceback() { - StringBuilder result = new StringBuilder(); - for (int i = nextChild - 1; i > -1; i--) { - PartialToken child = values[i]; - result.append(child.source()); - if (child.alternativesLeft() > 0) { - nextChild = i; - lastTokenEnd = child.location(); - break; + public void onChildFailed() { + PartialToken child = values[nextChild - 1]; + if (child.isOptional()) { + if (nextChild >= fields.length) { + onPopulated(lastTokenEnd); } + } else if (alternativesLeft() == 0) { + onFail(); } - return result; } private void set(Field field, Object value) { + log("Trying to set field ${} to '{}'", field.getName(), LoggerLayout.sanitize(value)); try { if (!Modifier.isStatic(field.getModifiers())) { - logger.debug("Populating field {}", field.getName(), value); + log("Setting field ${} to '{}'", field.getName(), LoggerLayout.sanitize(value)); field.setAccessible(true); field.set(token, convert(field.getType(), value)); try { token.reevaluate(); } catch (Exception e) { - logger.warn("Failed to reevaluate", e); + error("Failed to reevaluate", e); } + } else { + log("NOT Setting field {} to '{}' -- the field is static", field.getName(), LoggerLayout.sanitize(value)); } } catch (Exception e) { throw new RuntimeException("Failed to populate field " + field, e); @@ -236,84 +192,73 @@ protected T convert(Class into, Object what) { } @Override - public String toString() { + public String tag() { return tokenType.getName(); } @Override - public ParserLocation location() { - if (values.length > 0) { - for (int i = 0; i < values.length; i++) { - if (values[i] != null && values[i].isPopulated()) { - return values[i].location(); - } - } - } - return location; + public String toString() { + return String.format("%s || %s (positoin: %d)", tail(50), tokenType.getName() + (nextChild > - 1 ? "$" + (fields[nextChild - 1].getName()) : ""), position()); } @Override public boolean rotatable() { if (fields.length < 3) { - logger.debug("Not rotatable -- not enough fields"); + log("Not rotatable -- not enough fields"); return false; } if (!this.isPopulated()) { - logger.debug("Not rotatable -- not populated"); + log("Not rotatable -- not populated"); return false; } Field field = fields[0]; Class fieldType = field.getType(); if (!fieldType.isAssignableFrom(tokenType)) { - logger.debug("Not rotatable -- first field is not assignable from token type"); + log("Not rotatable -- first field is not assignable from token type"); return false; } field = fields[fields.length - 1]; fieldType = field.getType(); if (!fieldType.isAssignableFrom(tokenType)) { - logger.debug("Not rotatable -- last field is not assignable from token type"); + log("Not rotatable -- last field is not assignable from token type"); return false; } return true; } - public Field getCurrentField() { - return fields[nextField > 0 ? nextField - 1 : 0]; - } - @Override - public void rotate() { - logger.info("Rotating"); + public void rotateForth() { + log("Rotating"); token.invalidate(); - RuleToken wrap = new RuleToken(this, fields[0], fields[0].getType(), location); + RuleToken wrap = new RuleToken(this, fields[0], fields[0].getType(), location()); wrap.nextChild = nextChild; nextChild = 1; PartialToken[] wrapValues = wrap.values; wrap.values = values; values = wrapValues; values[0] = wrap; - X wrapToken = (X) wrap.token(); + X wrapToken = (X) wrap.token().orElse(null); wrap.token = token; token = wrapToken; } @Override - public void unrotate() { - logger.debug("Un-rotating"); + public void rotateBack() { + log("Un-rotating"); PartialToken firstToken = values[0]; CompoundToken kiddo; if (firstToken instanceof VariantToken) { - kiddo = (CompoundToken)((VariantToken)kiddo).resolvedAs(); + kiddo = (CompoundToken)((VariantToken)firstToken).resolvedAs().orElse(null); } else { kiddo = (CompoundToken) firstToken; } - Rule childToken = (Rule) kiddo.token(); + Rule childToken = (Rule) kiddo.token().orElse(null); Class childTokenType = kiddo.tokenType(); invalidate(); @@ -331,7 +276,7 @@ public void unrotate() { token = (X) childToken; tokenType = (Class) childTokenType; children(values); - set(fields[fields.length - 1], values[values.length - 1].token()); + set(fields[fields.length - 1], values[values.length - 1].token().orElse(null)); } @Override @@ -339,6 +284,21 @@ public PartialToken[] children() { return values; } + @Override + public int unfilledChildren() { + return fields.length - nextChild; + } + + @Override + public int currentChild() { + return nextChild - 1; + } + + @Override + public void nextChild(int newIndex) { + nextChild = newIndex; + } + @Override public void token(X token) { this.token = token; @@ -346,7 +306,17 @@ public void token(X token) { @Override public void children(PartialToken[] children) { - this.values = children; + lastTokenEnd = location(); + for (int i = 0; i < values.length; i++) { + if (i < children.length) { + values[i] = children[i]; + } else { + values[i] = null; + } + if (values[i] != null && values[i].isPopulated()) { + lastTokenEnd = values[i].end(); + } + } } @Override @@ -354,11 +324,6 @@ public void invalidate() { token.invalidate(); } - @Override - public ParserLocation end() { - return lastTokenEnd == null ? location : lastTokenEnd; - } - @Override public StringBuilder source() { StringBuilder result = new StringBuilder(); diff --git a/src/main/java/com/onkiup/linker/parser/token/TerminalToken.java b/src/main/java/com/onkiup/linker/parser/token/TerminalToken.java index 816509f..9392a58 100644 --- a/src/main/java/com/onkiup/linker/parser/token/TerminalToken.java +++ b/src/main/java/com/onkiup/linker/parser/token/TerminalToken.java @@ -14,195 +14,35 @@ import com.onkiup.linker.parser.TokenTestResult; import com.onkiup.linker.parser.annotation.OptionalToken; import com.onkiup.linker.parser.annotation.SkipIfFollowedBy; +import com.onkiup.linker.parser.token.CompoundToken; +import com.onkiup.linker.parser.util.LoggerLayout; -public class TerminalToken implements PartialToken, ConsumingToken { - - - private Field field; - private PartialToken parent; +public class TerminalToken extends AbstractToken implements ConsumingToken { private TokenMatcher matcher; - private TokenTestResult lastTestResult; private CharSequence token; - private boolean failed = false; - private boolean isOptional; - private boolean skipped; - private CharSequence optionalCondition = ""; - private final ParserLocation start; - private ParserLocation end; - private Logger logger; - public TerminalToken(PartialToken parent, Field field, ParserLocation location) { - this.parent = parent; - this.field = field; - this.logger = LoggerFactory.getLogger(field.getDeclaringClass().getName() + "$" + field.getName()); - this.start = this.end = location; - this.matcher = TokenMatcher.forField(field); - if (field.isAnnotationPresent(OptionalToken.class)) { - OptionalToken optional = field.getAnnotation(OptionalToken.class); - this.isOptional = true; - this.optionalCondition = optional.whenFollowedBy(); - } - - if (field.isAnnotationPresent(SkipIfFollowedBy.class)) { - if (isOptional) { - throw new IllegalStateException("Tokens cannot be both optional and skippable (" + field.getDeclaringClass().getSimpleName() + "." + field.getName() + ")"); - } - SkipIfFollowedBy condition = field.getAnnotation(SkipIfFollowedBy.class); - this.isOptional = true; - this.optionalCondition = condition.value(); - } + public TerminalToken(CompoundToken parent, Field field, Class tokenType, ParserLocation location) { + super(parent, field, location); + this.matcher = TokenMatcher.forField(field, tokenType); this.setTokenMatcher(matcher); } - public TerminalToken(PartialToken parent, TokenMatcher matcher) { - this.parent = parent; - this.matcher = matcher; - this.start = this.end = new ParserLocation("unknown", 0, 0, 0); - } - - @Override - public boolean isFailed() { - return failed; - } - - @Override - public Optional advance(boolean force) throws SyntaxError { - // returns next token from one of the parents - return parent.advance(force); - } - - @Override - public void onPopulate(CharSequence value) { - logger.debug("-- MATCHED"); - this.token = value; - this.end = - } - @Override - public boolean onFail(CharSequence on) { - logger.debug("-- FAILED"); - failed = true; - return !lookahead(on); + public void onConsumeSuccess(Object token) { + log("MATCHED '{}'", LoggerLayout.sanitize((String)token)); + this.token = (String) token; } @Override - public boolean lookahead(CharSequence buffer) { - if (optionalCondition == null || optionalCondition.length() == 0) { - logger.debug("the token is unconditionally optional"); - skipped = true; - return false; - } - - if (optionalCondition.length() > buffer.length()) { - logger.debug("unable to resolve optionality conditions -- not enough characters -- continuting lookahead"); - return true; - } - - if (Objects.equals(optionalCondition, buffer.subSequence(0, optionalCondition.length()))) { - logger.debug("parser input satisfies token optionality condition"); - skipped = true; - } - return false; - } - - private boolean isOptional() { - return skipped; - } - - @Override - public Optional pullback() { - logger.debug("Pullback request received"); - if (isPopulated()) { - logger.debug("Populated -- Rolling back characters '{}{}'", ignoredCharacters, token); - StringBuilder result = new StringBuilder() - .append(ignoredCharacters) - .append(token); - - token = null; - return Optional.of(result); - } - logger.debug("Not poppulated -- not returning any characters"); - return Optional.empty(); - } - - public boolean skipped() { - return skipped; + public Optional token() { + return Optional.ofNullable(token).map(CharSequence::toString); } @Override - public String getToken() { - return token; - } - - @Override - public Class getTokenType() { + public Class tokenType() { return String.class; } - @Override - public boolean isPopulated() { - return token != null; - } - - @Override - public Optional getParent() { - return Optional.ofNullable(parent); - } - - @Override - public String getIgnoredCharacters() { - return parent.getIgnoredCharacters(); - } - - private StringBuilder populate(TokenTestResult testResult) { - token = (String) testResult.getToken(); - int consumed = buffer.length() - cleanBuffer.length() + testResult.getTokenLength(); - buffer.delete(0, consumed); - return buffer; - } - - @Override - public String toString() { - return field.getDeclaringClass().getName() + "$" + field.getName(); - } - - @Override - public ParserLocation location() { - return location; - } - - @Override - public int consumed() { - return token != null ? ignoredCharacters.length() + token.length() : buffer.length(); - } - - @Override - public ParserLocation end() { - if (token == null) { - return location.advance(cleanBuffer); - } else { - return location.advance(token); - } - } - - @Override - public StringBuilder source() { - StringBuilder result = new StringBuilder(); - if (token != null && token.length() > 0) { - result.append(ignoredCharacters.toString()).append(token); - } - return result; - } - - @Override - public PartialToken expected() { - return this; - } - - @Override - public String tag() { - return field.getName(); - } } diff --git a/src/main/java/com/onkiup/linker/parser/token/VariantToken.java b/src/main/java/com/onkiup/linker/parser/token/VariantToken.java index e8901db..71c6247 100644 --- a/src/main/java/com/onkiup/linker/parser/token/VariantToken.java +++ b/src/main/java/com/onkiup/linker/parser/token/VariantToken.java @@ -1,5 +1,6 @@ package com.onkiup.linker.parser.token; +import java.lang.reflect.Field; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; @@ -12,42 +13,43 @@ import com.onkiup.linker.parser.ParserLocation; import com.onkiup.linker.parser.Rule; -import com.onkiup.linker.parser.SyntaxError; import com.onkiup.linker.parser.TokenGrammar; import com.onkiup.linker.parser.annotation.AdjustPriority; import com.onkiup.linker.parser.annotation.Alternatives; import com.onkiup.linker.parser.annotation.IgnoreCharacters; import com.onkiup.linker.parser.annotation.IgnoreVariant; -import com.onkiup.linker.parser.token.PartialToken; -import com.onkiup.linker.parser.util.ParserError; +import com.onkiup.linker.parser.util.LoggerLayout; -public class VariantToken implements PartialToken { +public class VariantToken extends AbstractToken implements CompoundToken { private static final Reflections reflections = new Reflections(new ConfigurationBuilder() .setUrls(ClasspathHelper.forClassLoader(TokenGrammar.class.getClassLoader())) .setScanners(new SubTypesScanner(true)) ); + private static final ConcurrentHashMap dynPriorities = new ConcurrentHashMap<>(); + private Class tokenType; private Class[] variants; - private PartialToken bestShot; - private int currentVariant; + private int nextVariant = 0; private PartialToken token; private String ignoreCharacters = ""; - private PartialToken parent; - private boolean rotated = false; - private ParserLocation location; - private final Logger logger; - public VariantToken(PartialToken parent, Class tokenType, ParserLocation location) { + public VariantToken(CompoundToken parent, Field field, Class tokenType, ParserLocation location) { + super(parent, field, location); + this.tokenType = tokenType; - this.logger = LoggerFactory.getLogger(tokenType); - this.parent = parent; - this.location = location; if (TokenGrammar.isConcrete(tokenType)) { throw new IllegalArgumentException("Variant token cannot handle concrete type " + tokenType); } + if (findInTree(token -> token != this && token.tokenType() == tokenType && token.position() == position()).isPresent()) { + log("---||| Not descending: already selecting same Variant sub-type at this location"); + variants = new Class[0]; + onFail(); + return; + } + if (tokenType.isAnnotationPresent(Alternatives.class)) { variants = tokenType.getAnnotation(Alternatives.class).value(); } else { @@ -55,12 +57,12 @@ public VariantToken(PartialToken parent, Class tokenType, ParserLocation loca variants = reflections.getSubTypesOf(tokenType).stream() .filter(type -> { if (type.isAnnotationPresent(IgnoreVariant.class)) { - logger.debug("Ignoring variant {} -- marked with @IgnoreVariant", type.getSimpleName()); + log("Ignoring variant {} -- marked with @IgnoreVariant", type.getSimpleName()); return false; } - boolean inTree = findInTree(token -> token != null && token.getTokenType() == type && token.location().position() == location.position()).isPresent(); + boolean inTree = findInTree(token -> token != null && token.tokenType() == type && token.location().position() == location.position()).isPresent(); if (inTree) { - logger.debug("Ignoring variant {} -- already in tree with same position ({})", type.getSimpleName(), location.position()); + log("Ignoring variant {} -- already in tree with same position ({})", type.getSimpleName(), location.position()); return false; } Class superClass = type.getSuperclass(); @@ -71,7 +73,7 @@ public VariantToken(PartialToken parent, Class tokenType, ParserLocation loca return true; } } - logger.debug("Ignoring " + type + " (extends: " + superClass + ") "); + log("Ignoring " + type + " (extends: " + superClass + ") "); return false; } return true; @@ -94,7 +96,7 @@ public VariantToken(PartialToken parent, Class tokenType, ParserLocation loca } if (parent != null) { - ignoreCharacters += parent.getIgnoredCharacters(); + ignoreCharacters = parent.ignoredCharacters(); } if (tokenType.isAnnotationPresent(IgnoreCharacters.class)) { @@ -103,156 +105,99 @@ public VariantToken(PartialToken parent, Class tokenType, ParserLocation loca } @Override - public Optional pushback(boolean force) { - StringBuilder result = null; - if (bestShot == null || token != null && bestShot.end().position() < token.end().position()) { - bestShot = token; - } - if (token != null && token.alternativesLeft() > 0) { - logger.info("Not pushing parent: token has alternatives: {}", token); - return token.pullback(); - } else if (currentVariant < variants.length) { - logger.debug("Not pushing parent back: not all options exhausted for {}", this); - result = (StringBuilder) (token == null ? null : token.pullback().orElse(null)); - token = null; - } else { - logger.debug("{}: Exhausted all variants on pushback, rollbacking parent token {}", this, parent); - result = (StringBuilder) getParent() - .flatMap(p -> p.pushback(false)) - .orElse(null); + public Optional> nextChild() { + if (nextVariant >= variants.length) { + return Optional.empty(); } - - return Optional.ofNullable(result); + return Optional.of(token = PartialToken.forField(this, targetField().orElse(null), variants[nextVariant++], location())); } @Override - public void rotate() { - if (token != null) { - token.rotate(); - rotated = true; - } + public PartialToken[] children() { + return new PartialToken[] {token}; } @Override - public void unrotate() { - if (token != null) { - token.unrotate(); - rotated = true; - } + public void children(PartialToken[] children) { + throw new RuntimeException("Unable to set children on VariantToken"); } @Override - public Optional pullback() { - logger.debug("Pullback request received"); - if (token != null) { - PartialToken discarded = token; - StringBuilder result = (StringBuilder) discarded.pullback().orElse(null); - token = null; - logger.debug("Pulled back from token: '{]}'", result); - return Optional.ofNullable(result); - } - logger.debug("No token present, nothing to delegate this pullback request to"); - return Optional.empty(); + public void onChildPopulated() { + updateDynPriority(variants[nextVariant - 1], -10); + onPopulated(token.end()); } @Override - public Optional advance(boolean forcePopulate) throws SyntaxError { - if (rotated) { - logger.debug("Token was rotated, not advancing"); - rotated = false; - return Optional.of(token); - } else if (isPopulated()) { - bestShot = token; - logger.debug("Populated {} as {}; Advancing to parent", this, token); - return parent == null ? Optional.empty() : parent.advance(forcePopulate); - } else if (token != null && token.alternativesLeft() > 0) { - logger.debug("{}: Advancing to child token with alternatives: {}", this, token); - return Optional.of(token); - } else if (currentVariant < variants.length) { - Class variant = variants[currentVariant++]; - - token = PartialToken.forClass(this, variant, location); - logger.info("Advancing to variant {}/{}: {}", currentVariant, variants.length, token); - return token.advance(false); - } - - token = null; - - logger.info("{}: Exhausted all variants on advance, returning to parent {}", this, parent); - if (parent == null) { - return Optional.empty(); + public void onChildFailed() { + updateDynPriority(variants[nextVariant - 1], 10); + if (nextVariant >= variants.length) { + onFail(); + } else { + dropPopulated(); } - - StringBuilder data = parent.pushback(forcePopulate).orElseGet(StringBuilder::new); - return Optional.of(new FailedToken(parent, data, location)); - } - - @Override - public boolean isPopulated() { - return token != null && token.isPopulated(); } @Override - public X getToken() { - return token.getToken(); + public Optional token() { + return (Optional) token.token(); } @Override - public Class getTokenType() { + public Class tokenType() { return tokenType; } - public PartialToken resolvedAs() { - return token; - } - @Override - public Optional getParent() { - return Optional.ofNullable(parent); - } - - @Override - public String getIgnoredCharacters() { - return ignoreCharacters; + public Optional traceback() { + Optional result = null; + if (token == null) { + return Optional.empty(); + } else if (token instanceof CompoundToken) { + result = ((CompoundToken)token).traceback(); + dropPopulated(); + } else { + result = Optional.of(token.source()); + dropPopulated(); + } + token = null; + return result; } private int calculatePriority(Class type) { - int result = 0; + int result = dynPriorities.getOrDefault(type, 0); if (!TokenGrammar.isConcrete(type)) { result += 1000; } - if (findInTree(other -> type == other.getTokenType()).isPresent()) { + if (findInTree(other -> type == other.tokenType()).isPresent()) { result += 1000; } if (type.isAnnotationPresent(AdjustPriority.class)) { AdjustPriority adjust = type.getAnnotation(AdjustPriority.class); result += adjust.value(); - logger.info("Adjusted priority by " + adjust.value()); + log("Adjusted priority by " + adjust.value()); } - logger.info(type.getSimpleName() + " priority " + result); + log(type.getSimpleName() + " priority " + result); return result; } @Override - public String toString() { + public String tag() { return "? extends " + tokenType.getName(); } @Override - public int consumed() { - if (token == null) { - return 0; - } - return token.consumed(); + public String toString() { + return String.format("%-50.50s || %s (%d/%d) (position: %d)", tail(50), tag(), nextVariant, variants.length, position()); } @Override public int alternativesLeft() { - return variants.length - currentVariant + (token == null ? 0 : token.alternativesLeft()); + return variants.length - nextVariant + (token == null ? 0 : token.alternativesLeft()); } @Override @@ -260,11 +205,6 @@ public void sortPriorities() { token.sortPriorities(); } - @Override - public PartialToken[] getChildren() { - return new PartialToken[] {token}; - } - @Override public boolean propagatePriority() { if (tokenType.isAnnotationPresent(AdjustPriority.class)) { @@ -285,27 +225,6 @@ public int basePriority() { return result; } - @Override - public boolean rotatable() { - return token != null && token.rotatable(); - } - - @Override - public ParserLocation end() { - if (token == null) { - return location; - } - return token.end(); - } - - @Override - public ParserLocation location() { - if (token != null) { - return token.location(); - } - return location; - } - @Override public StringBuilder source() { StringBuilder result = new StringBuilder(); @@ -316,17 +235,30 @@ public StringBuilder source() { return result; } - @Override - public PartialToken expected() { - if (bestShot != null) { - return bestShot.expected(); - } - return this; + public Optional> resolvedAs() { + return Optional.ofNullable(token); } @Override - public String tag() { - return "? extends " + tokenType.getSimpleName(); + public int unfilledChildren() { + return (token != null && token.isPopulated()) || variants.length == 0 ? 0 : 1; + } + + @Override + public int currentChild() { + return nextVariant - 1; + } + + @Override + public void nextChild(int newIndex) { + throw new UnsupportedOperationException(); + } + + private static void updateDynPriority(Class target, int change) { + if (!dynPriorities.containsKey(target)) { + dynPriorities.put(target, 0); + } + dynPriorities.put(target, dynPriorities.get(target) + change); } } diff --git a/src/main/java/com/onkiup/linker/parser/util/LoggerLayout.java b/src/main/java/com/onkiup/linker/parser/util/LoggerLayout.java new file mode 100644 index 0000000..0bc1b18 --- /dev/null +++ b/src/main/java/com/onkiup/linker/parser/util/LoggerLayout.java @@ -0,0 +1,52 @@ +package com.onkiup.linker.parser.util; + +import org.apache.log4j.Layout; + +import org.apache.log4j.spi.LoggingEvent; + +public class LoggerLayout extends Layout { + + private Layout parent; + private StringBuilder buffer; + + public LoggerLayout(Layout parent, StringBuilder buffer) { + this.parent = parent; + this.buffer = buffer; + } + + @Override + public String format(LoggingEvent event) { + CharSequence bufVal = sanitize(buffer.toString()); + return String.format("%s || %s :: %s\n", ralign(bufVal, 50), ralign(event.getLoggerName(), 50), event.getMessage()); + } + + @Override + public boolean ignoresThrowable() { + return parent.ignoresThrowable(); + } + + @Override + public void activateOptions() { + parent.activateOptions(); + } + + public Layout parent() { + return parent; + } + + public static String sanitize(Object what) { + return what == null ? "null" : sanitize(what.toString()); + } + public static String sanitize(String what) { + return what == null ? null : what.replaceAll("\n", "\\\\n").replaceAll("\t", "\\\\t"); + } + + public static String ralign(CharSequence what, int len) { + if (what.length() >= len) { + what = what.subSequence(what.length() - len, what.length()); + return what.toString(); + } + String format = String.format("%%%1$d.%1$ds%%2$s", len - what.length()); + return String.format(format, "", what); + } +} diff --git a/src/main/java/com/onkiup/linker/parser/util/ParserError.java b/src/main/java/com/onkiup/linker/parser/util/ParserError.java index 750b2d6..6ab2542 100644 --- a/src/main/java/com/onkiup/linker/parser/util/ParserError.java +++ b/src/main/java/com/onkiup/linker/parser/util/ParserError.java @@ -1,25 +1,35 @@ package com.onkiup.linker.parser.util; +import java.util.Optional; + import com.onkiup.linker.parser.token.PartialToken; public class ParserError extends RuntimeException { private PartialToken source; + + public ParserError(String msg, Optional> source) { + this(msg, source.orElse(null)); + } public ParserError(String msg, PartialToken source) { - super(message); + super(msg); this.source = source; } + public ParserError(String msg, Optional> source, Throwable cause) { + this(msg, source.orElse(null), cause); + } + public ParserError(String msg, PartialToken source, Throwable cause) { - super(message, cause); + super(msg, cause); this.source = source; } @Override public String toString() { StringBuilder result = new StringBuilder("Parser error at position "); - result.append(source.position()) + result.append(source == null ? "" : source.position()) .append(": ") .append(getMessage()) .append("\n"); @@ -29,7 +39,7 @@ public String toString() { result.append("\t") .append(parent.toString()) .append("\n"); - parent = (PartialToken) parent.getParent().orElse(null); + parent = (PartialToken) parent.parent().orElse(null); } return result.toString(); } diff --git a/src/main/java/resources/log4j.properties b/src/main/java/resources/log4j.properties new file mode 100644 index 0000000..518a38c --- /dev/null +++ b/src/main/java/resources/log4j.properties @@ -0,0 +1,48 @@ +# SLF4J's SimpleLogger configuration file +# Simple implementation of Logger that sends all enabled log messages, for all defined loggers, to System.err. + +# Default logging detail level for all instances of SimpleLogger. +# Must be one of ("trace", "debug", "info", "warn", or "error"). +# If not specified, defaults to "info". +org.slf4j.simpleLogger.defaultLogLevel=trace + +# Logging detail level for a SimpleLogger instance named "xxxxx". +# Must be one of ("trace", "debug", "info", "warn", or "error"). +# If not specified, the default logging detail level is used. +#org.slf4j.simpleLogger.log.xxxxx= + +# Set to true if you want the current date and time to be included in output messages. +# Default is false, and will output the number of milliseconds elapsed since startup. +#org.slf4j.simpleLogger.showDateTime=false + +# The date and time format to be used in the output messages. +# The pattern describing the date and time format is the same that is used in java.text.SimpleDateFormat. +# If the format is not specified or is invalid, the default format is used. +# The default format is yyyy-MM-dd HH:mm:ss:SSS Z. +#org.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd HH:mm:ss:SSS Z + +# Set to true if you want to output the current thread name. +# Defaults to true. +#org.slf4j.simpleLogger.showThreadName=true + +# Set to true if you want the Logger instance name to be included in output messages. +# Defaults to true. +#org.slf4j.simpleLogger.showLogName=true + +# Set to true if you want the last component of the name to be included in output messages. +# Defaults to false. +#org.slf4j.simpleLogger.showShortLogName=false + + +log4j.rootCategory=debug,console +log4j.logger.com.demo.package=debug,console +log4j.additivity.com.demo.package=false + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.out +log4j.appender.console.immediateFlush=true +log4j.appender.console.encoding=UTF-8 +#log4j.appender.console.threshold=warn + +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.conversionPattern=%d [%t] %-5p %c - %m%n diff --git a/src/test/java/com/onkiup/linker/parser/NumberMatcherTest.java b/src/test/java/com/onkiup/linker/parser/NumberMatcherTest.java deleted file mode 100644 index e69de29..0000000 diff --git a/src/test/java/com/onkiup/linker/parser/PatternMatcherTest.java b/src/test/java/com/onkiup/linker/parser/PatternMatcherTest.java deleted file mode 100644 index a7ea7cd..0000000 --- a/src/test/java/com/onkiup/linker/parser/PatternMatcherTest.java +++ /dev/null @@ -1,95 +0,0 @@ -package com.onkiup.linker.parser; -import org.junit.Assert; -import org.junit.Test; -import org.mockito.Mockito; - -import com.onkiup.linker.parser.annotation.CapturePattern; - -public class PatternMatcherTest { - - @Test - public void testMatch() { - CapturePattern pattern = Mockito.mock(CapturePattern.class); - Mockito.when(pattern.value()).thenReturn(""); - Mockito.when(pattern.pattern()).thenReturn("[^;\\s]+"); - Mockito.when(pattern.replacement()).thenReturn(""); - Mockito.when(pattern.until()).thenReturn(""); - PatternMatcher subject = new PatternMatcher(pattern); - - StringBuilder buffer = new StringBuilder(); - TokenTestResult result; - - buffer.append("a"); - result = subject.apply(buffer); - Assert.assertEquals(TestResult.MATCH_CONTINUE, result.getResult()); - Assert.assertEquals("a", result.getToken()); - - buffer.append(";"); - result = subject.apply(buffer); - Assert.assertEquals(TestResult.MATCH, result.getResult()); - Assert.assertEquals("a", result.getToken()); - - buffer = new StringBuilder("; test"); - result = subject.apply(buffer); - - Assert.assertEquals(TestResult.FAIL, result.getResult()); - } - - @Test - public void testReplace() { - CapturePattern pattern = Mockito.mock(CapturePattern.class); - Mockito.when(pattern.pattern()).thenReturn("([^;\\s]+)"); - Mockito.when(pattern.replacement()).thenReturn("$1--"); - Mockito.when(pattern.until()).thenReturn(""); - - PatternMatcher subject = new PatternMatcher(pattern); - StringBuilder buffer = new StringBuilder("test; data"); - - TokenTestResult result = subject.apply(buffer); - - Assert.assertTrue(result.isMatch()); - Assert.assertEquals(4, result.getTokenLength()); - Assert.assertEquals("test--", result.getToken()); - } - - @Test - public void testUntil() { - CapturePattern pattern = Mockito.mock(CapturePattern.class); - Mockito.when(pattern.pattern()).thenReturn(""); - Mockito.when(pattern.value()).thenReturn(""); - Mockito.when(pattern.replacement()).thenReturn(">($1)<"); - Mockito.when(pattern.until()).thenReturn("(until)"); - - PatternMatcher subject = new PatternMatcher(pattern); - StringBuilder buffer = new StringBuilder("s"); - - TokenTestResult result = subject.apply(buffer); - Assert.assertTrue(result.isMatchContinue()); - Assert.assertEquals(1, result.getTokenLength()); - Assert.assertEquals("s", result.getToken()); - - buffer.append("ome text unti"); - result = subject.apply(buffer); - Assert.assertTrue(result.isMatchContinue()); - Assert.assertEquals(buffer.length(), result.getTokenLength()); - Assert.assertEquals("some text unti", result.getToken()); - - buffer.append("l"); - result = subject.apply(buffer); - Assert.assertTrue(result.isMatch()); - Assert.assertEquals("some text >(until)<", result.getToken()); - Assert.assertEquals(15, result.getTokenLength()); - - // NO replacement - Mockito.when(pattern.replacement()).thenReturn(""); - Mockito.when(pattern.until()).thenReturn("until"); - subject = new PatternMatcher(pattern); - result = subject.apply(buffer); - Assert.assertTrue(result.isMatch()); - Assert.assertEquals("some text ", result.getToken()); - Assert.assertEquals(10, result.getTokenLength()); - - result = subject.apply(new StringBuilder("until")); - Assert.assertEquals(TestResult.FAIL, result.getResult()); - } -} diff --git a/src/test/java/com/onkiup/linker/parser/TerminalMatcherTest.java b/src/test/java/com/onkiup/linker/parser/TerminalMatcherTest.java deleted file mode 100644 index 54798eb..0000000 --- a/src/test/java/com/onkiup/linker/parser/TerminalMatcherTest.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.onkiup.linker.parser; - -import org.junit.Test; - -import static org.junit.Assert.*; - -public class TerminalMatcherTest { - - @Test - public void testMatching() { - StringBuilder source = new StringBuilder("test"); - TerminalMatcher matcher = new TerminalMatcher("test it"); - - assertTrue(matcher.apply(source).isMatchContinue()); - source.append(" it"); - assertTrue(matcher.apply(source).isMatch()); - } - -} - diff --git a/src/test/java/com/onkiup/linker/parser/TokenGrammarTest.java b/src/test/java/com/onkiup/linker/parser/TokenGrammarTest.java deleted file mode 100644 index 5dc3cd2..0000000 --- a/src/test/java/com/onkiup/linker/parser/TokenGrammarTest.java +++ /dev/null @@ -1,179 +0,0 @@ -package com.onkiup.linker.parser; - -import java.io.StringReader; -import java.util.HashMap; -import java.util.Map; - -import org.junit.Assert; -import org.junit.Test; - -import com.onkiup.linker.parser.annotation.CaptureLimit; -import com.onkiup.linker.parser.annotation.CapturePattern; -import com.onkiup.linker.parser.annotation.OptionalToken; - -public class TokenGrammarTest { - - public static interface Junction extends Rule { - - } - - public static class TestGrammarDefinition implements Junction { - private static final String marker = ":"; - @CapturePattern(pattern = "[^;\\n]+") - private String command; - } - - public static class CommentGrammarDefinition implements Junction { - private static final String marker = "//"; - @CapturePattern(pattern = "[^\\n]*") - private String comment; - @OptionalToken - private static final String tail = "\n"; - } - - public static class MultilineCommentGrammarDefinition implements Junction { - private static final String marker = "/*"; - @CapturePattern(until="\\*/") - private String comment; - private static final String tail = "*/"; - } - - public static class StatementSeparator implements Junction { - @CapturePattern(pattern = "[\\s\\n]*;[\\s\\n]*") - private String value; - } - - public static class ArrayToken implements Rule { - @CaptureLimit(min=2, max=4) - private Junction[] tokens = new Junction[3]; - } - - // bug in < 0.3.3 - public static class SubRuleGrammar implements Rule { - private static String MARKER = "!"; - private TestGrammarDefinition command; - } - - // bug in < 0.3.4 - public static class TestGrammarWithOptionalLastField implements Rule { - private static final String marker = ":"; - @OptionalToken - @CapturePattern(pattern="[^\\s\\n]+") - private String command; - } - - // bug in < 0.3.4 - public static class OptionalGrammarWrapper implements Rule { - private TestGrammarWithOptionalLastField test; - private static final String space = " "; - } - - // bug in < 0.3.4 - @Test - public void testOptionalFields() throws Exception { - TokenGrammar grammar = TokenGrammar.forClass(OptionalGrammarWrapper.class); - - OptionalGrammarWrapper result = grammar.parse(new StringReader(": ")); - Assert.assertNotNull(result); - Assert.assertNotNull(result.test); - Assert.assertEquals(null, result.test.command); - } - - // bug in < 0.3.3 - @Test - public void testSubRule() throws Exception { - TokenGrammar grammar = TokenGrammar.forClass(SubRuleGrammar.class); - Assert.assertNotNull(grammar); - - SubRuleGrammar result = grammar.parse(new StringReader("!:test")); - Assert.assertNotNull(result); - Assert.assertNotNull(result.command); - Assert.assertEquals("test", result.command.command); - } - - @Test - public void testGrammar() throws Exception { - TokenGrammar grammar = TokenGrammar.forClass(TestGrammarDefinition.class); - Assert.assertNotNull(grammar); - TestGrammarDefinition token = grammar.parse(new StringReader(":test")); - Assert.assertEquals("test", token.command); - } - - @Test - public void testTrailingCharactersException() throws Exception { - TokenGrammar grammar = TokenGrammar.forClass(TestGrammarDefinition.class); - Assert.assertNotNull(grammar); - try { - TestGrammarDefinition result = grammar.parse(new StringReader(":test; :another;")); - Assert.fail(); - } catch (Exception e) { - e.printStackTrace(); - } - } - - @Test - public void testJunction() throws Exception { - TokenGrammar grammar = TokenGrammar.forClass(Junction.class); - Assert.assertNotNull(grammar); - - Junction result = grammar.parse(new StringReader("// comment")); - Assert.assertTrue(result instanceof CommentGrammarDefinition); - CommentGrammarDefinition comment = (CommentGrammarDefinition) result; - Assert.assertEquals(" comment", comment.comment); - } - - @Test - public void testCapture() throws Exception { - TokenGrammar grammar = TokenGrammar.forClass(Junction.class); - Assert.assertNotNull(grammar); - - Junction result = grammar.parse(new StringReader("/* comment */")); - Assert.assertTrue(result instanceof MultilineCommentGrammarDefinition); - } - - @Test - public void testArrayCapture() throws Exception { - TokenGrammar grammar = TokenGrammar.forClass(ArrayToken.class); - String test = ":hello; // comment\n/* multiline\ncomment */"; - - ArrayToken result = grammar.parse(new StringReader(test)); - Assert.assertNotNull(result); - Assert.assertNotNull(result.tokens); - - Junction[] tokens = result.tokens; - Assert.assertEquals(4, tokens.length); - - Junction token = tokens[0]; - Assert.assertEquals(TestGrammarDefinition.class, token.getClass()); - Assert.assertEquals("hello", ((TestGrammarDefinition) token).command); - - token = tokens[1]; - Assert.assertEquals(StatementSeparator.class, token.getClass()); - - token = tokens[2]; - Assert.assertEquals(CommentGrammarDefinition.class, token.getClass()); - Assert.assertEquals(" comment", ((CommentGrammarDefinition)token).comment); - - token = tokens[3]; - Assert.assertEquals(MultilineCommentGrammarDefinition.class, token.getClass()); - Assert.assertEquals(" multiline\ncomment ", ((MultilineCommentGrammarDefinition)token).comment); - } - - @Test - public void testArrayCaptureLimit() throws Exception { - TokenGrammar grammar = TokenGrammar.forClass(ArrayToken.class); - try { - ArrayToken result = grammar.parse(new StringReader(":test")); - Assert.fail("CaptureLimit min is ignored"); - } catch (Exception e) { - // this is expected - } - - try { - grammar.parse(new StringReader(":a;:b;:c;:d;:e;:f;:g")); - Assert.fail("CaptureLimit max is ignored"); - } catch (Exception e) { - // this is expected - } - } -} diff --git a/src/test/java/com/onkiup/linker/parser/token/AbstractTokenTest.java b/src/test/java/com/onkiup/linker/parser/token/AbstractTokenTest.java new file mode 100644 index 0000000..3b7b890 --- /dev/null +++ b/src/test/java/com/onkiup/linker/parser/token/AbstractTokenTest.java @@ -0,0 +1,94 @@ +package com.onkiup.linker.parser.token; + +import static junit.framework.TestCase.assertTrue; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +import java.lang.reflect.Field; +import java.util.Optional; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.onkiup.linker.parser.ParserLocation; + +@PrepareForTest({PartialToken.class, LoggerFactory.class}) +@RunWith(PowerMockRunner.class) +public class AbstractTokenTest { + + @Test + public void testReadFlags() { + PowerMockito.mockStatic(PartialToken.class); + PowerMockito.when(PartialToken.getOptionalCondition(Mockito.any())).thenReturn(Optional.of("lalalalala")); + PowerMockito.when(PartialToken.hasOptionalAnnotation(Mockito.any())).thenReturn(true); + + AbstractToken token = Mockito.mock(AbstractToken.class); + Mockito.when(token.isOptional()).thenCallRealMethod(); + Mockito.when(token.optionalCondition()).thenCallRealMethod(); + Mockito.doCallRealMethod().when(token).readFlags(Mockito.any()); + + token.readFlags(null); + + assertFalse(token.isOptional()); + assertEquals("lalalalala", token.optionalCondition().get()); + + Mockito.when(PartialToken.getOptionalCondition(Mockito.any())).thenReturn(Optional.empty()); + token.readFlags(null); + assertTrue(token.isOptional()); + assertFalse(token.optionalCondition().isPresent()); + + Mockito.when(PartialToken.hasOptionalAnnotation(Mockito.any())).thenReturn(false); + token.readFlags(null); + assertFalse(token.isOptional()); + assertFalse(token.optionalCondition().isPresent()); + } + + @Test + public void testToString() throws Exception { + Field field = AbstractToken.class.getDeclaredField("field"); + AbstractToken token = Mockito.mock(AbstractToken.class); + Mockito.when(token.toString()).thenCallRealMethod(); + + // target field present + Mockito.when(token.targetField()).thenReturn(Optional.of(field)); + assertEquals(AbstractToken.class.getName() + "$" + "field", token.toString()); + // target field not present + Mockito.when(token.targetField()).thenReturn(Optional.empty()); + assertTrue(token.toString().startsWith(AbstractToken.class.getName() + "$MockitoMock$")); + } + + @Test + public void testOnPopulated() { + ParserLocation end = Mockito.mock(ParserLocation.class); + AbstractToken token = Mockito.mock(AbstractToken.class); + Mockito.doCallRealMethod().when(token).onPopulated(Mockito.any()); + Mockito.when(token.end()).thenCallRealMethod(); + + token.onPopulated(end); + assertEquals(end, token.end()); + } + + @Test + public void testLogging() { + Logger logger = Mockito.mock(Logger.class); + PowerMockito.mockStatic(LoggerFactory.class); + Mockito.when(LoggerFactory.getLogger(Mockito.anyString())).thenReturn(logger); + AbstractToken token = Mockito.mock(AbstractToken.class); + Mockito.when(token.logger()).thenReturn(logger); + Mockito.doCallRealMethod().when(token).log(Mockito.any(), Mockito.any()); + Mockito.doCallRealMethod().when(token).error(Mockito.any(), Mockito.any()); + + Object[] vararg = new Object[0]; + token.log("", vararg); + Mockito.verify(logger, Mockito.times(1)).debug("", vararg); + token.error("", null); + Mockito.verify(logger, Mockito.times(1)).error("", (Throwable)null); + } +} + diff --git a/src/test/java/com/onkiup/linker/parser/token/CollectionTokenTest.java b/src/test/java/com/onkiup/linker/parser/token/CollectionTokenTest.java index e3e754c..4dbb5f8 100644 --- a/src/test/java/com/onkiup/linker/parser/token/CollectionTokenTest.java +++ b/src/test/java/com/onkiup/linker/parser/token/CollectionTokenTest.java @@ -1,91 +1,109 @@ package com.onkiup.linker.parser.token; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + import java.lang.reflect.Field; -import org.junit.Assert; -import org.junit.Before; import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mockito; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; import com.onkiup.linker.parser.ParserLocation; import com.onkiup.linker.parser.annotation.CaptureLimit; +import com.onkiup.linker.parser.annotation.CapturePattern; -import static org.junit.Assert.*; - -@RunWith(PowerMockRunner.class) -@PrepareForTest(PartialToken.class) public class CollectionTokenTest { - - private String[] field; - @CaptureLimit(min=2, max=3) - private String[] limit; - - @Before - public void setup() { - PowerMockito.mockStatic(PartialToken.class); + @CapturePattern(".*") + private String[] stringField; + private Field arrayField = CollectionTokenTest.class.getDeclaredField("stringField"); + @CapturePattern(".*") + @CaptureLimit(min=1) + private String[] minLimitArray; + private Field minLimitField = CollectionTokenTest.class.getDeclaredField("minLimitArray"); + @CapturePattern(".*") + @CaptureLimit(max=2) + private String[] maxLimitArray; + private Field maxLimitField = CollectionTokenTest.class.getDeclaredField("maxLimitArray"); + + public CollectionTokenTest() throws NoSuchFieldException { } - @Test - public void testAdvanceAndSource() throws Exception { - Field field = getClass().getDeclaredField("field"); - - PartialToken member = Mockito.mock(PartialToken.class); - Mockito.when(member.isPopulated()).thenReturn(true); - Mockito.when(member.source()).thenReturn(new StringBuilder("123")); - PowerMockito.when(PartialToken.forClass(Mockito.any(), Mockito.any(), Mockito.any())).thenReturn(member); - - CollectionToken token = new CollectionToken(null, field, ParserLocation.ZERO); - - assertSame(member, token.advance(false).orElse(null)); - assertEquals(0, token.getChildren().length); - - member = Mockito.mock(PartialToken.class); - Mockito.when(member.isPopulated()).thenReturn(true); - Mockito.when(member.source()).thenReturn(new StringBuilder("456")); - PowerMockito.when(PartialToken.forClass(Mockito.any(), Mockito.any(), Mockito.any())).thenReturn(member); - - assertSame(member, token.advance(false).orElse(null)); - assertEquals(1, token.getChildren().length); - - member = Mockito.mock(PartialToken.class); - Mockito.when(member.isPopulated()).thenReturn(false); - Mockito.when(member.source()).thenReturn(new StringBuilder("789")); - PowerMockito.when(PartialToken.forClass(Mockito.any(), Mockito.any(), Mockito.any())).thenReturn(member); - - assertSame(member, token.advance(false).orElse(null)); - assertEquals(2, token.getChildren().length); - + public void onChildPopulated() { + CollectionToken token = new CollectionToken(null, arrayField, String[].class, null); + try { + token.onChildPopulated(); + fail(); + } catch (IllegalStateException ise) { + // this is expected + } + token = new CollectionToken<>(null, arrayField, String[].class, null); + PartialToken child = token.nextChild().get(); + token.onChildPopulated(); + PartialToken[] children = token.children(); + assertEquals(1, children.length); + assertSame(child, children[0]); + } - assertFalse(token.advance(false).isPresent()); + @Test + public void onChildFailed() { + CollectionToken token = new CollectionToken<>(null, arrayField, String[].class, null); + try { + token.onChildFailed(); + fail(); + } catch (IllegalStateException ise) { + // this is expected + } + token = new CollectionToken<>(null, arrayField, String[].class, null); + PartialToken child = token.nextChild().get(); + token.onChildFailed(); + PartialToken[] children = token.children(); + assertEquals(0, children.length); + assertFalse(token.isFailed()); assertTrue(token.isPopulated()); - assertEquals(2, token.getChildren().length); - assertEquals("123456", token.source().toString()); + token = new CollectionToken<>(null, minLimitField, String[].class, null); + child = token.nextChild().get(); + token.onChildFailed(); + assertTrue(token.isFailed()); + assertFalse(token.isPopulated()); } @Test - public void testLimits() throws Exception { - PartialToken member = Mockito.mock(PartialToken.class); - Mockito.when(member.isPopulated()).thenReturn(true); - - PowerMockito.when(PartialToken.forClass(Mockito.any(), Mockito.any(), Mockito.any())).thenReturn(member); - Field field = getClass().getDeclaredField("limit"); - - CollectionToken token = new CollectionToken(null, field, ParserLocation.ZERO); + public void source() { + CollectionToken token = new CollectionToken<>(null, maxLimitField, String[].class, null); + TerminalToken child = (TerminalToken)token.nextChild().get(); + ConsumingToken.ConsumptionState.inject(child, new ConsumingToken.ConsumptionState("token1", "token1|")); + child.onConsumeSuccess("token1"); + child.onPopulated(new ParserLocation("", 6, 0, 6)); + token.onChildPopulated(); + child = (TerminalToken)token.nextChild().get(); + ConsumingToken.ConsumptionState.inject(child, new ConsumingToken.ConsumptionState("token2", "token2")); + child.onConsumeSuccess("token2"); + child.onPopulated(new ParserLocation("", 12, 0, 12)); + token.onChildPopulated(); + assertEquals("token1|token2", token.source().toString()); + } - assertSame(member, token.advance(false).orElse(null)); - assertFalse(token.isPopulated()); - assertSame(member, token.advance(false).orElse(null)); - Assert.assertFalse(token.isPopulated()); - assertSame(member, token.advance(false).orElse(null)); - assertTrue(token.isPopulated()); - assertFalse(token.advance(false).isPresent()); + @Test + public void unfilledChildren() { + CollectionToken token = new CollectionToken<>(null, maxLimitField, String[].class, null); + assertEquals(2, token.unfilledChildren()); + token.nextChild(); + assertEquals(2, token.unfilledChildren()); + token.onChildPopulated(); + assertEquals(1, token.unfilledChildren()); + token.nextChild(); + assertEquals(1, token.unfilledChildren()); + token.onChildPopulated(); + assertEquals(0, token.unfilledChildren()); assertTrue(token.isPopulated()); } -} + @Test + public void alternativesLeft() { + // TODO + } +} diff --git a/src/test/java/com/onkiup/linker/parser/token/CompoundTokenTest.java b/src/test/java/com/onkiup/linker/parser/token/CompoundTokenTest.java new file mode 100644 index 0000000..51a588f --- /dev/null +++ b/src/test/java/com/onkiup/linker/parser/token/CompoundTokenTest.java @@ -0,0 +1,120 @@ +package com.onkiup.linker.parser.token; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.LinkedList; +import java.util.Optional; +import java.util.function.Consumer; + +import org.junit.Test; +import org.mockito.Mockito; + +import com.onkiup.linker.parser.Rule; +import com.onkiup.linker.parser.annotation.AdjustPriority; + +public class CompoundTokenTest { + + public static interface CttJunction extends Rule { + + } + + @AdjustPriority(100) + public static class CttConcrete implements CttJunction { + + } + + @Test + public void forClass() { + assertTrue(CompoundToken.forClass(CttJunction.class, null) instanceof VariantToken); + assertTrue(CompoundToken.forClass(CttConcrete.class, null) instanceof RuleToken); + } + + @Test + public void traceback() { + CompoundToken token = Mockito.mock(CompoundToken.class); + CompoundToken compoundChild = Mockito.mock(CompoundToken.class); + PartialToken sourceChild = Mockito.mock(PartialToken.class); + + Mockito.when(token.traceback()).thenCallRealMethod(); + Mockito.when(token.children()).thenReturn(new PartialToken[] {compoundChild, sourceChild}); + Mockito.when(compoundChild.traceback()).thenReturn(Optional.of("/COMPOUND_CHILD/")); + Mockito.when(sourceChild.source()).thenReturn("/SOURCE_CHILD/"); + Mockito.when(compoundChild.alternativesLeft()).thenReturn(1); + + assertEquals("/SOURCE_CHILD//COMPOUND_CHILD/", token.traceback().get().toString()); + Mockito.verify(token, Mockito.times(0)).onFail(); + Mockito.verify(compoundChild, Mockito.times(1)).invalidate(); + Mockito.verify(compoundChild, Mockito.times(0)).onFail(); + Mockito.verify(sourceChild, Mockito.times(1)).invalidate(); + Mockito.verify(sourceChild, Mockito.times(1)).onFail(); + + Mockito.when(token.children()).thenReturn(new PartialToken[] {sourceChild}); + assertEquals("/SOURCE_CHILD/", token.traceback().get().toString()); + Mockito.verify(token, Mockito.times(1)).onFail(); + Mockito.verify(token, Mockito.times(1)).invalidate(); + Mockito.verify(sourceChild, Mockito.times(2)).invalidate(); + Mockito.verify(sourceChild, Mockito.times(2)).onFail(); + } + + @Test + public void alternativesLeft() { + CompoundToken token = Mockito.mock(CompoundToken.class); + PartialToken child1 = Mockito.mock(PartialToken.class); + PartialToken child2 = Mockito.mock(PartialToken.class); + + Mockito.when(token.alternativesLeft()).thenCallRealMethod(); + Mockito.when(token.children()).thenReturn(new PartialToken[]{child1, child2}); + Mockito.when(child1.alternativesLeft()).thenReturn(3); + Mockito.when(child2.alternativesLeft()).thenReturn(5); + + assertEquals(8, token.alternativesLeft()); + } + + @Test + public void basePriority() { + CompoundToken token = Mockito.mock(CompoundToken.class); + PartialToken child = Mockito.mock(PartialToken.class); + + Mockito.when(token.basePriority()).thenCallRealMethod(); + Mockito.when(token.tokenType()).thenReturn(CttConcrete.class); + Mockito.when(token.children()).thenReturn(new PartialToken[]{child}); + Mockito.when(child.basePriority()).thenReturn(900); + Mockito.when(child.propagatePriority()).thenReturn(true); + + assertEquals(1000, token.basePriority()); + } + + @Test + public void source() { + CompoundToken token = Mockito.mock(CompoundToken.class); + CompoundToken compoundChild = Mockito.mock(CompoundToken.class); + PartialToken sourceChild = Mockito.mock(PartialToken.class); + + Mockito.when(token.traceback()).thenCallRealMethod(); + Mockito.when(token.children()).thenReturn(new PartialToken[] {compoundChild, sourceChild}); + Mockito.when(compoundChild.traceback()).thenReturn(Optional.of("/COMPOUND_CHILD/")); + Mockito.when(sourceChild.source()).thenReturn("/SOURCE_CHILD/"); + + assertEquals("/SOURCE_CHILD//COMPOUND_CHILD/", token.traceback().get().toString()); + } + + @Test + public void visit() { + CompoundToken token = Mockito.mock(CompoundToken.class); + PartialToken child1 = Mockito.mock(PartialToken.class); + PartialToken child2 = Mockito.mock(PartialToken.class); + final LinkedList> visited = new LinkedList<>(); + Consumer visitor = visited::add; + + Mockito.doCallRealMethod().when(token).visit(visitor); + Mockito.when(token.children()).thenReturn(new PartialToken[]{child1, child2}); + + token.visit(visitor); + + assertEquals(1, visited.size()); + assertEquals(token, visited.get(0)); + Mockito.verify(child1, Mockito.times(1)).visit(visitor); + Mockito.verify(child2, Mockito.times(1)).visit(visitor); + } +} diff --git a/src/test/java/com/onkiup/linker/parser/token/EnumTokenTest.java b/src/test/java/com/onkiup/linker/parser/token/EnumTokenTest.java new file mode 100644 index 0000000..172d9b4 --- /dev/null +++ b/src/test/java/com/onkiup/linker/parser/token/EnumTokenTest.java @@ -0,0 +1,51 @@ +package com.onkiup.linker.parser.token; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.fail; + +import java.lang.reflect.Field; + +import org.junit.Test; +import org.junit.validator.ValidateWith; +import org.mockito.Mockito; + +import com.onkiup.linker.parser.ParserLocation; +import com.onkiup.linker.parser.Rule; +import com.onkiup.linker.parser.annotation.CapturePattern; + +public class EnumTokenTest { + + public EnumTokenTest() throws NoSuchFieldException { + } + + public static enum EttEnum implements Rule { + ONE, + TWO, + THREE; + } + + public static class EttWrapper { + private EttEnum enumValue; + } + + private Field enumField = EttWrapper.class.getDeclaredField("enumValue"); + + @Test + public void testParsing() { + CompoundToken parent = Mockito.mock(CompoundToken.class); + EnumToken token = new EnumToken(parent, enumField, EttEnum.class, new ParserLocation(null, 0, 0, 0)); + String source = "TWO"; + + for (int i = 0; i < source.length(); i++) { + CharSequence returned = token.consume(source.charAt(i)).orElse(null); + if (returned != null && returned.length() > 0) { + fail("Unexpected buffer return at character#" + i + ": '" + returned); + } + } + + assertEquals(EttEnum.TWO, token.token().get()); + + assertEquals("z", token.consume('z').get()); + } +} diff --git a/src/test/java/com/onkiup/linker/parser/token/PartialTokenTest.java b/src/test/java/com/onkiup/linker/parser/token/PartialTokenTest.java new file mode 100644 index 0000000..eb81109 --- /dev/null +++ b/src/test/java/com/onkiup/linker/parser/token/PartialTokenTest.java @@ -0,0 +1,214 @@ +package com.onkiup.linker.parser.token; + +import static junit.framework.TestCase.assertEquals; +import static junit.framework.TestCase.assertFalse; +import static junit.framework.TestCase.assertSame; +import static junit.framework.TestCase.assertTrue; + +import java.lang.reflect.Field; +import java.util.LinkedList; +import java.util.Optional; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import com.onkiup.linker.parser.ParserLocation; +import com.onkiup.linker.parser.Rule; +import com.onkiup.linker.parser.annotation.AdjustPriority; +import com.onkiup.linker.parser.annotation.OptionalToken; +import com.onkiup.linker.parser.annotation.SkipIfFollowedBy; + +@RunWith(PowerMockRunner.class) +@PrepareForTest(PartialToken.class) +public class PartialTokenTest { + + private Object[] array; + + public PartialTokenTest() throws NoSuchFieldException { + } + + @AdjustPriority(value = 9000, propagate = true) + private static class TestRule implements Rule {} + @SkipIfFollowedBy("test") + private TestRule testRule; + @OptionalToken(whenFollowedBy = "hi") + private Rule rule; + @OptionalToken + private String string; + private enum TestEnum {}; + private TestEnum enumnumnum; + + private Field arrayField = PartialTokenTest.class.getDeclaredField("array"); + private Field testRuleField = PartialTokenTest.class.getDeclaredField("testRule"); + private Field junctionField = PartialTokenTest.class.getDeclaredField("rule"); + private Field stringField = PartialTokenTest.class.getDeclaredField("string"); + private Field enumField = PartialTokenTest.class.getDeclaredField("enumnumnum"); + + @Test + public void forField() throws Exception { + + Class[] constructorArguments = new Class[] { + CompoundToken.class, Field.class, Class.class, ParserLocation.class + }; + + PowerMockito.whenNew(CollectionToken.class).withAnyArguments().thenReturn(Mockito.mock(CollectionToken.class)); + PowerMockito.whenNew(VariantToken.class).withAnyArguments().thenReturn(Mockito.mock(VariantToken.class)); + PowerMockito.whenNew(RuleToken.class).withAnyArguments().thenReturn(Mockito.mock(RuleToken.class)); + PowerMockito.whenNew(TerminalToken.class).withAnyArguments().thenReturn(Mockito.mock(TerminalToken.class)); + PowerMockito.whenNew(EnumToken.class).withAnyArguments().thenReturn(Mockito.mock(EnumToken.class)); + + assertTrue(PartialToken.forField(null, arrayField, Object[].class, null) instanceof CollectionToken); + assertTrue(PartialToken.forField(null, testRuleField, TestRule.class, null) instanceof RuleToken); + assertTrue(PartialToken.forField(null, junctionField, Rule.class, null) instanceof VariantToken); + assertTrue(TerminalToken.class.isAssignableFrom(PartialToken.forField(null, stringField, String.class, null).getClass())); + assertTrue(PartialToken.forField(null, enumField, TestEnum.class, null) instanceof EnumToken); + + } + + @Test + public void getOptionalCondition() { + assertFalse(PartialToken.getOptionalCondition(enumField).isPresent()); + assertFalse(PartialToken.getOptionalCondition(stringField).isPresent()); + assertEquals("test", PartialToken.getOptionalCondition(testRuleField).get()); + assertEquals("hi", PartialToken.getOptionalCondition(junctionField).get()); + } + + @Test + public void isOptional() { + assertTrue(PartialToken.hasOptionalAnnotation(testRuleField)); + assertTrue(PartialToken.hasOptionalAnnotation(junctionField)); + assertTrue(PartialToken.hasOptionalAnnotation(stringField)); + assertFalse(PartialToken.hasOptionalAnnotation(enumField)); + } + + @Test + public void lookahead() { + PartialToken token = Mockito.mock(PartialToken.class); + CompoundToken parent = Mockito.mock(CompoundToken.class); + + Mockito.when(token.parent()).thenReturn(Optional.of(parent)); + Mockito.when(parent.parent()).thenReturn(Optional.empty()); + Mockito.when(token.lookahead(Mockito.any())).thenCallRealMethod(); + Mockito.when(parent.lookahead(Mockito.any())).thenCallRealMethod(); + Mockito.when(token.targetField()).thenReturn(Optional.of(testRuleField)); + Mockito.when(parent.targetField()).thenReturn(Optional.of(junctionField)); + + assertFalse(token.lookahead("m")); + assertTrue(token.lookahead("t")); + assertTrue(token.lookahead("h")); + assertTrue(token.lookahead("test")); + Mockito.verify(parent, Mockito.times(0)).markOptional(); + Mockito.verify(token, Mockito.times(1)).markOptional(); + assertFalse(token.lookahead("hi")); + Mockito.verify(parent, Mockito.times(1)).markOptional(); + assertFalse(token.lookahead("testt")); + Mockito.verify(token, Mockito.times(2)).markOptional(); + assertTrue(token.lookahead("testh")); + Mockito.verify(token, Mockito.times(3)).markOptional(); + Mockito.verify(parent, Mockito.times(1)).markOptional(); + assertFalse(token.lookahead("testhi")); + Mockito.verify(token, Mockito.times(4)).markOptional(); + Mockito.verify(parent, Mockito.times(2)).markOptional(); + } + + @Test + public void findInTree() { + PartialToken token = Mockito.mock(PartialToken.class); + CompoundToken parent = Mockito.mock(CompoundToken.class); + + Mockito.when(token.parent()).thenReturn(Optional.of(parent)); + Mockito.when(parent.parent()).thenReturn(Optional.empty()); + Mockito.when(token.findInTree(Mockito.any())).thenCallRealMethod(); + Mockito.when(parent.findInTree(Mockito.any())).thenCallRealMethod(); + + assertEquals(parent, token.findInTree(parent::equals).get()); + } + + @Test + public void position() { + PartialToken token = Mockito.mock(PartialToken.class); + Mockito.when(token.position()).thenCallRealMethod(); + + assertEquals(0, token.position()); + ParserLocation location = Mockito.mock(ParserLocation.class); + Mockito.when(location.position()).thenReturn(9000); + Mockito.when(token.location()).thenReturn(location); + assertEquals(9000, token.position()); + } + + @Test + public void basePriority() { + PartialToken token = Mockito.mock(PartialToken.class); + Mockito.when(token.tokenType()).thenReturn(TestRule.class); + Mockito.when(token.basePriority()).thenCallRealMethod(); + assertEquals(9000, token.basePriority()); + } + + @Test + public void propagatePriority() { + PartialToken token = Mockito.mock(PartialToken.class); + Mockito.when(token.tokenType()).thenReturn(TestRule.class); + Mockito.when(token.propagatePriority()).thenCallRealMethod(); + assertTrue(token.propagatePriority()); + } + + @Test + public void visit() { + PartialToken token = Mockito.mock(PartialToken.class); + Mockito.doCallRealMethod().when(token).visit(Mockito.any()); + PartialToken[] visited = new PartialToken[1]; + token.visit(t -> visited[0] = t); + assertEquals(token, visited[0]); + } + + @Test + public void alternativesLeft() { + PartialToken token = Mockito.mock(PartialToken.class); + Mockito.when(token.alternativesLeft()).thenCallRealMethod(); + assertEquals(0, token.alternativesLeft()); + } + + @Test + public void root() { + PartialToken parent = Mockito.mock(PartialToken.class); + PartialToken child = Mockito.mock(PartialToken.class); + + Mockito.when(child.parent()).thenReturn(Optional.of(parent)); + Mockito.when(parent.parent()).thenReturn(Optional.empty()); + Mockito.when(child.root()).thenCallRealMethod(); + Mockito.when(parent.root()).thenCallRealMethod(); + + assertEquals(parent, child.root()); + } + + @Test + public void tail() { + PartialToken token = Mockito.mock(PartialToken.class); + Mockito.when(token.tail(Mockito.anyInt())).thenCallRealMethod(); + Mockito.when(token.source()).thenReturn("source"); + + assertEquals("source", token.tail(6)); + assertEquals(" source", token.tail(7)); + assertEquals("rce", token.tail(3)); + } + + @Test + public void path() { + PartialToken token = Mockito.mock(PartialToken.class); + CompoundToken parent = Mockito.mock(CompoundToken.class); + + Mockito.when(token.parent()).thenReturn(Optional.of(parent)); + Mockito.when(parent.parent()).thenReturn(Optional.empty()); + Mockito.when(token.path()).thenCallRealMethod(); + Mockito.when(parent.path()).thenCallRealMethod(); + + LinkedList path = token.path(); + assertEquals(2, path.size()); + assertSame(parent, path.get(0)); + assertSame(token, path.get(1)); + } +} diff --git a/src/test/java/com/onkiup/linker/parser/token/RuleTokenTest.java b/src/test/java/com/onkiup/linker/parser/token/RuleTokenTest.java index a47cb32..7ac4bc4 100644 --- a/src/test/java/com/onkiup/linker/parser/token/RuleTokenTest.java +++ b/src/test/java/com/onkiup/linker/parser/token/RuleTokenTest.java @@ -1,69 +1,7 @@ package com.onkiup.linker.parser.token; -import java.lang.reflect.Field; -import java.util.Optional; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mockito; -import org.mockito.internal.verification.VerificationModeFactory; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.onkiup.linker.parser.ParserLocation; -import com.onkiup.linker.parser.Rule; -import com.onkiup.linker.parser.TokenGrammar; -import com.onkiup.linker.parser.annotation.CaptureLimit; -import com.onkiup.linker.parser.annotation.CapturePattern; -import com.onkiup.linker.parser.annotation.IgnoreCharacters; - import static org.junit.Assert.*; public class RuleTokenTest { - private static final Logger logger = LoggerFactory.getLogger(RuleTokenTest.class); - - public static class TestRule implements Rule { - private String first; - private String second; - } - -// @Test - public void testAdvance() throws Exception { - PartialToken child = Mockito.mock(PartialToken.class); - Mockito.when(child.isPopulated()).thenReturn(true); - Mockito.when(child.source()).thenReturn(new StringBuilder("123")); - PowerMockito.when(PartialToken.forField(Mockito.any(), Mockito.any(), Mockito.any())).thenReturn(child); - RuleToken token = new RuleToken(null, TestRule.class, ParserLocation.ZERO); - - assertSame(child, token.advance(false).get()); - assertSame(child, token.advance(false).get()); - assertFalse(token.advance(false).isPresent()); - PowerMockito.verifyStatic(PartialToken.class, Mockito.times(2)); - PartialToken.forField(Mockito.eq(token), Mockito.any(Field.class), Mockito.any()); - - assertEquals("123123", token.source().toString()); - } - -// @Test - public void testRollback() throws Exception { - PartialToken child = Mockito.mock(PartialToken.class); - Mockito.when(child.isPopulated()).thenReturn(true); - Mockito.when(child.pullback()).thenReturn(Optional.of(new StringBuilder("a"))); - PowerMockito.when(PartialToken.forField(Mockito.any(), Mockito.any(), Mockito.any())).thenReturn(child); - PartialToken parent = Mockito.mock(PartialToken.class); - Mockito.when(parent.pushback(false)).thenReturn(Optional.of(new StringBuilder("aa"))); - - RuleToken token = new RuleToken(parent, TestRule.class, ParserLocation.ZERO); - token.advance(false); - token.advance(false); - Optional rollbacked = token.pushback(false); - StringBuilder result = rollbacked.get(); - assertEquals("aa", result.toString()); - } } diff --git a/src/test/java/com/onkiup/linker/parser/token/TerminalTokenTest.java b/src/test/java/com/onkiup/linker/parser/token/TerminalTokenTest.java deleted file mode 100644 index ad44d36..0000000 --- a/src/test/java/com/onkiup/linker/parser/token/TerminalTokenTest.java +++ /dev/null @@ -1,50 +0,0 @@ -package com.onkiup.linker.parser.token; - -import org.junit.Test; -import org.mockito.Mockito; - -import com.onkiup.linker.parser.TestResult; -import com.onkiup.linker.parser.TokenMatcher; - -import static org.junit.Assert.*; - -public class TerminalTokenTest { - - @Test - public void testConsume() { - TokenMatcher matcher = Mockito.mock(TokenMatcher.class); - Mockito.when(matcher.apply(Mockito.any())).thenAnswer(invocation -> { - StringBuilder buffer = (StringBuilder) invocation.getArgument(0); - if ("test".equals(buffer.toString())) { - return TestResult.match(4, "test"); - } else if ("test".startsWith(buffer.toString())) { - return TestResult.matchContinue(buffer.length(), buffer.toString()); - } - return TestResult.fail(); - }); - - TerminalToken token = new TerminalToken(null, matcher); - assertFalse(token.consume('t', false).isPresent()); - assertFalse(token.consume('e', false).isPresent()); - assertFalse(token.consume('s', false).isPresent()); - assertEquals("", token.consume('t', false).get().toString()); - assertEquals("X", token.consume('X', true).get().toString()); - - assertEquals("test", token.source().toString()); - } - - @Test - public void testIgnoreCharacters() { - TokenMatcher matcher = Mockito.mock(TokenMatcher.class); - Mockito.when(matcher.apply(Mockito.any())).thenReturn(TestResult.match(1, "X")); - PartialToken parent = Mockito.mock(PartialToken.class); - Mockito.when(parent.getIgnoredCharacters()).thenReturn(" "); - - TerminalToken token = new TerminalToken(parent, matcher); - assertFalse(token.consume(' ', false).isPresent()); - assertEquals("", token.consume('X', false).get().toString()); - assertEquals("Y", token.consume('Y', true).get().toString()); - } - -} - diff --git a/src/test/java/com/onkiup/linker/parser/token/UnrotationTest.java b/src/test/java/com/onkiup/linker/parser/token/UnrotationTest.java deleted file mode 100644 index 0e1645e..0000000 --- a/src/test/java/com/onkiup/linker/parser/token/UnrotationTest.java +++ /dev/null @@ -1,100 +0,0 @@ -package com.onkiup.linker.parser.token; - -import java.lang.reflect.Field; -import java.util.Optional; - -import org.junit.Test; - -import com.onkiup.linker.parser.Rule; -import com.onkiup.linker.parser.TokenGrammar; -import com.onkiup.linker.parser.annotation.AdjustPriority; -import com.onkiup.linker.parser.annotation.CaptureLimit; -import com.onkiup.linker.parser.annotation.CapturePattern; -import com.onkiup.linker.parser.annotation.IgnoreCharacters; - -import static org.junit.Assert.*; - -public class UnrotationTest { - - public static class TestRule implements Rule { - private String first; - private String second; - } - - - public static interface UnrotationTestToken extends Rule { - - } - - public static interface UnrotationTestOperator extends Rule { - - } - - public static class UnrotationTestNumber implements UnrotationTestToken { - - @CapturePattern("\\d+") - private String value; - - } - - public static class UnrotationTestPlusOperator implements UnrotationTestOperator { - public static final String MARKER = "+"; - } - - - @AdjustPriority(value=10000, propagate=true) - public static class UnrotationTestStarOperator implements UnrotationTestOperator { - public static final String MARKER = "*"; - } - - @IgnoreCharacters(" ") - public static class UnrotationTestBinaryOperator implements UnrotationTestToken { - private UnrotationTestToken left; - private UnrotationTestOperator operator; - private UnrotationTestToken right; - } - - @Test - public void testUnrotation() throws Exception { - TokenGrammar grammar = TokenGrammar.forClass(UnrotationTestToken.class); - - UnrotationTestToken result = grammar.parse("1 + 2 * 3 + 4"); - assertNotNull(result); - assertTrue(result instanceof UnrotationTestBinaryOperator); - UnrotationTestBinaryOperator operator = (UnrotationTestBinaryOperator) result; - - UnrotationTestToken token = operator.right; - assertNotNull(token); - assertEquals(UnrotationTestNumber.class, token.getClass()); - UnrotationTestNumber number = (UnrotationTestNumber)token; - assertEquals("4", number.value); - - assertTrue(operator.operator instanceof UnrotationTestPlusOperator); - - token = operator.left; - assertTrue(token instanceof UnrotationTestBinaryOperator); - operator = (UnrotationTestBinaryOperator)token; - assertEquals(UnrotationTestPlusOperator.class, operator.operator.getClass()); - - token = operator.left; - assertTrue(token instanceof UnrotationTestNumber); - number = (UnrotationTestNumber)token; - assertEquals("1", number.value); - - token = operator.right; - assertTrue(token instanceof UnrotationTestBinaryOperator); - operator = (UnrotationTestBinaryOperator) token; - - token = operator.left; - assertEquals(UnrotationTestNumber.class, token.getClass()); - number = (UnrotationTestNumber)token; - assertEquals("2", number.value); - - assertEquals(UnrotationTestStarOperator.class, operator.operator.getClass()); - - token = operator.right; - assertEquals(UnrotationTestNumber.class, token.getClass()); - number = (UnrotationTestNumber)token; - assertEquals("3", number.value); - } -} diff --git a/src/test/java/com/onkiup/linker/parser/token/VariantTokenTest.java b/src/test/java/com/onkiup/linker/parser/token/VariantTokenTest.java deleted file mode 100644 index 963b6ad..0000000 --- a/src/test/java/com/onkiup/linker/parser/token/VariantTokenTest.java +++ /dev/null @@ -1,63 +0,0 @@ -package com.onkiup.linker.parser.token; - -import java.util.Optional; - -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mockito; -import org.mockito.internal.verification.VerificationModeFactory; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; - -import com.onkiup.linker.parser.ParserLocation; -import com.onkiup.linker.parser.Rule; - -import static org.junit.Assert.*; - -@RunWith(PowerMockRunner.class) -@PrepareForTest(PartialToken.class) -public class VariantTokenTest { - - public interface Variant extends Rule { - - } - - public static class VariantA implements Variant { - - } - - public static class VariantB implements Variant { - - } - - @Before - public void setup() { - PowerMockito.mockStatic(PartialToken.class); - } - - @Test - public void testAdvance() { - PartialToken target = Mockito.mock(PartialToken.class); - PartialToken child = Mockito.mock(PartialToken.class); - Mockito.when(child.isPopulated()).thenReturn(false); - Mockito.when(child.advance(Mockito.anyBoolean())).thenReturn(Optional.of(target)); - Mockito.when(child.source()).thenReturn(new StringBuilder("123")); - PowerMockito.when(PartialToken.forClass(Mockito.any(), Mockito.any(), Mockito.any())).thenReturn(child); - - VariantToken token = new VariantToken(null, Variant.class, ParserLocation.ZERO); - - assertSame(target, token.advance(false).get()); - assertSame(target, token.advance(false).get()); - Mockito.when(child.isPopulated()).thenReturn(true); - assertTrue(token.isPopulated()); - - PowerMockito.verifyStatic(PartialToken.class, Mockito.times(1)); - PartialToken.forClass(Mockito.eq(token), Mockito.eq(VariantA.class), Mockito.any()); - PartialToken.forClass(Mockito.eq(token), Mockito.eq(VariantB.class), Mockito.any()); - - assertEquals("123", token.source().toString()); - } - -} From e7f0c14fa9eb225aa42372d49ee4dcb034d0c7c8 Mon Sep 17 00:00:00 2001 From: Dima Chechetkin Date: Thu, 3 Oct 2019 09:36:58 -0400 Subject: [PATCH 3/3] 0.8 --- .../com/onkiup/linker/parser/NullMatcher.java | 19 + .../onkiup/linker/parser/ParserLocation.java | 13 + .../java/com/onkiup/linker/parser/Rule.java | 16 +- .../com/onkiup/linker/parser/SyntaxError.java | 22 +- .../onkiup/linker/parser/TokenGrammar.java | 274 ++++++++------- .../onkiup/linker/parser/TokenMatcher.java | 29 +- .../parser/annotation/ContextAware.java | 12 + .../linker/parser/annotation/MetaToken.java | 11 + .../parser/annotation/OptionalToken.java | 1 + .../linker/parser/token/AbstractToken.java | 38 +- .../linker/parser/token/CollectionToken.java | 143 ++++++-- .../linker/parser/token/CompoundToken.java | 70 ++-- .../linker/parser/token/ConsumingToken.java | 147 ++++---- .../onkiup/linker/parser/token/EnumToken.java | 5 + .../linker/parser/token/PartialToken.java | 131 ++++--- .../onkiup/linker/parser/token/RuleToken.java | 124 ++++++- .../linker/parser/token/TerminalToken.java | 3 +- .../linker/parser/token/VariantToken.java | 331 ++++++++++++++---- .../linker/parser/util/LoggerLayout.java | 35 +- .../parser/util/SelfPopulatingBuffer.java | 41 +++ .../onkiup/linker/parser/util/TextUtils.java | 28 ++ 21 files changed, 1078 insertions(+), 415 deletions(-) create mode 100644 src/main/java/com/onkiup/linker/parser/NullMatcher.java create mode 100644 src/main/java/com/onkiup/linker/parser/annotation/ContextAware.java create mode 100644 src/main/java/com/onkiup/linker/parser/annotation/MetaToken.java create mode 100644 src/main/java/com/onkiup/linker/parser/util/SelfPopulatingBuffer.java create mode 100644 src/main/java/com/onkiup/linker/parser/util/TextUtils.java diff --git a/src/main/java/com/onkiup/linker/parser/NullMatcher.java b/src/main/java/com/onkiup/linker/parser/NullMatcher.java new file mode 100644 index 0000000..5acdcd7 --- /dev/null +++ b/src/main/java/com/onkiup/linker/parser/NullMatcher.java @@ -0,0 +1,19 @@ +package com.onkiup.linker.parser; + +public class NullMatcher implements TokenMatcher { + + public NullMatcher() { + + } + + @Override + public TokenTestResult apply(CharSequence buffer) { + return TestResult.match(0, null); + } + + @Override + public String toString() { + return "NullMatcher"; + } +} + diff --git a/src/main/java/com/onkiup/linker/parser/ParserLocation.java b/src/main/java/com/onkiup/linker/parser/ParserLocation.java index a8a4110..88ae361 100644 --- a/src/main/java/com/onkiup/linker/parser/ParserLocation.java +++ b/src/main/java/com/onkiup/linker/parser/ParserLocation.java @@ -93,6 +93,19 @@ public ParserLocation advance(CharSequence source) { return result; } + public ParserLocation advance(char character) { + if (character < 0) { + return this; + } + int column = this.column + 1; + int line = this.line; + if (character == '\n') { + line++; + column = 0; + } + return new ParserLocation(name, position + 1, line, column); + } + public ParserLocation add(ParserLocation another) { if (another.name() != null && !Objects.equals(name(), another.name())) { throw new IllegalArgumentException("Unable to add parser location with a different name"); diff --git a/src/main/java/com/onkiup/linker/parser/Rule.java b/src/main/java/com/onkiup/linker/parser/Rule.java index e6dc93b..b5f179b 100644 --- a/src/main/java/com/onkiup/linker/parser/Rule.java +++ b/src/main/java/com/onkiup/linker/parser/Rule.java @@ -42,7 +42,7 @@ default Optional parent() { .map(meta -> { do { meta = (PartialToken) meta.parent().orElse(null); - } while (meta instanceof VariantToken); + } while (!(meta instanceof RuleToken)); return meta; }) .flatMap(PartialToken::token); @@ -57,12 +57,16 @@ default boolean populated() { .orElse(false); } - default PartialToken metadata() { - return Metadata.metadata(this).orElseThrow(() -> new RuntimeException("Failed to obtain metadata for " + Rule.this)); + default void onPopulated() { + + } + + default Optional metadata() { + return Metadata.metadata(this); } default ParserLocation location() { - return metadata().location(); + return metadata().map(PartialToken::location).orElse(null); } /** @@ -80,5 +84,9 @@ default void reevaluate() { default void invalidate() { } + + default CharSequence source() { + return metadata().map(PartialToken::source).orElse(null); + } } diff --git a/src/main/java/com/onkiup/linker/parser/SyntaxError.java b/src/main/java/com/onkiup/linker/parser/SyntaxError.java index 3786571..211bf33 100644 --- a/src/main/java/com/onkiup/linker/parser/SyntaxError.java +++ b/src/main/java/com/onkiup/linker/parser/SyntaxError.java @@ -11,11 +11,11 @@ public class SyntaxError extends RuntimeException { - private PartialToken expected; - private StringBuilder source; + private PartialToken expected; + private CharSequence source; private String message; - public SyntaxError(String message, PartialToken expected, StringBuilder source) { + public SyntaxError(String message, PartialToken expected, CharSequence source) { this.message = message; this.expected = expected; this.source = source; @@ -29,14 +29,18 @@ public String toString() { .append("\tExpected ") .append(expected) .append(" but got: '") - .append(expected != null && source != null && expected.position() < source.length() ? source.substring(expected.position()) : source) + .append(expected != null && source != null && expected.position() < source.length() ? source.subSequence(expected.position(), source.length()) : source) .append("'\n\tSource:\n\t\t") .append(source) - .append("\n\n\tTraceback:\n\t\t"); - - PartialToken parent = expected; - while (null != (parent = (PartialToken) parent.parent().orElse(null))) { - result.append(parent.toString().replace("\n", "\n\t\t")); + .append("\n\n\tTraceback:\n"); + + if (expected != null) { + expected.path().stream() + .map(PartialToken::toString) + .map(text -> text.replaceAll("\n", "\n\t\t") + '\n') + .forEach(result::append); + } else { + result.append("No traceback provided"); } return result.toString(); diff --git a/src/main/java/com/onkiup/linker/parser/TokenGrammar.java b/src/main/java/com/onkiup/linker/parser/TokenGrammar.java index 19027b1..f83e729 100644 --- a/src/main/java/com/onkiup/linker/parser/TokenGrammar.java +++ b/src/main/java/com/onkiup/linker/parser/TokenGrammar.java @@ -8,6 +8,7 @@ import java.util.Enumeration; import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; import java.util.stream.Collectors; import org.apache.log4j.Appender; @@ -20,6 +21,9 @@ import com.onkiup.linker.parser.token.PartialToken; import com.onkiup.linker.parser.util.LoggerLayout; import com.onkiup.linker.parser.util.ParserError; +import com.onkiup.linker.parser.util.SelfPopulatingBuffer; + +import sun.rmi.runtime.Log; // at 0.2.2: // - replaced all "evaluate" flags with context @@ -27,14 +31,20 @@ public class TokenGrammar { private static final Logger logger = LoggerFactory.getLogger("PARSER LOOP"); private static final ThreadLocal BUFFER = new ThreadLocal<>(); private Class type; + private Class metaType; private String ignoreTrail; public static TokenGrammar forClass(Class type) { - return new TokenGrammar<>(type); + return new TokenGrammar<>(type, null); + } + + public static TokenGrammar forClass(Class type, Class metaType) { + return new TokenGrammar<>(type, metaType); } - protected TokenGrammar(Class type) { + protected TokenGrammar(Class type, Class metaType) { this.type = type; + this.metaType = metaType; } public static boolean isConcrete(Class test) { @@ -83,91 +93,109 @@ public X tokenize(Reader source) throws SyntaxError { } public X tokenize(String sourceName, Reader source) throws SyntaxError { - CompoundToken rootToken = CompoundToken.forClass(type, new ParserLocation(sourceName, 0, 0, 0)); - CompoundToken parent = rootToken; - ConsumingToken consumer = nextConsumingToken(parent).orElseThrow(() -> new ParserError("No possible consuming tokens found", parent)); - StringBuilder buffer = new StringBuilder(); - int line = 0, col = 0; AtomicInteger position = new AtomicInteger(0); + SelfPopulatingBuffer buffer = null; + try { + buffer = new SelfPopulatingBuffer(sourceName, source); + } catch (IOException e) { + throw new RuntimeException("Failed to read source " + sourceName, e); + } try { - setupLoggingLayouts(buffer); + CompoundToken rootToken = CompoundToken.forClass(type, new ParserLocation(sourceName, 0, 0, 0)); + ConsumingToken.ConsumptionState.rootBuffer(rootToken, buffer); + CompoundToken parent = rootToken; + ConsumingToken consumer = nextConsumingToken(parent).orElseThrow(() -> new ParserError("No possible consuming tokens found", parent)); + ConsumingToken bestFail = consumer; + setupLoggingLayouts(buffer, position::get); do { if (logger.isDebugEnabled()) { + System.out.print("\u001B[H\u001Bc"); System.out.println("|----------------------------------------------------------------------------------------"); + System.out.println(consumer.location().toString()); System.out.println("|----------------------------------------------------------------------------------------"); - System.out.println("|----------------------------------------------------------------------------------------"); - Collection path = consumer.path(); - - String trace = path.stream() - .map(Object::toString) - .collect(Collectors.joining("\n|-")); - System.out.println("|-" + trace); + final ConsumingToken currentConsumer = consumer; + System.out.print(rootToken.dumpTree(token -> { + StringBuilder result = new StringBuilder(); + if (token == currentConsumer) { + result.append(">>> "); + } + return result + .append(token.getClass().getSimpleName()) + .append("(").append(token.position()).append(" - ").append(token.end().position()).append(")") + .append(" :: '") + .append(LoggerLayout.sanitize(token.head(50))) + .append("'"); + })); System.out.println("|----------------------------------------------------------------------------------------"); System.out.println("|----------------------------------------------------------------------------------------"); } ConsumingToken lastConsumer = consumer; - try { - boolean hitEnd = processConsumingToken(source, buffer, consumer); - if (hitEnd) { - logger.debug("Hit end while processing {}", consumer.tag()); - if (!consumer.isPopulated() && !consumer.isFailed()) { - consumer.onFail(); - } - } - - if (consumer.isFailed()) { - logger.debug("!!! CONSUMER FAILED !!! {}", consumer.tag()); - //if (!consumer.isOptional()) { - consumer = processTraceback(consumer, buffer).orElse(null); - //} - } else if (consumer.isPopulated()) { - logger.debug("consumer populated: {}", consumer.tag()); - consumer = onPopulated(consumer) - .flatMap(TokenGrammar::nextConsumingToken) - .orElse(null); - } + processConsumingToken(consumer, position); + boolean hitEnd = position.get() >= buffer.length(); + + if (consumer.isFailed()) { + logger.debug("!!! CONSUMER FAILED !!! {}", consumer.tag()); + bestFail = bestFail.position() > consumer.position() ? bestFail : consumer; + consumer = processTraceback(consumer).orElse(null); + } else if (consumer.isPopulated()) { + logger.debug("consumer populated: {}", consumer.tag()); + consumer = onPopulated(consumer, hitEnd).orElse(null); + } else if (hitEnd) { + logger.debug("Hit end while processing {}", consumer.tag()); + consumer.atEnd(); + consumer = nextConsumingToken(consumer).orElse(null); + } - if (consumer == lastConsumer) { - consumer = nextConsumingToken(consumer).orElse(null); - } + if (consumer != null) { + position.set(consumer.end().position()); + } - if (consumer == null || buffer.length() == 0) { - if (rootToken.isPopulated()) { - if (!hitEnd) { - consumer = processEarlyPopulation(rootToken, source, buffer).orElse(null); + if (consumer == null || hitEnd) { + logger.debug("attempting to recover; consumer == {}, buffer.length() == {}", consumer == null ? null : consumer.tag(), buffer.length()); + if (rootToken.isPopulated()) { + if (!hitEnd) { + if (!validateTrailingCharacters(buffer, position.get())) { + consumer = processEarlyPopulation(rootToken, buffer, position.get()).orElseThrow( + () -> new ParserError("Failed to recover from early population", lastConsumer)); + logger.debug("Recovered to {}", consumer.tag()); } else { - logger.debug("Perfectly parsed into: {}", rootToken.tag()); - return rootToken.token().get(); - } - } else if (consumer != null) { - logger.debug("Hit end and root token is not populated -- trying to traceback..."); - do { - consumer.onFail(); - consumer = processTraceback(consumer, buffer).orElse(null); - } while (buffer.length() == 0 && consumer != null); - - if (buffer.length() != 0 && rootToken.isPopulated()) { - consumer = processEarlyPopulation(rootToken, source, buffer).orElse(null); - } else if (rootToken.isPopulated()) { - return rootToken.token().get(); + logger.debug("Successfully parsed (with valid trailing characters '{}') into: {}", buffer.subSequence(position.get(), buffer.length()), rootToken.tag()); + return rootToken.token().orElse(null); } + } else { + logger.debug("Perfectly parsed into: {}", rootToken.tag()); + return rootToken.token().get(); } + } else if (consumer != null) { + logger.debug("Hit end and root token is not populated -- trying to traceback..."); + do { + consumer.onFail(); + consumer = processTraceback(consumer).orElse(null); + } while (buffer.length() == 0 && consumer != null); + + if (consumer != null && rootToken.isPopulated()) { + consumer = processEarlyPopulation(rootToken, buffer, position.get()).orElseThrow(() -> + new ParserError("Failed to recover from null consumer", lastConsumer)); + logger.debug("Recovered to {}", consumer.tag()); + } else if (rootToken.isPopulated()) { + return rootToken.token().get(); + } + } else { + throw new SyntaxError("Advanced up to this token and then failed", bestFail, buffer); } - } catch (IOException ioe) { - throw new RuntimeException("Failed to read source data", ioe); } - } while(consumer != null && buffer.length() > 0); + } while(consumer != null && position.get() < buffer.length()); if (rootToken.isPopulated()) { return rootToken.token().orElse(null); } - throw new SyntaxError("Unexpected end of input", rootToken, buffer); + throw new SyntaxError("Unexpected end of input", consumer, buffer); } catch (SyntaxError se) { - throw new RuntimeException("Syntax error at line " + line + ", column " + col, se); + throw new RuntimeException("Syntax error at position " + position.get(), se); } catch (Exception e) { throw new RuntimeException(e); } finally { @@ -175,48 +203,57 @@ public X tokenize(String sourceName, Reader source) throws SyntaxError { } } - private Optional> processEarlyPopulation(CompoundToken rootToken, Reader source, StringBuilder buffer) throws IOException { - if (validateTrailingCharacters(source, buffer)) { + private Optional> processEarlyPopulation(CompoundToken rootToken, CharSequence buffer, int position) { + logger.debug("Early population detected..."); + if (validateTrailingCharacters(buffer, position)) { logger.debug("Successfully parsed (with valid trailing characters '{}') into: {}", buffer, rootToken.tag()); - buffer.delete(0, buffer.length()); return Optional.empty(); } else if (rootToken.rotatable()) { + logger.debug("Rotating root token"); rootToken.rotate(); return nextConsumingToken(rootToken); - } else if (rootToken.alternativesLeft() > 0) { - logger.info("Root token populated too early, failing it..."); - rootToken.traceback().ifPresent(returned -> { - logger.debug("Returned by root token: '{}'", LoggerLayout.sanitize(returned)); - buffer.insert(0, returned); - }); - rootToken.onFail(); + } else if (rootToken.alternativesLeft()) { + logger.info("Root token populated too early, failing it... (Buffer left: '{}'", LoggerLayout.sanitize(buffer.subSequence(position, buffer.length()))); + rootToken.traceback(); return nextConsumingToken(rootToken); } else { return Optional.empty(); } } - private static Optional> onPopulated(PartialToken child) { + private static Optional> onPopulated(PartialToken child, boolean hitEnd) { return child.parent().flatMap(parent -> { parent.onChildPopulated(); + if (hitEnd) { + parent.atEnd(); + } if (parent.isPopulated()) { - return onPopulated(parent); + return onPopulated(parent, hitEnd); + } else if (parent.isFailed()) { + return processTraceback(parent); } - return Optional.of(parent); + return nextConsumingToken(parent); }); } - private static Optional> processTraceback(PartialToken child, StringBuilder buffer) { + private static Optional> processTraceback(PartialToken child) { return child.parent().flatMap(parent -> { if (child.isFailed()) { logger.debug("^^^--- TRACEBACK: {} <- {}", parent.tag(), child.tag()); parent.onChildFailed(); if (parent.isFailed() || parent.isPopulated()) { - return processTraceback(parent, buffer); + if (parent.isPopulated()) { + return onPopulated(parent, false); + } + return processTraceback(parent); } - parent.traceback().ifPresent(returned -> buffer.insert(0, returned.toString())); - return nextConsumingToken(parent); + if (!child.isOptional()) { + parent.traceback(); + } else { + child.traceback(); + } + return firstUnfilledParent(parent).flatMap(TokenGrammar::nextConsumingToken); } else { logger.debug("|||--- TRACEBACK: (self) <- {}", child.tag()); return firstUnfilledParent(child).flatMap(TokenGrammar::nextConsumingToken); @@ -226,7 +263,7 @@ private static Optional> processTraceback(PartialToken chil private static Optional> firstUnfilledParent(PartialToken child) { logger.debug("traversing back to first unfilled parent from {}", child.tag()); - if (child instanceof CompoundToken && ((CompoundToken)child).unfilledChildren() > 0) { + if (child instanceof CompoundToken && !child.isFailed() && ((CompoundToken)child).unfilledChildren() > 0) { logger.debug("<<<--- NEXT UNFILLED: (self) <--- {}", child.tag()); return Optional.of((CompoundToken)child); } @@ -256,18 +293,30 @@ private static Optional> firstUnfilledParent(PartialToken ch public static Optional> nextConsumingToken(CompoundToken from) { while (from != null) { PartialToken child = from.nextChild().orElse(null); + logger.debug("Searching for next consumer in child {}", child == null ? null : child.tag()); if (child instanceof ConsumingToken) { logger.debug("--->>> NEXT CONSUMER: {} ---> {}", from.tag(), child.tag()); return Optional.of((ConsumingToken)child); } else if (child instanceof CompoundToken) { - from = (CompoundToken)child; + logger.debug("--->>> searching for next consumer in {} --> {}", from.tag(), child.tag()); + from = (CompoundToken)child; } else if (child == null) { - from = from.parent().orElse(null); + CompoundToken parent = from.parent().orElse(null); + logger.debug("^^^--- searching for next consumer in parent {} <--- {}", parent == null ? null : parent.tag(), from.tag()); + if (from.isFailed()) { + logger.debug("notifying parent about child failure"); + return processTraceback(from); + } else if (from.isPopulated()) { + logger.debug("notifying parent about child population"); + return onPopulated(from, false); + } else { + throw new ParserError("next child == null but from is neither failed or populated", from); + } } else { throw new RuntimeException("Unknown child type: " + child.getClass()); } } - logger.debug("---XXX NEXT UNFILLED: {} ---> XXX (not found)", from); + logger.debug("---XXX NEXT CONSUMER: {} ---> XXX (not found)", from == null ? null : from.tag()); return Optional.empty(); } @@ -275,60 +324,33 @@ private static Optional> nextConsumingToken(ConsumingToken return from.parent().flatMap(TokenGrammar::nextConsumingToken); } - private boolean processConsumingToken(Reader source, StringBuilder buffer, ConsumingToken token) throws IOException { - boolean accepted = true; - while (accepted) { - if (populateBuffer(source, buffer)) { - char character = buffer.charAt(0); - accepted = token.consume(buffer.charAt(0)).map(CharSequence::toString).map(returned -> { - logger.debug("------ RETURN: '{}' +++ {} = '{}'", LoggerLayout.sanitize(String.valueOf(character)), token.tag(), LoggerLayout.sanitize(returned)); - buffer.replace(0, 1, returned); - return false; - }).orElseGet(() -> { - logger.debug("---+++ CONSUME: '{}' +++ {}", LoggerLayout.sanitize(String.valueOf(character)), token.tag()); - buffer.delete(0, 1); - return true; - }); - } else { - return true; - } + private void processConsumingToken(ConsumingToken token, AtomicInteger position) { + while (token.consume()) { + //position.incrementAndGet(); } - return !populateBuffer(source, buffer); + position.set(token.end().position()); } - private boolean populateBuffer(Reader source, StringBuilder buffer) throws IOException { - if (buffer.length() == 0) { - int character = source.read(); - if (character < 0) { - return false; - } else { - buffer.append((char)character); - } + private boolean validateTrailingCharacters(CharSequence buffer, int from) { + logger.debug("Validating trailing characters with pattern '{}' on '{}'", LoggerLayout.sanitize(ignoreTrail), LoggerLayout.sanitize(buffer.subSequence(from, buffer.length()))); + if (from >= buffer.length()) { + logger.debug("no trailing chars!"); + return true; } - return true; - } - - private boolean validateTrailingCharacters(Reader source, StringBuilder buffer) throws IOException { - boolean hitEnd = false; + char character; do { - populateBuffer(source, buffer); - if (buffer.length() > 0) { - char test = buffer.charAt(0); - if (ignoreTrail != null && ignoreTrail.indexOf(test) < 0) { - return false; - } - buffer.delete(0, 1); - populateBuffer(source, buffer); - } - } while (buffer.length() > 0); - return true; + character = buffer.charAt(from++); + } while (buffer.length() > from && ignoreTrail != null && ignoreTrail.indexOf(character) > -1); + boolean result = from >= buffer.length(); + logger.debug("Only valid trailing chars left? {}; from == {}; buffer.length == {}", result, from, buffer.length()); + return result; } - private void setupLoggingLayouts(StringBuilder buffer) { + private void setupLoggingLayouts(CharSequence buffer, Supplier position) { Enumeration appenders = org.apache.log4j.Logger.getRootLogger().getAllAppenders(); while(appenders.hasMoreElements()) { Appender appender = appenders.nextElement(); - LoggerLayout loggerLayout = new LoggerLayout(appender.getLayout(), buffer); + LoggerLayout loggerLayout = new LoggerLayout(appender.getLayout(), buffer, position); appender.setLayout(loggerLayout); } } diff --git a/src/main/java/com/onkiup/linker/parser/TokenMatcher.java b/src/main/java/com/onkiup/linker/parser/TokenMatcher.java index 547269d..7b95deb 100644 --- a/src/main/java/com/onkiup/linker/parser/TokenMatcher.java +++ b/src/main/java/com/onkiup/linker/parser/TokenMatcher.java @@ -5,16 +5,19 @@ import java.util.function.Function; import com.onkiup.linker.parser.annotation.CapturePattern; +import com.onkiup.linker.parser.annotation.ContextAware; +import com.onkiup.linker.parser.token.CompoundToken; +import com.onkiup.linker.parser.util.LoggerLayout; @FunctionalInterface public interface TokenMatcher extends Function { - public static TokenMatcher forField(Field field) { + public static TokenMatcher forField(CompoundToken parent, Field field) { Class type = field.getType(); - return forField(field, type); + return forField(parent, field, type); } - public static TokenMatcher forField(Field field, Class type) { + public static TokenMatcher forField(CompoundToken parent, Field field, Class type) { if (type.isArray()) { throw new IllegalArgumentException("Array fields should be handled as ArrayTokens"); } else if (Rule.class.isAssignableFrom(type)) { @@ -35,6 +38,26 @@ public static TokenMatcher forField(Field field, Class type) { } else if (field.isAnnotationPresent(CapturePattern.class)) { CapturePattern pattern = field.getAnnotation(CapturePattern.class); return new PatternMatcher(pattern); + } else if (field.isAnnotationPresent(ContextAware.class)) { + ContextAware contextAware = field.getAnnotation(ContextAware.class); + if (contextAware.matchField().length() > 0) { + Object token = parent.token().orElseThrow(() -> new IllegalStateException("Parent token is null")); + Field dependency = field.getDeclaringClass().getDeclaredField(contextAware.matchField()); + dependency.setAccessible(true); + Object fieldValue = dependency.get(token); + if (fieldValue instanceof String) { + parent.log("Creating context-aware matcher for field $" + field.getName() + " to be equal to '" + + LoggerLayout.sanitize(fieldValue) + "' value of target field $" + dependency.getName()); + return new TerminalMatcher((String)fieldValue); + } else if (fieldValue == null) { + parent.log("Creating context-aware null matcher for field $" + field.getName() + " to be equal to null value of target field $" + dependency.getName()); + return new NullMatcher(); + } else { + throw new IllegalArgumentException("Unable to create field matcher for target field value of type '" + fieldValue.getClass().getName() + "'"); + } + } else { + throw new IllegalArgumentException("Misconfigured ContextAware annotation?"); + } } else { throw new IllegalArgumentException("Non-static String fields MUST have CapturePattern annotation"); } diff --git a/src/main/java/com/onkiup/linker/parser/annotation/ContextAware.java b/src/main/java/com/onkiup/linker/parser/annotation/ContextAware.java new file mode 100644 index 0000000..a8a0d6e --- /dev/null +++ b/src/main/java/com/onkiup/linker/parser/annotation/ContextAware.java @@ -0,0 +1,12 @@ +package com.onkiup.linker.parser.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.FIELD) +public @interface ContextAware { + String matchField() default ""; +} diff --git a/src/main/java/com/onkiup/linker/parser/annotation/MetaToken.java b/src/main/java/com/onkiup/linker/parser/annotation/MetaToken.java new file mode 100644 index 0000000..6efe9b4 --- /dev/null +++ b/src/main/java/com/onkiup/linker/parser/annotation/MetaToken.java @@ -0,0 +1,11 @@ +package com.onkiup.linker.parser.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +public @interface MetaToken { +} diff --git a/src/main/java/com/onkiup/linker/parser/annotation/OptionalToken.java b/src/main/java/com/onkiup/linker/parser/annotation/OptionalToken.java index 6b9f6c8..24537f7 100644 --- a/src/main/java/com/onkiup/linker/parser/annotation/OptionalToken.java +++ b/src/main/java/com/onkiup/linker/parser/annotation/OptionalToken.java @@ -10,5 +10,6 @@ public @interface OptionalToken { String whenFollowedBy() default ""; String whenFieldIsNull() default ""; + String whenFieldNotNull() default ""; } diff --git a/src/main/java/com/onkiup/linker/parser/token/AbstractToken.java b/src/main/java/com/onkiup/linker/parser/token/AbstractToken.java index c50ecc8..2d33c97 100644 --- a/src/main/java/com/onkiup/linker/parser/token/AbstractToken.java +++ b/src/main/java/com/onkiup/linker/parser/token/AbstractToken.java @@ -1,6 +1,7 @@ package com.onkiup.linker.parser.token; import java.lang.reflect.Field; +import java.util.LinkedList; import java.util.Optional; import org.slf4j.Logger; @@ -16,6 +17,7 @@ public abstract class AbstractToken implements PartialToken { private boolean optional, populated, failed; private CharSequence optionalCondition; private Logger logger; + private LinkedList metatokens = new LinkedList(); public AbstractToken(CompoundToken parent, Field targetField, ParserLocation location) { this.parent = parent; @@ -41,8 +43,10 @@ public boolean isPopulated() { return populated; } - protected void dropPopulated() { + @Override + public void dropPopulated() { populated = false; + log("Dropped population flag"); } @Override @@ -55,6 +59,10 @@ public ParserLocation location() { return location; } + protected void location(ParserLocation location) { + this.location = location; + } + @Override public ParserLocation end() { return this.end == null ? this.location : this.end; @@ -72,8 +80,9 @@ public Optional targetField() { @Override public void onPopulated(ParserLocation end) { - log("populated"); + log("populated up to {}", end.position()); populated = true; + failed = false; this.end = end; } @@ -88,14 +97,23 @@ public Logger logger() { @Override public String tag() { return targetField() - .map(field -> field.getDeclaringClass().getName() + "$" + field.getName()) + .map(field -> field.getDeclaringClass().getName() + "$" + field.getName() + "(" + position() + ")") .orElseGet(super::toString); } @Override public String toString() { + ParserLocation location = location(); return targetField() - .map(field -> String.format("%-50.50s || %s (position: %d)", tail(50), field.getDeclaringClass().getName() + "$" + field.getName(), position())) + .map(field -> String.format( + "%50.50s || %s (%d:%d -- %d - %d)", + head(50), + field.getDeclaringClass().getName() + "$" + field.getName(), + location.line(), + location.column(), + location.position(), + end().position() + )) .orElseGet(super::toString); } @@ -107,11 +125,23 @@ protected void readFlags(Field field) { @Override public void onFail() { failed = true; + populated = false; + end = location; PartialToken.super.onFail(); } public Optional optionalCondition() { return Optional.ofNullable(optionalCondition); } + + @Override + public void addMetaToken(Object metatoken) { + metatokens.add(metatoken); + } + + @Override + public LinkedList metaTokens() { + return metatokens; + } } diff --git a/src/main/java/com/onkiup/linker/parser/token/CollectionToken.java b/src/main/java/com/onkiup/linker/parser/token/CollectionToken.java index 33111c4..6a5ec9d 100644 --- a/src/main/java/com/onkiup/linker/parser/token/CollectionToken.java +++ b/src/main/java/com/onkiup/linker/parser/token/CollectionToken.java @@ -5,17 +5,21 @@ import java.util.Arrays; import java.util.LinkedList; import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; import com.onkiup.linker.parser.ParserLocation; import com.onkiup.linker.parser.annotation.CaptureLimit; +import com.onkiup.linker.parser.util.LoggerLayout; +import com.onkiup.linker.parser.util.ParserError; public class CollectionToken extends AbstractToken implements CompoundToken { private Class fieldType; private Class memberType; private LinkedList children = new LinkedList<>(); - private PartialToken current; private CaptureLimit captureLimit; private ParserLocation lastTokenEnd; + private int nextMember = 0; public CollectionToken(CompoundToken parent, Field field, Class tokenType, ParserLocation location) { super(parent, field, location); @@ -29,29 +33,51 @@ public CollectionToken(CompoundToken parent, Field field, Class tokenType, Pa @Override public void onChildPopulated() { - if (current == null) { + if (children.size() == 0) { throw new RuntimeException("OnChildPopulated called when there is no child!"); } - children.add(current); - current = null; + PartialToken current = children.peekLast(); + if (current.isMetaToken()) { + addMetaToken(current.token()); + children.pollLast(); + return; + } + log("Populated collection token #{}: {}", children.size(), current.tag()); + lastTokenEnd = current.end(); if (captureLimit != null && children.size() >= captureLimit.max()) { - onPopulated(current.end()); + onPopulated(lastTokenEnd); + } + } + + @Override + public void atEnd() { + log("Force-populating..."); + if (captureLimit == null || children.size() >= captureLimit.min()) { + onPopulated(lastTokenEnd); + } else { + onFail(); } } @Override public void onChildFailed() { - if (current == null) { - throw new IllegalStateException("No child is currently populated yet onChildFailed was called"); + if (children.size() == 0) { + throw new ParserError("No child is currently populated yet onChildFailed was called", this); } + children.pollLast(); + lastTokenEnd = children.size() > 0 ? children.peekLast().end() : location(); int size = children.size(); if (captureLimit != null && size < captureLimit.min()) { log("Child failed and collection is underpopulated -- failing the whole collection"); - onFail(); + if (!alternativesLeft()) { + onFail(); + } else { + log("Not failing -- have some alternatives left"); + } } else { log("Child failed and collection has enough elements (or no lower limit) -- marking collection as populated"); - onPopulated(lastTokenEnd); + onPopulated(children.size() == 0 ? location() : lastTokenEnd); } } @@ -78,46 +104,54 @@ private static final M[] newArray(Class memberType, int size) { @Override public String tag() { - return fieldType.getName() + "[]"; + return fieldType.getName() + "[]("+position()+")"; } @Override public String toString() { - return String.format("%-50.50s || %s[%d] (position: %d)", tail(50), fieldType.getName(), children.size(), position()); + ParserLocation location = location(); + return String.format( + "%50.50s || %s[%d] (%d:%d -- %d - %d)", + head(50), + fieldType.getName(), + children.size(), + location.line(), + location.column(), + location.position(), + end().position() + ); } @Override - public CharSequence source() { - StringBuilder result = new StringBuilder(); - for (int i = 0; i < children.size(); i++) { - result.append(children.get(i).source()); - } - if (current != null) { - result.append(current.source()); - } - return result; + public ParserLocation end() { + return isFailed() ? location() : children.size() > 0 ? children.peekLast().end() : lastTokenEnd; } @Override public Optional> nextChild() { + if (isFailed() || isPopulated()) { + return Optional.empty(); + } + + PartialToken current = null; if (captureLimit == null || captureLimit.max() > children.size()) { - return Optional.of(current = PartialToken.forField(this, targetField().orElse(null), memberType, lastTokenEnd)); + if (nextMember == children.size()) { + log("creating partial token for member#{}", children.size()); + current = PartialToken.forField(this, targetField().orElse(null), memberType, lastTokenEnd); + children.add(current); + } else if (nextMember < children.size()) { + current = children.get(nextMember); + } + nextMember++; + log("nextChild = [{}]{}", children.size(), current.tag()); + return Optional.of(current); } return Optional.empty(); } @Override public PartialToken[] children() { - PartialToken[] result; - if (current != null) { - result = new PartialToken[children.size() + 1]; - result = children.toArray(result); - result[result.length - 1] = current; - } else { - result = new PartialToken[children.size()]; - result = children.toArray(result); - } - return result; + return children.toArray(new PartialToken[children.size()]); } @Override @@ -139,18 +173,55 @@ public int currentChild() { @Override public void nextChild(int newIndex) { - children = new LinkedList<>(children.subList(0, newIndex)); - current = children.peekLast(); + nextMember = newIndex; + log("next child set to {}/{} ({})", newIndex, children.size(), children.get(newIndex)); } @Override public void children(PartialToken[] children) { this.children = new LinkedList<>(Arrays.asList(children)); - current = null; } @Override - public int alternativesLeft() { - return current == null ? 0 : current.alternativesLeft(); + public boolean alternativesLeft() { + for (int i = children.size() - 1; i > -1; i--) { + PartialToken child = children.get(i); + log("getting alternatives from [{}]{}", i, child.tag()); + if (child.alternativesLeft()) { + log("found alternatives at [{}]{}", i, child.tag()); + return true; + } + } + + return false; + } + + @Override + public CharSequence dumpTree(int offset, CharSequence prefix, CharSequence childPrefix, Function, CharSequence> formatter) { + final int childOffset = offset + 1; + String insideFormat = "%s ├─%s #%s : %s"; + String lastFormat = "%s └─%s #%s : %s"; + StringBuilder result = new StringBuilder(super.dumpTree(offset, prefix, childPrefix, formatter)); + if (!isPopulated()) { + int last = children.size() - 1; + for (int i = 0; i < children.size(); i++) { + PartialToken child = children.get(i); + String format = i == last ? lastFormat : insideFormat; + if (child == null) { + result.append(String.format(format, childPrefix, "[N]", i, null)); + result.append('\n'); + } else if (child.isPopulated()) { + result.append(child.dumpTree(childOffset, String.format(format, childPrefix, "[+]", i, ""), + childPrefix + " │", formatter)); + } else if (child.isFailed()) { + result.append(child.dumpTree(childOffset, String.format(format, childPrefix, "[F]", i, ""), + childPrefix + " │", formatter)); + } else { + result.append(child.dumpTree(childOffset, String.format(format, childPrefix, ">>>", i, ""), + childPrefix + " │", formatter)); + } + } + } + return result; } } diff --git a/src/main/java/com/onkiup/linker/parser/token/CompoundToken.java b/src/main/java/com/onkiup/linker/parser/token/CompoundToken.java index 63cb795..6c5a759 100644 --- a/src/main/java/com/onkiup/linker/parser/token/CompoundToken.java +++ b/src/main/java/com/onkiup/linker/parser/token/CompoundToken.java @@ -4,6 +4,8 @@ import java.util.Objects; import java.util.Optional; import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Collectors; import com.onkiup.linker.parser.ParserLocation; import com.onkiup.linker.parser.Rule; @@ -27,6 +29,13 @@ static CompoundToken forClass(Class type, ParserLocation positio void onChildFailed(); int unfilledChildren(); + default boolean hasUnfilledChildren() { + return unfilledChildren() > 0; + } + + default boolean onlyOneUnfilledChildLeft() { + return unfilledChildren() == 1; + } int currentChild(); void nextChild(int newIndex); @@ -37,40 +46,36 @@ static CompoundToken forClass(Class type, ParserLocation positio PartialToken[] children(); /** - * @param children an array of PartialToken objects to replace current token's children with + * @param children an array of PartialToken objects to replace current token's children with */ void children(PartialToken[] children); Optional> nextChild(); - + /** * Walks through token's children in reverse order removing them until the first child with alternativesLeft() > 0 * If no such child found, then returns full token source * @return source for removed tokens */ - default Optional traceback() { + default void traceback() { + log("!!! TRACING BACK"); PartialToken[] children = children(); if (children.length == 0) { invalidate(); onFail(); - return Optional.empty(); + return; } - StringBuilder result = new StringBuilder(); int newSize = 0; for (int i = children.length - 1; i > -1; i--) { PartialToken child = children[i]; if (child == null) { continue; } - if (child instanceof CompoundToken) { - ((CompoundToken)child).traceback().ifPresent(returned -> result.insert(0, returned.toString())); - } else { - result.insert(0, child.source().toString()); - } - child.invalidate(); + child.traceback(); - if (child.alternativesLeft() > 0) { + if (!child.isFailed()) { + log("found alternatives at child#{}", i); newSize = i + 1; break; } @@ -83,31 +88,30 @@ default Optional traceback() { System.arraycopy(children, 0, newChildren, 0, newSize); children(newChildren); nextChild(newSize - 1); - log("Traced back to child #{}: {}", newSize, newChildren[newSize-1]); + dropPopulated(); + log("Traced back to child #{}: {}", newSize - 1, newChildren[newSize-1].tag()); } else { - invalidate(); onFail(); } - - return Optional.of(result); - } - - /** - * @return String containing all characters to ignore for this token - */ - default String ignoredCharacters() { - return ""; } /** * @return number of alternatives for this token, including its children */ @Override - default int alternativesLeft() { - return Arrays.stream(children()) - .filter(Objects::nonNull) - .mapToInt(PartialToken::alternativesLeft) - .sum(); + default boolean alternativesLeft() { + PartialToken[] children = children(); + for (int i = 0; i < children.length; i++) { + PartialToken child = children[i]; + if (child != null) { + log("getting alternatives from child#{} {}", i, child.tag()); + if (child.alternativesLeft()) { + log("child#{} {} reported that it has alternatives", i, child.tag()); + return true; + } + } + } + return false; } @Override @@ -133,16 +137,6 @@ default boolean rotatable() { default void unrotate() { } - @Override - default CharSequence source() { - StringBuilder result = new StringBuilder(); - Arrays.stream(children()) - .filter(Objects::nonNull) - .map(PartialToken::source) - .forEach(result::append); - return result; - } - @Override default void visit(Consumer> visitor) { Arrays.stream(children()) diff --git a/src/main/java/com/onkiup/linker/parser/token/ConsumingToken.java b/src/main/java/com/onkiup/linker/parser/token/ConsumingToken.java index d36deae..99f9c4e 100644 --- a/src/main/java/com/onkiup/linker/parser/token/ConsumingToken.java +++ b/src/main/java/com/onkiup/linker/parser/token/ConsumingToken.java @@ -4,67 +4,61 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; +import javax.swing.text.html.parser.Parser; + +import org.apache.log4j.Layout; + import com.onkiup.linker.parser.ParserLocation; +import com.onkiup.linker.parser.Rule; import com.onkiup.linker.parser.TestResult; import com.onkiup.linker.parser.TokenMatcher; import com.onkiup.linker.parser.TokenTestResult; +import com.onkiup.linker.parser.util.LoggerLayout; import com.onkiup.linker.parser.util.ParserError; public interface ConsumingToken extends PartialToken { default void setTokenMatcher(TokenMatcher matcher) { - CharSequence ignoredCharacters = parent().map(CompoundToken::ignoredCharacters).orElse(null); - ConsumptionState.create(this, ignoredCharacters, matcher); + ConsumptionState.create(this, matcher); } void onConsumeSuccess(Object token); /** * Attempts to consume next character - * @return null if character was consumed, otherwise returns a CharSequence with failed characters + * @return true if consumption should continue */ - default Optional consume(char character) { + default boolean consume() { ConsumptionState consumption = ConsumptionState.of(this).orElseThrow(() -> new ParserError("No consumption state found (call ConsumingToken::setTokenMatcher to create it first)", this)); - consumption.consume(character); - - if (consumption.failed()) { - if (!lookahead(consumption.buffer())) { - log("Lokahead complete"); - onFail(); - CharSequence consumed = consumption.consumed(); - consumption.clear(); - return Optional.of(consumed); - } - log("performing lookahead so, reporting successfull consumption (despite that the token has already failed)"); - return Optional.empty(); - } + boolean doNext = consumption.consume(); TokenTestResult result = consumption.test(); if (result.isFailed()) { log("failed; switching to lookahead mode"); consumption.setFailed(); - return Optional.empty(); + consumption.lookahead(); + consumption.clear(); + onFail(); + return false; } else if (result.isMatch()) { - int tokenLength = result.getTokenLength(); - CharSequence excess = consumption.trim(tokenLength); - log("matched"); + consumption.trim(result.getTokenLength()); + log("matched at position {}", consumption.end().position()); onConsumeSuccess(result.getToken()); - onPopulated(location().add(consumption.end())); - parent() - .filter(p -> p.unfilledChildren() == 0) - .map(p -> p.lookahead(excess)); - return Optional.of(excess); + onPopulated(consumption.end()); + return false; } if (result.isMatchContinue()) { log("matched; continuing..."); onConsumeSuccess(result.getToken()); onPopulated(consumption.end()); + } else if (consumption.hitEnd()) { + onFail(); } - return Optional.empty(); + return doNext; } @Override @@ -74,28 +68,20 @@ default void invalidate() { } @Override - default Optional traceback() { - return ConsumptionState.of(this).map(ConsumptionState::consumed) - .filter(consumed -> consumed.length() > 0); - } - - @Override - default CharSequence source() { - if (isFailed()) { - return ""; - } - return ConsumptionState.of(this).map(ConsumptionState::consumed).orElse(""); + default void atEnd() { + parent().ifPresent(CompoundToken::atEnd); } class ConsumptionState { private static final ConcurrentHashMap states = new ConcurrentHashMap<>(); + private static final ConcurrentHashMap buffers = new ConcurrentHashMap<>(); private static synchronized Optional of(ConsumingToken token) { return Optional.ofNullable(states.get(token)); } - private static void create(ConsumingToken token, CharSequence ignoredCharacters, Function tester) { - states.put(token, new ConsumptionState(ignoredCharacters, tester)); + private static void create(ConsumingToken token, Function tester) { + states.put(token, new ConsumptionState(token, tester)); } static void inject(ConsumingToken token, ConsumptionState state) { @@ -106,53 +92,81 @@ private static void discard(ConsumingToken token) { states.remove(token); } - private final StringBuilder buffer = new StringBuilder(); - private final StringBuilder consumed = new StringBuilder(); - private final CharSequence ignoredCharacters; + private final String ignoredCharacters; private final Function tester; - private ParserLocation end; + private ParserLocation start, end, ignored; private boolean failed; + private ConsumingToken token; + private CharSequence buffer; + private boolean hitEnd = false; - private ConsumptionState(CharSequence ignoredCharacters, Function tester) { - this.ignoredCharacters = ignoredCharacters; + private ConsumptionState(ConsumingToken token, Function tester) { + this.token = token; + this.ignoredCharacters = token.ignoredCharacters(); this.tester = tester; + this.start = this.end = this.ignored = token.location(); + this.buffer = rootBuffer(token.root()).orElseThrow(() -> + new RuntimeException("No root buffer registered for token " + token)); } - ConsumptionState(CharSequence buffer, CharSequence consumed) { + ConsumptionState(ParserLocation start, ParserLocation ignored, ParserLocation end) { this.ignoredCharacters = ""; this.tester = null; - this.buffer.append(buffer); - this.consumed.append(consumed); + this.start = start; + this.end = end; + this.ignored = ignored; + } + + public static void rootBuffer(PartialToken rootToken, CharSequence buffer) { + buffers.put(rootToken, buffer); + } + + public static Optional rootBuffer(PartialToken root) { + return Optional.ofNullable(buffers.get(root)); } protected CharSequence buffer() { - return buffer.subSequence(0, buffer.length()); + return buffer.subSequence(ignored.position(), end.position()); } protected CharSequence consumed() { - return consumed.subSequence(0, consumed.length()); + return buffer.subSequence(start.position(), end.position()); } protected ParserLocation end() { - return ParserLocation.endOf(consumed()); + return end; } private boolean ignored(int character) { return ignoredCharacters != null && ignoredCharacters.chars().anyMatch(ignored -> ignored == character); } - private void consume(char character) { - consumed.append(character); - if (buffer.length() > 0 || !ignored(character)) { - buffer.append(character); + private boolean consume() { + if (end.position() < buffer.length()) { + char consumed = buffer.charAt(end.position()); + end = end.advance(consumed); + if (end.position() - ignored.position() < 2 && ignored(consumed)) { + ignored = ignored.advance(consumed); + token.log("Ignored '{}' ({} - {} - {})", LoggerLayout.sanitize(consumed), start.position(), ignored.position(), end.position()); + return true; + } + token.log("Consumed '{}' ({} - {} - {})", LoggerLayout.sanitize(consumed), start.position(), ignored.position(), end.position()); + return true; + } else { + hitEnd = true; } + return false; + } + + private boolean hitEnd() { + return hitEnd; } private TokenTestResult test() { - if (buffer.length() == 0) { + if (end.position() - ignored.position() == 0) { return TestResult.continueNoMatch(); } - return tester.apply(buffer); + return tester.apply(buffer()); } private void setFailed() { @@ -163,17 +177,20 @@ private boolean failed() { return failed; } - private CharSequence trim(int size) { - CharSequence trimmed = buffer.subSequence(size, buffer.length()); - buffer.delete(size, buffer.length()); - consumed.delete(consumed.length() - trimmed.length(), consumed.length()); - return trimmed; + private void trim(int size) { + end = ignored.advance(buffer().subSequence(0, size)); } private void clear() { - consumed.delete(0, consumed.length()); - buffer.delete(0, buffer.length()); + end = ignored = start; + } + + private void lookahead() { + token.lookahead(buffer, ignored.position()); + token.log("Lookahead complete"); + token.onFail(); } + } } diff --git a/src/main/java/com/onkiup/linker/parser/token/EnumToken.java b/src/main/java/com/onkiup/linker/parser/token/EnumToken.java index 92d68ea..e038478 100644 --- a/src/main/java/com/onkiup/linker/parser/token/EnumToken.java +++ b/src/main/java/com/onkiup/linker/parser/token/EnumToken.java @@ -78,6 +78,11 @@ public Class tokenType() { return enumType; } + @Override + public void atEnd() { + + } + @Override public void onConsumeSuccess(Object value) { token = (X) value; diff --git a/src/main/java/com/onkiup/linker/parser/token/PartialToken.java b/src/main/java/com/onkiup/linker/parser/token/PartialToken.java index 6873724..b695992 100644 --- a/src/main/java/com/onkiup/linker/parser/token/PartialToken.java +++ b/src/main/java/com/onkiup/linker/parser/token/PartialToken.java @@ -1,11 +1,12 @@ package com.onkiup.linker.parser.token; import java.lang.reflect.Field; -import java.util.Arrays; import java.util.LinkedList; +import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.function.Consumer; +import java.util.function.Function; import java.util.function.Predicate; import org.slf4j.Logger; @@ -14,10 +15,12 @@ import com.onkiup.linker.parser.Rule; import com.onkiup.linker.parser.TokenGrammar; import com.onkiup.linker.parser.annotation.AdjustPriority; +import com.onkiup.linker.parser.annotation.MetaToken; import com.onkiup.linker.parser.annotation.OptionalToken; import com.onkiup.linker.parser.annotation.SkipIfFollowedBy; import com.onkiup.linker.parser.util.LoggerLayout; import com.onkiup.linker.parser.util.ParserError; +import com.onkiup.linker.parser.util.TextUtils; public interface PartialToken { @@ -75,15 +78,22 @@ static boolean hasOptionalAnnotation(Field field) { static boolean isOptional(CompoundToken owner, Field field) { try { if (field.isAnnotationPresent(OptionalToken.class)) { + owner.log("Performing context-aware optionality check for field ${}", field); OptionalToken optionalToken = field.getAnnotation(OptionalToken.class); + boolean result; if (optionalToken.whenFieldIsNull().length() != 0) { final String fieldName = optionalToken.whenFieldIsNull(); - Field targetField = owner.tokenType().getField(fieldName); - targetField.setAccessible(true); - return targetField.get(owner.token()) == null; + result = testContextField(owner, fieldName, Objects::isNull); + owner.log("whenFieldIsNull({}) == {}", fieldName, result); + } else if (optionalToken.whenFieldNotNull().length() != 0) { + final String fieldName = optionalToken.whenFieldNotNull(); + result = testContextField(owner, fieldName, Objects::nonNull); + owner.log("whenFieldNotNull({}) == {}", fieldName, result); + } else { + result = optionalToken.whenFollowedBy().length() == 0; + owner.log("No context-aware conditions found; isOptional = {}", result); } - - return optionalToken.whenFollowedBy().length() != 0; + return result; } return false; @@ -92,6 +102,14 @@ static boolean isOptional(CompoundToken owner, Field field) { } } + static boolean testContextField(CompoundToken owner, String fieldName, Predicate tester) + throws NoSuchFieldException, IllegalAccessException { + Field targetField = owner.tokenType().getField(fieldName); + targetField.setAccessible(true); + boolean result = tester.test(targetField.get(owner.token())); + return result; + } + /** * @return Java representation of populated token */ @@ -104,6 +122,9 @@ static boolean isOptional(CompoundToken owner, Field field) { * The result of this method should always be calculated */ boolean isPopulated(); + + void dropPopulated(); + boolean isFailed(); boolean isOptional(); @@ -116,13 +137,28 @@ static boolean isOptional(CompoundToken owner, Field field) { void markOptional(); void onPopulated(ParserLocation end); String tag(); + void atEnd(); - Optional traceback(); + default void traceback() { + onFail(); + } + + List metaTokens(); + void addMetaToken(Object metatoken); + + default boolean isMetaToken() { + return tokenType().isAnnotationPresent(MetaToken.class); + } /** * @return all characters consumed by the token and its children */ - CharSequence source(); + default CharSequence source() { + PartialToken root = root(); + return ConsumingToken.ConsumptionState.rootBuffer(root) + .map(buffer -> buffer.subSequence(position(), end().position())) + .orElse("?!"); + } Logger logger(); @@ -147,39 +183,27 @@ default void onFail() { * Called on failed tokens * @return true if the token should continue consumption, false otherwise */ - default boolean lookahead(CharSequence buffer) { - log("performing lookahead"); - return targetField() + default void lookahead(CharSequence source, int from) { + log("performing lookahead at position {}", from); + targetField() .flatMap(PartialToken::getOptionalCondition) - .map(condition -> { - log("Loookahead '{}' on '{}'", condition, buffer); - final CharSequence[] parentBuffer = new CharSequence[] { buffer }; - boolean myResult = true; - if (!isOptional()) { - if (buffer.length() >= condition.length()) { - CharSequence test = buffer.subSequence(0, condition.length()); - if (Objects.equals(test, condition)) { - log("Optional condition match: '{}' == '{}'", condition, test); - parentBuffer[0] = buffer.subSequence(condition.length(), buffer.length()); - markOptional(); - } - myResult = false; - } else if (!condition.subSequence(0, buffer.length()).equals(buffer)) { - parentBuffer[0] = buffer; - myResult = false; - } - } else { - parentBuffer[0] = buffer.subSequence(condition.length(), buffer.length()); + .ifPresent(condition -> { + int start = TextUtils.firstNonIgnoredCharacter(this, source, from); + CharSequence buffer = source.subSequence(start, start + condition.length()); + log("Loookahead '{}' on '{}'", LoggerLayout.sanitize(condition), LoggerLayout.sanitize(buffer)); + if (!isOptional() && Objects.equals(condition, buffer)) { + log("Optional condition match: '{}' == '{}'", LoggerLayout.sanitize(condition), LoggerLayout.sanitize(buffer)); + markOptional(); } - - return myResult || isOptional() && parent() - .filter(p -> p.unfilledChildren() == 1) - .filter(p -> p.lookahead(parentBuffer[0])) - .isPresent(); - }).orElseGet(() -> targetField() - .flatMap(field -> parent().map(parent -> isOptional(parent, field))) - .orElse(false) - ); + }); + + parent() + .filter(CompoundToken::onlyOneUnfilledChildLeft) + .filter(p -> p != this) + .ifPresent(p -> { + log("Delegating lookahead to parent {}", p.tag()); + p.lookahead(source, from); + }); } default Optional> findInTree(Predicate comparator) { @@ -233,8 +257,15 @@ default void visit(Consumer> visitor) { visitor.accept(this); } - default int alternativesLeft() { - return 0; + /** + * @return String containing all characters to ignore for this token + */ + default String ignoredCharacters() { + return parent().map(CompoundToken::ignoredCharacters).orElse(""); + } + + default boolean alternativesLeft() { + return false; } default PartialToken root() { @@ -252,12 +283,28 @@ default CharSequence tail(int length) { return LoggerLayout.ralign(LoggerLayout.sanitize(source().toString()), length); } - default LinkedList path() { + default CharSequence head(int length) { + return LoggerLayout.head(LoggerLayout.sanitize(source()), 50); + } + + default LinkedList> path() { LinkedList path = parent() .map(PartialToken::path) .orElseGet(LinkedList::new); path.add(this); return path; } + + default CharSequence dumpTree() { + return dumpTree(PartialToken::tag); + } + + default CharSequence dumpTree(Function, CharSequence> formatter) { + return dumpTree(0, "", "", formatter); + } + + default CharSequence dumpTree(int offset, CharSequence prefix, CharSequence childPrefix, Function, CharSequence> formatter) { + return String.format("%s%s\n", prefix, formatter.apply(this)); + } } diff --git a/src/main/java/com/onkiup/linker/parser/token/RuleToken.java b/src/main/java/com/onkiup/linker/parser/token/RuleToken.java index cb8b266..66df0e7 100644 --- a/src/main/java/com/onkiup/linker/parser/token/RuleToken.java +++ b/src/main/java/com/onkiup/linker/parser/token/RuleToken.java @@ -7,6 +7,7 @@ import java.lang.reflect.Modifier; import java.util.Arrays; import java.util.Optional; +import java.util.function.Function; import com.onkiup.linker.parser.ParserLocation; import com.onkiup.linker.parser.Rule; @@ -30,6 +31,7 @@ public RuleToken(CompoundToken parent, Field field, Class type, ParserLocatio try { this.token = type.newInstance(); + Rule.Metadata.metadata(token, this); } catch (Exception e) { throw new IllegalArgumentException("Failed to instantiate rule token " + type, e); } @@ -73,10 +75,7 @@ public void sortPriorities() { @Override public Optional token() { - return Optional.ofNullable(token).map(token -> { - Rule.Metadata.metadata(token, this); - return token; - }); + return Optional.ofNullable(token); } @Override @@ -92,20 +91,31 @@ public String ignoredCharacters() { @Override public Optional> nextChild() { if (nextChild >= fields.length) { + log("No next child (nextChild = {}; fields = {})", nextChild, fields.length); return Optional.empty(); } if (values[nextChild] == null || values[nextChild].isFailed() || values[nextChild].isPopulated()) { Field childField = fields[nextChild]; - PartialToken result = PartialToken.forField(this, childField, lastTokenEnd); - return Optional.of(values[nextChild++] = result); - } else { - return Optional.of(values[nextChild++]); + log("Creating partial token for child#{} at position {}", nextChild, lastTokenEnd.position()); + values[nextChild] = PartialToken.forField(this, childField, lastTokenEnd); } + log("nextChild#{} = {}", nextChild, values[nextChild].tag()); + return Optional.of(values[nextChild++]); } @Override public void onChildPopulated() { PartialToken child = values[nextChild - 1]; + + if (child.isMetaToken()) { + // woopsie... + addMetaToken(child.token()); + /* TODO: handle metatokens properly r + values[--nextChild] = null; + return; + */ + } + Field field = fields[nextChild - 1]; set(field, child.token().orElse(null)); lastTokenEnd = child.end(); @@ -123,10 +133,15 @@ public void onChildFailed() { PartialToken child = values[nextChild - 1]; if (child.isOptional()) { if (nextChild >= fields.length) { + log("Optional last child failed -- marking as populated"); onPopulated(lastTokenEnd); + } else { + log ("Ignoring optional child failure"); } - } else if (alternativesLeft() == 0) { + } else if (!alternativesLeft()) { onFail(); + } else { + log("not failing -- alternatives left"); } } @@ -193,12 +208,64 @@ protected T convert(Class into, Object what) { @Override public String tag() { - return tokenType.getName(); + return tokenType.getName() + "(" + position() + ")"; + } + + @Override + public void atEnd() { + log("Trying to force-populate..."); + for (int i = Math.max(0, nextChild - 1); i < fields.length; i++) { + if (!PartialToken.isOptional(this, fields[i])) { + if (values[i] == null || !values[i].isPopulated()) { + onFail(); + return; + } + } + } + onPopulated(lastTokenEnd); + } + + @Override + public void onPopulated(ParserLocation end) { + super.onPopulated(end); + try { + token.onPopulated(); + } catch (Throwable e) { + error("Failed to reevaluate on population", e); + } + } + + @Override + public void onFail() { + super.onFail(); + try { + token.reevaluate(); + } catch (Throwable e) { + error("Failed to reevaluate on failure", e); + } } @Override public String toString() { - return String.format("%s || %s (positoin: %d)", tail(50), tokenType.getName() + (nextChild > - 1 ? "$" + (fields[nextChild - 1].getName()) : ""), position()); + ParserLocation location = location(); + return String.format( + "%50.50s || %s (%d:%d -- %d - %d)", + head(50), + tokenType.getName() + (nextChild > 0 ? ">>" + (fields[nextChild - 1].getName()) : ""), + location.line(), + location.column(), + location.position(), + end().position() + ); + } + + @Override + public ParserLocation end() { + return isFailed() ? + location() : + nextChild > 0 &&values[nextChild - 1] != null ? + values[nextChild -1].end() : + lastTokenEnd; } @Override @@ -296,6 +363,7 @@ public int currentChild() { @Override public void nextChild(int newIndex) { + log("next child set to {}/{}", newIndex, fields.length - 1); nextChild = newIndex; } @@ -325,15 +393,37 @@ public void invalidate() { } @Override - public StringBuilder source() { - StringBuilder result = new StringBuilder(); - for (int i = 0; i < values.length; i++) { - if (values[i] != null) { - result.append(values[i].source()); + public CharSequence dumpTree(int offset, CharSequence prefix, CharSequence childPrefix, Function, CharSequence> formatter) { + final int childOffset = offset + 1; + String insideFormat = "%s ├─%s %s : %s"; + String lastFormat = "%s └─%s %s : %s"; + + StringBuilder result = new StringBuilder(super.dumpTree(offset, prefix, childPrefix, formatter)); + if (!isPopulated()) { + for (int i = 0; i <= nextChild; i++) { + if (i < fields.length) { + boolean nextToLast = i == fields.length - 2 || i == nextChild - 1; + boolean last = i == fields.length - 1 || i == nextChild || (nextToLast && values[i + 1] == null); + String format = i == nextChild ? lastFormat : insideFormat; + PartialToken child = values[i]; + String fieldName = fields[i].getName(); + if (child == null) { + result.append(String.format(format, childPrefix, "[N]", fieldName, null)); + result.append('\n'); + } else if (child.isFailed()) { + result.append(child.dumpTree(childOffset, String.format(format, childPrefix, "[F]", fieldName, ""), + childPrefix + " │", formatter)); + } else if (child.isPopulated()) { + result.append(child.dumpTree(childOffset, String.format(format, childPrefix, "[+]", fieldName, ""), + childPrefix + " │", formatter)); + } else { + result.append(child.dumpTree(childOffset, String.format(format, childPrefix, ">>>", fieldName, ""), + childPrefix + " │", formatter)); + } + } } } return result; } - } diff --git a/src/main/java/com/onkiup/linker/parser/token/TerminalToken.java b/src/main/java/com/onkiup/linker/parser/token/TerminalToken.java index 9392a58..8bd711a 100644 --- a/src/main/java/com/onkiup/linker/parser/token/TerminalToken.java +++ b/src/main/java/com/onkiup/linker/parser/token/TerminalToken.java @@ -23,7 +23,7 @@ public class TerminalToken extends AbstractToken implements ConsumingTok public TerminalToken(CompoundToken parent, Field field, Class tokenType, ParserLocation location) { super(parent, field, location); - this.matcher = TokenMatcher.forField(field, tokenType); + this.matcher = TokenMatcher.forField(parent, field, tokenType); this.setTokenMatcher(matcher); } @@ -43,6 +43,5 @@ public Optional token() { public Class tokenType() { return String.class; } - } diff --git a/src/main/java/com/onkiup/linker/parser/token/VariantToken.java b/src/main/java/com/onkiup/linker/parser/token/VariantToken.java index 71c6247..1cfe36e 100644 --- a/src/main/java/com/onkiup/linker/parser/token/VariantToken.java +++ b/src/main/java/com/onkiup/linker/parser/token/VariantToken.java @@ -1,15 +1,18 @@ package com.onkiup.linker.parser.token; import java.lang.reflect.Field; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; +import java.util.stream.Collectors; import org.reflections.Reflections; import org.reflections.scanners.SubTypesScanner; import org.reflections.util.ClasspathHelper; import org.reflections.util.ConfigurationBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import com.onkiup.linker.parser.ParserLocation; import com.onkiup.linker.parser.Rule; @@ -18,10 +21,12 @@ import com.onkiup.linker.parser.annotation.Alternatives; import com.onkiup.linker.parser.annotation.IgnoreCharacters; import com.onkiup.linker.parser.annotation.IgnoreVariant; -import com.onkiup.linker.parser.util.LoggerLayout; +import com.onkiup.linker.parser.util.ParserError; public class VariantToken extends AbstractToken implements CompoundToken { + private static boolean excludeMatchingParents = true; + private static final Reflections reflections = new Reflections(new ConfigurationBuilder() .setUrls(ClasspathHelper.forClassLoader(TokenGrammar.class.getClassLoader())) .setScanners(new SubTypesScanner(true)) @@ -29,11 +34,14 @@ public class VariantToken extends AbstractToken implements Co private static final ConcurrentHashMap dynPriorities = new ConcurrentHashMap<>(); + private static final ConcurrentHashMap>> tags = new ConcurrentHashMap<>(); + private Class tokenType; private Class[] variants; + private PartialToken[] values; private int nextVariant = 0; - private PartialToken token; private String ignoreCharacters = ""; + private List> tried = new LinkedList<>(); public VariantToken(CompoundToken parent, Field field, Class tokenType, ParserLocation location) { super(parent, field, location); @@ -43,37 +51,34 @@ public VariantToken(CompoundToken parent, Field field, Class tokenType, Parse throw new IllegalArgumentException("Variant token cannot handle concrete type " + tokenType); } - if (findInTree(token -> token != this && token.tokenType() == tokenType && token.position() == position()).isPresent()) { - log("---||| Not descending: already selecting same Variant sub-type at this location"); - variants = new Class[0]; - onFail(); - return; - } - if (tokenType.isAnnotationPresent(Alternatives.class)) { variants = tokenType.getAnnotation(Alternatives.class).value(); } else { final ConcurrentHashMap typePriorities = new ConcurrentHashMap<>(); - variants = reflections.getSubTypesOf(tokenType).stream() + variants = (reflections.getSubTypesOf(tokenType).stream() + .filter(TokenGrammar::isConcrete) .filter(type -> { if (type.isAnnotationPresent(IgnoreVariant.class)) { log("Ignoring variant {} -- marked with @IgnoreVariant", type.getSimpleName()); return false; } - boolean inTree = findInTree(token -> token != null && token.tokenType() == type && token.location().position() == location.position()).isPresent(); - if (inTree) { - log("Ignoring variant {} -- already in tree with same position ({})", type.getSimpleName(), location.position()); + if (isLeftRecursive(type)) { + log("Ignoring variant {} -- left recursive", type.getSimpleName()); return false; } - Class superClass = type.getSuperclass(); - if (superClass != tokenType) { - Class[] interfaces = type.getInterfaces(); - for (Class iface : interfaces) { - if (iface == tokenType) { - return true; - } + if (excludeMatchingParents) { + boolean inTree = findInTree(token -> token != null && token.tokenType() == type && + token.location().position() == location.position()).isPresent(); + if (inTree) { + log("Ignoring variant {} -- already in tree with same position ({})", type.getSimpleName(), + location.position()); + return false; } - log("Ignoring " + type + " (extends: " + superClass + ") "); + } + + Boolean tagged = getTag(type).orElse(null); + if (tagged != null && !tagged) { + log("Ignoring " + type + " (tagged as failed for this position)"); return false; } return true; @@ -92,8 +97,9 @@ public VariantToken(CompoundToken parent, Field field, Class tokenType, Parse } return result; }) - .toArray(Class[]::new); + .toArray(Class[]::new)); } + values = new PartialToken[variants.length]; if (parent != null) { ignoreCharacters = parent.ignoredCharacters(); @@ -104,17 +110,50 @@ public VariantToken(CompoundToken parent, Field field, Class tokenType, Parse } } + private boolean isLeftRecursive(Class target) { + return parent().map(p -> p.tokenType() == target && p.position() == position()).orElse(false); + } + + private boolean willLeftRecurse(Class target) { + Field[] fields = target.getDeclaredFields(); + return fields.length > 0 && tokenType.isAssignableFrom(fields[0].getType()); + } + @Override public Optional> nextChild() { if (nextVariant >= variants.length) { + log("Unable to return next child: variants exhausted (nextVariant = {}, variants total = {})", nextVariant, variants.length); + onFail(); return Optional.empty(); } - return Optional.of(token = PartialToken.forField(this, targetField().orElse(null), variants[nextVariant++], location())); + Boolean tag = getTag(variants[nextVariant]).orElse(null); + while (Boolean.FALSE.equals(tag) && ++nextVariant < variants.length) { + log("Skipping variant {} -- tagged as failed for position {}", variants[nextVariant], position()); + tag = getTag(variants[nextVariant]).orElse(null); + } + + if (nextVariant >= variants.length) { + onFail(); + return Optional.empty(); + } + + if (values[nextVariant] == null || values[nextVariant].isFailed() || values[nextVariant].isPopulated()) { + log("Creating partial token for nextChild#{}", nextVariant); + updateDynPriority(variants[nextVariant], 10); + tried.add(variants[nextVariant]); + values[nextVariant] = PartialToken.forField(this, targetField().orElse(null), variants[nextVariant], location()); + } + + log("nextChild#{} = {}", nextVariant, values[nextVariant].tag()); + return Optional.of(values[nextVariant++]); } @Override public PartialToken[] children() { - return new PartialToken[] {token}; + if (nextVariant >= values.length) { + return new PartialToken[0]; + } + return new PartialToken[] {values[currentChild()]}; } @Override @@ -124,13 +163,65 @@ public void children(PartialToken[] children) { @Override public void onChildPopulated() { - updateDynPriority(variants[nextVariant - 1], -10); - onPopulated(token.end()); + int current = currentChild(); + updateDynPriority(variants[current], -20); + if (values[current] == null) { + throw new ParserError("No current token but onChildToken was called...", this); + } + if (TokenGrammar.isConcrete(variants[current])) { + storeTag(values[current],true); + } + if (values[current].isMetaToken()) { + log("Metatoken detected"); + addMetaToken(values[current].token()); + location(values[current].end()); + values[current] = null; + nextVariant = 0; + return; + } + onPopulated(values[current].end()); + } + + private void storeTag(PartialToken token, boolean result) { + PartialToken root = root(); + int position = token.position(); + Class ofType = token.tokenType(); + if (!tags.containsKey(root)) { + tags.put(root, new ConcurrentHashMap<>()); + } + ConcurrentHashMap> myTags = tags.get(root); + if (!myTags.containsKey(position)) { + myTags.put(position, new ConcurrentHashMap<>()); + } + if (result || !myTags.get(position).containsKey(ofType)) { + myTags.get(position).put(ofType, result); + log("Tagged position {} as {} with type {}", position, result ? "compatible" : "incompatible", ofType.getName()); + } + } + + private Optional getTag(Class forType) { + log("Searching for tags on {}", forType.getName()); + return getTags().map(tags -> tags.get(forType)); + } + + private Optional> getTags() { + int position = location().position(); + PartialToken root = root(); + log("Searching tags for position {}", position); + if (!tags.containsKey(root)) { + log("Did not find tags for root token"); + return Optional.empty(); + } + return Optional.ofNullable(tags.get(root).get(position)); } @Override public void onChildFailed() { - updateDynPriority(variants[nextVariant - 1], 10); + int current = currentChild(); + updateDynPriority(variants[current], 30); + if (TokenGrammar.isConcrete(variants[current])) { + storeTag(values[current],false); + } if (nextVariant >= variants.length) { onFail(); } else { @@ -140,7 +231,11 @@ public void onChildFailed() { @Override public Optional token() { - return (Optional) token.token(); + int current = currentChild(); + if (values[current] == null) { + return Optional.empty(); + } + return (Optional) values[current].token(); } @Override @@ -149,19 +244,41 @@ public Class tokenType() { } @Override - public Optional traceback() { - Optional result = null; - if (token == null) { - return Optional.empty(); - } else if (token instanceof CompoundToken) { - result = ((CompoundToken)token).traceback(); - dropPopulated(); - } else { - result = Optional.of(token.source()); - dropPopulated(); + public void onFail() { + log("Tried: {}", tried.stream().map(Class::getSimpleName).collect(Collectors.joining(", "))); + super.onFail(); + } + + @Override + public void traceback() { + log("!!! TRACING BACK"); + if (variants.length == 0) { + onFail(); + return; } - token = null; - return result; + int current = currentChild(); + for (int i = currentChild(); i > -1; i--) { + PartialToken token = values[i]; + if (token != null) { + token.traceback(); + dropPopulated(); + if (token.alternativesLeft()){ + nextVariant = i; + break; + } + } + nextVariant = i; + } + + if (nextVariant == 0) { + nextVariant = current + 1; + } + + if (nextVariant >= variants.length) { + onFail(); + return; + } + log("Traced back fro variant#{} to variant#{}: {}", current, nextVariant, values[nextVariant]); } private int calculatePriority(Class type) { @@ -174,10 +291,14 @@ private int calculatePriority(Class type) { result += 1000; } + if (willLeftRecurse(type)) { + result += 99999; + } + if (type.isAnnotationPresent(AdjustPriority.class)) { AdjustPriority adjust = type.getAnnotation(AdjustPriority.class); result += adjust.value(); - log("Adjusted priority by " + adjust.value()); + log("Adjusted priority by " + adjust.value()); } log(type.getSimpleName() + " priority " + result); @@ -187,22 +308,76 @@ private int calculatePriority(Class type) { @Override public String tag() { - return "? extends " + tokenType.getName(); + return "? extends " + tokenType.getName() + "(" + position() + ")"; + } + + @Override + public ParserLocation end() { + int current = currentChild(); + return isFailed() || values[current] == null ? location() : values[current].end(); + } + + @Override + public void atEnd() { + int current = currentChild(); + if (values[current] == null) { + onFail(); + } + values[current].atEnd(); + if (values[current].isPopulated()) { + onPopulated(values[current].end()); + } else { + onFail(); + } } @Override public String toString() { - return String.format("%-50.50s || %s (%d/%d) (position: %d)", tail(50), tag(), nextVariant, variants.length, position()); + ParserLocation location = location(); + return String.format( + "%50.50s || %s (%d/%d) (%d:%d -- %d - %d)", + head(50), + tag(), + nextVariant, + variants.length, + location.line(), + location.column(), + location.position(), + end().position() + ); } @Override - public int alternativesLeft() { - return variants.length - nextVariant + (token == null ? 0 : token.alternativesLeft()); + public boolean alternativesLeft() { + if (isFailed() || variants.length == 0) { + log("failed -- no alternatives"); + return false; + } + if (nextVariant < variants.length) { + log("some untested variants left -- counting as alternatives"); + return true; + } + for (int i = currentChild(); i > -1; i--) { + if (values[i] != null) { + if (values[i].alternativesLeft()) { + log("found alternatives at value#{}: {}", i, values[i]); + return true; + } + } else { + log("value#{} is null -- counting as an alternative", i); + return true; + } + } + log("-- no alternatives left in any of {} variants", variants.length); + return false; } @Override public void sortPriorities() { - token.sortPriorities(); + int current = currentChild(); + if (values[current] != null) { + values[currentChild()].sortPriorities(); + } } @Override @@ -210,7 +385,11 @@ public boolean propagatePriority() { if (tokenType.isAnnotationPresent(AdjustPriority.class)) { return tokenType.getAnnotation(AdjustPriority.class).propagate(); } - return token.propagatePriority(); + int current = currentChild(); + if (values[current] != null) { + return values[current].propagatePriority(); + } + return false; } @Override @@ -219,34 +398,25 @@ public int basePriority() { if (tokenType.isAnnotationPresent(AdjustPriority.class)) { result += tokenType.getAnnotation(AdjustPriority.class).value(); } - if (token.propagatePriority()) { - result += token.basePriority(); + int current = currentChild(); + if (values[current].propagatePriority()) { + result += values[current].basePriority(); } return result; } - @Override - public StringBuilder source() { - StringBuilder result = new StringBuilder(); - if (token != null) { - result.append(token.source()); - } - - return result; - } - public Optional> resolvedAs() { - return Optional.ofNullable(token); + return Optional.ofNullable(values[currentChild()]); } - + @Override public int unfilledChildren() { - return (token != null && token.isPopulated()) || variants.length == 0 ? 0 : 1; + return variants.length - currentChild(); } @Override public int currentChild() { - return nextVariant - 1; + return nextVariant == 0 ? 0 : nextVariant - 1; } @Override @@ -260,5 +430,36 @@ private static void updateDynPriority(Class target, int change) { } dynPriorities.put(target, dynPriorities.get(target) + change); } + + @Override + public CharSequence dumpTree(int offset, CharSequence prefix, CharSequence childPrefix, Function, CharSequence> formatter) { + final int childOffset = offset + 1; + String insideFormat = "%s ├─%s %s: %s"; + String lastFormat = "%s └─%s %s: %s"; + StringBuilder result = new StringBuilder(super.dumpTree(offset, prefix, childPrefix, formatter)); + for (int i = 0; i <= nextVariant; i++) { + if (i < variants.length) { + boolean last = i == variants.length - 1 || i == nextVariant || (values[i+1] == null && i == nextVariant - 1); + String format = last ? lastFormat : insideFormat; + PartialToken child = values[i]; + String variantName = variants[i].getSimpleName(); + if (child == null && !isPopulated()) { + if (i < nextVariant) { + result.append(String.format(format, childPrefix, "", variantName, null)); + result.append('\n'); + } else { + continue; + } + } else if (child != null && child.isFailed() && !isPopulated()) { + result.append(child.dumpTree(childOffset, String.format(format, childPrefix, "[F]", variantName, ""), childPrefix + (last ? " " : " │"), formatter)); + } else if (child != null && child.isPopulated()) { + result.append(child.dumpTree(childOffset, String.format(format, childPrefix, "[+]", variantName, ""), childPrefix + (last ? " " : " │"), formatter)); + } else if (child != null) { + result.append(child.dumpTree(childOffset, String.format(format, childPrefix, ">>>", variantName, ""), childPrefix + (last ? " " : " │"), formatter)); + } + } + } + return result; + } } diff --git a/src/main/java/com/onkiup/linker/parser/util/LoggerLayout.java b/src/main/java/com/onkiup/linker/parser/util/LoggerLayout.java index 0bc1b18..7193dad 100644 --- a/src/main/java/com/onkiup/linker/parser/util/LoggerLayout.java +++ b/src/main/java/com/onkiup/linker/parser/util/LoggerLayout.java @@ -1,5 +1,9 @@ package com.onkiup.linker.parser.util; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + import org.apache.log4j.Layout; import org.apache.log4j.spi.LoggingEvent; @@ -7,17 +11,33 @@ public class LoggerLayout extends Layout { private Layout parent; - private StringBuilder buffer; + private CharSequence buffer; + private Supplier position; - public LoggerLayout(Layout parent, StringBuilder buffer) { + public LoggerLayout(Layout parent, CharSequence buffer, Supplier position) { this.parent = parent; this.buffer = buffer; + this.position = position; + } + + public static CharSequence repeat(CharSequence s, int times) { + if (times < 1) { + return ""; + } + return IntStream.of(times).boxed() + .map(i -> s) + .collect(Collectors.joining()); } @Override public String format(LoggingEvent event) { - CharSequence bufVal = sanitize(buffer.toString()); - return String.format("%s || %s :: %s\n", ralign(bufVal, 50), ralign(event.getLoggerName(), 50), event.getMessage()); + int position = this.position.get(); + CharSequence bufVal = buffer; + if (position < buffer.length()) { + bufVal = buffer.subSequence(Math.max(0, position - 50), position); + } + bufVal = String.format("'%s'", ralign(sanitize(bufVal), 48)); + return String.format("%50.50s || %s :: %s\n", bufVal, ralign(event.getLoggerName(), 50), event.getMessage()); } @Override @@ -41,6 +61,13 @@ public static String sanitize(String what) { return what == null ? null : what.replaceAll("\n", "\\\\n").replaceAll("\t", "\\\\t"); } + public static CharSequence head(CharSequence what, int len) { + if (what.length() < len) { + return what; + } + return what.subSequence(0, len); + } + public static String ralign(CharSequence what, int len) { if (what.length() >= len) { what = what.subSequence(what.length() - len, what.length()); diff --git a/src/main/java/com/onkiup/linker/parser/util/SelfPopulatingBuffer.java b/src/main/java/com/onkiup/linker/parser/util/SelfPopulatingBuffer.java new file mode 100644 index 0000000..d2cd10b --- /dev/null +++ b/src/main/java/com/onkiup/linker/parser/util/SelfPopulatingBuffer.java @@ -0,0 +1,41 @@ +package com.onkiup.linker.parser.util; + +import java.io.IOException; +import java.io.Reader; + +public class SelfPopulatingBuffer implements CharSequence { + + private final StringBuilder buffer = new StringBuilder(); + private final String name; + + public SelfPopulatingBuffer(String name, Reader reader) throws IOException { + this.name = name; + for (int character = reader.read(); character > -1; character = reader.read()) { + buffer.append((char)character); + } + } + + public String name() { + return name; + } + + @Override + public int length() { + return buffer.length(); + } + + @Override + public char charAt(int index) { + return buffer.charAt(index); + } + + @Override + public CharSequence subSequence(int start, int end) { + return buffer.subSequence(start, end); + } + + @Override + public String toString() { + return buffer.toString(); + } +} diff --git a/src/main/java/com/onkiup/linker/parser/util/TextUtils.java b/src/main/java/com/onkiup/linker/parser/util/TextUtils.java new file mode 100644 index 0000000..a2b5aff --- /dev/null +++ b/src/main/java/com/onkiup/linker/parser/util/TextUtils.java @@ -0,0 +1,28 @@ +package com.onkiup.linker.parser.util; + +import com.onkiup.linker.parser.token.PartialToken; + +public interface TextUtils { + static CharSequence removeIgnoredCharacters(PartialToken token, CharSequence from) { + String ignoredCharacters = token.ignoredCharacters(); + token.log("Removing ignored characters '{}' from '{}'", LoggerLayout.sanitize(ignoredCharacters), LoggerLayout.sanitize(from)); + if (ignoredCharacters.length() == 0) { + return from; + } + for (int i = 0; i < from.length(); i++) { + if (ignoredCharacters.indexOf(from.charAt(i)) < 0) { + return from.subSequence(i, from.length()); + } + } + return ""; + } + + static int firstNonIgnoredCharacter(PartialToken token, CharSequence buffer, int from) { + String ignoredCharacters = token.ignoredCharacters(); + char character; + do { + character = buffer.charAt(from++); + } while (ignoredCharacters.indexOf(character) > -1); + return from - 1; + } +}