Current iteration

core/src/main/java/io/parsingdata/metal/Shorthand.java

@@ -44,6 +44,7 @@
 import io.parsingdata.metal.expression.value.Cat;
 import io.parsingdata.metal.expression.value.Const;
 import io.parsingdata.metal.expression.value.ConstantFactory;
+import io.parsingdata.metal.expression.value.reference.CurrentIteration;
 import io.parsingdata.metal.expression.value.Elvis;
 import io.parsingdata.metal.expression.value.Expand;
 import io.parsingdata.metal.expression.value.FoldLeft;
@@ -89,6 +90,7 @@ public final class Shorthand {
     public static final Token EMPTY = def(EMPTY_NAME, 0L);
     public static final ValueExpression SELF = new Self();
     public static final ValueExpression CURRENT_OFFSET = new CurrentOffset();
+    public static final ValueExpression CURRENT_ITERATION = new CurrentIteration();
     public static final Expression TRUE = new True();
 
     private Shorthand() {}

core/src/main/java/io/parsingdata/metal/data/ParseState.java

@@ -16,6 +16,7 @@
 
 package io.parsingdata.metal.data;
 
+import static java.math.BigInteger.ONE;
 import static java.math.BigInteger.ZERO;
 
 import static io.parsingdata.metal.Util.checkNotNegative;
@@ -36,43 +37,55 @@ public class ParseState {
     public final ParseGraph order;
     public final BigInteger offset;
     public final Source source;
+    public final ImmutableList<BigInteger> iterations;
 
-    public ParseState(final ParseGraph order, final Source source, final BigInteger offset) {
+    public ParseState(final ParseGraph order, final Source source, final BigInteger offset, final ImmutableList<BigInteger> iterations) {
         this.order = checkNotNull(order, "order");
         this.source = checkNotNull(source, "source");
         this.offset = checkNotNegative(offset, "offset");
+        this.iterations = checkNotNull(iterations, "iteration");
     }
 
     public static ParseState createFromByteStream(final ByteStream input, final BigInteger offset) {
-        return new ParseState(ParseGraph.EMPTY, new ByteStreamSource(input), offset);
+        return new ParseState(ParseGraph.EMPTY, new ByteStreamSource(input), offset, new ImmutableList<>());
     }
 
     public static ParseState createFromByteStream(final ByteStream input) {
         return createFromByteStream(input, ZERO);
     }
 
     public ParseState addBranch(final Token token) {
-        return new ParseState(order.addBranch(token), source, offset);
+        if (token.isIterable()) {
+            return new ParseState(order.addBranch(token), source, offset, iterations.add(ZERO));
+        }
+        return new ParseState(order.addBranch(token), source, offset, iterations);
     }
 
-    public ParseState closeBranch() {
-        return new ParseState(order.closeBranch(), source, offset);
+    public ParseState closeBranch(final Token token) {
+        if (token.isIterable()) {
+            return new ParseState(order.closeBranch(), source, offset, iterations.tail);
+        }
+        return new ParseState(order.closeBranch(), source, offset, iterations);
     }
 
     public ParseState add(final ParseValue parseValue) {
-        return new ParseState(order.add(parseValue), source, offset);
+        return new ParseState(order.add(parseValue), source, offset, iterations);
     }
 
     public ParseState add(final ParseReference parseReference) {
-        return new ParseState(order.add(parseReference), source, offset);
+        return new ParseState(order.add(parseReference), source, offset, iterations);
+    }
+
+    public ParseState iter() {
+        return new ParseState(order, source, offset, iterations.tail.add(iterations.head.add(ONE)));
     }
 
     public Optional<ParseState> seek(final BigInteger newOffset) {
-        return newOffset.compareTo(ZERO) >= 0 ? Optional.of(new ParseState(order, source, newOffset)) : Optional.empty();
+        return newOffset.compareTo(ZERO) >= 0 ? Optional.of(new ParseState(order, source, newOffset, iterations)) : Optional.empty();
     }
 
     public ParseState source(final ValueExpression dataExpression, final int index, final ParseState parseState, final Encoding encoding) {
-        return new ParseState(order, new DataExpressionSource(dataExpression, index, parseState, encoding), ZERO);
+        return new ParseState(order, new DataExpressionSource(dataExpression, index, parseState, encoding), ZERO, iterations);
     }
 
     public Optional<Slice> slice(final BigInteger length) {
@@ -81,20 +94,22 @@ public Optional<Slice> slice(final BigInteger length) {
 
     @Override
     public String toString() {
-        return getClass().getSimpleName() + "(source:" + source + ";offset:" + offset + ";order:" + order + ")";
+        final String iterationString = iterations.isEmpty() ? "" : ";iterations:" + iterations.toString();
+        return getClass().getSimpleName() + "(source:" + source + ";offset:" + offset + ";order:" + order + iterationString + ")";
     }
 
     @Override
     public boolean equals(final Object obj) {
         return Util.notNullAndSameClass(this, obj)
             && Objects.equals(order, ((ParseState)obj).order)
             && Objects.equals(offset, ((ParseState)obj).offset)
-            && Objects.equals(source, ((ParseState)obj).source);
+            && Objects.equals(source, ((ParseState)obj).source)
+            && Objects.equals(iterations, ((ParseState)obj).iterations);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(getClass(), order, offset, source);
+        return Objects.hash(getClass(), order, offset, source, iterations);
     }
 
 }

core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java

@@ -0,0 +1,62 @@
+/*
+ * Copyright 2013-2018 Netherlands Forensic Institute
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.parsingdata.metal.expression.value.reference;
+
+import static io.parsingdata.metal.expression.value.ConstantFactory.createFromNumeric;
+
+import java.util.Optional;
+
+import io.parsingdata.metal.Util;
+import io.parsingdata.metal.data.ImmutableList;
+import io.parsingdata.metal.data.ParseState;
+import io.parsingdata.metal.encoding.Encoding;
+import io.parsingdata.metal.expression.value.Value;
+import io.parsingdata.metal.expression.value.ValueExpression;
+import io.parsingdata.metal.token.Rep;
+import io.parsingdata.metal.token.RepN;
+import io.parsingdata.metal.token.Token;
+
+/**
+ * A {@link ValueExpression} that represents the 0-based current iteration in an
+ * iterable {@link Token} (e.g. when inside a {@link Rep} or {@link RepN}).
+ */
+public class CurrentIteration implements ValueExpression {
+
+    @Override
+    public ImmutableList<Optional<Value>> eval(final ParseState parseState, final Encoding encoding) {
+        if (parseState.iterations.head == null) {
+            return ImmutableList.create(Optional.empty());
+        }
+        return ImmutableList.create(Optional.of(createFromNumeric(parseState.iterations.head, new Encoding())));
+    }
+
+    @Override
+    public String toString() {
+        return getClass().getSimpleName();
+    }
+
+    @Override
+    public boolean equals(final Object obj) {
+        return Util.notNullAndSameClass(this, obj);
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode();
+    }
+
+}

core/src/main/java/io/parsingdata/metal/token/Cho.java

@@ -62,7 +62,7 @@ private Trampoline<Optional<ParseState>> iterate(final Environment environment,
         }
         return list.head
             .parse(environment)
-            .map(result -> complete(() -> success(result.closeBranch())))
+            .map(result -> complete(() -> success(result.closeBranch(this))))
             .orElseGet(() -> intermediate(() -> iterate(environment, list.tail)));
     }
 

core/src/main/java/io/parsingdata/metal/token/IterableToken.java

@@ -0,0 +1,79 @@
+/*
+ * Copyright 2013-2018 Netherlands Forensic Institute
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.parsingdata.metal.token;
+
+import io.parsingdata.metal.Trampoline;
+import io.parsingdata.metal.data.Environment;
+import io.parsingdata.metal.data.ParseState;
+import io.parsingdata.metal.encoding.Encoding;
+
+import java.util.Objects;
+import java.util.Optional;
+import java.util.function.Function;
+
+import static io.parsingdata.metal.Trampoline.complete;
+import static io.parsingdata.metal.Trampoline.intermediate;
+import static io.parsingdata.metal.Util.checkNotNull;
+import static io.parsingdata.metal.Util.success;
+
+public abstract class IterableToken extends Token {
+
+    public final Token token;
+
+    IterableToken(String name, final Token token, Encoding encoding) {
+        super(name, encoding);
+        this.token = checkNotNull(token, "token");
+    }
+
+    protected final Optional<ParseState> parse(final Environment environment, final Function<Environment, Boolean> stopCondition, final Function<Environment, Optional<ParseState>> ifIterationFails) {
+        return iterate(environment.addBranch(this), stopCondition, ifIterationFails).computeResult();
+    }
+
+    /**
+     * Iteratively parse iterations of the token, given a stop condition and the logic how to handle a failed parse.
+     *
+     * @param environment the environment to apply the parse to
+     * @param stopCondition a function to determine when to stop the iteration
+     * @param ifIterationFails a function to determine how to handle a failed parse
+     * @return a trampolined {@code Optional<ParseState>}
+     */
+    private Trampoline<Optional<ParseState>> iterate(final Environment environment, final Function<Environment, Boolean> stopCondition, final Function<Environment, Optional<ParseState>> ifIterationFails) {
+        if (stopCondition.apply(environment)) {
+            return complete(() -> success(environment.parseState.closeBranch(this)));
+        }
+        return token
+                .parse(environment)
+                .map(nextParseState -> intermediate(() -> iterate(environment.withParseState(nextParseState.iter()), stopCondition, ifIterationFails)))
+                .orElseGet(() -> complete(() -> ifIterationFails.apply(environment)));
+    }
+
+    @Override
+    public boolean isIterable() {
+        return true;
+    }
+
+    @Override
+    public boolean equals(final Object obj) {
+        return super.equals(obj)
+                && Objects.equals(token, ((IterableToken)obj).token);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), token);
+    }
+
+}

core/src/main/java/io/parsingdata/metal/token/Post.java

@@ -55,7 +55,7 @@ public Post(final String name, final Token token, final Expression predicate, fi
     protected Optional<ParseState> parseImpl(final Environment environment) {
         return token
             .parse(environment.addBranch(this))
-            .map(nextParseState -> predicate.eval(nextParseState, environment.encoding) ? success(nextParseState.closeBranch()) : failure())
+            .map(nextParseState -> predicate.eval(nextParseState, environment.encoding) ? success(nextParseState.closeBranch(this)) : failure())
             .orElseGet(Util::failure);
     }
 

core/src/main/java/io/parsingdata/metal/token/Pre.java

@@ -59,7 +59,7 @@ protected Optional<ParseState> parseImpl(final Environment environment) {
         }
         return token
             .parse(environment.addBranch(this))
-            .map(resultParseState -> success(resultParseState.closeBranch()))
+            .map(resultParseState -> success(resultParseState.closeBranch(this)))
             .orElseGet(Util::failure);
     }
 

core/src/main/java/io/parsingdata/metal/token/Rep.java

@@ -16,15 +16,10 @@
 
 package io.parsingdata.metal.token;
 
-import static io.parsingdata.metal.Trampoline.complete;
-import static io.parsingdata.metal.Trampoline.intermediate;
-import static io.parsingdata.metal.Util.checkNotNull;
 import static io.parsingdata.metal.Util.success;
 
-import java.util.Objects;
 import java.util.Optional;
 
-import io.parsingdata.metal.Trampoline;
 import io.parsingdata.metal.data.Environment;
 import io.parsingdata.metal.data.ParseState;
 import io.parsingdata.metal.encoding.Encoding;
@@ -38,41 +33,20 @@
  *
  * @see RepN
  */
-public class Rep extends Token {
-
-    public final Token token;
+public class Rep extends IterableToken {
 
     public Rep(final String name, final Token token, final Encoding encoding) {
-        super(name, encoding);
-        this.token = checkNotNull(token, "token");
+        super(name, token, encoding);
     }
 
     @Override
-    protected Optional<ParseState> parseImpl(final Environment environment) {
-        return iterate(environment.addBranch(this)).computeResult();
-    }
-
-    private Trampoline<Optional<ParseState>> iterate(final Environment environment) {
-        return token
-            .parse(environment)
-            .map(nextParseState -> intermediate(() -> iterate(environment.withParseState(nextParseState))))
-            .orElseGet(() -> complete(() -> success(environment.parseState.closeBranch())));
+    protected Optional<ParseState> parseImpl(Environment environment) {
+        return parse(environment, env -> false, env -> success(env.parseState.closeBranch(this)));
     }
 
     @Override
     public String toString() {
         return getClass().getSimpleName() + "(" + makeNameFragment() + token + ")";
     }
 
-    @Override
-    public boolean equals(final Object obj) {
-        return super.equals(obj)
-            && Objects.equals(token, ((Rep)obj).token);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(super.hashCode(), token);
-    }
-
 }