remove consumeEOF

Author: Marcisbee

Diff for: README.md

@@ -11,7 +11,7 @@ npm i nanolex
 ```
 
 ```ts
-import { createToken, nanolex, getComposedTokens } from "nanolex";
+import { EOF, createToken, nanolex, getComposedTokens } from "nanolex";
 
 // Define tokens
 const Whitespace = createToken(/[ \t\n\r]+/, "WhiteSpace");
@@ -35,7 +35,6 @@ export function parser(value: string) {
   // Initiate grammar
   const {
     consume,
-    consumeEOF,
     zeroOrOne,
     zeroOrMany,
     zeroOrManySep,
@@ -83,7 +82,7 @@ export function parser(value: string) {
   }
 
   // Run the grammar
-  const [output] = throwIfError(and([FUNCTION, consumeEOF()]));
+  const [output] = throwIfError(and([FUNCTION, consume(EOF)]));
 
   return output;
 }

Diff for: deno.json

@@ -1,5 +1,5 @@
 {
   "name": "@marcisbee/nanolex",
-  "version": "0.3.1",
+  "version": "0.4.0",
   "exports": "./src/nanolex.ts"
 }

Diff for: package.json

@@ -1,6 +1,6 @@
 {
   "name": "nanolex",
-  "version": "0.3.1",
+  "version": "0.4.0",
   "description": "Parser grammar builder",
   "main": "./dist/nanolex.js",
   "module": "./dist/nanolex.mjs",

Diff for: src/nanolex.ts

@@ -4,6 +4,8 @@ type ErrorToken = {
 	i: number;
 };
 
+export const EOF = Symbol("EOF") as any as TokenLike;
+
 export function createToken(
 	token: RegExp | string,
 	name: string = typeof token === "string" ? token : token.source,
@@ -59,7 +61,6 @@ export function nanolex(
 	return {
 		consume,
 		consumeUntil,
-		consumeEOF,
 		peek,
 		oneOrMany: (rule: GrammarLike, transformer?: (value: any) => any) =>
 			many(rule, 1, transformer),
@@ -307,7 +308,7 @@ export function nanolex(
 					continue;
 				}
 
-				if (!token.test(c)) {
+				if (token === EOF || !token.test(c)) {
 					i += 1;
 
 					output.push(c);
@@ -317,6 +318,10 @@ export function nanolex(
 				return output;
 			}
 
+			if (EOF) {
+				return output;
+			}
+
 			saveError({
 				got: c,
 				token,
@@ -339,7 +344,7 @@ export function nanolex(
 					continue;
 				}
 
-				if (token.test(c)) {
+				if (token !== EOF && token.test(c)) {
 					i += 1;
 
 					if (transform) {
@@ -358,6 +363,10 @@ export function nanolex(
 				break;
 			}
 
+			if (token === EOF && i >= chunksLength) {
+				return;
+			}
+
 			if (c == null) {
 				saveError({
 					got: c,
@@ -375,31 +384,6 @@ export function nanolex(
 			return;
 		};
 	}
-	function consumeEOF(): GrammarLike<undefined> {
-		return (): any => {
-			// consumeTimes += 1;
-			let c: string;
-
-			while (((c = chunks[i]), i < chunksLength)) {
-				if (!c) {
-					i += 1;
-					continue;
-				}
-
-				const currI = i;
-				if (!skipCheck && tokensSkip() !== undefined) {
-					continue;
-				}
-				i = currI;
-
-				saveError({
-					got: c,
-					i,
-				});
-				return;
-			}
-		};
-	}
 	function throwIfError<T extends GrammarLike>(rule: T): any {
 		const output = rule();
 

Diff for: tests/css.ts

@@ -1,4 +1,4 @@
-import { createToken, nanolex, getComposedTokens } from "../src/nanolex.ts";
+import { EOF, createToken, nanolex, getComposedTokens } from "../src/nanolex.ts";
 
 const LineBreak = createToken(/[\n\r]/, "LineBreak");
 const Whitespace = createToken(/[ \t]+/, "Whitespace");
@@ -47,7 +47,6 @@ const tokens = getComposedTokens([
 export function parser(value: string) {
 	const {
 		consume,
-		consumeEOF,
 		zeroOrOne,
 		zeroOrMany,
 		zeroOrManySep,
@@ -154,7 +153,7 @@ export function parser(value: string) {
 		return zeroOrMany(RULESET)();
 	}
 
-	const [output] = throwIfError(and([PROGRAM, consumeEOF()]));
+	const [output] = throwIfError(and([PROGRAM, consume(EOF)]));
 
 	return output;
 }

Diff for: tests/jsexpression.ts

@@ -1,4 +1,4 @@
-import { createToken, nanolex, getComposedTokens } from "../src/nanolex.ts";
+import { EOF, createToken, nanolex, getComposedTokens } from "../src/nanolex.ts";
 
 const Whitespace = createToken(/[ \t\n\r]+/, "WhiteSpace");
 const OperatorPlus = createToken("+");
@@ -9,7 +9,6 @@ const tokens = getComposedTokens([Whitespace, OperatorPlus, Integer]);
 export function parser(value: string) {
 	const {
 		consume,
-		consumeEOF,
 		zeroOrOne,
 		zeroOrMany,
 		zeroOrManySep,
@@ -59,7 +58,7 @@ export function parser(value: string) {
 		return or([BinaryExpression, Literal])();
 	}
 
-	const [output] = throwIfError(and([Expression, consumeEOF()]));
+	const [output] = throwIfError(and([Expression, consume(EOF)]));
 
 	return output;
 }

Diff for: tests/json.ts

@@ -1,4 +1,4 @@
-import { createToken, nanolex, getComposedTokens } from "../src/nanolex.ts";
+import { EOF, createToken, nanolex, getComposedTokens } from "../src/nanolex.ts";
 
 const Whitespace = createToken(/[ \t\n\r]+/, "WhiteSpace");
 const True = createToken("true");
@@ -37,7 +37,6 @@ const tokens = getComposedTokens([
 export function parser(value: string) {
 	const {
 		consume,
-		consumeEOF,
 		zeroOrOne,
 		zeroOrMany,
 		zeroOrManySep,
@@ -112,7 +111,7 @@ export function parser(value: string) {
 		))();
 	}
 
-	const [output] = throwIfError(and([Json, consumeEOF()])) as any;
+	const [output] = throwIfError(and([Json, consume(EOF)])) as any;
 
 	return output;
 }