Latest compliance

composer.json

@@ -13,7 +13,8 @@
   ],
 
   "require": {
-    "php": ">=5.4.0"
+    "php": ">=5.4.0",
+    "symfony/polyfill-mbstring": "^1.4"
   },
 
   "require-dev": {

src/FnDispatcher.php

@@ -62,7 +62,7 @@ private function fn_contains(array $args)
         if (is_array($args[0])) {
             return in_array($args[1], $args[0]);
         } elseif (is_string($args[1])) {
-            return strpos($args[0], $args[1]) !== false;
+            return mb_strpos($args[0], $args[1], 0, 'UTF-8') !== false;
         } else {
             return null;
         }
@@ -72,7 +72,7 @@ private function fn_ends_with(array $args)
     {
         $this->validate('ends_with', $args, [['string'], ['string']]);
         list($search, $suffix) = $args;
-        return $suffix === '' || substr($search, -strlen($suffix)) === $suffix;
+        return $suffix === '' || mb_substr($search, -mb_strlen($suffix, 'UTF-8'), null, 'UTF-8') === $suffix;
     }
 
     private function fn_floor(array $args)
@@ -112,7 +112,7 @@ private function fn_keys(array $args)
     private function fn_length(array $args)
     {
         $this->validate('length', $args, [['string', 'array', 'object']]);
-        return is_string($args[0]) ? strlen($args[0]) : count((array) $args[0]);
+        return is_string($args[0]) ? mb_strlen($args[0], 'UTF-8') : count((array) $args[0]);
     }
 
     private function fn_max(array $args)
@@ -207,7 +207,7 @@ private function fn_starts_with(array $args)
     {
         $this->validate('starts_with', $args, [['string'], ['string']]);
         list($search, $prefix) = $args;
-        return $prefix === '' || strpos($search, $prefix) === 0;
+        return $prefix === '' || mb_strpos($search, $prefix, 0, 'UTF-8') === 0;
     }
 
     private function fn_type(array $args)
@@ -240,7 +240,7 @@ private function fn_to_number(array $args)
         if ($type == 'number') {
             return $value;
         } elseif ($type == 'string' && is_numeric($value)) {
-            return strpos($value, '.') ? (float) $value : (int) $value;
+            return mb_strpos($value, '.', 0, 'UTF-8') ? (float) $value : (int) $value;
         } else {
             return null;
         }
@@ -282,7 +282,7 @@ private function fn_map(array $args)
 
     private function typeError($from, $msg)
     {
-        if (strpos($from, ':')) {
+        if (mb_strpos($from, ':', 0, 'UTF-8')) {
             list($fn, $pos) = explode(':', $from);
             throw new \RuntimeException(
                 sprintf('Argument %d of %s %s', $pos, $fn, $msg)

src/Lexer.php

@@ -340,7 +340,7 @@ public function tokenize($input)
         eof:
         $tokens[] = [
             'type'  => self::T_EOF,
-            'pos'   => strlen($input),
+            'pos'   => mb_strlen($input, 'UTF-8'),
             'value' => null
         ];
 

src/Utils.php

@@ -214,7 +214,7 @@ private static function adjustSlice($length, $start, $stop, $step)
     private static function sliceIndices($subject, $start, $stop, $step)
     {
         $type = gettype($subject);
-        $len = $type == 'string' ? strlen($subject) : count($subject);
+        $len = $type == 'string' ? mb_strlen($subject, 'UTF-8') : count($subject);
         list($start, $stop, $step) = self::adjustSlice($len, $start, $stop, $step);
 
         $result = [];

tests/ComplianceTest.php

@@ -61,6 +61,7 @@ public function testPassesCompliance(
 
         $file = __DIR__ . '/compliance/' . $file . '.json';
         $failure .= "\n{$compiledStr}php bin/jp.php --file {$file} --suite {$suite} --case {$case}\n\n"
+            . "Result: " . $this->prettyJson($evalResult) . "\n\n"
             . "Expected: " . $this->prettyJson($result) . "\n\n";
         $failure .= 'Associative? ' . var_export($asAssoc, true) . "\n\n";
 

tests/compliance/basic.json

@@ -18,6 +18,22 @@
             "expression": "foo\n.\nbar\n.baz",
             "result": "correct"
          },
+         {
+            "expression": "foo . bar . baz",
+            "result": "correct"
+         },
+         {
+            "expression": "foo\t.\tbar\t.\tbaz",
+            "result": "correct"
+         },
+         {
+            "expression": "foo\r.\rbar\r.\rbaz",
+            "result": "correct"
+         },
+         {
+            "expression": "foo\r\n.\r\nbar\r\n.\r\nbaz",
+            "result": "correct"
+         },
          {
             "expression": "foo.bar.baz.bad",
             "result": null

tests/compliance/boolean.json

@@ -205,7 +205,9 @@
     "given": {
       "one": 1,
       "two": 2,
-      "three": 3
+      "three": 3,
+      "emptylist": [],
+      "boolvalue": false
     },
     "cases": [
       {
@@ -236,6 +238,22 @@
         "expression": "one != two",
         "result": true
       },
+      {
+        "expression": "emptylist < one",
+        "result": null
+      },
+      {
+        "expression": "emptylist < nullvalue",
+        "result": null
+      },
+      {
+        "expression": "emptylist < boolvalue",
+        "result": null
+      },
+      {
+        "expression": "one < boolvalue",
+        "result": null
+      },
       {
         "expression": "one < two && three > one",
         "result": true

tests/compliance/functions.json

@@ -83,6 +83,10 @@
       "expression": "avg(strings)",
       "error": "invalid-type"
     },
+    {
+      "expression": "avg(empty_list)",
+      "result": null
+    },
     {
       "expression": "ceil(`1.2`)",
       "result": 2
@@ -175,6 +179,10 @@
       "expression": "length('abc')",
       "result": 3
     },
+    {
+      "expression": "length('✓foo')",
+      "result": 4
+    },
     {
       "expression": "length('')",
       "result": 0
@@ -568,12 +576,12 @@
       "error": "invalid-arity"
     },
     {
-      "description": "function projection on single arg function",
+      "comment": "function projection on single arg function",
       "expression": "numbers[].to_string(@)",
       "result": ["-1", "3", "4", "5"]
     },
     {
-      "description": "function projection on single arg function",
+      "comment": "function projection on single arg function",
       "expression": "array[].to_number(@)",
       "result": [-1, 3, 4, 5, 100]
     }
@@ -591,7 +599,7 @@
   },
   "cases": [
     {
-      "description": "function projection on variadic function",
+      "comment": "function projection on variadic function",
       "expression": "foo[].not_null(f, e, d, c, b, a)",
       "result": ["b", "c", "d", "e", "f"]
     }
@@ -609,7 +617,7 @@
   },
   "cases": [
     {
-      "description": "sort by field expression",
+      "comment": "sort by field expression",
       "expression": "sort_by(people, &age)",
       "result": [
          {"age": 10, "age_str": "10", "bool": true, "name": 3},
@@ -630,7 +638,7 @@
       ]
     },
     {
-      "description": "sort by function expression",
+      "comment": "sort by function expression",
       "expression": "sort_by(people, &to_number(age_str))",
       "result": [
          {"age": 10, "age_str": "10", "bool": true, "name": 3},
@@ -641,7 +649,7 @@
       ]
     },
     {
-      "description": "function projection on sort_by function",
+      "comment": "function projection on sort_by function",
       "expression": "sort_by(people, &age)[].name",
       "result": [3, "a", "c", "b", "d"]
     },
@@ -689,6 +697,10 @@
       "expression": "max_by(people, &to_number(age_str))",
       "result": {"age": 50, "age_str": "50", "bool": false, "name": "d"}
     },
+    {
+      "expression": "max_by(`[]`, &age)",
+      "result": null
+    },
     {
       "expression": "min_by(people, &age)",
       "result": {"age": 10, "age_str": "10", "bool": true, "name": 3}
@@ -708,6 +720,10 @@
     {
       "expression": "min_by(people, &to_number(age_str))",
       "result": {"age": 10, "age_str": "10", "bool": true, "name": 3}
+    },
+    {
+      "expression": "min_by(`[]`, &age)",
+      "result": null
     }
   ]
 }, {
@@ -729,7 +745,7 @@
   },
   "cases": [
     {
-      "description": "stable sort order",
+      "comment": "stable sort order",
       "expression": "sort_by(people, &age)",
       "result": [
          {"age": 10, "order": "1"},

tests/compliance/literal.json

@@ -184,6 +184,16 @@
           "comment": "Can escape the single quote",
           "expression": "'foo\\'bar'",
           "result": "foo'bar"
+        },
+        {
+          "comment": "Backslash not followed by single quote is treated as any other character",
+          "expression": "'\\z'",
+          "result": "\\z"
+        },
+        {
+          "comment": "Backslash not followed by single quote is treated as any other character",
+          "expression": "'\\\\'",
+          "result": "\\\\"
         }
       ]
     }

tests/compliance/multiselect.json

@@ -387,6 +387,11 @@
           "comment": "Nested multiselect",
           "expression": "[[*]]",
           "result": [[]]
+        },
+        {
+          "comment": "Select on null",
+          "expression": "missing.{foo: bar}",
+          "result": null
         }
     ]
 }